Add transcription accuracy and latency tests

TODO.md

@@ -32,6 +32,13 @@ is explicitly binary-safe, broken worker pipes no longer terminate Lumi, the
 worker restarts and reloads the selected model automatically, and the Companion
 test reports the server failure at Speech recognition instead of hanging.
 
+Experimental.5 retries and acknowledges the saved OBS source attachment after
+reconnects, waits for finalized path-test captions, and adds a separate
+user-controlled accuracy/latency test with a ten-second silence cutoff. The host
+retains per-word latency/confidence inspection data for one hour, while paired
+device management now defaults to active credentials and purges revoked history
+after 30 days.
+
 Release-blocking work remains:
 
 - Package the native whisper.cpp worker as the Windows CUDA runtime, then benchmark

companion/native/obs-bridge/src/plugin.cpp

@@ -13,6 +13,7 @@
 #include <cstdint>
 #include <cstring>
 #include <mutex>
+#include <optional>
 #include <set>
 #include <string>
 #include <thread>
@@ -334,7 +335,7 @@ static bool output_caption(const std::string &text, double display_seconds)
     return true;
 }
 
-static void handle_command(const json &message)
+static std::optional<json> handle_command(const json &message)
 {
     const auto type = message.value("type", "");
     if (type == "select_sources") {
@@ -343,12 +344,14 @@ static void handle_command(const json &message)
         source_list_dirty.store(true, std::memory_order_release);
         blog(installed ? LOG_INFO : LOG_WARNING, "[Lumi Companion] %s selected OBS audio source %s",
              installed ? "Attached" : "Could not attach", uuid.c_str());
+        return json{{"type", "selection_state"}, {"protocol_version", protocol_version}, {"source_uuid", uuid}, {"attached", installed}};
     } else if (type == "caption" && message.contains("payload")) {
         const auto &payload = message["payload"];
         const auto text = payload.value("stable_text", "");
         const auto duration = payload.value("display_seconds", 2.0);
         output_caption(text, duration);
     }
+    return std::nullopt;
 }
 
 static bool read_available_command(HANDLE pipe)
@@ -363,7 +366,10 @@ static bool read_available_command(HANDLE pipe)
     if (size == 0 || size > max_json_bytes) return false;
     std::vector<uint8_t> body(size);
     if (!read_exact(pipe, body.data(), body.size())) return false;
-    try { handle_command(json::parse(body.begin(), body.end())); }
+    try {
+        const auto response = handle_command(json::parse(body.begin(), body.end()));
+        if (response && !write_json(pipe, *response)) return false;
+    }
     catch (const std::exception &error) { blog(LOG_WARNING, "[Lumi Companion] Ignored invalid IPC command: %s", error.what()); }
     return true;
 }

companion/plugins/Lumi.Companion.Transcription/ObsBridgePipe.cs

@@ -7,7 +7,7 @@ namespace Lumi.Companion.Transcription;
 
 public sealed class ObsBridgePipe : IAsyncDisposable
 {
-    private static readonly HashSet<string> AllowedTypes = ["hello", "source_list", "source_state", "obs_state", "health"];
+    private static readonly HashSet<string> AllowedTypes = ["hello", "source_list", "source_state", "selection_state", "obs_state", "health"];
     private readonly string _pipeName;
     private readonly Func<JsonElement, Task> _onMessage;
     private readonly Func<ReadOnlyMemory<byte>, Task> _onAudio;

companion/scripts/build-obs-bridge.ps1

@@ -1,6 +1,6 @@
 param(
   [string]$ObsVersion = "31.1.1",
-  [string]$BridgeVersion = "0.1.0-experimental.3",
+  [string]$BridgeVersion = "0.1.0-experimental.5",
   [string]$CacheRoot = "$env:LOCALAPPDATA\LumiCompanionBuild"
 )
 

companion/scripts/publish-companion.ps1

@@ -1,6 +1,6 @@
 param(
-  [string]$Version = "0.1.0-experimental.4",
-  [string]$BridgeVersion = "0.1.0-experimental.3"
+  [string]$Version = "0.1.0-experimental.5",
+  [string]$BridgeVersion = "0.1.0-experimental.5"
 )
 
 $ErrorActionPreference = "Stop"

companion/src/Lumi.Companion.App/App.axaml.cs

@@ -82,8 +82,8 @@ public partial class App : Application
             health.Header = $"Health: {state.Summary}";
             update.Header = state.UpdateAvailable ? $"Update available: {state.AvailableVersion}" : "Updates: Current";
             update.IsEnabled = state.UpdateAvailable;
-            test.Header = state.TestRunning ? "Transcription test running…" : "Run transcription test";
-            test.IsEnabled = !state.TestRunning;
+            test.Header = state.TestRunning || state.BenchmarkRunning ? "Transcription test running…" : "Run transcription test";
+            test.IsEnabled = !state.TestRunning && !state.BenchmarkRunning;
         });
     }
 

companion/src/Lumi.Companion.App/CompanionRuntime.cs

@@ -1,3 +1,4 @@
+using System.Buffers.Binary;
 using System.Diagnostics;
 using System.Reflection;
 using System.Security.Cryptography;
@@ -26,6 +27,13 @@ public sealed class CompanionRuntime : IAsyncDisposable
     private TaskCompletionSource<bool>? _captionSignal;
     private TaskCompletionSource<bool>? _sessionStartSignal;
     private TaskCompletionSource<string>? _testFailure;
+    private TaskCompletionSource<(string SourceUuid, bool Attached)>? _bridgeSelectionSignal;
+    private readonly SemaphoreSlim _bridgeSelectionGate = new(1, 1);
+    private readonly Dictionary<string, BenchmarkCaptionRevision> _benchmarkCaptions = [];
+    private CancellationTokenSource? _benchmarkLifetime;
+    private TaskCompletionSource<bool>? _benchmarkCompleteSignal;
+    private DateTimeOffset? _benchmarkLastVoiceAt;
+    private int _benchmarkStopping;
     private bool _disposed;
     private CompanionUpdate? _availableUpdate;
 
@@ -38,14 +46,17 @@ public sealed class CompanionRuntime : IAsyncDisposable
         State = new CompanionState();
         TestStages = CreateInitialTestStages();
         ObsSources = [];
+        Benchmark = EmptyBenchmark("Not started");
     }
 
     public CompanionState State { get; private set; }
     public IReadOnlyList<TestStage> TestStages { get; private set; }
     public IReadOnlyList<ObsSource> ObsSources { get; private set; }
+    public BenchmarkSnapshot Benchmark { get; private set; }
     public event Action<CompanionState>? StateChanged;
     public event Action<IReadOnlyList<TestStage>>? TestStagesChanged;
     public event Action<IReadOnlyList<ObsSource>>? ObsSourcesChanged;
+    public event Action<BenchmarkSnapshot>? BenchmarkChanged;
     public event Action<string, bool>? CaptionReceived;
     public event Action<string>? LogAdded;
     public event Func<Task>? UpdateRestartRequested;
@@ -130,7 +141,8 @@ public sealed class CompanionRuntime : IAsyncDisposable
         {
             if (_disposed) return;
             _testFailure?.TrySetResult(error is null ? "The Lumi connection closed during the test." : $"The Lumi connection closed during the test: {Friendly(error)}");
-            SetState(State with { Connected = false, Health = TrayHealth.Degraded, Detail = "The secure Lumi connection closed. Retry when the host is available." });
+            _benchmarkLifetime?.Cancel();
+            SetState(State with { Connected = false, BenchmarkRunning = false, Health = TrayHealth.Degraded, Detail = "The secure Lumi connection closed. Retry when the host is available.", BenchmarkDetail = State.BenchmarkRunning ? "The benchmark was aborted because the Lumi connection closed." : State.BenchmarkDetail });
             _ = WriteLogAsync("disconnected", error?.Message ?? "Connection closed.");
         };
         SetState(State with { Connected = false, Detail = "Connecting securely to Lumi…" });
@@ -163,7 +175,7 @@ public sealed class CompanionRuntime : IAsyncDisposable
 
     public async Task RunTestAsync(CancellationToken cancellationToken = default)
     {
-        if (State.TestRunning) return;
+        if (State.TestRunning || State.BenchmarkRunning) return;
         SetState(State with { TestRunning = true, Detail = "Running the transcription path check…" });
         var stages = CreateInitialTestStages().ToArray();
         TestStages = stages;
@@ -175,8 +187,9 @@ public sealed class CompanionRuntime : IAsyncDisposable
             await EvaluateStageAsync(stages, 1, State.ObsConnected, "OBS reported a healthy local connection.", "Open OBS 31 or newer and retry.", cancellationToken);
             if (!State.ObsConnected) return;
             var selected = ObsSources.FirstOrDefault(source => source.Uuid == _settings.Current.PrimarySourceUuid);
-            await EvaluateStageAsync(stages, 2, selected is { Missing: false, Active: true }, "The selected microphone is available and active in Program.", selected is null ? "Choose a microphone on the Transcription page." : selected.Missing ? "The selected microphone is missing from OBS." : "Put the selected microphone in the active Program scene, then retry.", cancellationToken);
-            if (selected is not { Missing: false, Active: true }) return;
+            var bridgeAttached = selected is { Missing: false } && await SyncBridgeSelectionAsync(cancellationToken);
+            await EvaluateStageAsync(stages, 2, selected is { Missing: false, Active: true } && bridgeAttached, "The selected microphone is available, active in Program, and attached for capture.", selected is null ? "Choose a microphone on the Transcription page." : selected.Missing ? "The selected microphone is missing from OBS." : !bridgeAttached ? "Companion could not attach the selected OBS source. Keep OBS open and retry." : "Put the selected microphone in the active Program scene, then retry.", cancellationToken);
+            if (selected is not { Missing: false, Active: true } || !bridgeAttached) return;
             await EvaluateStageAsync(stages, 4, State.Connected, "The secure Lumi transport is ready.", "Reconnect to Lumi before testing.", cancellationToken);
             if (!State.Connected) return;
             _audioSignal = new(TaskCreationOptions.RunContinuationsAsynchronously);
@@ -224,6 +237,87 @@ public sealed class CompanionRuntime : IAsyncDisposable
         }
     }
 
+    public async Task StartBenchmarkAsync(CancellationToken cancellationToken = default)
+    {
+        if (State.BenchmarkRunning || State.TestRunning) return;
+        if (!State.Connected || _socket is null) throw new InvalidOperationException("Reconnect to Lumi before starting the transcription test.");
+        if (!State.ObsConnected) throw new InvalidOperationException("Open OBS and wait for the managed integration to connect.");
+        var selected = ObsSources.FirstOrDefault(source => source.Uuid == _settings.Current.PrimarySourceUuid);
+        if (selected is not { Missing: false, Active: true }) throw new InvalidOperationException("Choose an available microphone that is active in the OBS Program scene.");
+        if (!await SyncBridgeSelectionAsync(cancellationToken)) throw new InvalidOperationException("Companion could not attach the selected OBS source. Keep OBS open and retry.");
+
+        _benchmarkCaptions.Clear();
+        _sessionStartSignal = new(TaskCreationOptions.RunContinuationsAsynchronously);
+        _testFailure = new(TaskCreationOptions.RunContinuationsAsynchronously);
+        _benchmarkCompleteSignal = new(TaskCreationOptions.RunContinuationsAsynchronously);
+        _benchmarkLastVoiceAt = DateTimeOffset.UtcNow;
+        Interlocked.Exchange(ref _benchmarkStopping, 0);
+        Benchmark = EmptyBenchmark("Starting", DateTimeOffset.UtcNow);
+        BenchmarkChanged?.Invoke(Benchmark);
+        SetState(State with { BenchmarkRunning = true, BenchmarkDetail = "Starting server-hosted accuracy and latency measurement…", Health = TrayHealth.Operating });
+        try
+        {
+            await _socket.SendAsync("start", new { mode = "benchmark" }, _socket.SessionId, cancellationToken);
+            var started = await WaitForTestSignalAsync(_sessionStartSignal.Task, TimeSpan.FromSeconds(8), cancellationToken);
+            if (!started.Completed)
+                throw new InvalidOperationException(started.Failure ?? "Lumi did not start the benchmark within 8 seconds.");
+
+            _benchmarkLifetime?.Cancel();
+            _benchmarkLifetime?.Dispose();
+            _benchmarkLifetime = CancellationTokenSource.CreateLinkedTokenSource(_maintenanceLifetime.Token);
+            _ = MonitorBenchmarkSilenceAsync(_benchmarkLifetime.Token);
+            SetState(State with { BenchmarkDetail = "Listening. Speak naturally; end the test manually, or remain silent for 10 seconds." });
+            await WriteLogAsync("benchmark_started", $"Started transcription benchmark for {selected.Name}.");
+        }
+        catch (Exception error)
+        {
+            _benchmarkLifetime?.Cancel();
+            _sessionStartSignal = null;
+            _testFailure = null;
+            _benchmarkCompleteSignal = null;
+            SetState(State with { BenchmarkRunning = false, BenchmarkDetail = Friendly(error), Health = TrayHealth.Degraded });
+            throw;
+        }
+    }
+
+    public async Task StopBenchmarkAsync(string reason = "benchmark_complete", CancellationToken cancellationToken = default)
+    {
+        if (!State.BenchmarkRunning || _socket is null || Interlocked.Exchange(ref _benchmarkStopping, 1) != 0) return;
+        _benchmarkLifetime?.Cancel();
+        SetState(State with { BenchmarkDetail = reason == "silence_timeout" ? "Ten seconds of silence detected. Finalizing the test…" : "Finalizing the test…" });
+        try
+        {
+            await _socket.SendAsync("stop", new { reason }, _socket.SessionId, cancellationToken);
+            if (_benchmarkCompleteSignal is not null)
+                await Task.WhenAny(_benchmarkCompleteSignal.Task, Task.Delay(TimeSpan.FromSeconds(7), cancellationToken));
+        }
+        finally
+        {
+            SetState(State with { BenchmarkRunning = false, BenchmarkDetail = Benchmark.Words.Count > 0 ? $"Test complete with {Benchmark.Words.Count} measured words or phrases." : "The test ended without a finalized transcript." });
+            _sessionStartSignal = null;
+            _testFailure = null;
+            Interlocked.Exchange(ref _benchmarkStopping, 0);
+        }
+    }
+
+    private async Task MonitorBenchmarkSilenceAsync(CancellationToken cancellationToken)
+    {
+        try
+        {
+            using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(500));
+            while (await timer.WaitForNextTickAsync(cancellationToken))
+            {
+                if (!State.BenchmarkRunning) return;
+                if (_benchmarkLastVoiceAt is { } lastVoice && DateTimeOffset.UtcNow - lastVoice >= TimeSpan.FromSeconds(10))
+                {
+                    await StopBenchmarkAsync("silence_timeout", CancellationToken.None);
+                    return;
+                }
+            }
+        }
+        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { }
+    }
+
     private async Task EvaluateStageAsync(TestStage[] stages, int index, bool passed, string success, string blocked, CancellationToken cancellationToken)
     {
         MarkStage(stages, index, TestStageState.Running, "Checking…");
@@ -359,23 +453,40 @@ public sealed class CompanionRuntime : IAsyncDisposable
     {
         if (message.Type == "status" && message.Payload.TryGetProperty("kind", out var kind) && kind.GetString() == "session" &&
             message.Payload.TryGetProperty("state", out var state) && state.GetString() == "running")
+        {
             _sessionStartSignal?.TrySetResult(true);
+            if (message.Payload.TryGetProperty("benchmark_id", out var benchmarkId) && benchmarkId.ValueKind == JsonValueKind.String)
+            {
+                Benchmark = Benchmark with { Id = benchmarkId.GetString(), Status = "Running" };
+                BenchmarkChanged?.Invoke(Benchmark);
+            }
+        }
         if (message.Type == "caption")
         {
-            var text = message.Payload.TryGetProperty("stable_text", out var stable) ? stable.GetString() : null;
+            var stableText = message.Payload.TryGetProperty("stable_text", out var stable) ? stable.GetString() : null;
+            var uncertainText = message.Payload.TryGetProperty("uncertain_text", out var uncertain) ? uncertain.GetString() : null;
+            var text = string.Join(" ", new[] { stableText, uncertainText }.Where(value => !string.IsNullOrWhiteSpace(value)));
             var simulated = message.Payload.TryGetProperty("delivery", out var delivery) && delivery.TryGetProperty("disposition", out var disposition) && disposition.GetString() == "simulated";
             if (!string.IsNullOrWhiteSpace(text))
             {
-                if (simulated) _captionSignal?.TrySetResult(true);
+                var final = ReadBoolean(message.Payload, "final");
+                if (simulated && final) _captionSignal?.TrySetResult(true);
                 CaptionReceived?.Invoke(text, simulated);
                 if (!simulated) _ = _obsBridge?.SendAsync(new { type = "caption", payload = message.Payload });
+                if (State.BenchmarkRunning) UpdateBenchmarkCaption(message.Payload, text, final);
             }
         }
+        if (message.Type == "benchmark_complete")
+        {
+            ApplyBenchmarkCompletion(message.Payload);
+            _benchmarkCompleteSignal?.TrySetResult(true);
+        }
         if (message.Type == "error")
         {
             var serverMessage = message.Payload.TryGetProperty("message", out var value) ? value.GetString() : "Lumi reported an error.";
             _testFailure?.TrySetResult(serverMessage ?? "Lumi reported an inference error.");
-            SetState(State with { Health = TrayHealth.Degraded, Detail = serverMessage ?? "Lumi reported an error." });
+            _benchmarkLifetime?.Cancel();
+            SetState(State with { Health = TrayHealth.Degraded, Detail = serverMessage ?? "Lumi reported an error.", BenchmarkRunning = false, BenchmarkDetail = serverMessage ?? "Lumi reported an inference error." });
         }
         return Task.CompletedTask;
     }
@@ -416,6 +527,7 @@ public sealed class CompanionRuntime : IAsyncDisposable
                 .Where(source => Guid.TryParse(source.Uuid, out _)).ToArray();
             ObsSourcesChanged?.Invoke(ObsSources);
             foreach (var source in ObsSources) await SendSourceUpdateAsync(source);
+            _ = SyncBridgeSelectionAsync(_lifetimeToken());
         }
         else if (type == "source_state")
         {
@@ -427,11 +539,19 @@ public sealed class CompanionRuntime : IAsyncDisposable
                 await SendSourceUpdateAsync(source);
             }
         }
+        else if (type == "selection_state")
+        {
+            _bridgeSelectionSignal?.TrySetResult((ReadString(message, "source_uuid") ?? string.Empty, ReadBoolean(message, "attached")));
+        }
     }
 
     private Task OnObsAudioAsync(ReadOnlyMemory<byte> frame)
+    {
+        if (HasVoice(frame.Span))
         {
             _audioSignal?.TrySetResult(true);
+            if (State.BenchmarkRunning) _benchmarkLastVoiceAt = DateTimeOffset.UtcNow;
+        }
         if (_socket is not null && State.Connected && !_socket.QueueBridgeAudio(frame))
             _ = WriteLogAsync("audio_dropped", "An obsolete audio frame was dropped before network delivery.");
         return Task.CompletedTask;
@@ -453,13 +573,137 @@ public sealed class CompanionRuntime : IAsyncDisposable
         }, _socket.SessionId, _lifetimeToken());
     }
 
-    private Task<bool> SyncBridgeSelectionAsync(CancellationToken cancellationToken = default) => _obsBridge?.SendAsync(new
+    private async Task<bool> SyncBridgeSelectionAsync(CancellationToken cancellationToken = default)
+    {
+        if (_obsBridge is null || string.IsNullOrWhiteSpace(_settings.Current.PrimarySourceUuid)) return false;
+        await _bridgeSelectionGate.WaitAsync(cancellationToken);
+        try
+        {
+            var sourceUuid = _settings.Current.PrimarySourceUuid;
+            var signal = new TaskCompletionSource<(string SourceUuid, bool Attached)>(TaskCreationOptions.RunContinuationsAsynchronously);
+            _bridgeSelectionSignal = signal;
+            var sent = await _obsBridge.SendAsync(new
             {
                 type = "select_sources",
                 protocol_version = 1,
-        source_uuids = string.IsNullOrWhiteSpace(_settings.Current.PrimarySourceUuid) ? Array.Empty<string>() : new[] { _settings.Current.PrimarySourceUuid },
-        primary_source_uuid = _settings.Current.PrimarySourceUuid
-    }, cancellationToken) ?? Task.FromResult(false);
+                source_uuids = new[] { sourceUuid },
+                primary_source_uuid = sourceUuid
+            }, cancellationToken);
+            if (!sent) return false;
+            var completed = await Task.WhenAny(signal.Task, Task.Delay(TimeSpan.FromSeconds(2), cancellationToken));
+            if (completed != signal.Task) { cancellationToken.ThrowIfCancellationRequested(); return true; }
+            var result = await signal.Task;
+            return result.Attached && string.Equals(result.SourceUuid, sourceUuid, StringComparison.OrdinalIgnoreCase);
+        }
+        finally
+        {
+            _bridgeSelectionSignal = null;
+            _bridgeSelectionGate.Release();
+        }
+    }
+
+    private void UpdateBenchmarkCaption(JsonElement payload, string text, bool final)
+    {
+        var captionId = ReadString(payload, "caption_id") ?? Guid.NewGuid().ToString();
+        var revision = payload.TryGetProperty("revision", out var revisionValue) && revisionValue.TryGetInt32(out var parsedRevision) ? parsedRevision : 0;
+        if (_benchmarkCaptions.TryGetValue(captionId, out var current) && current.Revision >= revision) return;
+        var words = new List<BenchmarkWord>();
+        if (payload.TryGetProperty("analysis", out var analysis) && analysis.TryGetProperty("words", out var wordValues) && wordValues.ValueKind == JsonValueKind.Array)
+        {
+            foreach (var word in wordValues.EnumerateArray())
+            {
+                var wordText = ReadString(word, "text");
+                if (string.IsNullOrWhiteSpace(wordText)) continue;
+                words.Add(new BenchmarkWord(wordText, ReadDouble(word, "latency_ms"), final ? Math.Clamp(ReadDouble(word, "confidence"), 0, 1) : null, final));
+            }
+        }
+        _benchmarkCaptions[captionId] = new BenchmarkCaptionRevision(revision, final, text, words, DateTimeOffset.UtcNow);
+        PublishLiveBenchmark();
+    }
+
+    private void PublishLiveBenchmark()
+    {
+        var captions = _benchmarkCaptions.Values.OrderBy(value => value.ReceivedAt).ToArray();
+        var words = captions.SelectMany(value => value.Words).ToArray();
+        Benchmark = Benchmark with
+        {
+            Transcript = string.Join(" ", captions.Select(value => value.Text).Where(value => !string.IsNullOrWhiteSpace(value))),
+            Words = words,
+            Latency = CalculateMetric(words.Select(word => word.LatencyMs)),
+            Confidence = CalculateMetric(words.Where(word => word.Confidence.HasValue).Select(word => word.Confidence!.Value)),
+            Status = State.BenchmarkRunning ? "Running" : Benchmark.Status
+        };
+        BenchmarkChanged?.Invoke(Benchmark);
+    }
+
+    private void ApplyBenchmarkCompletion(JsonElement payload)
+    {
+        var words = new List<BenchmarkWord>();
+        if (payload.TryGetProperty("words", out var values) && values.ValueKind == JsonValueKind.Array)
+        {
+            foreach (var word in values.EnumerateArray())
+            {
+                var text = ReadString(word, "text");
+                if (string.IsNullOrWhiteSpace(text)) continue;
+                var confidence = word.TryGetProperty("confidence", out var confidenceValue) && confidenceValue.ValueKind == JsonValueKind.Number ? confidenceValue.GetDouble() : (double?)null;
+                words.Add(new BenchmarkWord(text, ReadDouble(word, "latency_ms"), confidence, ReadBoolean(word, "final")));
+            }
+        }
+        var finalWords = words.Count > 0 ? words : Benchmark.Words;
+        Benchmark = new BenchmarkSnapshot(
+            ReadString(payload, "id") ?? Benchmark.Id,
+            ReadString(payload, "transcript") ?? Benchmark.Transcript,
+            finalWords,
+            ParseMetric(payload, "latency") ?? CalculateMetric(finalWords.Select(word => word.LatencyMs)),
+            ParseMetric(payload, "confidence") ?? CalculateMetric(finalWords.Where(word => word.Confidence.HasValue).Select(word => word.Confidence!.Value)),
+            Benchmark.StartedAt,
+            ReadString(payload, "status") ?? "Completed");
+        BenchmarkChanged?.Invoke(Benchmark);
+    }
+
+    private static MetricStatistics? ParseMetric(JsonElement payload, string name)
+    {
+        if (!payload.TryGetProperty("stats", out var stats) || !stats.TryGetProperty(name, out var value)) return null;
+        return new MetricStatistics(ReadInt(value, "count"), ReadNullableDouble(value, "min"), ReadNullableDouble(value, "low_1_average"),
+            ReadNullableDouble(value, "median"), ReadNullableDouble(value, "average"), ReadNullableDouble(value, "p99"),
+            ReadNullableDouble(value, "high_1_average"), ReadNullableDouble(value, "max"));
+    }
+
+    private static MetricStatistics CalculateMetric(IEnumerable<double> input)
+    {
+        var values = input.Where(double.IsFinite).Order().ToArray();
+        if (values.Length == 0) return new(0, null, null, null, null, null, null, null);
+        var tail = Math.Max(1, (int)Math.Ceiling(values.Length * 0.01));
+        return new(values.Length, values[0], values.Take(tail).Average(), Percentile(values, 0.5), values.Average(), Percentile(values, 0.99), values.TakeLast(tail).Average(), values[^1]);
+    }
+
+    private static double Percentile(double[] values, double ratio)
+    {
+        var position = (values.Length - 1) * ratio;
+        var lower = (int)Math.Floor(position);
+        var upper = (int)Math.Ceiling(position);
+        return values[lower] + (values[upper] - values[lower]) * (position - lower);
+    }
+
+    private static bool HasVoice(ReadOnlySpan<byte> frame)
+    {
+        if (frame.Length <= ProtocolV1.AudioHeaderBytes || (frame[5] & 1) == 0 || (frame[5] & 2) != 0) return false;
+        var samples = frame[ProtocolV1.AudioHeaderBytes..];
+        double squares = 0;
+        var count = samples.Length / 2;
+        for (var index = 0; index < count; index += 1)
+        {
+            var value = BinaryPrimitives.ReadInt16LittleEndian(samples.Slice(index * 2, 2)) / 32768.0;
+            squares += value * value;
+        }
+        return count > 0 && Math.Sqrt(squares / count) >= 0.012;
+    }
+
+    private static BenchmarkSnapshot EmptyBenchmark(string status, DateTimeOffset? startedAt = null) =>
+        new(null, string.Empty, [], CalculateMetric([]), CalculateMetric([]), startedAt, status);
+    private static double ReadDouble(JsonElement value, string name) => value.TryGetProperty(name, out var property) && property.ValueKind == JsonValueKind.Number ? property.GetDouble() : 0;
+    private static double? ReadNullableDouble(JsonElement value, string name) => value.TryGetProperty(name, out var property) && property.ValueKind == JsonValueKind.Number ? property.GetDouble() : null;
+    private static int ReadInt(JsonElement value, string name) => value.TryGetProperty(name, out var property) && property.TryGetInt32(out var result) ? result : 0;
 
     private static bool ReadBoolean(JsonElement value, string name) => value.TryGetProperty(name, out var property) && property.ValueKind == JsonValueKind.True;
     private static string? ReadString(JsonElement value, string name) => value.TryGetProperty(name, out var property) && property.ValueKind == JsonValueKind.String ? property.GetString() : null;
@@ -533,6 +777,7 @@ public sealed class CompanionRuntime : IAsyncDisposable
         TaskCanceledException => "The connection timed out. Check that Lumi is reachable.",
         _ => error.Message
     };
+    private sealed record BenchmarkCaptionRevision(int Revision, bool Final, string Text, IReadOnlyList<BenchmarkWord> Words, DateTimeOffset ReceivedAt);
     private static TestStage[] CreateInitialTestStages() =>
     [
         new("OBS integration", "Waiting to check the managed bridge.", TestStageState.Waiting),
@@ -550,10 +795,13 @@ public sealed class CompanionRuntime : IAsyncDisposable
     public async ValueTask DisposeAsync()
     {
         if (_disposed) return;
+        if (State.BenchmarkRunning) try { await StopBenchmarkAsync("disconnect", CancellationToken.None); } catch { }
         _disposed = true;
         _maintenanceLifetime.Cancel();
+        _benchmarkLifetime?.Cancel();
         if (_socket is not null) await _socket.DisposeAsync();
         if (_obsBridge is not null) await _obsBridge.DisposeAsync();
+        _benchmarkLifetime?.Dispose(); _bridgeSelectionGate.Dispose();
         _http.Dispose(); _maintenanceLifetime.Dispose();
     }
 }

companion/src/Lumi.Companion.App/CompanionState.cs

@@ -12,6 +12,7 @@ public sealed record CompanionState(
     bool ObsStreaming = false,
     bool ObsRecording = false,
     bool TestRunning = false,
+    bool BenchmarkRunning = false,
     TrayHealth Health = TrayHealth.Degraded,
     string Detail = "Pair Lumi Companion to get started.",
     string? DeviceName = null,
@@ -20,6 +21,7 @@ public sealed record CompanionState(
     bool UpdateAvailable = false,
     string? AvailableVersion = null,
     string UpdateDetail = "Checking for updates…",
+    string BenchmarkDetail = "Start a dedicated test, speak naturally, then end it when you have enough material.",
     bool ObsBridgeRepairNeeded = false,
     bool ObsBridgePackageAvailable = false,
     string ObsBridgeDetail = "Checking the managed OBS integration…")
@@ -35,13 +37,18 @@ public sealed record CompanionState(
         _ => "Partially ready"
     };
 
-    public bool RequiresQuitConfirmation => ObsStreaming || ObsRecording;
-    public string QuitWarning => ObsStreaming
+    public bool RequiresQuitConfirmation => ObsStreaming || ObsRecording || BenchmarkRunning;
+    public string QuitWarning => BenchmarkRunning
+        ? "A transcription accuracy and latency test is running. Quitting will safely end and mark the test as aborted."
+        : ObsStreaming
         ? "OBS is streaming. Quitting Lumi Companion will stop transcription and closed captions, but it will not stop the OBS stream."
         : ObsRecording ? "OBS is recording. Quitting Lumi Companion will stop active companion features." : string.Empty;
 }
 
 public sealed record TestStage(string Name, string Detail, TestStageState State);
+public sealed record BenchmarkWord(string Text, double LatencyMs, double? Confidence, bool Final);
+public sealed record MetricStatistics(int Count, double? Minimum, double? LowOnePercentAverage, double? Median, double? Average, double? P99, double? HighOnePercentAverage, double? Maximum);
+public sealed record BenchmarkSnapshot(string? Id, string Transcript, IReadOnlyList<BenchmarkWord> Words, MetricStatistics Latency, MetricStatistics Confidence, DateTimeOffset? StartedAt, string Status);
 public sealed record ObsSource(string Uuid, string Name, bool Active, bool Missing)
 {
     public override string ToString() => Missing ? $"{Name} (missing)" : Active ? $"{Name} (active)" : Name;

companion/src/Lumi.Companion.App/Lumi.Companion.App.csproj

@@ -5,7 +5,7 @@
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
     <ApplicationManifest>app.manifest</ApplicationManifest>
-    <Version>0.1.0-experimental.4</Version>
+    <Version>0.1.0-experimental.5</Version>
     <AssemblyVersion>0.1.0.0</AssemblyVersion>
   </PropertyGroup>
   <ItemGroup>

companion/src/Lumi.Companion.App/MainWindow.axaml

@@ -138,6 +138,35 @@
               <TextBlock x:Name="SimulatedCaption" Text="A real caption will appear here only after every upstream stage passes." Classes="muted" FontStyle="Italic" />
             </StackPanel>
           </Border>
+          <Border Classes="card">
+            <StackPanel Spacing="16">
+              <StackPanel Spacing="5">
+                <TextBlock Text="ACCURACY &amp; LATENCY" Classes="eyebrow" />
+                <TextBlock Text="Dedicated transcription test" Classes="sectionTitle" />
+                <TextBlock Text="Speak naturally for as long as needed. End the test yourself, or Lumi will finalize it after 10 seconds of silence. Nothing is sent to Twitch." Classes="muted" />
+              </StackPanel>
+              <StackPanel Orientation="Horizontal" Spacing="10">
+                <Button x:Name="StartBenchmarkButton" Classes="primary" Content="Start dedicated test" />
+                <Button x:Name="StopBenchmarkButton" Classes="secondary" Content="End test" IsEnabled="False" />
+              </StackPanel>
+              <TextBlock x:Name="BenchmarkStatusText" Text="Not started." Classes="muted" />
+              <Border Classes="soft">
+                <StackPanel Spacing="5">
+                  <TextBlock Text="Measured transcript" FontWeight="SemiBold" />
+                  <TextBlock x:Name="BenchmarkTranscript" Text="Finalized speech will appear here." Classes="muted" TextWrapping="Wrap" />
+                </StackPanel>
+              </Border>
+              <StackPanel Spacing="8">
+                <TextBlock Text="Latency" FontWeight="SemiBold" />
+                <WrapPanel x:Name="LatencyStatsPanel" Orientation="Horizontal" />
+              </StackPanel>
+              <StackPanel Spacing="8">
+                <TextBlock Text="Final confidence" FontWeight="SemiBold" />
+                <WrapPanel x:Name="ConfidenceStatsPanel" Orientation="Horizontal" />
+              </StackPanel>
+              <TextBlock Text="Low and high 1% values are tail averages. Confidence is included only after an utterance is finalized." Classes="muted" FontSize="12" />
+            </StackPanel>
+          </Border>
         </StackPanel>
       </ScrollViewer>
 

companion/src/Lumi.Companion.App/MainWindow.axaml.cs

@@ -28,10 +28,12 @@ public partial class MainWindow : Window
         WireActions();
         RenderState(runtime.State);
         RenderTestStages(runtime.TestStages);
+        RenderBenchmark(runtime.Benchmark);
         Closing += OnClosing;
         runtime.StateChanged += state => Dispatcher.UIThread.Post(() => RenderState(state));
         runtime.TestStagesChanged += stages => Dispatcher.UIThread.Post(() => RenderTestStages(stages));
         runtime.ObsSourcesChanged += sources => Dispatcher.UIThread.Post(() => RenderSources(sources));
+        runtime.BenchmarkChanged += benchmark => Dispatcher.UIThread.Post(() => RenderBenchmark(benchmark));
         runtime.CaptionReceived += (text, simulated) => Dispatcher.UIThread.Post(() =>
         {
             if (simulated) SimulatedCaption.Text = text;
@@ -49,6 +51,8 @@ public partial class MainWindow : Window
         OpenWebButton.Click += (_, _) => _runtime.OpenLumiWebUi();
         OpenLogsButton.Click += (_, _) => _runtime.OpenLogsDirectory();
         RunTestButton.Click += async (_, _) => await RunUiActionAsync(() => _runtime.RunTestAsync(), RunTestButton);
+        StartBenchmarkButton.Click += async (_, _) => await RunUiActionAsync(() => _runtime.StartBenchmarkAsync(), StartBenchmarkButton);
+        StopBenchmarkButton.Click += async (_, _) => await RunUiActionAsync(() => _runtime.StopBenchmarkAsync(), StopBenchmarkButton);
         ForgetButton.Click += async (_, _) => await ForgetDeviceAsync();
         SaveSettingsButton.Click += async (_, _) => await SaveSettingsAsync();
         CheckUpdateButton.Click += async (_, _) => await RunUiActionAsync(() => _runtime.CheckForUpdatesAsync(), CheckUpdateButton);
@@ -175,7 +179,10 @@ public partial class MainWindow : Window
         ReconnectButton.IsEnabled = state.Paired && !state.Connected;
         ForgetButton.IsEnabled = state.Paired;
         RunTestButton.Content = state.TestRunning ? "Testing…" : "Run full path test";
-        RunTestButton.IsEnabled = !state.TestRunning;
+        RunTestButton.IsEnabled = !state.TestRunning && !state.BenchmarkRunning;
+        StartBenchmarkButton.IsEnabled = !state.TestRunning && !state.BenchmarkRunning;
+        StopBenchmarkButton.IsEnabled = state.BenchmarkRunning;
+        BenchmarkStatusText.Text = state.BenchmarkDetail;
         UpdatePanel.IsVisible = state.UpdateAvailable;
         UpdateTitle.Text = state.UpdateAvailable ? $"Lumi Companion {state.AvailableVersion} is available" : "Lumi Companion is current";
         UpdateDetail.Text = state.UpdateDetail;
@@ -252,7 +259,32 @@ public partial class MainWindow : Window
         TestStepSymbol.Text = _testPassed ? "✓" : "○";
         TestStepDetail.Text = _testPassed ? "Passed" : "Not passed";
         RunTestButton.Content = _runtime.State.TestRunning ? "Testing…" : "Run full path test";
-        RunTestButton.IsEnabled = !_runtime.State.TestRunning;
+        RunTestButton.IsEnabled = !_runtime.State.TestRunning && !_runtime.State.BenchmarkRunning;
+    }
+
+    private void RenderBenchmark(BenchmarkSnapshot benchmark)
+    {
+        BenchmarkTranscript.Text = string.IsNullOrWhiteSpace(benchmark.Transcript) ? "Finalized speech will appear here." : benchmark.Transcript;
+        RenderMetric(LatencyStatsPanel, benchmark.Latency, value => $"{value:0} ms");
+        RenderMetric(ConfidenceStatsPanel, benchmark.Confidence, value => $"{value:P0}");
+    }
+
+    private static void RenderMetric(Panel panel, MetricStatistics metric, Func<double, string> formatter)
+    {
+        panel.Children.Clear();
+        var values = new (string Label, double? Value)[]
+        {
+            ("Minimum", metric.Minimum), ("Low 1% avg", metric.LowOnePercentAverage),
+            ("Median", metric.Median), ("Average", metric.Average), ("P99", metric.P99),
+            ("High 1% avg", metric.HighOnePercentAverage), ("Maximum", metric.Maximum)
+        };
+        foreach (var item in values)
+        {
+            var content = new StackPanel { Spacing = 2 };
+            content.Children.Add(new TextBlock { Text = item.Label, FontSize = 11, Foreground = new SolidColorBrush(Color.Parse("#5A6872")) });
+            content.Children.Add(new TextBlock { Text = item.Value.HasValue ? formatter(item.Value.Value) : "—", FontWeight = FontWeight.SemiBold });
+            panel.Children.Add(new Border { Classes = { "soft" }, Child = content, MinWidth = 96, Margin = new Thickness(0, 0, 8, 8), Padding = new Thickness(12, 9) });
+        }
     }
 
     private void AddLog(string line)

companion/src/Lumi.Companion.Core/CompanionSocket.cs

@@ -1,3 +1,4 @@
+using System.Buffers.Binary;
 using System.Net.WebSockets;
 using System.Threading.Channels;
 using Lumi.Companion.Protocol;
@@ -58,6 +59,7 @@ public sealed class CompanionSocket : IAsyncDisposable
         if (encoded.Length < ProtocolV1.AudioHeaderBytes || !encoded.Span[..4].SequenceEqual("LACP"u8)) return false;
         var normalized = encoded.ToArray();
         SessionId.TryWriteBytes(normalized.AsSpan(20, 16), bigEndian: true, out _);
+        BinaryPrimitives.WriteUInt64LittleEndian(normalized.AsSpan(12, 8), (ulong)DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() * 1000UL);
         return QueueEncodedAudio(normalized);
     }
     public async Task SendAsync(string type, object payload, Guid? sessionId, CancellationToken cancellationToken)

knowledge/plugins/lumi-transcription.md

@@ -14,7 +14,7 @@ editable: false
 Server-hosted whisper.cpp transcription for Lumi Companion and OBS closed captions.
 ## Metadata
 Plugin ID: lumi_transcription
-Version: 0.1.0-experimental.3
+Version: 0.1.0-experimental.5
 Default state: enabled
 ## Web Routes
 - /plugins/lumi_transcription
@@ -27,6 +27,8 @@ Default state: enabled
 - GET /plugins/lumi_transcription/api/devices
 - POST /plugins/lumi_transcription/api/devices/:id/revoke
 - POST /plugins/lumi_transcription/api/devices/:id/capabilities
+- GET /plugins/lumi_transcription/api/tests
+- GET /plugins/lumi_transcription/api/tests/:id
 - GET /plugins/lumi_transcription/api/settings
 - PATCH /plugins/lumi_transcription/api/settings
 - POST /plugins/lumi_transcription/api/models/:id/download
@@ -100,7 +102,7 @@ Default state: enabled
 ### GET /plugins/lumi_transcription/api/devices
 
 - Purpose: Renders or serves the lumi_transcription plugin page for api devices.
-- Inputs: No request parameters detected by static analysis.
+- Inputs: query: `status`
 - Response format: JSON response
 - Access: admin access expected
 - Side effects: Usually read-only.
@@ -124,6 +126,24 @@ Default state: enabled
 - Side effects: writes or mutates server-side state
 - Limits/notes: Generated from static route source analysis; confirm exact behavior in the handler before changing integrations. Most non-API POST routes are browser form submissions and usually redirect after completion. API consumers should expect JSON unless the response format says otherwise.
 
+### GET /plugins/lumi_transcription/api/tests
+
+- Purpose: Renders or serves the lumi_transcription plugin page for api tests.
+- Inputs: No request parameters detected by static analysis.
+- Response format: JSON response
+- Access: admin access expected
+- Side effects: Usually read-only.
+- Limits/notes: Generated from static route source analysis; confirm exact behavior in the handler before changing integrations. API consumers should expect JSON unless the response format says otherwise.
+
+### GET /plugins/lumi_transcription/api/tests/:id
+
+- Purpose: Renders or serves the lumi_transcription plugin page for api tests id.
+- Inputs: path params: `id`
+- Response format: JSON response
+- Access: admin access expected
+- Side effects: Usually read-only.
+- Limits/notes: Generated from static route source analysis; confirm exact behavior in the handler before changing integrations. API consumers should expect JSON unless the response format says otherwise.
+
 ### GET /plugins/lumi_transcription/api/settings
 
 - Purpose: Renders or serves the lumi_transcription plugin page for api settings.

plugins/lumi_transcription/backend/companion/device_store.js

@@ -8,6 +8,7 @@ class DeviceStore {
     this.now = options.now || Date.now;
     this.randomBytes = options.randomBytes || crypto.randomBytes;
     this.migrate();
+    this.cleanup();
   }
 
   migrate() {
@@ -70,8 +71,13 @@ class DeviceStore {
     return { allowed: true, device: serialize(row, capabilities) };
   }
 
-  list() { return this.db.prepare("SELECT * FROM transcription_devices ORDER BY last_connected_at DESC").all().map((row) => serialize(row, parseArray(row.capabilities_json))); }
+  list(options = {}) {
+    const status = options.status === "revoked" ? "revoked" : options.status === "all" ? "all" : "active";
+    const where = status === "revoked" ? "WHERE revoked_at IS NOT NULL" : status === "active" ? "WHERE revoked_at IS NULL" : "";
+    return this.db.prepare(`SELECT * FROM transcription_devices ${where} ORDER BY last_connected_at DESC`).all().map((row) => serialize(row, parseArray(row.capabilities_json)));
+  }
   revoke(deviceId) { return this.db.prepare("UPDATE transcription_devices SET revoked_at = ? WHERE id = ? AND revoked_at IS NULL").run(this.now(), deviceId).changes === 1; }
+  cleanup(now = this.now()) { return this.db.prepare("DELETE FROM transcription_devices WHERE revoked_at IS NOT NULL AND revoked_at <= ?").run(now - 30 * 86400000).changes; }
   setCapabilities(deviceId, capabilities) {
     const allowed = DEFAULT_CAPABILITIES.filter((capability) => new Set(capabilities || []).has(capability));
     const changed = this.db.prepare("UPDATE transcription_devices SET capabilities_json = ? WHERE id = ? AND revoked_at IS NULL").run(JSON.stringify(allowed), deviceId).changes;

plugins/lumi_transcription/backend/companion/gateway.js

@@ -84,7 +84,7 @@ class CompanionGateway {
       case "source_update": send("status", { kind: "source", source: this.sessions.updateSource(session.id, message.payload || {}) }); break;
       case "obs_state": send("status", { kind: "obs", ...(await this.sessions.updateObsState(session.id, message.payload || {})) }); break;
       case "start": send("status", { kind: "session", ...(await this.sessions.start(session.id, message.payload || {})) }); break;
-      case "stop": send("status", { kind: "session", ...(await this.sessions.stop(session.id)) }); break;
+      case "stop": send("status", { kind: "session", ...(await this.sessions.stop(session.id, cleanReason(message.payload?.reason))) }); break;
       case "ack": break;
       default: throw coded("UNEXPECTED_MESSAGE", `Message ${message.type} is not valid after the handshake.`);
     }
@@ -99,5 +99,6 @@ function reject(socket, status, reason) { const labels = { 401: "Unauthorized",
 function closeWith(socket, code, reason) { if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) socket.close(code, String(reason).slice(0, 120)); }
 function sameHostOrigin(origin, host) { try { return new URL(origin).host === host; } catch { return false; } }
 function coded(code, message) { return Object.assign(new Error(message), { code }); }
+function cleanReason(value) { return ["requested", "test_complete", "benchmark_complete", "silence_timeout", "disconnect"].includes(String(value)) ? String(value) : "requested"; }
 
 module.exports = { CompanionGateway, sameHostOrigin };

plugins/lumi_transcription/backend/sessions/session_coordinator.js

@@ -9,6 +9,7 @@ class SessionCoordinator {
     this.log = options.log || { append() {} };
     this.now = options.now || Date.now;
     this.graceMs = options.graceMs || 30000;
+    this.benchmarks = options.benchmarks || null;
     this.sessions = new Map();
     this.stabilizer = new CaptionStabilizer({ now: this.now });
     this.provider.on?.("hypothesis", (event) => Promise.resolve(this.onHypothesis(event)).catch((error) => {
@@ -33,7 +34,7 @@ class SessionCoordinator {
     const session = {
       id: crypto.randomUUID(), deviceId: device.id, connected: true, send,
       state: "idle", mode: null, obs: { streaming: false, recording: false },
-      tracks: new Map(), delivery: this.deliveryFactory(send), createdAt: this.now(), graceUntil: 0, graceTimer: null
+      tracks: new Map(), delivery: this.deliveryFactory(send), createdAt: this.now(), graceUntil: 0, graceTimer: null, benchmarkId: null
     };
     this.sessions.set(session.id, session);
     this.log.append({ kind: "session", state: "created", session_id: session.id, device_id: device.id });
@@ -66,7 +67,7 @@ class SessionCoordinator {
   }
   async start(sessionId, options = {}) {
     const session = this.require(sessionId);
-    const mode = options.mode === "test" ? "test" : "live";
+    const mode = options.mode === "test" ? "test" : options.mode === "benchmark" ? "benchmark" : "live";
     if (mode === "live" && !session.obs.streaming) throw Object.assign(new Error("Live transcription requires an active OBS stream."), { code: "OBS_NOT_STREAMING" });
     const tracks = Array.from(session.tracks.values()).filter((track) => track.enabled && !track.source_missing);
     if (!tracks.length) throw Object.assign(new Error("Select an available OBS audio source first."), { code: "NO_TRACKS" });
@@ -79,9 +80,10 @@ class SessionCoordinator {
     }
     await this.provider.startSession({ id: session.id, mode });
     for (const track of tracks) await this.provider.addTrack(session.id, serializeTrack(track));
-    await session.delivery.start({ testMode: mode === "test" });
+    await session.delivery.start({ testMode: mode !== "live" });
     session.mode = mode;
     session.state = "running";
+    if (mode === "benchmark" && this.benchmarks) session.benchmarkId = this.benchmarks.start({ sessionId: session.id, deviceId: session.deviceId, source: serializeTrack(tracks[0]) });
     this.log.append({ kind: "session", state: "running", mode, session_id: session.id, tracks: tracks.map((track) => track.source_uuid) });
     return this.status(session.id);
   }
@@ -91,6 +93,12 @@ class SessionCoordinator {
     session.graceTimer = null;
     await session.delivery.stop();
     if (session.state !== "idle") await this.provider.stopSession(session.id);
+    if (session.benchmarkId && this.benchmarks) {
+      const benchmarkStatus = reason === "silence_timeout" ? "silence_timeout" : reason === "provider_failed" ? "failed" : reason === "disconnect" || reason === "plugin_shutdown" ? "aborted" : "completed";
+      const result = this.benchmarks.finish(session.id, benchmarkStatus);
+      if (result) session.send("benchmark_complete", benchmarkCompletion(result), session.id);
+      session.benchmarkId = null;
+    }
     session.state = "idle";
     session.mode = null;
     for (const track of session.tracks.values()) track.buffer.clear();
@@ -114,12 +122,13 @@ class SessionCoordinator {
     const session = this.sessions.get(sessionId);
     if (!session) return;
     session.connected = false;
-    if (session.state === "running") this.beginGrace(session);
+    if (session.state === "running" && session.mode === "benchmark") this.stop(session.id, "disconnect").catch(() => {});
+    else if (session.state === "running") this.beginGrace(session);
     else this.expireLater(session);
   }
   status(sessionId) {
     const session = this.require(sessionId);
-    return { session_id: session.id, state: session.state, mode: session.mode, obs: { ...session.obs }, grace_until: session.graceUntil || null, tracks: Array.from(session.tracks.values()).map(serializeTrack) };
+    return { session_id: session.id, state: session.state, mode: session.mode, benchmark_id: session.benchmarkId, obs: { ...session.obs }, grace_until: session.graceUntil || null, tracks: Array.from(session.tracks.values()).map(serializeTrack) };
   }
   summary() {
     const sessions = Array.from(this.sessions.values());
@@ -159,8 +168,10 @@ class SessionCoordinator {
       latency: { capture_ms: raw.capture_ms || 0, network_ms: raw.network_ms || 0, queue_ms: raw.queue_ms || 0, inference_ms: raw.inference_ms || 0, stabilization_ms: stable.stabilization_ms, total_ms: raw.total_ms || 0 },
       model: { id: raw.model_id || "unknown", provider: "whisper.cpp", backend: raw.backend || "unknown" }
     };
+    event.analysis = { transcript: String(raw.text || "").slice(0, 8000), words: Array.isArray(raw.words) ? raw.words.slice(0, 500) : [], emitted_at_ms: raw.emitted_at_ms || this.now() };
+    if (session.mode === "benchmark") this.benchmarks?.record(session.id, event);
     const result = await session.delivery.deliver(event);
-    this.log.append({ kind: "caption", session_id: session.id, source_uuid: track.source_uuid, caption_text: event.stable_text, uncertain_text: event.uncertain_text, revision: event.revision, final: event.final, delivery: result.disposition, latency: event.latency, model: event.model });
+    this.log.append({ kind: "caption", session_id: session.id, source_uuid: track.source_uuid, caption_text: session.mode === "benchmark" ? undefined : event.stable_text, uncertain_text: session.mode === "benchmark" ? undefined : event.uncertain_text, revision: event.revision, final: event.final, delivery: result.disposition, latency: event.latency, model: event.model, benchmark_id: session.benchmarkId });
   }
   async onProviderError(error) {
     const affected = Array.from(this.sessions.values()).filter((session) => session.state === "running" || session.state === "grace");
@@ -172,6 +183,11 @@ class SessionCoordinator {
       session.mode = null;
       for (const track of session.tracks.values()) track.buffer.clear();
       try { await session.delivery.stop(); } catch { }
+      if (session.benchmarkId && this.benchmarks) {
+        const result = this.benchmarks.finish(session.id, "failed");
+        if (result) session.send("benchmark_complete", benchmarkCompletion(result), session.id);
+        session.benchmarkId = null;
+      }
       session.send("error", {
         code: error.code || "PROVIDER_FAILED",
         message: `Speech recognition stopped: ${error.message}`,
@@ -186,5 +202,11 @@ function speakerLabel(session, track) { const enabled = Array.from(session.track
 function serializeTrack(track) { return { source_uuid: track.source_uuid, display_name: track.display_name, speaker_label: track.speaker_label, enabled: track.enabled, primary: track.primary, single_speaker: track.single_speaker, delivery_enabled: track.delivery_enabled, program_active: track.program_active, source_missing: track.source_missing, last_activity_at: track.last_activity_at, sequence: track.sequence?.metrics?.() }; }
 function text(value, max) { return String(value || "").trim().slice(0, max); }
 function isUuid(value) { return /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(String(value || "")); }
+function benchmarkCompletion(result) {
+  return {
+    id: result.id, status: result.status, transcript: result.transcript, stats: result.stats,
+    words: result.words.slice(0, 250).map((word) => ({ text: word.text, latency_ms: word.latency_ms, confidence: word.confidence, final: word.final }))
+  };
+}
 
 module.exports = { SessionCoordinator, serializeTrack };

plugins/lumi_transcription/backend/tests/benchmark_store.js

@@ -0,0 +1,134 @@
+const crypto = require("crypto");
+
+class BenchmarkStore {
+  constructor(db, options = {}) {
+    this.db = db;
+    this.now = options.now || Date.now;
+    this.retentionMs = options.retentionMs || 60 * 60 * 1000;
+    this.migrate();
+    this.db.prepare("UPDATE transcription_benchmark_tests SET status = 'aborted', ended_at = ? WHERE status = 'running'").run(this.now());
+    this.cleanup();
+  }
+
+  migrate() {
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS transcription_benchmark_tests (
+        id TEXT PRIMARY KEY, session_id TEXT NOT NULL UNIQUE, device_id TEXT NOT NULL,
+        source_uuid TEXT NOT NULL, source_name TEXT NOT NULL, started_at INTEGER NOT NULL,
+        ended_at INTEGER, status TEXT NOT NULL, model_id TEXT, backend TEXT
+      );
+      CREATE TABLE IF NOT EXISTS transcription_benchmark_revisions (
+        id INTEGER PRIMARY KEY AUTOINCREMENT, test_id TEXT NOT NULL, caption_id TEXT NOT NULL,
+        revision INTEGER NOT NULL, received_at INTEGER NOT NULL, final INTEGER NOT NULL,
+        text TEXT NOT NULL, words_json TEXT NOT NULL, inference_ms REAL NOT NULL,
+        model_id TEXT, backend TEXT,
+        UNIQUE(test_id, caption_id, revision)
+      );
+      CREATE INDEX IF NOT EXISTS transcription_benchmark_started_idx ON transcription_benchmark_tests(started_at DESC);
+      CREATE INDEX IF NOT EXISTS transcription_benchmark_revision_test_idx ON transcription_benchmark_revisions(test_id, received_at);
+    `);
+  }
+
+  start({ sessionId, deviceId, source }) {
+    this.cleanup();
+    const id = crypto.randomUUID();
+    this.db.prepare(`INSERT INTO transcription_benchmark_tests
+      (id, session_id, device_id, source_uuid, source_name, started_at, ended_at, status, model_id, backend)
+      VALUES (?, ?, ?, ?, ?, ?, NULL, 'running', NULL, NULL)`)
+      .run(id, sessionId, deviceId, source.source_uuid, source.display_name || "OBS source", this.now());
+    return id;
+  }
+
+  record(sessionId, event) {
+    const test = this.db.prepare("SELECT id FROM transcription_benchmark_tests WHERE session_id = ? AND status = 'running'").get(sessionId);
+    if (!test) return false;
+    const count = this.db.prepare("SELECT COUNT(*) AS count FROM transcription_benchmark_revisions WHERE test_id = ?").get(test.id).count;
+    if (count >= 10000) return false;
+    const words = Array.isArray(event.analysis?.words) ? event.analysis.words.slice(0, 500).map(safeWord) : [];
+    this.db.prepare(`INSERT OR IGNORE INTO transcription_benchmark_revisions
+      (test_id, caption_id, revision, received_at, final, text, words_json, inference_ms, model_id, backend)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
+      .run(test.id, event.caption_id, event.revision, this.now(), event.final ? 1 : 0,
+        String(event.analysis?.transcript || event.stable_text || event.uncertain_text || "").slice(0, 8000), JSON.stringify(words),
+        finite(event.latency?.inference_ms), event.model?.id || null, event.model?.backend || null);
+    this.db.prepare("UPDATE transcription_benchmark_tests SET model_id = COALESCE(?, model_id), backend = COALESCE(?, backend) WHERE id = ?")
+      .run(event.model?.id || null, event.model?.backend || null, test.id);
+    return true;
+  }
+
+  finish(sessionId, status = "completed") {
+    const allowed = ["completed", "silence_timeout", "aborted", "failed"].includes(status) ? status : "completed";
+    this.db.prepare("UPDATE transcription_benchmark_tests SET ended_at = ?, status = ? WHERE session_id = ? AND status = 'running'")
+      .run(this.now(), allowed, sessionId);
+    return this.bySession(sessionId);
+  }
+
+  bySession(sessionId) {
+    const row = this.db.prepare("SELECT * FROM transcription_benchmark_tests WHERE session_id = ?").get(sessionId);
+    return row ? this.snapshot(row) : null;
+  }
+
+  list() {
+    this.cleanup();
+    return this.db.prepare("SELECT * FROM transcription_benchmark_tests ORDER BY started_at DESC").all().map((row) => this.snapshot(row));
+  }
+
+  snapshot(row) {
+    const revisions = this.db.prepare("SELECT * FROM transcription_benchmark_revisions WHERE test_id = ? ORDER BY received_at, revision").all(row.id);
+    const captions = new Map();
+    for (const revision of revisions) {
+      const existing = captions.get(revision.caption_id);
+      if (!existing || (revision.final && !existing.final) || revision.final === existing.final && revision.revision > existing.revision) captions.set(revision.caption_id, revision);
+    }
+    const selected = Array.from(captions.values()).sort((a, b) => a.received_at - b.received_at);
+    const words = [];
+    for (const revision of selected) {
+      for (const word of parseWords(revision.words_json)) words.push({ ...word, confidence: revision.final ? word.confidence : null, final: Boolean(revision.final) });
+    }
+    const latency = metricStats(words.map((word) => word.latency_ms));
+    const confidence = metricStats(words.map((word) => word.confidence).filter(Number.isFinite));
+    return {
+      id: row.id, session_id: row.session_id, device_id: row.device_id,
+      source_uuid: row.source_uuid, source_name: row.source_name, started_at: row.started_at,
+      ended_at: row.ended_at, duration_ms: Math.max(0, (row.ended_at || this.now()) - row.started_at),
+      status: row.status, model_id: row.model_id, backend: row.backend,
+      transcript: selected.map((revision) => revision.text).filter(Boolean).join(" "), words,
+      stats: { latency, confidence }, revisions: revisions.length
+    };
+  }
+
+  cleanup(now = this.now()) {
+    const cutoff = now - this.retentionMs;
+    const ids = this.db.prepare("SELECT id FROM transcription_benchmark_tests WHERE ended_at IS NOT NULL AND ended_at < ?").all(cutoff).map((row) => row.id);
+    const removeRevisions = this.db.prepare("DELETE FROM transcription_benchmark_revisions WHERE test_id = ?");
+    const removeTest = this.db.prepare("DELETE FROM transcription_benchmark_tests WHERE id = ?");
+    this.db.transaction(() => { for (const id of ids) { removeRevisions.run(id); removeTest.run(id); } })();
+    return ids.length;
+  }
+}
+
+function safeWord(word) {
+  return {
+    text: String(word?.text || "").slice(0, 120),
+    latency_ms: finite(word?.latency_ms), confidence: clamp(word?.confidence, 0, 1),
+    audio_start_ms: finite(word?.audio_start_ms), audio_end_ms: finite(word?.audio_end_ms),
+    captured_at_ms: finite(word?.captured_at_ms)
+  };
+}
+function parseWords(value) { try { const parsed = JSON.parse(value); return Array.isArray(parsed) ? parsed.map(safeWord).filter((word) => word.text) : []; } catch { return []; } }
+function finite(value) { const number = Number(value); return Number.isFinite(number) ? Math.max(0, number) : 0; }
+function clamp(value, min, max) { return Math.min(max, Math.max(min, finite(value))); }
+function metricStats(input) {
+  const values = input.map(Number).filter(Number.isFinite).sort((a, b) => a - b);
+  if (!values.length) return { count: 0, min: null, low_1_average: null, median: null, average: null, p99: null, high_1_average: null, max: null };
+  const tail = Math.max(1, Math.ceil(values.length * 0.01));
+  return {
+    count: values.length, min: values[0], low_1_average: average(values.slice(0, tail)),
+    median: percentile(values, 0.5), average: average(values), p99: percentile(values, 0.99),
+    high_1_average: average(values.slice(-tail)), max: values.at(-1)
+  };
+}
+function average(values) { return values.reduce((sum, value) => sum + value, 0) / values.length; }
+function percentile(values, ratio) { const position = (values.length - 1) * ratio; const lower = Math.floor(position); const upper = Math.ceil(position); return values[lower] + (values[upper] - values[lower]) * (position - lower); }
+
+module.exports = { BenchmarkStore, metricStats };

plugins/lumi_transcription/backend/transcription/provider.js

@@ -186,9 +186,19 @@ class WhisperCppServerProvider extends TranscriptionProvider {
     this.sessions.add(session.id); this.worker.send({ type: "start_session", session });
   }
   async addTrack(sessionId, track) { this.worker.send({ type: "add_track", session_id: sessionId, track }); }
-  async pushAudio(sessionId, trackId, frame) { return this.worker.send({ type: "audio", session_id: sessionId, track_id: trackId, sequence: frame.sequence, capture_timestamp_us: frame.capture_timestamp_us, captured_at: Date.now() }, frame.pcm); }
+  async pushAudio(sessionId, trackId, frame) { return this.worker.send({ type: "audio", session_id: sessionId, track_id: trackId, sequence: frame.sequence, capture_timestamp_us: frame.capture_timestamp_us, captured_at: Math.floor(frame.capture_timestamp_us / 1000) }, frame.pcm); }
   async removeTrack(sessionId, trackId) { this.worker.send({ type: "remove_track", session_id: sessionId, track_id: trackId }); }
-  async stopSession(sessionId) { this.sessions.delete(sessionId); this.worker.send({ type: "stop_session", session_id: sessionId }); }
+  async stopSession(sessionId) {
+    this.sessions.delete(sessionId);
+    const stopped = new Promise((resolve) => {
+      const timer = setTimeout(() => { cleanup(); resolve(false); }, 5000);
+      const onStopped = (event) => { if (event.session_id !== sessionId) return; cleanup(); resolve(true); };
+      const cleanup = () => { clearTimeout(timer); this.off("session_stopped", onStopped); };
+      this.on("session_stopped", onStopped);
+    });
+    this.worker.send({ type: "stop_session", session_id: sessionId });
+    await stopped;
+  }
   async health() {
     const worker = this.worker.health();
     return {

plugins/lumi_transcription/backend/transcription/worker-native/src/main.cpp

@@ -35,12 +35,26 @@ struct track_state {
     std::vector<float> samples;
     std::uint64_t audio_start_us = 0;
     std::uint64_t audio_end_us = 0;
+    std::uint64_t captured_at_ms = 0;
     std::uint64_t last_voice_us = 0;
     std::uint64_t last_decode_us = 0;
     bool voiced = false;
     bool new_utterance = true;
 };
 
+struct word_analysis {
+    std::string text;
+    double confidence_sum = 0;
+    int confidence_pieces = 0;
+    std::int64_t t0 = 0;
+    std::int64_t t1 = 0;
+};
+
+struct transcription_result {
+    std::string text;
+    std::vector<word_analysis> words;
+};
+
 whisper_pointer context;
 std::string model_id = "unknown";
 std::string backend = "cpu";
@@ -112,7 +126,7 @@ void load_model(const json & packet) {
     emit({{"type", "model_loaded"}, {"model_id", model_id}, {"backend", backend}});
 }
 
-std::string transcribe(const std::vector<float> & samples, double & inference_ms) {
+transcription_result transcribe(const std::vector<float> & samples, double & inference_ms) {
     if (!context) return {};
     auto parameters = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);
     parameters.print_realtime = false;
@@ -122,31 +136,70 @@ std::string transcribe(const std::vector<float> & samples, double & inference_ms
     parameters.translate = false;
     parameters.no_context = true;
     parameters.single_segment = true;
+    parameters.token_timestamps = true;
+    parameters.split_on_word = true;
     parameters.language = "en";
     parameters.n_threads = std::max(1U, std::min(4U, std::thread::hardware_concurrency()));
     const auto started = clock_type::now();
     const auto result = whisper_full(context.get(), parameters, samples.data(), static_cast<int>(samples.size()));
     inference_ms = std::chrono::duration<double, std::milli>(clock_type::now() - started).count();
     if (result != 0) return {};
-    std::string text;
+    transcription_result output;
     const int segments = whisper_full_n_segments(context.get());
-    for (int index = 0; index < segments; ++index) text += whisper_full_get_segment_text(context.get(), index);
-    const auto first = text.find_first_not_of(" \t\r\n");
-    const auto last = text.find_last_not_of(" \t\r\n");
-    return first == std::string::npos ? std::string{} : text.substr(first, last - first + 1);
+    for (int segment = 0; segment < segments; ++segment) {
+        output.text += whisper_full_get_segment_text(context.get(), segment);
+        const int tokens = whisper_full_n_tokens(context.get(), segment);
+        for (int index = 0; index < tokens; ++index) {
+            const char * value = whisper_full_get_token_text(context.get(), segment, index);
+            if (!value || !*value) continue;
+            std::string raw(value);
+            if (raw.starts_with("<|")) continue;
+            const auto first = raw.find_first_not_of(" \t\r\n");
+            if (first == std::string::npos) continue;
+            const auto last = raw.find_last_not_of(" \t\r\n");
+            const bool begins_word = first > 0 || output.words.empty();
+            const auto data = whisper_full_get_token_data(context.get(), segment, index);
+            if (begins_word) output.words.push_back({});
+            auto & word = output.words.back();
+            word.text += raw.substr(first, last - first + 1);
+            word.confidence_sum += std::clamp(static_cast<double>(data.p), 0.0, 1.0);
+            word.confidence_pieces += 1;
+            if (word.confidence_pieces == 1) word.t0 = std::max<std::int64_t>(0, data.t0);
+            word.t1 = std::max(word.t0, data.t1);
+        }
+    }
+    const auto first = output.text.find_first_not_of(" \t\r\n");
+    const auto last = output.text.find_last_not_of(" \t\r\n");
+    output.text = first == std::string::npos ? std::string{} : output.text.substr(first, last - first + 1);
+    return output;
 }
 
 void decode_track(const std::string & session_id, const std::string & track_id, track_state & track, bool final) {
     if (!context || track.samples.size() < min_decode_samples) return;
     double inference_ms = 0;
-    const auto text = transcribe(track.samples, inference_ms);
-    if (!text.empty()) {
-        const bool incomplete = !final && std::isalnum(static_cast<unsigned char>(text.back())) != 0;
+    const auto result = transcribe(track.samples, inference_ms);
+    if (!result.text.empty()) {
+        const bool incomplete = !final && std::isalnum(static_cast<unsigned char>(result.text.back())) != 0;
+        const auto emitted_at_ms = static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
+            std::chrono::system_clock::now().time_since_epoch()).count());
+        json words = json::array();
+        for (const auto & word : result.words) {
+            const auto captured_at_ms = track.captured_at_ms + static_cast<std::uint64_t>(word.t0) * 10ULL;
+            words.push_back({
+                {"text", word.text},
+                {"confidence", word.confidence_pieces ? word.confidence_sum / word.confidence_pieces : 0.0},
+                {"audio_start_ms", static_cast<double>(word.t0) * 10.0},
+                {"audio_end_ms", static_cast<double>(word.t1) * 10.0},
+                {"captured_at_ms", captured_at_ms},
+                {"latency_ms", emitted_at_ms >= captured_at_ms ? emitted_at_ms - captured_at_ms : 0}
+            });
+        }
         emit({
-            {"type", "hypothesis"}, {"session_id", session_id}, {"track_id", track_id}, {"text", text},
+            {"type", "hypothesis"}, {"session_id", session_id}, {"track_id", track_id}, {"text", result.text},
             {"final", final}, {"new_utterance", track.new_utterance}, {"incomplete_word", incomplete},
             {"audio_start_us", track.audio_start_us}, {"audio_end_us", track.audio_end_us},
-            {"inference_ms", inference_ms}, {"model_id", model_id}, {"backend", backend}
+            {"inference_ms", inference_ms}, {"model_id", model_id}, {"backend", backend},
+            {"emitted_at_ms", emitted_at_ms}, {"words", std::move(words)}
         });
         track.new_utterance = false;
     }
@@ -156,6 +209,7 @@ void decode_track(const std::string & session_id, const std::string & track_id,
         track.voiced = false;
         track.new_utterance = true;
         track.audio_start_us = 0;
+        track.captured_at_ms = 0;
     }
 }
 
@@ -166,7 +220,10 @@ void audio(const json & packet, const std::vector<std::uint8_t> & pcm) {
     if (session_id.empty() || track_id.empty()) return;
     auto & track = tracks[session_id + "\n" + track_id];
     const auto captured_us = packet.value("capture_timestamp_us", std::uint64_t{0});
-    if (track.samples.empty()) track.audio_start_us = captured_us;
+    if (track.samples.empty()) {
+        track.audio_start_us = captured_us;
+        track.captured_at_ms = packet.value("captured_at", std::uint64_t{0});
+    }
     const auto count = pcm.size() / 2;
     double squares = 0;
     for (std::size_t index = 0; index < count; ++index) {
@@ -176,16 +233,12 @@ void audio(const json & packet, const std::vector<std::uint8_t> & pcm) {
         track.samples.push_back(normalized);
         squares += static_cast<double>(normalized) * normalized;
     }
-    if (track.samples.size() > max_samples) {
-        const auto removed = track.samples.size() - max_samples;
-        track.samples.erase(track.samples.begin(), track.samples.begin() + static_cast<std::ptrdiff_t>(removed));
-        track.audio_start_us += static_cast<std::uint64_t>(removed * 1000000ULL / sample_rate);
-    }
     const auto duration_us = static_cast<std::uint64_t>(count * 1000000ULL / sample_rate);
     track.audio_end_us = captured_us + duration_us;
     const auto rms = std::sqrt(squares / static_cast<double>(std::max<std::size_t>(1, count)));
     if (rms >= 0.012) { track.voiced = true; track.last_voice_us = track.audio_end_us; }
-    const bool final = track.voiced && track.audio_end_us > track.last_voice_us && track.audio_end_us - track.last_voice_us >= finalize_silence_us;
+    const bool final = track.voiced && (track.samples.size() >= max_samples ||
+        (track.audio_end_us > track.last_voice_us && track.audio_end_us - track.last_voice_us >= finalize_silence_us));
     const bool decode_due = track.last_decode_us == 0 || track.audio_end_us - track.last_decode_us >= static_cast<std::uint64_t>(decode_interval.count()) * 1000ULL;
     if (final || (track.voiced && decode_due)) decode_track(session_id, track_id, track, final);
 }
@@ -199,8 +252,13 @@ void handle(const json & packet, const std::vector<std::uint8_t> & pcm, bool & r
     else if (type == "audio") audio(packet, pcm);
     else if (type == "remove_track") tracks.erase(packet.value("session_id", "") + "\n" + packet.value("track_id", ""));
     else if (type == "stop_session") {
-        const auto prefix = packet.value("session_id", "") + "\n";
+        const auto session_id = packet.value("session_id", "");
+        const auto prefix = session_id + "\n";
+        for (auto & [key, track] : tracks) {
+            if (key.starts_with(prefix) && track.voiced) decode_track(session_id, key.substr(prefix.size()), track, true);
+        }
         std::erase_if(tracks, [&](const auto & item) { return item.first.starts_with(prefix); });
+        emit({{"type", "session_stopped"}, {"session_id", session_id}});
     }
     else if (type == "shutdown") running = false;
     else diagnostic("packet_rejected", "Unknown worker message type.");

plugins/lumi_transcription/companion_manifest.json

@@ -1,8 +1,8 @@
 {
   "schema_version": 1,
-  "version": "0.1.0-experimental.4",
+  "version": "0.1.0-experimental.5",
   "signed": false,
-  "release_notes": "Reports server inference failures at Speech recognition and handles abrupt WebSocket disconnects without leaving the test hanging.",
+  "release_notes": "Reliably reattaches the saved OBS source after startup, waits for finalized path-test captions, and adds a dedicated accuracy and latency benchmark with live statistics.",
   "artifacts": [
     {
       "id": "windows-x64-self-contained",
@@ -10,9 +10,9 @@
       "architecture": "x64",
       "label": "Windows x64 self-contained",
       "filename": "Lumi.Companion-win-x64.zip",
-      "url": "https://git.rolfsvaag.no/Rolfsvaag_Datateknikk/Lumi/releases/download/companion-v0.1.0-experimental.4/Lumi.Companion-win-x64.zip",
-      "sha256": "20b73e929c6eba816b5ff5737ced053cdcb1e737fac831bad7260f22d3cd2fe3",
-      "bytes": 41666903,
+      "url": "https://git.rolfsvaag.no/Rolfsvaag_Datateknikk/Lumi/releases/download/companion-v0.1.0-experimental.5/Lumi.Companion-win-x64.zip",
+      "sha256": "5728700f5559f957853b5e49f7f0efc3352ba631ee57526cc0fd56b96b24b733",
+      "bytes": 41676248,
       "entrypoint": "Lumi.Companion.App.exe"
     }
   ]

plugins/lumi_transcription/index.js

@@ -12,6 +12,7 @@ const { JsonlDiagnosticLog } = require("./backend/logs/jsonl_log");
 const { ArtifactManager } = require("./backend/models/artifact_manager");
 const { SessionCoordinator } = require("./backend/sessions/session_coordinator");
 const { WhisperWorkerSupervisor, WhisperCppServerProvider } = require("./backend/transcription/provider");
+const { BenchmarkStore } = require("./backend/tests/benchmark_store");
 const { ensureDataDirs, dataPath, resolveWorkerExecutable } = require("./backend/paths");
 const modelManifest = require("./models_manifest.json");
 const runtimeManifest = require("./runtime_manifest.json");
@@ -26,9 +27,10 @@ module.exports = {
     ensureDataDirs();
     const devices = new DeviceStore(db);
     const revisions = new RevisionStore(db);
+    const benchmarks = new BenchmarkStore(db);
     const diagnosticLog = new JsonlDiagnosticLog(dataPath("logs"));
     diagnosticLog.cleanup();
-    const cleanupTimer = setInterval(() => diagnosticLog.cleanup(), 60 * 60 * 1000);
+    const cleanupTimer = setInterval(() => { diagnosticLog.cleanup(); devices.cleanup(); benchmarks.cleanup(); }, 60 * 60 * 1000);
     cleanupTimer.unref?.();
     const supervisor = new WhisperWorkerSupervisor({
       executable: resolveWorkerExecutable(process.env.LUMI_TRANSCRIPTION_WORKER),
@@ -45,7 +47,8 @@ module.exports = {
     const sessions = new SessionCoordinator({
       provider,
       deliveryFactory: (send) => new CompanionCaptionDeliveryAdapter(send),
-      log: diagnosticLog
+      log: diagnosticLog,
+      benchmarks
     });
     const gateway = new CompanionGateway({ devices, sessions, log: diagnosticLog });
     const unregisterUpgrade = web.addUpgradeHandler("/plugins/lumi_transcription/live", (request, socket, head) => gateway.upgrade(request, socket, head));
@@ -74,6 +77,8 @@ module.exports = {
         pluginVersion: manifest.version,
         providerHealth: await provider.health(),
         devices: devices.list(),
+        revokedDevices: devices.list({ status: "revoked" }),
+        benchmarks: benchmarks.list(),
         settings: revisions.list(),
         models: modelManifest.models.map((entry) => ({ ...entry, status: models.status(entry) })),
         runtimeManifest,
@@ -83,7 +88,7 @@ module.exports = {
     });
     router.get("/api/status", requireAdmin, async (_req, res) => res.json({
       ok: true, plugin: { id: PLUGIN_ID, version: manifest.version }, protocol_version: 1,
-      provider: await provider.health(), devices: devices.list(), settings: revisions.list(),
+      provider: await provider.health(), devices: devices.list(), revoked_devices: devices.list({ status: "revoked" }), settings: revisions.list(),
       models: modelManifest.models.map((entry) => ({ id: entry.id, ...models.status(entry) })),
       runtime: runtimeManifest
     }));
@@ -127,13 +132,19 @@ module.exports = {
       try { res.set("Cache-Control", "no-store"); res.status(201).json({ ok: true, ...devices.exchange(req.body || {}) }); }
       catch (error) { res.status(error.code === "PAIRING_ALREADY_USED" ? 409 : 400).json({ ok: false, code: error.code, error: error.message }); }
     });
-    router.get("/api/devices", requireAdmin, (_req, res) => res.json({ devices: devices.list() }));
+    router.get("/api/devices", requireAdmin, (req, res) => res.json({ devices: devices.list({ status: req.query.status }) }));
     router.post("/api/devices/:id/revoke", requireAdmin, (req, res) => res.json({ ok: devices.revoke(req.params.id) }));
     router.post("/api/devices/:id/capabilities", requireAdmin, (req, res) => {
       const capabilities = devices.setCapabilities(req.params.id, req.body.capabilities);
       if (!capabilities) return res.status(404).json({ ok: false, error: "Device was not found or is revoked." });
       res.json({ ok: true, capabilities });
     });
+    router.get("/api/tests", requireAdmin, (_req, res) => res.json({ tests: benchmarks.list(), retention_ms: benchmarks.retentionMs }));
+    router.get("/api/tests/:id", requireAdmin, (req, res) => {
+      const test = benchmarks.list().find((entry) => entry.id === req.params.id);
+      if (!test) return res.status(404).json({ error: "Transcription test was not found or has expired." });
+      res.json({ test });
+    });
     router.get("/api/settings", requireSettingsAccess(devices), (_req, res) => res.json({ fields: revisions.list() }));
     router.patch("/api/settings", requireSettingsAccess(devices), (req, res) => {
       try {
@@ -221,7 +232,7 @@ function compareVersions(left, right) {
 }
 async function buildDashboardSummary({ provider, devices, sessions, companionPackages }) {
   const inference = await provider.health();
-  const activeDevices = devices.list().filter((device) => !device.revoked_at);
+  const activeDevices = devices.list();
   const connections = sessions.summary();
   const packageStatus = companionPackages.status();
   const issues = [];

plugins/lumi_transcription/plugin.json

@@ -1,7 +1,7 @@
 {
   "id": "lumi_transcription",
   "name": "Lumi Transcription",
-  "version": "0.1.0-experimental.4",
+  "version": "0.1.0-experimental.5",
   "description": "Server-hosted whisper.cpp transcription for Lumi Companion and OBS closed captions.",
   "main": "index.js",
   "channel": "experimental",

plugins/lumi_transcription/public/transcription.css

@@ -1 +1,2 @@
 .transcription-overview > .section-header{align-items:flex-start}.transcription-overview .page-header{margin:0}.transcription-overview .page-header h1{font-size:clamp(2rem,5vw,3.7rem)}.setup-path ol{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));list-style:none;padding:0;margin:0 0 var(--lumi-space-5);counter-reset:steps;gap:var(--lumi-space-3)}.setup-path li{counter-increment:steps;padding:var(--lumi-space-4);border-radius:var(--lumi-radius-md);background:var(--lumi-surface-subtle);min-width:0}.setup-path li::before{content:counter(steps);display:grid;place-items:center;width:1.8rem;height:1.8rem;margin-bottom:var(--lumi-space-3);border:1px solid var(--lumi-border);border-radius:50%;font-weight:700}.setup-path li.is-current::before{background:var(--lumi-primary);border-color:var(--lumi-primary);color:var(--lumi-button-text)}.setup-path li.is-complete::before{content:"✓";background:var(--lumi-success-bg);border-color:var(--lumi-success);color:var(--lumi-success)}.setup-path li span{display:block;margin-top:var(--lumi-space-2);color:var(--lumi-text-muted);font-size:.9rem}.transcription-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:var(--lumi-space-5)}.transcription-metrics{grid-template-columns:repeat(2,minmax(0,1fr));margin-bottom:var(--lumi-space-4)}.transcription-device-list{list-style:none;padding:0;margin:0;display:grid;gap:var(--lumi-space-2)}.transcription-device-list li,.model-row article{display:flex;align-items:center;justify-content:space-between;gap:var(--lumi-space-4);padding:var(--lumi-space-3) 0;border-bottom:1px solid var(--lumi-border)}.transcription-device-list li:last-child,.model-row article:last-child{border-bottom:0}.transcription-device-list .hint{display:block;margin-top:var(--lumi-space-1)}.model-row h3,.model-row p{margin:0}.model-actions{display:flex;align-items:center;justify-content:flex-end;gap:var(--lumi-space-2);flex-wrap:wrap}.empty-state{padding:var(--lumi-space-5);border-radius:var(--lumi-radius-md);background:var(--lumi-surface-subtle)}.empty-state p{margin-bottom:0;color:var(--lumi-text-muted)}@media(max-width:860px){.setup-path ol,.transcription-grid{grid-template-columns:1fr}.transcription-overview > .section-header{align-items:stretch}.transcription-metrics{grid-template-columns:1fr}}@media(max-width:560px){.model-row article,.transcription-device-list li{align-items:flex-start;flex-direction:column}.model-actions{justify-content:flex-start}}
+.benchmark-history{margin-top:var(--lumi-space-5)}.benchmark-test{border:1px solid var(--lumi-border);border-radius:var(--lumi-radius-md);margin-top:var(--lumi-space-3);overflow:visible;background:var(--lumi-surface)}.benchmark-test>summary{display:flex;align-items:center;justify-content:space-between;gap:var(--lumi-space-4);padding:var(--lumi-space-4);cursor:pointer;list-style:none}.benchmark-test>summary::-webkit-details-marker{display:none}.benchmark-test>summary span:first-child{display:grid;gap:var(--lumi-space-1)}.benchmark-test>summary small{color:var(--lumi-text-muted);font-weight:400}.benchmark-body{display:grid;gap:var(--lumi-space-4);padding:0 var(--lumi-space-4) var(--lumi-space-4);border-top:1px solid var(--lumi-border)}.analysis-mode{display:flex;gap:var(--lumi-space-2);padding-top:var(--lumi-space-4);flex-wrap:wrap}.device-view-toggle{padding:0 0 var(--lumi-space-3)}.benchmark-legends{display:grid;gap:var(--lumi-space-2)}.benchmark-legend{display:flex;align-items:center;gap:var(--lumi-space-2);flex-wrap:wrap;font-size:.75rem}.benchmark-legend strong{min-width:5rem}.benchmark-legend span{padding:.25rem .5rem;border-radius:999px;color:#182026}.legend-fast{background:#6f9f82}.legend-good{background:#6f91a3}.legend-trouble{background:#c0a65b}.legend-borderline{background:#c18152}.legend-critical{background:#b5686b}.benchmark-transcript{line-height:2.2;padding:var(--lumi-space-4);border-radius:var(--lumi-radius-md);background:var(--lumi-surface-subtle)}.benchmark-word{position:relative;display:inline-block;margin:.12rem .14rem;padding:.08rem .3rem;border-radius:.35rem;color:#172126;outline-offset:2px;transition:background .16s ease}.benchmark-word:hover::after,.benchmark-word:focus-visible::after{content:attr(data-tooltip);position:absolute;z-index:20;left:50%;bottom:calc(100% + .45rem);transform:translateX(-50%);width:max-content;max-width:min(22rem,80vw);padding:.45rem .6rem;border-radius:.45rem;background:#182026;color:#fff;font-size:.75rem;line-height:1.35;white-space:normal;box-shadow:0 6px 24px rgba(0,0,0,.2);pointer-events:none}.benchmark-stat-columns{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:var(--lumi-space-4)}.benchmark-stat-columns h3{margin:0 0 var(--lumi-space-3)}.benchmark-stat-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:var(--lumi-space-2)}.benchmark-stat-grid>div{padding:var(--lumi-space-3);border-radius:var(--lumi-radius-sm);background:var(--lumi-surface-subtle)}.benchmark-stat-grid span,.benchmark-stat-grid strong{display:block}.benchmark-stat-grid span{color:var(--lumi-text-muted);font-size:.78rem}.benchmark-stat-grid strong{margin-top:.2rem}@media(max-width:760px){.benchmark-stat-columns{grid-template-columns:1fr}.benchmark-test>summary{align-items:flex-start;flex-direction:column}}@media(prefers-reduced-motion:reduce){.benchmark-word{transition:none}}

plugins/lumi_transcription/public/transcription.js

@@ -3,6 +3,46 @@
   if (!root) return;
   const button = root.querySelector("[data-create-pairing]");
   const status = root.querySelector("[data-status]");
+  const activeList = root.querySelector("[data-active-devices]");
+  const revokedList = root.querySelector("[data-revoked-devices]");
+  root.querySelectorAll("[data-device-view]").forEach((button) => button.addEventListener("click", () => {
+    const revoked = button.dataset.deviceView === "revoked";
+    activeList.hidden = revoked; revokedList.hidden = !revoked;
+    root.querySelectorAll("[data-device-view]").forEach((choice) => {
+      const selected = choice === button; choice.setAttribute("aria-pressed", String(selected)); choice.classList.toggle("subtle", !selected);
+    });
+  }));
+
+  const latencyStops = [[0, "#6f9f82"], [750, "#6f9f82"], [1250, "#6f91a3"], [2000, "#c0a65b"], [3000, "#c18152"], [4500, "#b5686b"]];
+  const confidenceStops = [[0, "#b5686b"], [.45, "#c18152"], [.65, "#c0a65b"], [.8, "#6f91a3"], [.92, "#6f9f82"], [1, "#6f9f82"]];
+  const colorAt = (value, stops) => {
+    const bounded = Math.max(stops[0][0], Math.min(stops.at(-1)[0], Number(value)));
+    let upper = stops.findIndex(([point]) => point >= bounded);
+    if (upper <= 0) return stops[0][1];
+    const [lowPoint, lowColor] = stops[upper - 1]; const [highPoint, highColor] = stops[upper];
+    const ratio = highPoint === lowPoint ? 0 : (bounded - lowPoint) / (highPoint - lowPoint);
+    const rgb = [1, 3, 5].map((offset) => Math.round(parseInt(lowColor.slice(offset, offset + 2), 16) + (parseInt(highColor.slice(offset, offset + 2), 16) - parseInt(lowColor.slice(offset, offset + 2), 16)) * ratio));
+    return `rgb(${rgb.join(",")})`;
+  };
+  const paintTest = (test, mode) => {
+    test.dataset.analysisMode = mode;
+    test.querySelectorAll("[data-analysis-select]").forEach((button) => {
+      const selected = button.dataset.analysisSelect === mode;
+      button.setAttribute("aria-pressed", String(selected)); button.classList.toggle("subtle", !selected);
+    });
+    test.querySelectorAll(".benchmark-word").forEach((word) => {
+      const latency = colorAt(word.dataset.latency, latencyStops);
+      const confidence = word.dataset.confidence === "" ? "#aeb8bc" : colorAt(word.dataset.confidence, confidenceStops);
+      word.style.background = mode === "latency" ? latency : mode === "confidence" ? confidence : `linear-gradient(to bottom, ${latency} 0 60%, ${confidence} 60% 100%)`;
+    });
+  };
+  root.querySelectorAll("[data-benchmark-test]").forEach((test) => {
+    paintTest(test, test.dataset.analysisMode || "combined");
+    test.addEventListener("click", (event) => {
+      const choice = event.target.closest("[data-analysis-select]");
+      if (choice) paintTest(test, choice.dataset.analysisSelect);
+    });
+  });
   button?.addEventListener("click", async () => {
     button.disabled = true;
     status.textContent = "Creating a one-time pairing package…";

plugins/lumi_transcription/tests/verify.js

@@ -22,6 +22,7 @@ const { BoundedQueue, SequenceTracker, RollingPcmBuffer } = require("../backend/
 const { SessionCoordinator } = require("../backend/sessions/session_coordinator");
 const { CaptionStabilizer, LatestCaptionGate } = require("../backend/transcription/stabilizer");
 const { WhisperWorkerSupervisor, WhisperCppServerProvider } = require("../backend/transcription/provider");
+const { BenchmarkStore, metricStats } = require("../backend/tests/benchmark_store");
 const plugin = require("../index");
 const { createWebUpgradeRegistry } = require("../../../src/services/web-upgrades");
 
@@ -36,6 +37,7 @@ async function run() {
     verifyRevisions();
     verifyQueues();
     verifyStabilization();
+    verifyBenchmarkRetention();
     await verifySessionLifecycle();
     await verifyProviderFailureFeedback();
     await verifyWorkerRestart();
@@ -77,9 +79,13 @@ function verifyPairingAndRevocation() {
   assert.equal(store.authenticate(`LumiDevice ${issued.device_id}.${issued.device_secret}`, "transcription.capture.v1").reason, "capability_revoked");
   assert.equal(store.revoke(issued.device_id), true);
   assert.equal(store.authenticate(`LumiDevice ${issued.device_id}.${issued.device_secret}`).reason, "device_revoked");
+  assert.equal(store.list().length, 0);
+  assert.equal(store.list({ status: "revoked" }).length, 1);
   const expired = store.issuePairing({ userId: "admin", host: "https://lumi.example", ttlMs: 50 });
   now += 51;
   assert.throws(() => store.exchange({ token: expired.token, device: {} }), (error) => error.code === "PAIRING_INVALID");
+  now += 30 * 86400000;
+  assert.equal(store.cleanup(), 1);
   db.close();
 }
 
@@ -99,6 +105,7 @@ function verifyLocalhostTransportPolicy() {
 }
 
 function verifyCompanionVersionOrdering() {
+  assert.equal(plugin.compareVersions("0.1.0-experimental.5", "0.1.0-experimental.4"), 1);
   assert.equal(plugin.compareVersions("0.1.0-experimental.4", "0.1.0-experimental.3"), 1);
   assert.equal(plugin.compareVersions("0.1.0-experimental.3", "0.1.0-experimental.2"), 1);
   assert.equal(plugin.compareVersions("0.1.0-experimental.2", "0.1.0-experimental.2"), 0);
@@ -161,6 +168,30 @@ function verifyStabilization() {
   assert.equal(gate.accept({ session_id: source, caption_id: fragment.caption_id, revision: 1 }), false);
 }
 
+function verifyBenchmarkRetention() {
+  const db = new Database(":memory:");
+  let now = 1000;
+  const store = new BenchmarkStore(db, { now: () => now, retentionMs: 3600000 });
+  const source = crypto.randomUUID();
+  const id = store.start({ sessionId: "benchmark-session", deviceId: "device", source: { source_uuid: source, display_name: "Mic" } });
+  store.record("benchmark-session", {
+    caption_id: "caption", revision: 1, final: true, stable_text: "hello world",
+    analysis: { transcript: "hello world", words: [
+      { text: "hello", latency_ms: 700, confidence: .96, audio_start_ms: 0, audio_end_ms: 300 },
+      { text: "world", latency_ms: 1400, confidence: .82, audio_start_ms: 300, audio_end_ms: 700 }
+    ] }, latency: { inference_ms: 500 }, model: { id: "small.en", backend: "cpu" }
+  });
+  const result = store.finish("benchmark-session", "completed");
+  assert.equal(result.id, id);
+  assert.equal(result.words.length, 2);
+  assert.equal(result.stats.latency.average, 1050);
+  assert.equal(metricStats([1, 2, 3]).median, 2);
+  now += 3600001;
+  assert.equal(store.cleanup(), 1);
+  assert.equal(store.list().length, 0);
+  db.close();
+}
+
 async function verifySessionLifecycle() {
   class Provider extends EventEmitter {
     constructor() { super(); this.audio = []; this.stops = 0; }
@@ -253,6 +284,10 @@ async function verifyNativeWorkerBoundary() {
   assert.match(cmake, /f049fff95a089aa9969deb009cdd4892b3e74916/);
   assert.match(source, /max_samples = sample_rate \* 6/);
   assert.match(source, /_setmode\(_fileno\(stdin\), _O_BINARY\)/);
+  assert.match(source, /token_timestamps = true/);
+  assert.match(source, /session_stopped/);
+  const bridge = fs.readFileSync(path.join(__dirname, "../../../companion/native/obs-bridge/src/plugin.cpp"), "utf8");
+  assert.match(bridge, /selection_state/);
   assert.doesNotMatch(source, /ofstream|fwrite|WriteAllBytes/);
 }
 

plugins/lumi_transcription/views/settings.ejs

@@ -48,18 +48,77 @@
   </section>
 
   <section class="card" aria-labelledby="devices-title">
-    <div class="section-header"><div><span class="eyebrow">Access</span><h2 id="devices-title">Paired devices</h2><p class="hint">Credentials are revocable and scoped to Companion capabilities.</p></div><span class="badge"><%= devices.filter((device) => !device.revoked_at).length %> active</span></div>
-    <% if (!devices.length) { %><div class="empty-state"><strong>No Companion is paired</strong><p>Use Download Companion to create a package that expires after 15 minutes and works once.</p></div><% } %>
+    <div class="section-header"><div><span class="eyebrow">Access</span><h2 id="devices-title">Paired devices</h2><p class="hint">Active credentials are shown first. Switch to revoked history when needed; revoked records are permanently removed after 30 days.</p></div><span class="badge"><%= devices.length %> active</span></div>
+    <div class="analysis-mode device-view-toggle" role="group" aria-label="Paired device status">
+      <button class="button" type="button" data-device-view="active" aria-pressed="true">Active (<%= devices.length %>)</button>
+      <button class="button subtle" type="button" data-device-view="revoked" aria-pressed="false">Revoked (<%= revokedDevices.length %>)</button>
+    </div>
+    <div data-active-devices>
+      <% if (!devices.length) { %><div class="empty-state"><strong>No active Companion is paired</strong><p>Use Download Companion to create a package that expires after 15 minutes and works once.</p></div><% } %>
       <% if (devices.length) { %>
         <ul class="transcription-device-list">
           <% devices.forEach((device) => { %>
-      <li><div><strong><%= device.name %></strong><span class="hint"><%= device.revoked_at ? 'Revoked' : 'Last connected ' + new Date(device.last_connected_at).toLocaleString() %></span></div><span class="badge <%= device.revoked_at ? 'danger' : 'success' %>"><%= device.revoked_at ? 'Revoked' : 'Allowed' %></span></li>
+            <li><div><strong><%= device.name %></strong><span class="hint">Last connected <%= new Date(device.last_connected_at).toLocaleString() %><% if (device.metadata?.companion_version) { %> · Companion <%= device.metadata.companion_version %><% } %></span></div><span class="badge success">Allowed</span></li>
           <% }) %>
         </ul>
       <% } %>
+    </div>
+    <div data-revoked-devices hidden>
+      <% if (!revokedDevices.length) { %><div class="empty-state"><strong>No revoked device history</strong><p>Revoked credentials will appear here for up to 30 days.</p></div><% } %>
+      <% if (revokedDevices.length) { %>
+        <ul class="transcription-device-list">
+          <% revokedDevices.forEach((device) => { %><li><div><strong><%= device.name %></strong><span class="hint">Revoked <%= new Date(device.revoked_at).toLocaleString() %> · permanent removal by <%= new Date(device.revoked_at + 30 * 86400000).toLocaleDateString() %></span></div><span class="badge danger">Revoked</span></li><% }) %>
+        </ul>
+      <% } %>
+      </div>
   </section>
 </div>
 
+<section class="card benchmark-history" aria-labelledby="benchmark-history-title" data-transcription-admin>
+  <div class="section-header">
+    <div><span class="eyebrow">Last hour</span><h2 id="benchmark-history-title">Transcription test analysis</h2><p class="hint">Inspect each dedicated Companion test by latency, finalized confidence, or both. Test transcripts and measurements are permanently deleted after one hour.</p></div>
+    <span class="badge"><%= benchmarks.length %> retained</span>
+  </div>
+  <% if (!benchmarks.length) { %>
+    <div class="empty-state"><strong>No dedicated tests in the last hour</strong><p>Start Accuracy &amp; latency test from the Companion Test page. Full-path checks are intentionally not retained here.</p></div>
+  <% } %>
+  <% benchmarks.forEach((test) => { const metricRows = [
+    ['Minimum', 'min'], ['Low 1% avg', 'low_1_average'], ['Median', 'median'], ['Average', 'average'],
+    ['P99', 'p99'], ['High 1% avg', 'high_1_average'], ['Maximum', 'max']
+  ]; %>
+    <details class="benchmark-test" data-benchmark-test data-analysis-mode="combined">
+      <summary>
+        <span><strong><%= new Date(test.started_at).toLocaleString() %></strong><small><%= test.source_name %> · <%= test.model_id || 'model pending' %> / <%= test.backend || 'backend pending' %> · <%= Math.round(test.duration_ms / 1000) %>s</small></span>
+        <span class="badge <%= test.status === 'completed' || test.status === 'silence_timeout' ? 'success' : test.status === 'running' ? 'info' : 'warning' %>"><%= test.status.replaceAll('_', ' ') %></span>
+      </summary>
+      <div class="benchmark-body">
+        <div class="analysis-mode" role="group" aria-label="Transcript analysis coloring">
+          <button class="button subtle" type="button" data-analysis-select="latency" aria-pressed="false">Latency</button>
+          <button class="button subtle" type="button" data-analysis-select="confidence" aria-pressed="false">Confidence</button>
+          <button class="button" type="button" data-analysis-select="combined" aria-pressed="true">Combined</button>
+        </div>
+        <div class="benchmark-legends">
+          <div class="benchmark-legend" aria-label="Latency color meaning"><strong>Latency</strong><span class="legend-fast">Green &lt;750 ms</span><span class="legend-good">Blue &lt;1,250 ms</span><span class="legend-trouble">Yellow &lt;2,000 ms</span><span class="legend-borderline">Orange &lt;3,000 ms</span><span class="legend-critical">Red ≥3,000 ms</span></div>
+          <div class="benchmark-legend" aria-label="Confidence color meaning"><strong>Confidence</strong><span class="legend-fast">Green ≥92%</span><span class="legend-good">Blue ≥80%</span><span class="legend-trouble">Yellow ≥65%</span><span class="legend-borderline">Orange ≥45%</span><span class="legend-critical">Red &lt;45%</span></div>
+        </div>
+        <div class="benchmark-transcript" aria-label="Measured transcript">
+          <% if (!test.words.length) { %><p class="hint"><%= test.transcript || 'No finalized words were measured.' %></p><% } %>
+          <% test.words.forEach((word) => { const confidenceLabel = Number.isFinite(word.confidence) ? `${(word.confidence * 100).toFixed(1)}%` : 'not finalized'; %>
+            <span class="benchmark-word" tabindex="0" data-latency="<%= word.latency_ms %>" data-confidence="<%= Number.isFinite(word.confidence) ? word.confidence : '' %>" data-tooltip="Latency <%= Math.round(word.latency_ms) %> ms · Confidence <%= confidenceLabel %> · Audio <%= Math.round(word.audio_start_ms) %>–<%= Math.round(word.audio_end_ms) %> ms"><%= word.text %></span>
+          <% }) %>
+        </div>
+        <div class="benchmark-stat-columns">
+          <% [['Latency', test.stats.latency, (value) => `${Math.round(value)} ms`], ['Final confidence', test.stats.confidence, (value) => `${(value * 100).toFixed(1)}%`]].forEach(([label, stats, format]) => { %>
+            <section><h3><%= label %></h3><div class="benchmark-stat-grid">
+              <% metricRows.forEach(([metricLabel, key]) => { %><div><span><%= metricLabel %></span><strong><%= Number.isFinite(stats[key]) ? format(stats[key]) : '—' %></strong></div><% }) %>
+            </div><p class="hint"><%= stats.count %> measured values</p></section>
+          <% }) %>
+        </div>
+      </div>
+    </details>
+  <% }) %>
+</section>
+
 <section class="card" id="speech-models" aria-labelledby="models-title" data-transcription-admin>
   <div class="section-header"><div><span class="eyebrow">Curated choices</span><h2 id="models-title">Speech models</h2><p class="hint">Downloads require confirmation and checksum verification before a model can load.</p></div></div>
   <div class="model-row">