diff --git a/TODO.md b/TODO.md index 39ad72a..7d7cd5d 100644 --- a/TODO.md +++ b/TODO.md @@ -32,6 +32,13 @@ is explicitly binary-safe, broken worker pipes no longer terminate Lumi, the worker restarts and reloads the selected model automatically, and the Companion test reports the server failure at Speech recognition instead of hanging. +Experimental.5 retries and acknowledges the saved OBS source attachment after +reconnects, waits for finalized path-test captions, and adds a separate +user-controlled accuracy/latency test with a ten-second silence cutoff. The host +retains per-word latency/confidence inspection data for one hour, while paired +device management now defaults to active credentials and purges revoked history +after 30 days. + Release-blocking work remains: - Package the native whisper.cpp worker as the Windows CUDA runtime, then benchmark diff --git a/companion/native/obs-bridge/src/plugin.cpp b/companion/native/obs-bridge/src/plugin.cpp index 5f9fb87..c4455db 100644 --- a/companion/native/obs-bridge/src/plugin.cpp +++ b/companion/native/obs-bridge/src/plugin.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -334,7 +335,7 @@ static bool output_caption(const std::string &text, double display_seconds) return true; } -static void handle_command(const json &message) +static std::optional handle_command(const json &message) { const auto type = message.value("type", ""); if (type == "select_sources") { @@ -343,12 +344,14 @@ static void handle_command(const json &message) source_list_dirty.store(true, std::memory_order_release); blog(installed ? LOG_INFO : LOG_WARNING, "[Lumi Companion] %s selected OBS audio source %s", installed ? "Attached" : "Could not attach", uuid.c_str()); + return json{{"type", "selection_state"}, {"protocol_version", protocol_version}, {"source_uuid", uuid}, {"attached", installed}}; } else if (type == "caption" && message.contains("payload")) { const auto &payload = message["payload"]; const auto text = payload.value("stable_text", ""); const auto duration = payload.value("display_seconds", 2.0); output_caption(text, duration); } + return std::nullopt; } static bool read_available_command(HANDLE pipe) @@ -363,7 +366,10 @@ static bool read_available_command(HANDLE pipe) if (size == 0 || size > max_json_bytes) return false; std::vector body(size); if (!read_exact(pipe, body.data(), body.size())) return false; - try { handle_command(json::parse(body.begin(), body.end())); } + try { + const auto response = handle_command(json::parse(body.begin(), body.end())); + if (response && !write_json(pipe, *response)) return false; + } catch (const std::exception &error) { blog(LOG_WARNING, "[Lumi Companion] Ignored invalid IPC command: %s", error.what()); } return true; } diff --git a/companion/plugins/Lumi.Companion.Transcription/ObsBridgePipe.cs b/companion/plugins/Lumi.Companion.Transcription/ObsBridgePipe.cs index b6ee5b3..26dfe57 100644 --- a/companion/plugins/Lumi.Companion.Transcription/ObsBridgePipe.cs +++ b/companion/plugins/Lumi.Companion.Transcription/ObsBridgePipe.cs @@ -7,7 +7,7 @@ namespace Lumi.Companion.Transcription; public sealed class ObsBridgePipe : IAsyncDisposable { - private static readonly HashSet AllowedTypes = ["hello", "source_list", "source_state", "obs_state", "health"]; + private static readonly HashSet AllowedTypes = ["hello", "source_list", "source_state", "selection_state", "obs_state", "health"]; private readonly string _pipeName; private readonly Func _onMessage; private readonly Func, Task> _onAudio; diff --git a/companion/scripts/build-obs-bridge.ps1 b/companion/scripts/build-obs-bridge.ps1 index a3c9e22..6eeacff 100644 --- a/companion/scripts/build-obs-bridge.ps1 +++ b/companion/scripts/build-obs-bridge.ps1 @@ -1,6 +1,6 @@ param( [string]$ObsVersion = "31.1.1", - [string]$BridgeVersion = "0.1.0-experimental.3", + [string]$BridgeVersion = "0.1.0-experimental.5", [string]$CacheRoot = "$env:LOCALAPPDATA\LumiCompanionBuild" ) diff --git a/companion/scripts/publish-companion.ps1 b/companion/scripts/publish-companion.ps1 index b8cfc3c..73359e6 100644 --- a/companion/scripts/publish-companion.ps1 +++ b/companion/scripts/publish-companion.ps1 @@ -1,6 +1,6 @@ param( - [string]$Version = "0.1.0-experimental.4", - [string]$BridgeVersion = "0.1.0-experimental.3" + [string]$Version = "0.1.0-experimental.5", + [string]$BridgeVersion = "0.1.0-experimental.5" ) $ErrorActionPreference = "Stop" diff --git a/companion/src/Lumi.Companion.App/App.axaml.cs b/companion/src/Lumi.Companion.App/App.axaml.cs index 42b79d7..9553d74 100644 --- a/companion/src/Lumi.Companion.App/App.axaml.cs +++ b/companion/src/Lumi.Companion.App/App.axaml.cs @@ -82,8 +82,8 @@ public partial class App : Application health.Header = $"Health: {state.Summary}"; update.Header = state.UpdateAvailable ? $"Update available: {state.AvailableVersion}" : "Updates: Current"; update.IsEnabled = state.UpdateAvailable; - test.Header = state.TestRunning ? "Transcription test running…" : "Run transcription test"; - test.IsEnabled = !state.TestRunning; + test.Header = state.TestRunning || state.BenchmarkRunning ? "Transcription test running…" : "Run transcription test"; + test.IsEnabled = !state.TestRunning && !state.BenchmarkRunning; }); } diff --git a/companion/src/Lumi.Companion.App/CompanionRuntime.cs b/companion/src/Lumi.Companion.App/CompanionRuntime.cs index 8986e59..101a846 100644 --- a/companion/src/Lumi.Companion.App/CompanionRuntime.cs +++ b/companion/src/Lumi.Companion.App/CompanionRuntime.cs @@ -1,3 +1,4 @@ +using System.Buffers.Binary; using System.Diagnostics; using System.Reflection; using System.Security.Cryptography; @@ -26,6 +27,13 @@ public sealed class CompanionRuntime : IAsyncDisposable private TaskCompletionSource? _captionSignal; private TaskCompletionSource? _sessionStartSignal; private TaskCompletionSource? _testFailure; + private TaskCompletionSource<(string SourceUuid, bool Attached)>? _bridgeSelectionSignal; + private readonly SemaphoreSlim _bridgeSelectionGate = new(1, 1); + private readonly Dictionary _benchmarkCaptions = []; + private CancellationTokenSource? _benchmarkLifetime; + private TaskCompletionSource? _benchmarkCompleteSignal; + private DateTimeOffset? _benchmarkLastVoiceAt; + private int _benchmarkStopping; private bool _disposed; private CompanionUpdate? _availableUpdate; @@ -38,14 +46,17 @@ public sealed class CompanionRuntime : IAsyncDisposable State = new CompanionState(); TestStages = CreateInitialTestStages(); ObsSources = []; + Benchmark = EmptyBenchmark("Not started"); } public CompanionState State { get; private set; } public IReadOnlyList TestStages { get; private set; } public IReadOnlyList ObsSources { get; private set; } + public BenchmarkSnapshot Benchmark { get; private set; } public event Action? StateChanged; public event Action>? TestStagesChanged; public event Action>? ObsSourcesChanged; + public event Action? BenchmarkChanged; public event Action? CaptionReceived; public event Action? LogAdded; public event Func? UpdateRestartRequested; @@ -130,7 +141,8 @@ public sealed class CompanionRuntime : IAsyncDisposable { if (_disposed) return; _testFailure?.TrySetResult(error is null ? "The Lumi connection closed during the test." : $"The Lumi connection closed during the test: {Friendly(error)}"); - SetState(State with { Connected = false, Health = TrayHealth.Degraded, Detail = "The secure Lumi connection closed. Retry when the host is available." }); + _benchmarkLifetime?.Cancel(); + SetState(State with { Connected = false, BenchmarkRunning = false, Health = TrayHealth.Degraded, Detail = "The secure Lumi connection closed. Retry when the host is available.", BenchmarkDetail = State.BenchmarkRunning ? "The benchmark was aborted because the Lumi connection closed." : State.BenchmarkDetail }); _ = WriteLogAsync("disconnected", error?.Message ?? "Connection closed."); }; SetState(State with { Connected = false, Detail = "Connecting securely to Lumi…" }); @@ -163,7 +175,7 @@ public sealed class CompanionRuntime : IAsyncDisposable public async Task RunTestAsync(CancellationToken cancellationToken = default) { - if (State.TestRunning) return; + if (State.TestRunning || State.BenchmarkRunning) return; SetState(State with { TestRunning = true, Detail = "Running the transcription path check…" }); var stages = CreateInitialTestStages().ToArray(); TestStages = stages; @@ -175,8 +187,9 @@ public sealed class CompanionRuntime : IAsyncDisposable await EvaluateStageAsync(stages, 1, State.ObsConnected, "OBS reported a healthy local connection.", "Open OBS 31 or newer and retry.", cancellationToken); if (!State.ObsConnected) return; var selected = ObsSources.FirstOrDefault(source => source.Uuid == _settings.Current.PrimarySourceUuid); - await EvaluateStageAsync(stages, 2, selected is { Missing: false, Active: true }, "The selected microphone is available and active in Program.", selected is null ? "Choose a microphone on the Transcription page." : selected.Missing ? "The selected microphone is missing from OBS." : "Put the selected microphone in the active Program scene, then retry.", cancellationToken); - if (selected is not { Missing: false, Active: true }) return; + var bridgeAttached = selected is { Missing: false } && await SyncBridgeSelectionAsync(cancellationToken); + await EvaluateStageAsync(stages, 2, selected is { Missing: false, Active: true } && bridgeAttached, "The selected microphone is available, active in Program, and attached for capture.", selected is null ? "Choose a microphone on the Transcription page." : selected.Missing ? "The selected microphone is missing from OBS." : !bridgeAttached ? "Companion could not attach the selected OBS source. Keep OBS open and retry." : "Put the selected microphone in the active Program scene, then retry.", cancellationToken); + if (selected is not { Missing: false, Active: true } || !bridgeAttached) return; await EvaluateStageAsync(stages, 4, State.Connected, "The secure Lumi transport is ready.", "Reconnect to Lumi before testing.", cancellationToken); if (!State.Connected) return; _audioSignal = new(TaskCreationOptions.RunContinuationsAsynchronously); @@ -224,6 +237,87 @@ public sealed class CompanionRuntime : IAsyncDisposable } } + public async Task StartBenchmarkAsync(CancellationToken cancellationToken = default) + { + if (State.BenchmarkRunning || State.TestRunning) return; + if (!State.Connected || _socket is null) throw new InvalidOperationException("Reconnect to Lumi before starting the transcription test."); + if (!State.ObsConnected) throw new InvalidOperationException("Open OBS and wait for the managed integration to connect."); + var selected = ObsSources.FirstOrDefault(source => source.Uuid == _settings.Current.PrimarySourceUuid); + if (selected is not { Missing: false, Active: true }) throw new InvalidOperationException("Choose an available microphone that is active in the OBS Program scene."); + if (!await SyncBridgeSelectionAsync(cancellationToken)) throw new InvalidOperationException("Companion could not attach the selected OBS source. Keep OBS open and retry."); + + _benchmarkCaptions.Clear(); + _sessionStartSignal = new(TaskCreationOptions.RunContinuationsAsynchronously); + _testFailure = new(TaskCreationOptions.RunContinuationsAsynchronously); + _benchmarkCompleteSignal = new(TaskCreationOptions.RunContinuationsAsynchronously); + _benchmarkLastVoiceAt = DateTimeOffset.UtcNow; + Interlocked.Exchange(ref _benchmarkStopping, 0); + Benchmark = EmptyBenchmark("Starting", DateTimeOffset.UtcNow); + BenchmarkChanged?.Invoke(Benchmark); + SetState(State with { BenchmarkRunning = true, BenchmarkDetail = "Starting server-hosted accuracy and latency measurement…", Health = TrayHealth.Operating }); + try + { + await _socket.SendAsync("start", new { mode = "benchmark" }, _socket.SessionId, cancellationToken); + var started = await WaitForTestSignalAsync(_sessionStartSignal.Task, TimeSpan.FromSeconds(8), cancellationToken); + if (!started.Completed) + throw new InvalidOperationException(started.Failure ?? "Lumi did not start the benchmark within 8 seconds."); + + _benchmarkLifetime?.Cancel(); + _benchmarkLifetime?.Dispose(); + _benchmarkLifetime = CancellationTokenSource.CreateLinkedTokenSource(_maintenanceLifetime.Token); + _ = MonitorBenchmarkSilenceAsync(_benchmarkLifetime.Token); + SetState(State with { BenchmarkDetail = "Listening. Speak naturally; end the test manually, or remain silent for 10 seconds." }); + await WriteLogAsync("benchmark_started", $"Started transcription benchmark for {selected.Name}."); + } + catch (Exception error) + { + _benchmarkLifetime?.Cancel(); + _sessionStartSignal = null; + _testFailure = null; + _benchmarkCompleteSignal = null; + SetState(State with { BenchmarkRunning = false, BenchmarkDetail = Friendly(error), Health = TrayHealth.Degraded }); + throw; + } + } + + public async Task StopBenchmarkAsync(string reason = "benchmark_complete", CancellationToken cancellationToken = default) + { + if (!State.BenchmarkRunning || _socket is null || Interlocked.Exchange(ref _benchmarkStopping, 1) != 0) return; + _benchmarkLifetime?.Cancel(); + SetState(State with { BenchmarkDetail = reason == "silence_timeout" ? "Ten seconds of silence detected. Finalizing the test…" : "Finalizing the test…" }); + try + { + await _socket.SendAsync("stop", new { reason }, _socket.SessionId, cancellationToken); + if (_benchmarkCompleteSignal is not null) + await Task.WhenAny(_benchmarkCompleteSignal.Task, Task.Delay(TimeSpan.FromSeconds(7), cancellationToken)); + } + finally + { + SetState(State with { BenchmarkRunning = false, BenchmarkDetail = Benchmark.Words.Count > 0 ? $"Test complete with {Benchmark.Words.Count} measured words or phrases." : "The test ended without a finalized transcript." }); + _sessionStartSignal = null; + _testFailure = null; + Interlocked.Exchange(ref _benchmarkStopping, 0); + } + } + + private async Task MonitorBenchmarkSilenceAsync(CancellationToken cancellationToken) + { + try + { + using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(500)); + while (await timer.WaitForNextTickAsync(cancellationToken)) + { + if (!State.BenchmarkRunning) return; + if (_benchmarkLastVoiceAt is { } lastVoice && DateTimeOffset.UtcNow - lastVoice >= TimeSpan.FromSeconds(10)) + { + await StopBenchmarkAsync("silence_timeout", CancellationToken.None); + return; + } + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { } + } + private async Task EvaluateStageAsync(TestStage[] stages, int index, bool passed, string success, string blocked, CancellationToken cancellationToken) { MarkStage(stages, index, TestStageState.Running, "Checking…"); @@ -359,23 +453,40 @@ public sealed class CompanionRuntime : IAsyncDisposable { if (message.Type == "status" && message.Payload.TryGetProperty("kind", out var kind) && kind.GetString() == "session" && message.Payload.TryGetProperty("state", out var state) && state.GetString() == "running") + { _sessionStartSignal?.TrySetResult(true); + if (message.Payload.TryGetProperty("benchmark_id", out var benchmarkId) && benchmarkId.ValueKind == JsonValueKind.String) + { + Benchmark = Benchmark with { Id = benchmarkId.GetString(), Status = "Running" }; + BenchmarkChanged?.Invoke(Benchmark); + } + } if (message.Type == "caption") { - var text = message.Payload.TryGetProperty("stable_text", out var stable) ? stable.GetString() : null; + var stableText = message.Payload.TryGetProperty("stable_text", out var stable) ? stable.GetString() : null; + var uncertainText = message.Payload.TryGetProperty("uncertain_text", out var uncertain) ? uncertain.GetString() : null; + var text = string.Join(" ", new[] { stableText, uncertainText }.Where(value => !string.IsNullOrWhiteSpace(value))); var simulated = message.Payload.TryGetProperty("delivery", out var delivery) && delivery.TryGetProperty("disposition", out var disposition) && disposition.GetString() == "simulated"; if (!string.IsNullOrWhiteSpace(text)) { - if (simulated) _captionSignal?.TrySetResult(true); + var final = ReadBoolean(message.Payload, "final"); + if (simulated && final) _captionSignal?.TrySetResult(true); CaptionReceived?.Invoke(text, simulated); if (!simulated) _ = _obsBridge?.SendAsync(new { type = "caption", payload = message.Payload }); + if (State.BenchmarkRunning) UpdateBenchmarkCaption(message.Payload, text, final); } } + if (message.Type == "benchmark_complete") + { + ApplyBenchmarkCompletion(message.Payload); + _benchmarkCompleteSignal?.TrySetResult(true); + } if (message.Type == "error") { var serverMessage = message.Payload.TryGetProperty("message", out var value) ? value.GetString() : "Lumi reported an error."; _testFailure?.TrySetResult(serverMessage ?? "Lumi reported an inference error."); - SetState(State with { Health = TrayHealth.Degraded, Detail = serverMessage ?? "Lumi reported an error." }); + _benchmarkLifetime?.Cancel(); + SetState(State with { Health = TrayHealth.Degraded, Detail = serverMessage ?? "Lumi reported an error.", BenchmarkRunning = false, BenchmarkDetail = serverMessage ?? "Lumi reported an inference error." }); } return Task.CompletedTask; } @@ -416,6 +527,7 @@ public sealed class CompanionRuntime : IAsyncDisposable .Where(source => Guid.TryParse(source.Uuid, out _)).ToArray(); ObsSourcesChanged?.Invoke(ObsSources); foreach (var source in ObsSources) await SendSourceUpdateAsync(source); + _ = SyncBridgeSelectionAsync(_lifetimeToken()); } else if (type == "source_state") { @@ -427,11 +539,19 @@ public sealed class CompanionRuntime : IAsyncDisposable await SendSourceUpdateAsync(source); } } + else if (type == "selection_state") + { + _bridgeSelectionSignal?.TrySetResult((ReadString(message, "source_uuid") ?? string.Empty, ReadBoolean(message, "attached"))); + } } private Task OnObsAudioAsync(ReadOnlyMemory frame) { - _audioSignal?.TrySetResult(true); + if (HasVoice(frame.Span)) + { + _audioSignal?.TrySetResult(true); + if (State.BenchmarkRunning) _benchmarkLastVoiceAt = DateTimeOffset.UtcNow; + } if (_socket is not null && State.Connected && !_socket.QueueBridgeAudio(frame)) _ = WriteLogAsync("audio_dropped", "An obsolete audio frame was dropped before network delivery."); return Task.CompletedTask; @@ -453,13 +573,137 @@ public sealed class CompanionRuntime : IAsyncDisposable }, _socket.SessionId, _lifetimeToken()); } - private Task SyncBridgeSelectionAsync(CancellationToken cancellationToken = default) => _obsBridge?.SendAsync(new + private async Task SyncBridgeSelectionAsync(CancellationToken cancellationToken = default) { - type = "select_sources", - protocol_version = 1, - source_uuids = string.IsNullOrWhiteSpace(_settings.Current.PrimarySourceUuid) ? Array.Empty() : new[] { _settings.Current.PrimarySourceUuid }, - primary_source_uuid = _settings.Current.PrimarySourceUuid - }, cancellationToken) ?? Task.FromResult(false); + if (_obsBridge is null || string.IsNullOrWhiteSpace(_settings.Current.PrimarySourceUuid)) return false; + await _bridgeSelectionGate.WaitAsync(cancellationToken); + try + { + var sourceUuid = _settings.Current.PrimarySourceUuid; + var signal = new TaskCompletionSource<(string SourceUuid, bool Attached)>(TaskCreationOptions.RunContinuationsAsynchronously); + _bridgeSelectionSignal = signal; + var sent = await _obsBridge.SendAsync(new + { + type = "select_sources", + protocol_version = 1, + source_uuids = new[] { sourceUuid }, + primary_source_uuid = sourceUuid + }, cancellationToken); + if (!sent) return false; + var completed = await Task.WhenAny(signal.Task, Task.Delay(TimeSpan.FromSeconds(2), cancellationToken)); + if (completed != signal.Task) { cancellationToken.ThrowIfCancellationRequested(); return true; } + var result = await signal.Task; + return result.Attached && string.Equals(result.SourceUuid, sourceUuid, StringComparison.OrdinalIgnoreCase); + } + finally + { + _bridgeSelectionSignal = null; + _bridgeSelectionGate.Release(); + } + } + + private void UpdateBenchmarkCaption(JsonElement payload, string text, bool final) + { + var captionId = ReadString(payload, "caption_id") ?? Guid.NewGuid().ToString(); + var revision = payload.TryGetProperty("revision", out var revisionValue) && revisionValue.TryGetInt32(out var parsedRevision) ? parsedRevision : 0; + if (_benchmarkCaptions.TryGetValue(captionId, out var current) && current.Revision >= revision) return; + var words = new List(); + if (payload.TryGetProperty("analysis", out var analysis) && analysis.TryGetProperty("words", out var wordValues) && wordValues.ValueKind == JsonValueKind.Array) + { + foreach (var word in wordValues.EnumerateArray()) + { + var wordText = ReadString(word, "text"); + if (string.IsNullOrWhiteSpace(wordText)) continue; + words.Add(new BenchmarkWord(wordText, ReadDouble(word, "latency_ms"), final ? Math.Clamp(ReadDouble(word, "confidence"), 0, 1) : null, final)); + } + } + _benchmarkCaptions[captionId] = new BenchmarkCaptionRevision(revision, final, text, words, DateTimeOffset.UtcNow); + PublishLiveBenchmark(); + } + + private void PublishLiveBenchmark() + { + var captions = _benchmarkCaptions.Values.OrderBy(value => value.ReceivedAt).ToArray(); + var words = captions.SelectMany(value => value.Words).ToArray(); + Benchmark = Benchmark with + { + Transcript = string.Join(" ", captions.Select(value => value.Text).Where(value => !string.IsNullOrWhiteSpace(value))), + Words = words, + Latency = CalculateMetric(words.Select(word => word.LatencyMs)), + Confidence = CalculateMetric(words.Where(word => word.Confidence.HasValue).Select(word => word.Confidence!.Value)), + Status = State.BenchmarkRunning ? "Running" : Benchmark.Status + }; + BenchmarkChanged?.Invoke(Benchmark); + } + + private void ApplyBenchmarkCompletion(JsonElement payload) + { + var words = new List(); + if (payload.TryGetProperty("words", out var values) && values.ValueKind == JsonValueKind.Array) + { + foreach (var word in values.EnumerateArray()) + { + var text = ReadString(word, "text"); + if (string.IsNullOrWhiteSpace(text)) continue; + var confidence = word.TryGetProperty("confidence", out var confidenceValue) && confidenceValue.ValueKind == JsonValueKind.Number ? confidenceValue.GetDouble() : (double?)null; + words.Add(new BenchmarkWord(text, ReadDouble(word, "latency_ms"), confidence, ReadBoolean(word, "final"))); + } + } + var finalWords = words.Count > 0 ? words : Benchmark.Words; + Benchmark = new BenchmarkSnapshot( + ReadString(payload, "id") ?? Benchmark.Id, + ReadString(payload, "transcript") ?? Benchmark.Transcript, + finalWords, + ParseMetric(payload, "latency") ?? CalculateMetric(finalWords.Select(word => word.LatencyMs)), + ParseMetric(payload, "confidence") ?? CalculateMetric(finalWords.Where(word => word.Confidence.HasValue).Select(word => word.Confidence!.Value)), + Benchmark.StartedAt, + ReadString(payload, "status") ?? "Completed"); + BenchmarkChanged?.Invoke(Benchmark); + } + + private static MetricStatistics? ParseMetric(JsonElement payload, string name) + { + if (!payload.TryGetProperty("stats", out var stats) || !stats.TryGetProperty(name, out var value)) return null; + return new MetricStatistics(ReadInt(value, "count"), ReadNullableDouble(value, "min"), ReadNullableDouble(value, "low_1_average"), + ReadNullableDouble(value, "median"), ReadNullableDouble(value, "average"), ReadNullableDouble(value, "p99"), + ReadNullableDouble(value, "high_1_average"), ReadNullableDouble(value, "max")); + } + + private static MetricStatistics CalculateMetric(IEnumerable input) + { + var values = input.Where(double.IsFinite).Order().ToArray(); + if (values.Length == 0) return new(0, null, null, null, null, null, null, null); + var tail = Math.Max(1, (int)Math.Ceiling(values.Length * 0.01)); + return new(values.Length, values[0], values.Take(tail).Average(), Percentile(values, 0.5), values.Average(), Percentile(values, 0.99), values.TakeLast(tail).Average(), values[^1]); + } + + private static double Percentile(double[] values, double ratio) + { + var position = (values.Length - 1) * ratio; + var lower = (int)Math.Floor(position); + var upper = (int)Math.Ceiling(position); + return values[lower] + (values[upper] - values[lower]) * (position - lower); + } + + private static bool HasVoice(ReadOnlySpan frame) + { + if (frame.Length <= ProtocolV1.AudioHeaderBytes || (frame[5] & 1) == 0 || (frame[5] & 2) != 0) return false; + var samples = frame[ProtocolV1.AudioHeaderBytes..]; + double squares = 0; + var count = samples.Length / 2; + for (var index = 0; index < count; index += 1) + { + var value = BinaryPrimitives.ReadInt16LittleEndian(samples.Slice(index * 2, 2)) / 32768.0; + squares += value * value; + } + return count > 0 && Math.Sqrt(squares / count) >= 0.012; + } + + private static BenchmarkSnapshot EmptyBenchmark(string status, DateTimeOffset? startedAt = null) => + new(null, string.Empty, [], CalculateMetric([]), CalculateMetric([]), startedAt, status); + private static double ReadDouble(JsonElement value, string name) => value.TryGetProperty(name, out var property) && property.ValueKind == JsonValueKind.Number ? property.GetDouble() : 0; + private static double? ReadNullableDouble(JsonElement value, string name) => value.TryGetProperty(name, out var property) && property.ValueKind == JsonValueKind.Number ? property.GetDouble() : null; + private static int ReadInt(JsonElement value, string name) => value.TryGetProperty(name, out var property) && property.TryGetInt32(out var result) ? result : 0; private static bool ReadBoolean(JsonElement value, string name) => value.TryGetProperty(name, out var property) && property.ValueKind == JsonValueKind.True; private static string? ReadString(JsonElement value, string name) => value.TryGetProperty(name, out var property) && property.ValueKind == JsonValueKind.String ? property.GetString() : null; @@ -533,6 +777,7 @@ public sealed class CompanionRuntime : IAsyncDisposable TaskCanceledException => "The connection timed out. Check that Lumi is reachable.", _ => error.Message }; + private sealed record BenchmarkCaptionRevision(int Revision, bool Final, string Text, IReadOnlyList Words, DateTimeOffset ReceivedAt); private static TestStage[] CreateInitialTestStages() => [ new("OBS integration", "Waiting to check the managed bridge.", TestStageState.Waiting), @@ -550,10 +795,13 @@ public sealed class CompanionRuntime : IAsyncDisposable public async ValueTask DisposeAsync() { if (_disposed) return; + if (State.BenchmarkRunning) try { await StopBenchmarkAsync("disconnect", CancellationToken.None); } catch { } _disposed = true; _maintenanceLifetime.Cancel(); + _benchmarkLifetime?.Cancel(); if (_socket is not null) await _socket.DisposeAsync(); if (_obsBridge is not null) await _obsBridge.DisposeAsync(); + _benchmarkLifetime?.Dispose(); _bridgeSelectionGate.Dispose(); _http.Dispose(); _maintenanceLifetime.Dispose(); } } diff --git a/companion/src/Lumi.Companion.App/CompanionState.cs b/companion/src/Lumi.Companion.App/CompanionState.cs index 5be8e52..3034542 100644 --- a/companion/src/Lumi.Companion.App/CompanionState.cs +++ b/companion/src/Lumi.Companion.App/CompanionState.cs @@ -12,6 +12,7 @@ public sealed record CompanionState( bool ObsStreaming = false, bool ObsRecording = false, bool TestRunning = false, + bool BenchmarkRunning = false, TrayHealth Health = TrayHealth.Degraded, string Detail = "Pair Lumi Companion to get started.", string? DeviceName = null, @@ -20,6 +21,7 @@ public sealed record CompanionState( bool UpdateAvailable = false, string? AvailableVersion = null, string UpdateDetail = "Checking for updates…", + string BenchmarkDetail = "Start a dedicated test, speak naturally, then end it when you have enough material.", bool ObsBridgeRepairNeeded = false, bool ObsBridgePackageAvailable = false, string ObsBridgeDetail = "Checking the managed OBS integration…") @@ -35,13 +37,18 @@ public sealed record CompanionState( _ => "Partially ready" }; - public bool RequiresQuitConfirmation => ObsStreaming || ObsRecording; - public string QuitWarning => ObsStreaming + public bool RequiresQuitConfirmation => ObsStreaming || ObsRecording || BenchmarkRunning; + public string QuitWarning => BenchmarkRunning + ? "A transcription accuracy and latency test is running. Quitting will safely end and mark the test as aborted." + : ObsStreaming ? "OBS is streaming. Quitting Lumi Companion will stop transcription and closed captions, but it will not stop the OBS stream." : ObsRecording ? "OBS is recording. Quitting Lumi Companion will stop active companion features." : string.Empty; } public sealed record TestStage(string Name, string Detail, TestStageState State); +public sealed record BenchmarkWord(string Text, double LatencyMs, double? Confidence, bool Final); +public sealed record MetricStatistics(int Count, double? Minimum, double? LowOnePercentAverage, double? Median, double? Average, double? P99, double? HighOnePercentAverage, double? Maximum); +public sealed record BenchmarkSnapshot(string? Id, string Transcript, IReadOnlyList Words, MetricStatistics Latency, MetricStatistics Confidence, DateTimeOffset? StartedAt, string Status); public sealed record ObsSource(string Uuid, string Name, bool Active, bool Missing) { public override string ToString() => Missing ? $"{Name} (missing)" : Active ? $"{Name} (active)" : Name; diff --git a/companion/src/Lumi.Companion.App/Lumi.Companion.App.csproj b/companion/src/Lumi.Companion.App/Lumi.Companion.App.csproj index 400a9ae..f4bab1c 100644 --- a/companion/src/Lumi.Companion.App/Lumi.Companion.App.csproj +++ b/companion/src/Lumi.Companion.App/Lumi.Companion.App.csproj @@ -5,7 +5,7 @@ enable enable app.manifest - 0.1.0-experimental.4 + 0.1.0-experimental.5 0.1.0.0 diff --git a/companion/src/Lumi.Companion.App/MainWindow.axaml b/companion/src/Lumi.Companion.App/MainWindow.axaml index 2100afc..299f779 100644 --- a/companion/src/Lumi.Companion.App/MainWindow.axaml +++ b/companion/src/Lumi.Companion.App/MainWindow.axaml @@ -138,6 +138,35 @@ + + + + + + + + + + + +
+ <% if (!devices.length) { %>
No active Companion is paired

Use Download Companion to create a package that expires after 15 minutes and works once.

<% } %> + <% if (devices.length) { %> +
    + <% devices.forEach((device) => { %> +
  • <%= device.name %>Last connected <%= new Date(device.last_connected_at).toLocaleString() %><% if (device.metadata?.companion_version) { %> · Companion <%= device.metadata.companion_version %><% } %>
    Allowed
  • + <% }) %> +
+ <% } %> +
+ +
+
+
Last hour

Transcription test analysis

Inspect each dedicated Companion test by latency, finalized confidence, or both. Test transcripts and measurements are permanently deleted after one hour.

+ <%= benchmarks.length %> retained +
+ <% if (!benchmarks.length) { %> +
No dedicated tests in the last hour

Start Accuracy & latency test from the Companion Test page. Full-path checks are intentionally not retained here.

+ <% } %> + <% benchmarks.forEach((test) => { const metricRows = [ + ['Minimum', 'min'], ['Low 1% avg', 'low_1_average'], ['Median', 'median'], ['Average', 'average'], + ['P99', 'p99'], ['High 1% avg', 'high_1_average'], ['Maximum', 'max'] + ]; %> +
+ + <%= new Date(test.started_at).toLocaleString() %><%= test.source_name %> · <%= test.model_id || 'model pending' %> / <%= test.backend || 'backend pending' %> · <%= Math.round(test.duration_ms / 1000) %>s + <%= test.status.replaceAll('_', ' ') %> + +
+
+ + + +
+
+
LatencyGreen <750 msBlue <1,250 msYellow <2,000 msOrange <3,000 msRed ≥3,000 ms
+
ConfidenceGreen ≥92%Blue ≥80%Yellow ≥65%Orange ≥45%Red <45%
+
+
+ <% if (!test.words.length) { %>

<%= test.transcript || 'No finalized words were measured.' %>

<% } %> + <% test.words.forEach((word) => { const confidenceLabel = Number.isFinite(word.confidence) ? `${(word.confidence * 100).toFixed(1)}%` : 'not finalized'; %> + <%= word.text %> + <% }) %> +
+
+ <% [['Latency', test.stats.latency, (value) => `${Math.round(value)} ms`], ['Final confidence', test.stats.confidence, (value) => `${(value * 100).toFixed(1)}%`]].forEach(([label, stats, format]) => { %> +

<%= label %>

+ <% metricRows.forEach(([metricLabel, key]) => { %>
<%= metricLabel %><%= Number.isFinite(stats[key]) ? format(stats[key]) : '—' %>
<% }) %> +

<%= stats.count %> measured values

+ <% }) %> +
+
+
+ <% }) %> +
+
Curated choices

Speech models

Downloads require confirmation and checksum verification before a model can load.