using System.Buffers.Binary; using System.Diagnostics; using System.Reflection; using System.Security.Cryptography; using System.Text; using System.Text.Json; using Lumi.Companion.Core; using Lumi.Companion.Protocol; using Lumi.Companion.Transcription; using Microsoft.Win32; namespace Lumi.Companion.App; public sealed class CompanionRuntime : IAsyncDisposable { private const string RunKey = @"Software\Microsoft\Windows\CurrentVersion\Run"; private const string PathValidationContract = "voice-free-path-v1"; private readonly CompanionPaths _paths; private readonly CompanionSettingsStore _settings; private readonly SecureCredentialStore _credentials; private readonly HttpClient _http = new() { Timeout = TimeSpan.FromSeconds(15) }; private readonly UpdateService _updates; private readonly ObsBridgeManager _bridgeManager = new(); private readonly CancellationTokenSource _maintenanceLifetime = new(); private CompanionSocket? _socket; private ObsBridgePipe? _obsBridge; private TaskCompletionSource? _captionSignal; private TaskCompletionSource? _sessionStartSignal; private TaskCompletionSource? _sessionStopSignal; private TaskCompletionSource? _testFailure; private TaskCompletionSource<(string SourceUuid, bool Attached)>? _bridgeSelectionSignal; private bool? _bridgeSelectionAttached; private readonly SemaphoreSlim _bridgeSelectionGate = new(1, 1); private readonly Dictionary _benchmarkCaptions = []; private CancellationTokenSource? _benchmarkLifetime; private TaskCompletionSource? _benchmarkCompleteSignal; private DateTimeOffset? _benchmarkLastVoiceAt; private int _benchmarkStopping; private long _lastVoiceMeterAt; private bool _disposed; private CompanionUpdate? _availableUpdate; private bool _serverReady; private string? _serverReadinessFingerprint; private string _serverReadinessDetail = "Waiting for Lumi to report speech recognition readiness."; public CompanionRuntime(CompanionPaths paths, CompanionSettingsStore settings) { _paths = paths; _settings = settings; _credentials = new SecureCredentialStore(paths.Root); _updates = new UpdateService(_http, paths); State = new CompanionState(); TestStages = CreateInitialTestStages(); ObsSources = []; Benchmark = EmptyBenchmark("Not started"); } public CompanionState State { get; private set; } public IReadOnlyList TestStages { get; private set; } public IReadOnlyList ObsSources { get; private set; } public BenchmarkSnapshot Benchmark { get; private set; } public event Action? StateChanged; public event Action>? TestStagesChanged; public event Action>? ObsSourcesChanged; public event Action? BenchmarkChanged; public event Action? VoiceLevelChanged; public event Action? CaptionReceived; public event Action? LogAdded; public event Func? UpdateRestartRequested; public async Task InitializeAsync() { try { await InitializeCoreAsync(); } catch (Exception error) { SetState(State with { Health = TrayHealth.Failed, Detail = $"Companion startup could not finish. {Friendly(error)}" }); await WriteLogAsync("startup_failed", error.Message); } } private async Task InitializeCoreAsync() { await _settings.LoadAsync(); StartObsBridgeBoundary(); _ = RunUpdateChecksAsync(_maintenanceLifetime.Token); ApplyAutoStart(_settings.Current.AutoStartWithWindows); DeviceCredential? credential; try { credential = _credentials.Load(); } catch (Exception error) { SetState(State with { Health = TrayHealth.Failed, Detail = "The saved device credential could not be opened. Pair this computer again." }); await WriteLogAsync("credential_load_failed", error.Message); return; } if (credential is null) { var bundledPairing = FindBundledPairingPackage(); if (bundledPairing is not null) { await PairAsync(bundledPairing); try { File.Delete(bundledPairing); } catch { } return; } SetState(WithBridgeState(State with { Detail = "Download a pairing package from Lumi, then open it here." })); return; } // An installer migration preserves the current-user DPAPI credential. If a // bootstrap was copied alongside it, it is unnecessary and should not remain // on disk where a later local reset could accidentally consume it. var redundantPairing = FindBundledPairingPackage(); if (redundantPairing is not null) try { File.Delete(redundantPairing); } catch { } SetState(WithBridgeState(State with { Paired = true, DeviceName = Environment.MachineName, Host = credential.Host, Detail = "Connecting securely to Lumi…" })); await ConnectAsync(credential); } public async Task PairAsync(string packagePath, CancellationToken cancellationToken = default) { SetState(State with { Health = TrayHealth.Degraded, Detail = "Validating the pairing package…" }); var client = new PairingClient(_http); try { var credential = await client.PairAsync(packagePath, new { install_id = _credentials.GetOrCreateInstallId(), name = Environment.MachineName, companion_version = Version }, cancellationToken); _credentials.Save(credential); SetState(State with { Paired = true, DeviceName = Environment.MachineName, Host = credential.Host, Detail = "Pairing complete. Connecting to Lumi…" }); await WriteLogAsync("paired", $"Paired {Environment.MachineName} with {credential.Host}."); await ConnectAsync(credential, cancellationToken); } catch (Exception error) { SetState(State with { Health = TrayHealth.Failed, Detail = Friendly(error) }); await WriteLogAsync("pairing_failed", error.Message); throw; } } public async Task RetryConnectionAsync(CancellationToken cancellationToken = default) { var credential = _credentials.Load() ?? throw new InvalidOperationException("Pair this computer before reconnecting."); await ConnectAsync(credential, cancellationToken); } private async Task ConnectAsync(DeviceCredential credential, CancellationToken cancellationToken = default) { if (_socket is not null) await _socket.DisposeAsync(); _serverReady = false; _serverReadinessFingerprint = null; _serverReadinessDetail = "Waiting for Lumi to report speech recognition readiness."; _socket = new CompanionSocket(); _socket.MessageReceived += OnServerMessageAsync; _socket.Disconnected += error => { if (_disposed) return; _testFailure?.TrySetResult(error is null ? "The Lumi connection closed during the test." : $"The Lumi connection closed during the test: {Friendly(error)}"); _benchmarkLifetime?.Cancel(); SetState(State with { Connected = false, BenchmarkRunning = false, Health = TrayHealth.Degraded, Detail = "The secure Lumi connection closed. Retry when the host is available.", BenchmarkDetail = State.BenchmarkRunning ? "The benchmark was aborted because the Lumi connection closed." : State.BenchmarkDetail }); _ = WriteLogAsync("disconnected", error?.Message ?? "Connection closed."); }; SetState(State with { Connected = false, Detail = "Connecting securely to Lumi…" }); try { await _socket.ConnectAsync(credential, Version, "0.1.0", null, cancellationToken); var bridgeInstalled = State.ObsBridgeInstalled || DetectBridgeInstallation(); SetState(State with { Paired = true, Connected = true, ObsBridgeInstalled = bridgeInstalled, Health = bridgeInstalled ? TrayHealth.Ready : TrayHealth.Degraded, Detail = bridgeInstalled ? "Lumi is connected. Waiting for OBS." : "Lumi is connected. Install the managed OBS integration to continue setup.", Host = credential.Host, LastConnectedAt = DateTimeOffset.Now }); foreach (var source in ObsSources) await SendSourceUpdateAsync(source); await SendRuntimeStateAsync(cancellationToken); await _socket.SendAsync("readiness", new { }, _socket.SessionId, cancellationToken); await WriteLogAsync("connected", $"Connected to {credential.Host}."); await CheckForUpdatesAsync(cancellationToken); } catch (Exception error) { SetState(State with { Paired = true, Connected = false, Health = TrayHealth.Degraded, Detail = $"Lumi could not be reached. {Friendly(error)}" }); await WriteLogAsync("connection_failed", error.Message); } } public async Task RunTestAsync(CancellationToken cancellationToken = default) { if (State.TestRunning || State.BenchmarkRunning) return; SetState(State with { TestRunning = true, Detail = "Running the transcription path check…" }); var stages = CreateInitialTestStages().ToArray(); TestStages = stages; TestStagesChanged?.Invoke(TestStages); try { await EvaluateStageAsync(stages, 0, State.ObsBridgeInstalled, "The managed OBS integration is installed.", "Install or repair the OBS integration before testing.", cancellationToken); if (!State.ObsBridgeInstalled) return; await EvaluateStageAsync(stages, 1, State.ObsConnected, "OBS reported a healthy local connection.", "Open OBS 31 or newer and retry.", cancellationToken); if (!State.ObsConnected) return; var selected = ObsSources.FirstOrDefault(source => source.Uuid == _settings.Current.PrimarySourceUuid); var bridgeAttached = selected is { Missing: false } && await SyncBridgeSelectionAsync(cancellationToken); await EvaluateStageAsync(stages, 2, selected is { Missing: false, Active: true } && bridgeAttached, "The selected microphone is available, active in Program, and attached for capture.", selected is null ? "Choose a microphone on the Transcription page." : selected.Missing ? "The selected microphone is missing from OBS." : !bridgeAttached ? "Companion could not attach the selected OBS source. Keep OBS open and retry." : "Put the selected microphone in the active Program scene, then retry.", cancellationToken); if (selected is not { Missing: false, Active: true } || !bridgeAttached) return; await EvaluateStageAsync(stages, 4, State.Connected, "The secure Lumi transport is ready.", "Reconnect to Lumi before testing.", cancellationToken); if (!State.Connected) return; _captionSignal = new(TaskCreationOptions.RunContinuationsAsynchronously); _sessionStartSignal = new(TaskCreationOptions.RunContinuationsAsynchronously); _sessionStopSignal = new(TaskCreationOptions.RunContinuationsAsynchronously); _testFailure = new(TaskCreationOptions.RunContinuationsAsynchronously); await _socket!.SendAsync("start", new { mode = "test" }, _socket.SessionId, cancellationToken); var sessionStart = await WaitForTestSignalAsync(_sessionStartSignal.Task, TimeSpan.FromSeconds(8), cancellationToken); if (!sessionStart.Completed) { MarkStage(stages, 3, TestStageState.Blocked, "Audio capture was not attempted because the server inference session did not start."); MarkStage(stages, 5, TestStageState.Blocked, sessionStart.Failure ?? "Lumi did not confirm that speech recognition started within 8 seconds."); return; } MarkStage(stages, 3, TestStageState.Passed, "The selected OBS source is attached. You do not need to speak for this check."); MarkStage(stages, 5, TestStageState.Passed, "Lumi reports that the selected speech model is loaded and ready."); MarkStage(stages, 6, TestStageState.Running, "Checking the safe caption return path…"); var caption = await WaitForTestSignalAsync(_captionSignal.Task, TimeSpan.FromSeconds(8), cancellationToken); if (!caption.Completed) { MarkStage(stages, 6, TestStageState.Blocked, caption.Failure ?? "The safe test caption did not return within 8 seconds."); return; } MarkStage(stages, 6, TestStageState.Passed, "A safe test caption returned over the secure connection."); MarkStage(stages, 7, TestStageState.Passed, "The delivery adapter stayed in safe simulation mode."); MarkStage(stages, 8, TestStageState.Passed, "The exact outgoing caption was rendered locally, not sent to Twitch."); var fingerprint = CurrentPathFingerprint() ?? throw new InvalidOperationException("Lumi readiness changed while the check was running. Try the check again."); await _settings.SaveAsync(_settings.Current with { PathTestPassedAt = DateTimeOffset.UtcNow, PathTestFingerprint = fingerprint }, cancellationToken); RefreshPathReadiness("The voice-free full-path check passed. It stays valid until a relevant setup or model configuration changes."); await SendRuntimeStateAsync(cancellationToken); SetState(State with { Detail = "Everything is ready. The dedicated transcript test is available when you want to measure real speech.", Health = TrayHealth.Ready }); } finally { if (_socket is not null && State.Connected) try { await _socket.SendAsync("stop", new { reason = "test_complete" }, _socket.SessionId, CancellationToken.None); if (_sessionStopSignal is not null) await Task.WhenAny(_sessionStopSignal.Task, Task.Delay(TimeSpan.FromSeconds(5))); } catch { } _captionSignal = null; _sessionStartSignal = null; _sessionStopSignal = null; _testFailure = null; SetState(State with { TestRunning = false, Detail = TestStages.Any(stage => stage.State == TestStageState.Blocked) ? "The test stopped at the first unavailable real boundary." : State.Detail }); } } public async Task StartBenchmarkAsync(CancellationToken cancellationToken = default) { if (State.BenchmarkRunning || State.TestRunning) return; if (!State.Connected || _socket is null) throw new InvalidOperationException("Reconnect to Lumi before starting the transcription test."); if (!State.ObsConnected) throw new InvalidOperationException("Open OBS and wait for the managed integration to connect."); var selected = ObsSources.FirstOrDefault(source => source.Uuid == _settings.Current.PrimarySourceUuid); if (selected is not { Missing: false, Active: true }) throw new InvalidOperationException("Choose an available microphone that is active in the OBS Program scene."); if (!await SyncBridgeSelectionAsync(cancellationToken)) throw new InvalidOperationException("Companion could not attach the selected OBS source. Keep OBS open and retry."); _benchmarkCaptions.Clear(); _sessionStartSignal = new(TaskCreationOptions.RunContinuationsAsynchronously); _testFailure = new(TaskCreationOptions.RunContinuationsAsynchronously); _benchmarkCompleteSignal = new(TaskCreationOptions.RunContinuationsAsynchronously); _benchmarkLastVoiceAt = DateTimeOffset.UtcNow; Interlocked.Exchange(ref _benchmarkStopping, 0); Benchmark = EmptyBenchmark("Starting", DateTimeOffset.UtcNow); BenchmarkChanged?.Invoke(Benchmark); VoiceLevelChanged?.Invoke(-60); SetState(State with { BenchmarkRunning = true, BenchmarkDetail = "Starting server-hosted accuracy and latency measurement…", Health = TrayHealth.Operating }); try { await _socket.SendAsync("start", new { mode = "benchmark" }, _socket.SessionId, cancellationToken); var started = await WaitForTestSignalAsync(_sessionStartSignal.Task, TimeSpan.FromSeconds(8), cancellationToken); if (!started.Completed) throw new InvalidOperationException(started.Failure ?? "Lumi did not start the benchmark within 8 seconds."); _benchmarkLifetime?.Cancel(); _benchmarkLifetime?.Dispose(); _benchmarkLifetime = CancellationTokenSource.CreateLinkedTokenSource(_maintenanceLifetime.Token); _ = MonitorBenchmarkSilenceAsync(_benchmarkLifetime.Token); SetState(State with { BenchmarkDetail = "Listening. Speak naturally; end the test manually, or remain silent for 10 seconds." }); await WriteLogAsync("benchmark_started", $"Started transcription benchmark for {selected.Name}."); } catch (Exception error) { _benchmarkLifetime?.Cancel(); _sessionStartSignal = null; _testFailure = null; _benchmarkCompleteSignal = null; SetState(State with { BenchmarkRunning = false, BenchmarkDetail = Friendly(error), Health = TrayHealth.Degraded }); throw; } } public async Task StopBenchmarkAsync(string reason = "benchmark_complete", CancellationToken cancellationToken = default) { if (!State.BenchmarkRunning || _socket is null || Interlocked.Exchange(ref _benchmarkStopping, 1) != 0) return; _benchmarkLifetime?.Cancel(); SetState(State with { BenchmarkDetail = reason == "silence_timeout" ? "Ten seconds of silence detected. Finalizing the test…" : "Finalizing the test…" }); try { await _socket.SendAsync("stop", new { reason }, _socket.SessionId, cancellationToken); if (_benchmarkCompleteSignal is not null) await Task.WhenAny(_benchmarkCompleteSignal.Task, Task.Delay(TimeSpan.FromSeconds(7), cancellationToken)); } finally { VoiceLevelChanged?.Invoke(-60); SetState(State with { BenchmarkRunning = false, BenchmarkDetail = Benchmark.Words.Count > 0 ? $"Test complete with {Benchmark.Words.Count} measured words or phrases." : "The test ended without a finalized transcript." }); _sessionStartSignal = null; _testFailure = null; Interlocked.Exchange(ref _benchmarkStopping, 0); } } private async Task MonitorBenchmarkSilenceAsync(CancellationToken cancellationToken) { try { using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(500)); while (await timer.WaitForNextTickAsync(cancellationToken)) { if (!State.BenchmarkRunning) return; if (_benchmarkLastVoiceAt is { } lastVoice && DateTimeOffset.UtcNow - lastVoice >= TimeSpan.FromSeconds(10)) { await StopBenchmarkAsync("silence_timeout", CancellationToken.None); return; } } } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { } } private async Task EvaluateStageAsync(TestStage[] stages, int index, bool passed, string success, string blocked, CancellationToken cancellationToken) { MarkStage(stages, index, TestStageState.Running, "Checking…"); await Task.Delay(180, cancellationToken); MarkStage(stages, index, passed ? TestStageState.Passed : TestStageState.Blocked, passed ? success : blocked); } private void MarkStage(TestStage[] stages, int index, TestStageState state, string detail) { stages[index] = stages[index] with { State = state, Detail = detail }; TestStages = stages.ToArray(); TestStagesChanged?.Invoke(TestStages); } public async Task SaveSettingsAsync(CompanionSettings value, CancellationToken cancellationToken = default) { await _settings.SaveAsync(value, cancellationToken); ApplyAutoStart(value.AutoStartWithWindows); await WriteLogAsync("settings_saved", "Local companion preferences updated."); } public async Task CheckForUpdatesAsync(CancellationToken cancellationToken = default) { var credential = _credentials.Load(); if (credential is null) { SetState(State with { UpdateDetail = "Pair this computer before checking for updates." }); return; } try { _availableUpdate = await _updates.CheckAsync(credential, Version, cancellationToken); SetState(State with { UpdateAvailable = _availableUpdate is not null, AvailableVersion = _availableUpdate?.Version, UpdateDetail = _availableUpdate is null ? $"Lumi Companion {Version} is current." : $"Lumi Companion {_availableUpdate.Version} is ready to install when OBS is idle." }); } catch (Exception error) { SetState(State with { UpdateDetail = $"Update check could not finish. {Friendly(error)}" }); await WriteLogAsync("update_check_failed", error.Message); } } public async Task ApplyUpdateAsync(CancellationToken cancellationToken = default) { if (State.ObsStreaming || State.ObsRecording) throw new InvalidOperationException("Stop streaming and recording before updating Lumi Companion."); var update = _availableUpdate ?? throw new InvalidOperationException("No Companion update is ready to install."); try { SetState(State with { UpdateDetail = $"Downloading and verifying {update.Version}…" }); var staged = await _updates.StageAsync(update, cancellationToken); if (State.ObsStreaming || State.ObsRecording) throw new InvalidOperationException("OBS started output while the update was downloading. Stop streaming and recording, then try again."); _updates.LaunchApplier(staged); SetState(State with { UpdateDetail = "Update verified. Restarting Lumi Companion…" }); await WriteLogAsync("update_staged", $"Verified Companion {update.Version}; restarting to apply it."); if (UpdateRestartRequested is { } restart) await restart(); } catch (Exception error) { SetState(State with { UpdateDetail = $"Update could not finish. {Friendly(error)}" }); await WriteLogAsync("update_failed", error.Message); throw; } } public async Task InstallOrRepairObsBridgeAsync(CancellationToken cancellationToken = default) { try { var result = await _bridgeManager.InstallOrRepairAsync(cancellationToken); SetState(WithBridgeState(State with { Detail = "OBS integration installed. Start or restart OBS to connect it to Companion." })); RefreshPathReadiness(); await SendRuntimeStateAsync(cancellationToken); await WriteLogAsync("obs_bridge_installed", $"Installed managed OBS integration {result.Version}."); } catch (Exception error) { SetState(State with { ObsBridgeDetail = $"OBS integration maintenance could not finish. {Friendly(error)}" }); await WriteLogAsync("obs_bridge_install_failed", error.Message); throw; } } public async Task RemoveObsBridgeAsync() { try { await _bridgeManager.RemoveAsync(); SetState(WithBridgeState(State with { ObsConnected = false, Detail = "OBS integration removed. Other Companion features remain installed." })); RefreshPathReadiness(); await SendRuntimeStateAsync(CancellationToken.None); await WriteLogAsync("obs_bridge_removed", "Removed the managed OBS integration."); } catch (Exception error) { SetState(State with { ObsBridgeDetail = $"OBS integration removal could not finish. {Friendly(error)}" }); await WriteLogAsync("obs_bridge_remove_failed", error.Message); throw; } } private async Task RunUpdateChecksAsync(CancellationToken cancellationToken) { using var timer = new PeriodicTimer(TimeSpan.FromHours(6)); try { while (await timer.WaitForNextTickAsync(cancellationToken)) await CheckForUpdatesAsync(cancellationToken); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { } } public async Task SelectSourceAsync(ObsSource source, CancellationToken cancellationToken = default) { await _settings.SaveAsync(_settings.Current with { PrimarySourceUuid = source.Uuid, PrimarySourceName = source.Name }, cancellationToken); await SyncBridgeSelectionAsync(cancellationToken); foreach (var item in ObsSources) await SendSourceUpdateAsync(item); RefreshPathReadiness(); await SendRuntimeStateAsync(cancellationToken); await WriteLogAsync("source_selected", $"Selected OBS source {source.Name}."); } public void OpenLumiWebUi() { if (!Uri.TryCreate(State.Host, UriKind.Absolute, out var uri)) return; Process.Start(new ProcessStartInfo(uri.ToString()) { UseShellExecute = true }); } public void OpenLogsDirectory() { Directory.CreateDirectory(_paths.LogsDirectory); Process.Start(new ProcessStartInfo(_paths.LogsDirectory) { UseShellExecute = true }); } public async Task ForgetDeviceAsync() { if (_socket is not null) { await _socket.DisposeAsync(); _socket = null; } _credentials.Remove(); SetState(WithBridgeState(new CompanionState(Detail: "Device removed. Pair this computer to reconnect."))); await WriteLogAsync("device_removed", "Saved device credential removed locally."); } private Task OnServerMessageAsync(ServerEnvelope message) { if (message.Type == "status" && message.Payload.TryGetProperty("kind", out var statusKind) && statusKind.GetString() == "readiness") { _serverReady = ReadBoolean(message.Payload, "ready"); _serverReadinessFingerprint = ReadString(message.Payload, "fingerprint"); _serverReadinessDetail = ReadString(message.Payload, "detail") ?? "Lumi did not provide speech recognition readiness details."; RefreshPathReadiness(); _ = SendRuntimeStateAsync(_lifetimeToken()); } if (message.Type == "status" && message.Payload.TryGetProperty("kind", out var kind) && kind.GetString() == "session" && message.Payload.TryGetProperty("state", out var state)) { if (state.GetString() == "running") { _sessionStartSignal?.TrySetResult(true); if (message.Payload.TryGetProperty("benchmark_id", out var benchmarkId) && benchmarkId.ValueKind == JsonValueKind.String) { Benchmark = Benchmark with { Id = benchmarkId.GetString(), Status = "Running" }; BenchmarkChanged?.Invoke(Benchmark); } } else if (state.GetString() == "idle") _sessionStopSignal?.TrySetResult(true); } if (message.Type == "caption") { var stableText = message.Payload.TryGetProperty("stable_text", out var stable) ? stable.GetString() : null; var uncertainText = message.Payload.TryGetProperty("uncertain_text", out var uncertain) ? uncertain.GetString() : null; var text = string.Join(" ", new[] { stableText, uncertainText }.Where(value => !string.IsNullOrWhiteSpace(value))); var simulated = message.Payload.TryGetProperty("delivery", out var delivery) && delivery.TryGetProperty("disposition", out var disposition) && disposition.GetString() == "simulated"; if (!string.IsNullOrWhiteSpace(text)) { var final = ReadBoolean(message.Payload, "final"); if (simulated && final) _captionSignal?.TrySetResult(true); CaptionReceived?.Invoke(text, simulated); if (!simulated) _ = _obsBridge?.SendAsync(new { type = "caption", payload = message.Payload }); if (State.BenchmarkRunning) UpdateBenchmarkCaption(message.Payload, text, final); } } if (message.Type == "benchmark_complete") { ApplyBenchmarkCompletion(message.Payload); _benchmarkCompleteSignal?.TrySetResult(true); } if (message.Type == "error") { var serverMessage = message.Payload.TryGetProperty("message", out var value) ? value.GetString() : "Lumi reported an error."; _testFailure?.TrySetResult(serverMessage ?? "Lumi reported an inference error."); _benchmarkLifetime?.Cancel(); SetState(State with { Health = TrayHealth.Degraded, Detail = serverMessage ?? "Lumi reported an error.", BenchmarkRunning = false, BenchmarkDetail = serverMessage ?? "Lumi reported an inference error." }); } return Task.CompletedTask; } private void StartObsBridgeBoundary() { if (_obsBridge is not null) return; var userKey = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes($"{Environment.UserDomainName}\\{Environment.UserName}"))).Substring(0, 16); _obsBridge = new ObsBridgePipe(userKey, OnObsMessageAsync, OnObsAudioAsync); _obsBridge.ConnectionChanged += connected => { var health = connected && State.Connected ? TrayHealth.Ready : State.Health; if (!connected) _bridgeSelectionAttached = null; SetState(State with { ObsBridgeInstalled = connected || State.ObsBridgeInstalled, ObsConnected = connected, Health = health, Detail = connected ? "OBS and Lumi are connected. Choose a microphone and run a safe test." : State.Detail }); RefreshPathReadiness(); _ = WriteLogAsync("obs_connection", connected ? "OBS bridge connected." : "OBS bridge disconnected."); if (connected) _ = SyncBridgeSelectionAsync(); _ = SendRuntimeStateAsync(_lifetimeToken()); }; _obsBridge.Start(); } private async Task OnObsMessageAsync(JsonElement message) { var type = message.GetProperty("type").GetString(); if (type == "obs_state") { var streaming = ReadBoolean(message, "streaming"); var recording = ReadBoolean(message, "recording"); SetState(State with { ObsConnected = true, ObsStreaming = streaming, ObsRecording = recording, Health = streaming ? TrayHealth.Operating : TrayHealth.Ready, Detail = streaming ? "OBS is live and companion services are active." : "OBS is connected and ready." }); RefreshPathReadiness(); await SendRuntimeStateAsync(_lifetimeToken(), ReadString(message, "version")); } else if (type == "source_list" && message.TryGetProperty("sources", out var sources) && sources.ValueKind == JsonValueKind.Array) { ObsSources = sources.EnumerateArray().Select(source => new ObsSource( ReadString(source, "source_uuid") ?? ReadString(source, "uuid") ?? string.Empty, ReadString(source, "display_name") ?? ReadString(source, "name") ?? "OBS source", ReadBoolean(source, "program_active") || ReadBoolean(source, "active"), ReadBoolean(source, "source_missing") || ReadBoolean(source, "missing"))) .Where(source => Guid.TryParse(source.Uuid, out _)) .GroupBy(source => source.Uuid) .Select(group => group.Last()) .ToArray(); _bridgeSelectionAttached = null; ObsSourcesChanged?.Invoke(ObsSources); foreach (var source in ObsSources) await SendSourceUpdateAsync(source); RefreshPathReadiness(); _ = SyncBridgeSelectionAsync(_lifetimeToken()); } else if (type == "source_state") { var source = new ObsSource(ReadString(message, "source_uuid") ?? string.Empty, ReadString(message, "display_name") ?? "OBS source", ReadBoolean(message, "program_active"), ReadBoolean(message, "source_missing")); if (Guid.TryParse(source.Uuid, out _)) { var previous = ObsSources.FirstOrDefault(item => item.Uuid == source.Uuid); if (previous == source) return; ObsSources = ObsSources.Where(item => item.Uuid != source.Uuid).Append(source).OrderBy(item => item.Name).ToArray(); ObsSourcesChanged?.Invoke(ObsSources); await SendSourceUpdateAsync(source); RefreshPathReadiness(); } } else if (type == "selection_state") { _bridgeSelectionSignal?.TrySetResult((ReadString(message, "source_uuid") ?? string.Empty, ReadBoolean(message, "attached"))); } } private Task OnObsAudioAsync(ReadOnlyMemory frame) { var level = MeasureDbfs(frame.Span); if (State.BenchmarkRunning && Environment.TickCount64 - Interlocked.Read(ref _lastVoiceMeterAt) >= 50) { Interlocked.Exchange(ref _lastVoiceMeterAt, Environment.TickCount64); VoiceLevelChanged?.Invoke(level); } if (level >= -38.5) { if (State.BenchmarkRunning) _benchmarkLastVoiceAt = DateTimeOffset.UtcNow; } var audioNeeded = State.TestRunning || State.BenchmarkRunning || State.ObsStreaming; if (audioNeeded && _socket is not null && State.Connected && !_socket.QueueBridgeAudio(frame)) _ = WriteLogAsync("audio_dropped", "An obsolete audio frame was dropped before network delivery."); return Task.CompletedTask; } private async Task SendSourceUpdateAsync(ObsSource source) { if (_socket is null || !State.Connected) return; await _socket.SendAsync("source_update", new { source_uuid = source.Uuid, display_name = source.Name, enabled = source.Uuid == _settings.Current.PrimarySourceUuid, primary = source.Uuid == _settings.Current.PrimarySourceUuid, program_active = source.Active, source_missing = source.Missing, single_speaker = true, delivery_enabled = true }, _socket.SessionId, _lifetimeToken()); } private async Task SyncBridgeSelectionAsync(CancellationToken cancellationToken = default) { if (_obsBridge is null || string.IsNullOrWhiteSpace(_settings.Current.PrimarySourceUuid)) { _bridgeSelectionAttached = false; RefreshPathReadiness(); return false; } await _bridgeSelectionGate.WaitAsync(cancellationToken); try { var sourceUuid = _settings.Current.PrimarySourceUuid; var signal = new TaskCompletionSource<(string SourceUuid, bool Attached)>(TaskCreationOptions.RunContinuationsAsynchronously); _bridgeSelectionSignal = signal; var sent = await _obsBridge.SendAsync(new { type = "select_sources", protocol_version = 1, source_uuids = new[] { sourceUuid }, primary_source_uuid = sourceUuid }, cancellationToken); var attached = false; if (sent) { var completed = await Task.WhenAny(signal.Task, Task.Delay(TimeSpan.FromSeconds(2), cancellationToken)); if (completed != signal.Task) { cancellationToken.ThrowIfCancellationRequested(); attached = true; } else { var result = await signal.Task; attached = result.Attached && string.Equals(result.SourceUuid, sourceUuid, StringComparison.OrdinalIgnoreCase); } } _bridgeSelectionAttached = attached; RefreshPathReadiness(); return attached; } finally { _bridgeSelectionSignal = null; _bridgeSelectionGate.Release(); } } private void UpdateBenchmarkCaption(JsonElement payload, string text, bool final) { var captionId = ReadString(payload, "caption_id") ?? Guid.NewGuid().ToString(); var revision = payload.TryGetProperty("revision", out var revisionValue) && revisionValue.TryGetInt32(out var parsedRevision) ? parsedRevision : 0; if (_benchmarkCaptions.TryGetValue(captionId, out var current) && current.Revision >= revision) return; var words = new List(); if (payload.TryGetProperty("analysis", out var analysis) && analysis.TryGetProperty("words", out var wordValues) && wordValues.ValueKind == JsonValueKind.Array) { foreach (var word in wordValues.EnumerateArray()) { var wordText = ReadString(word, "text"); if (string.IsNullOrWhiteSpace(wordText)) continue; words.Add(new BenchmarkWord(wordText, ReadDouble(word, "latency_ms"), final ? Math.Clamp(ReadDouble(word, "confidence"), 0, 1) : null, final)); } } _benchmarkCaptions[captionId] = new BenchmarkCaptionRevision(revision, final, text, words, DateTimeOffset.UtcNow); PublishLiveBenchmark(); } private void PublishLiveBenchmark() { var captions = _benchmarkCaptions.Values.OrderBy(value => value.ReceivedAt).ToArray(); var words = captions.SelectMany(value => value.Words).ToArray(); Benchmark = Benchmark with { Transcript = string.Join(" ", captions.Select(value => value.Text).Where(value => !string.IsNullOrWhiteSpace(value))), Words = words, Latency = CalculateMetric(words.Select(word => word.LatencyMs)), Confidence = CalculateMetric(words.Where(word => word.Confidence.HasValue).Select(word => word.Confidence!.Value)), Status = State.BenchmarkRunning ? "Running" : Benchmark.Status }; BenchmarkChanged?.Invoke(Benchmark); } private void ApplyBenchmarkCompletion(JsonElement payload) { var words = new List(); if (payload.TryGetProperty("words", out var values) && values.ValueKind == JsonValueKind.Array) { foreach (var word in values.EnumerateArray()) { var text = ReadString(word, "text"); if (string.IsNullOrWhiteSpace(text)) continue; var confidence = word.TryGetProperty("confidence", out var confidenceValue) && confidenceValue.ValueKind == JsonValueKind.Number ? confidenceValue.GetDouble() : (double?)null; words.Add(new BenchmarkWord(text, ReadDouble(word, "latency_ms"), confidence, ReadBoolean(word, "final"))); } } var finalWords = words.Count > 0 ? words : Benchmark.Words; Benchmark = new BenchmarkSnapshot( ReadString(payload, "id") ?? Benchmark.Id, ReadString(payload, "transcript") ?? Benchmark.Transcript, finalWords, ParseMetric(payload, "latency") ?? CalculateMetric(finalWords.Select(word => word.LatencyMs)), ParseMetric(payload, "confidence") ?? CalculateMetric(finalWords.Where(word => word.Confidence.HasValue).Select(word => word.Confidence!.Value)), Benchmark.StartedAt, ReadString(payload, "status") ?? "Completed"); BenchmarkChanged?.Invoke(Benchmark); } private static MetricStatistics? ParseMetric(JsonElement payload, string name) { if (!payload.TryGetProperty("stats", out var stats) || !stats.TryGetProperty(name, out var value)) return null; return new MetricStatistics(ReadInt(value, "count"), ReadNullableDouble(value, "min"), ReadNullableDouble(value, "low_1_average"), ReadNullableDouble(value, "median"), ReadNullableDouble(value, "average"), ReadNullableDouble(value, "p99"), ReadNullableDouble(value, "high_1_average"), ReadNullableDouble(value, "max")); } private static MetricStatistics CalculateMetric(IEnumerable input) { var values = input.Where(double.IsFinite).Order().ToArray(); if (values.Length == 0) return new(0, null, null, null, null, null, null, null); var tail = Math.Max(1, (int)Math.Ceiling(values.Length * 0.01)); return new(values.Length, values[0], values.Take(tail).Average(), Percentile(values, 0.5), values.Average(), Percentile(values, 0.99), values.TakeLast(tail).Average(), values[^1]); } private static double Percentile(double[] values, double ratio) { var position = (values.Length - 1) * ratio; var lower = (int)Math.Floor(position); var upper = (int)Math.Ceiling(position); return values[lower] + (values[upper] - values[lower]) * (position - lower); } private async Task SendRuntimeStateAsync(CancellationToken cancellationToken, string? obsVersion = null) { if (_socket is null || !State.Connected) return; var bridge = _bridgeManager.Inspect(); var fingerprint = CurrentPathFingerprint(); var pathValid = _serverReady && fingerprint is not null && _settings.Current.PathTestPassedAt.HasValue && string.Equals(_settings.Current.PathTestFingerprint, fingerprint, StringComparison.Ordinal); await _socket.SendAsync("obs_state", new { streaming = State.ObsStreaming, recording = State.ObsRecording, version = obsVersion, auto_start = _settings.Current.StartWithObs, bridge_installed = bridge.Valid, bridge_connected = State.ObsConnected, bridge_version = bridge.Version, path_test_valid = pathValid, path_test_at = pathValid ? _settings.Current.PathTestPassedAt?.ToUnixTimeMilliseconds() : null }, _socket.SessionId, cancellationToken); } private string? CurrentPathFingerprint() { var bridge = _bridgeManager.Inspect(); if (!_serverReady || string.IsNullOrWhiteSpace(_serverReadinessFingerprint) || !bridge.Valid || string.IsNullOrWhiteSpace(State.Host) || string.IsNullOrWhiteSpace(_settings.Current.PrimarySourceUuid)) return null; if (State.ObsConnected && _bridgeSelectionAttached == false) return null; if (State.ObsConnected && ObsSources.FirstOrDefault(source => source.Uuid == _settings.Current.PrimarySourceUuid) is not { Missing: false, Active: true }) return null; var value = string.Join("|", PathValidationContract, State.Host, _settings.Current.PrimarySourceUuid, bridge.Version, _serverReadinessFingerprint); return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value))).ToLowerInvariant(); } private void RefreshPathReadiness(string? validDetail = null) { var current = CurrentPathFingerprint(); var valid = current is not null && _settings.Current.PathTestPassedAt.HasValue && string.Equals(_settings.Current.PathTestFingerprint, current, StringComparison.Ordinal); var detail = valid ? validDetail ?? $"Passed {_settings.Current.PathTestPassedAt!.Value.LocalDateTime:g}; no relevant setup or model changes detected." : !_serverReady ? _serverReadinessDetail : !_bridgeManager.Inspect().Valid ? "Install or repair the managed OBS integration." : string.IsNullOrWhiteSpace(_settings.Current.PrimarySourceUuid) ? "Choose the primary OBS microphone first." : State.ObsConnected && _bridgeSelectionAttached == false ? "Companion could not attach the saved OBS microphone. Re-select it or restart OBS." : State.ObsConnected && ObsSources.FirstOrDefault(source => source.Uuid == _settings.Current.PrimarySourceUuid) is not { Missing: false, Active: true } ? "The saved microphone is not currently available in the active OBS Program scene." : _settings.Current.PathTestPassedAt.HasValue ? "A relevant source, model, or integration setting changed. Run the short voice-free check when convenient." : "Run this short check once. It does not require speaking and remains valid until a relevant configuration changes."; var health = State.ObsStreaming ? TrayHealth.Operating : valid && State.Connected && State.ObsConnected ? TrayHealth.Ready : State.Connected ? TrayHealth.Degraded : State.Health; var overview = valid && State.Connected && State.ObsConnected && !State.ObsStreaming ? "Everything is ready. Lumi will let you know if a relevant connection or configuration needs attention." : State.Detail; SetState(State with { PathTestValid = valid, PathTestDetail = detail, Health = health, Detail = overview }); } private static double MeasureDbfs(ReadOnlySpan frame) { if (frame.Length <= ProtocolV1.AudioHeaderBytes || (frame[5] & 1) == 0 || (frame[5] & 2) != 0) return -60; var samples = frame[ProtocolV1.AudioHeaderBytes..]; double squares = 0; var count = samples.Length / 2; for (var index = 0; index < count; index += 1) { var value = BinaryPrimitives.ReadInt16LittleEndian(samples.Slice(index * 2, 2)) / 32768.0; squares += value * value; } if (count == 0) return -60; var rms = Math.Sqrt(squares / count); return Math.Clamp(rms <= 0 ? -60 : 20 * Math.Log10(rms), -60, 0); } private static BenchmarkSnapshot EmptyBenchmark(string status, DateTimeOffset? startedAt = null) => new(null, string.Empty, [], CalculateMetric([]), CalculateMetric([]), startedAt, status); private static double ReadDouble(JsonElement value, string name) => value.TryGetProperty(name, out var property) && property.ValueKind == JsonValueKind.Number ? property.GetDouble() : 0; private static double? ReadNullableDouble(JsonElement value, string name) => value.TryGetProperty(name, out var property) && property.ValueKind == JsonValueKind.Number ? property.GetDouble() : null; private static int ReadInt(JsonElement value, string name) => value.TryGetProperty(name, out var property) && property.TryGetInt32(out var result) ? result : 0; private static bool ReadBoolean(JsonElement value, string name) => value.TryGetProperty(name, out var property) && property.ValueKind == JsonValueKind.True; private static string? ReadString(JsonElement value, string name) => value.TryGetProperty(name, out var property) && property.ValueKind == JsonValueKind.String ? property.GetString() : null; private CancellationToken _lifetimeToken() => _disposed ? new CancellationToken(true) : CancellationToken.None; private async Task<(bool Completed, string? Failure)> WaitForTestSignalAsync(Task signal, TimeSpan timeout, CancellationToken cancellationToken) { var failure = _testFailure?.Task ?? throw new InvalidOperationException("The test failure signal is unavailable."); using var timeoutSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); var delay = Task.Delay(timeout, timeoutSource.Token); var completed = await Task.WhenAny(signal, failure, delay); if (completed == signal) { timeoutSource.Cancel(); await signal; return (true, null); } if (completed == failure) { timeoutSource.Cancel(); return (false, await failure); } cancellationToken.ThrowIfCancellationRequested(); return (false, null); } private bool DetectBridgeInstallation() => _bridgeManager.Inspect().Valid; private CompanionState WithBridgeState(CompanionState state) { var bridge = _bridgeManager.Inspect(); return state with { ObsBridgeInstalled = bridge.Valid, ObsBridgeRepairNeeded = bridge.Installed && !bridge.Valid, ObsBridgePackageAvailable = bridge.PackageAvailable, ObsBridgeDetail = bridge.Detail }; } private static string? FindBundledPairingPackage() { try { return Directory.EnumerateFiles(AppContext.BaseDirectory, "*.lumi-pairing.json", SearchOption.TopDirectoryOnly).Take(2).SingleOrDefault(); } catch { return null; } } private void ApplyAutoStart(bool enabled) { if (!OperatingSystem.IsWindows()) return; try { using var key = Registry.CurrentUser.CreateSubKey(RunKey); if (enabled) key.SetValue("Lumi Companion", $"\"{Environment.ProcessPath}\" --background"); else key.DeleteValue("Lumi Companion", false); } catch (Exception error) { _ = WriteLogAsync("autostart_failed", error.Message); } } private async Task WriteLogAsync(string kind, string message) { try { Directory.CreateDirectory(_paths.LogsDirectory); PruneLogs(); var path = Path.Combine(_paths.LogsDirectory, $"companion-{DateTime.UtcNow:yyyy-MM-dd}.jsonl"); var line = JsonSerializer.Serialize(new { timestamp = DateTimeOffset.UtcNow, kind, message }, ProtocolV1.JsonOptions); await File.AppendAllTextAsync(path, line + Environment.NewLine); LogAdded?.Invoke($"{DateTime.Now:HH:mm:ss} {message}"); } catch { } } private void PruneLogs() { var files = new DirectoryInfo(_paths.LogsDirectory).GetFiles("*.jsonl").OrderBy(file => file.CreationTimeUtc).ToList(); foreach (var file in files.Where(file => file.CreationTimeUtc < DateTime.UtcNow.AddDays(-7))) file.Delete(); const long cap = 256L * 1024 * 1024; var total = files.Where(file => file.Exists).Sum(file => file.Length); foreach (var file in files.Where(file => file.Exists)) { if (total <= cap) break; total -= file.Length; file.Delete(); } } private void SetState(CompanionState state) { State = state; StateChanged?.Invoke(state); } private static string Friendly(Exception error) => error switch { InvalidDataException => error.Message, System.Net.WebSockets.WebSocketException => "The Lumi server connection ended unexpectedly. The host may have restarted; reconnect and review transcription diagnostics.", EndOfStreamException => error.Message, HttpRequestException => "Check the Lumi address, TLS certificate, and network connection.", TaskCanceledException => "The connection timed out. Check that Lumi is reachable.", _ => error.Message }; private sealed record BenchmarkCaptionRevision(int Revision, bool Final, string Text, IReadOnlyList Words, DateTimeOffset ReceivedAt); private static TestStage[] CreateInitialTestStages() => [ new("OBS integration", "Waiting to check the managed bridge.", TestStageState.Waiting), new("OBS connection", "Waiting for OBS.", TestStageState.Waiting), new("Microphone", "Waiting for a selected source.", TestStageState.Waiting), new("Audio source", "Waiting to validate the selected OBS source. Speaking is not required.", TestStageState.Waiting), new("Lumi connection", "Waiting to verify secure transport.", TestStageState.Waiting), new("Speech recognition", "Waiting for the server-hosted model readiness report.", TestStageState.Waiting), new("Caption return", "Waiting for a safe test caption.", TestStageState.Waiting), new("Delivery adapter", "Waiting for safe simulation mode.", TestStageState.Waiting), new("Simulated output", "Nothing is sent to Twitch during this test.", TestStageState.Waiting) ]; public static string Version => (Assembly.GetExecutingAssembly().GetCustomAttribute()?.InformationalVersion ?? "0.1.0").Split('+')[0]; public async ValueTask DisposeAsync() { if (_disposed) return; if (State.BenchmarkRunning) try { await StopBenchmarkAsync("disconnect", CancellationToken.None); } catch { } _disposed = true; _maintenanceLifetime.Cancel(); _benchmarkLifetime?.Cancel(); if (_socket is not null) await _socket.DisposeAsync(); if (_obsBridge is not null) await _obsBridge.DisposeAsync(); _benchmarkLifetime?.Dispose(); _bridgeSelectionGate.Dispose(); _http.Dispose(); _maintenanceLifetime.Dispose(); } }