Make transcription readiness reusable
This commit is contained in:
Franz Rolfsvaag
parent
16f5f1fbb6
commit
2e6de64e61
6
TODO.md
6
TODO.md
@ -39,6 +39,12 @@ retains per-word latency/confidence inspection data for one hour, while paired
|
||||
device management now defaults to active credentials and purges revoked history
|
||||
after 30 days.
|
||||
|
||||
Experimental.6 makes the full-path readiness check voice-free, preserves a valid
|
||||
pass across restarts until a relevant source/model/bridge configuration changes,
|
||||
adds a live dBFS meter to the dedicated speech test, and fixes reusable benchmark
|
||||
sessions so failed starts cannot leave worker activity behind. Companion now also
|
||||
reports verified OBS integration and path readiness to the WebUI setup progress.
|
||||
|
||||
Release-blocking work remains:
|
||||
|
||||
- Package the native whisper.cpp worker as the Windows CUDA runtime, then benchmark
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
param(
|
||||
[string]$Version = "0.1.0-experimental.5",
|
||||
[string]$Version = "0.1.0-experimental.6",
|
||||
[string]$BridgeVersion = "0.1.0-experimental.5"
|
||||
)
|
||||
|
||||
|
||||
@ -14,6 +14,7 @@ namespace Lumi.Companion.App;
|
||||
public sealed class CompanionRuntime : IAsyncDisposable
|
||||
{
|
||||
private const string RunKey = @"Software\Microsoft\Windows\CurrentVersion\Run";
|
||||
private const string PathValidationContract = "voice-free-path-v1";
|
||||
private readonly CompanionPaths _paths;
|
||||
private readonly CompanionSettingsStore _settings;
|
||||
private readonly SecureCredentialStore _credentials;
|
||||
@ -23,19 +24,24 @@ public sealed class CompanionRuntime : IAsyncDisposable
|
||||
private readonly CancellationTokenSource _maintenanceLifetime = new();
|
||||
private CompanionSocket? _socket;
|
||||
private ObsBridgePipe? _obsBridge;
|
||||
private TaskCompletionSource<bool>? _audioSignal;
|
||||
private TaskCompletionSource<bool>? _captionSignal;
|
||||
private TaskCompletionSource<bool>? _sessionStartSignal;
|
||||
private TaskCompletionSource<bool>? _sessionStopSignal;
|
||||
private TaskCompletionSource<string>? _testFailure;
|
||||
private TaskCompletionSource<(string SourceUuid, bool Attached)>? _bridgeSelectionSignal;
|
||||
private bool? _bridgeSelectionAttached;
|
||||
private readonly SemaphoreSlim _bridgeSelectionGate = new(1, 1);
|
||||
private readonly Dictionary<string, BenchmarkCaptionRevision> _benchmarkCaptions = [];
|
||||
private CancellationTokenSource? _benchmarkLifetime;
|
||||
private TaskCompletionSource<bool>? _benchmarkCompleteSignal;
|
||||
private DateTimeOffset? _benchmarkLastVoiceAt;
|
||||
private int _benchmarkStopping;
|
||||
private long _lastVoiceMeterAt;
|
||||
private bool _disposed;
|
||||
private CompanionUpdate? _availableUpdate;
|
||||
private bool _serverReady;
|
||||
private string? _serverReadinessFingerprint;
|
||||
private string _serverReadinessDetail = "Waiting for Lumi to report speech recognition readiness.";
|
||||
|
||||
public CompanionRuntime(CompanionPaths paths, CompanionSettingsStore settings)
|
||||
{
|
||||
@ -57,6 +63,7 @@ public sealed class CompanionRuntime : IAsyncDisposable
|
||||
public event Action<IReadOnlyList<TestStage>>? TestStagesChanged;
|
||||
public event Action<IReadOnlyList<ObsSource>>? ObsSourcesChanged;
|
||||
public event Action<BenchmarkSnapshot>? BenchmarkChanged;
|
||||
public event Action<double>? VoiceLevelChanged;
|
||||
public event Action<string, bool>? CaptionReceived;
|
||||
public event Action<string>? LogAdded;
|
||||
public event Func<Task>? UpdateRestartRequested;
|
||||
@ -135,6 +142,9 @@ public sealed class CompanionRuntime : IAsyncDisposable
|
||||
private async Task ConnectAsync(DeviceCredential credential, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (_socket is not null) await _socket.DisposeAsync();
|
||||
_serverReady = false;
|
||||
_serverReadinessFingerprint = null;
|
||||
_serverReadinessDetail = "Waiting for Lumi to report speech recognition readiness.";
|
||||
_socket = new CompanionSocket();
|
||||
_socket.MessageReceived += OnServerMessageAsync;
|
||||
_socket.Disconnected += error =>
|
||||
@ -161,8 +171,8 @@ public sealed class CompanionRuntime : IAsyncDisposable
|
||||
LastConnectedAt = DateTimeOffset.Now
|
||||
});
|
||||
foreach (var source in ObsSources) await SendSourceUpdateAsync(source);
|
||||
if (State.ObsConnected)
|
||||
await _socket.SendAsync("obs_state", new { streaming = State.ObsStreaming, recording = State.ObsRecording, auto_start = _settings.Current.StartWithObs }, _socket.SessionId, cancellationToken);
|
||||
await SendRuntimeStateAsync(cancellationToken);
|
||||
await _socket.SendAsync("readiness", new { }, _socket.SessionId, cancellationToken);
|
||||
await WriteLogAsync("connected", $"Connected to {credential.Host}.");
|
||||
await CheckForUpdatesAsync(cancellationToken);
|
||||
}
|
||||
@ -192,9 +202,9 @@ public sealed class CompanionRuntime : IAsyncDisposable
|
||||
if (selected is not { Missing: false, Active: true } || !bridgeAttached) return;
|
||||
await EvaluateStageAsync(stages, 4, State.Connected, "The secure Lumi transport is ready.", "Reconnect to Lumi before testing.", cancellationToken);
|
||||
if (!State.Connected) return;
|
||||
_audioSignal = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
_captionSignal = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
_sessionStartSignal = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
_sessionStopSignal = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
_testFailure = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
await _socket!.SendAsync("start", new { mode = "test" }, _socket.SessionId, cancellationToken);
|
||||
var sessionStart = await WaitForTestSignalAsync(_sessionStartSignal.Task, TimeSpan.FromSeconds(8), cancellationToken);
|
||||
@ -204,34 +214,36 @@ public sealed class CompanionRuntime : IAsyncDisposable
|
||||
MarkStage(stages, 5, TestStageState.Blocked, sessionStart.Failure ?? "Lumi did not confirm that speech recognition started within 8 seconds.");
|
||||
return;
|
||||
}
|
||||
MarkStage(stages, 3, TestStageState.Running, "Speak normally into the selected microphone…");
|
||||
var audio = await WaitForTestSignalAsync(_audioSignal.Task, TimeSpan.FromSeconds(10), cancellationToken);
|
||||
if (!audio.Completed)
|
||||
{
|
||||
MarkStage(stages, 3, TestStageState.Blocked, audio.Failure ?? "No active microphone audio reached the companion within 10 seconds.");
|
||||
return;
|
||||
}
|
||||
MarkStage(stages, 3, TestStageState.Passed, "Live microphone audio reached the companion.");
|
||||
MarkStage(stages, 5, TestStageState.Running, "Waiting for server-hosted speech recognition…");
|
||||
var caption = await WaitForTestSignalAsync(_captionSignal.Task, TimeSpan.FromSeconds(30), cancellationToken);
|
||||
MarkStage(stages, 3, TestStageState.Passed, "The selected OBS source is attached. You do not need to speak for this check.");
|
||||
MarkStage(stages, 5, TestStageState.Passed, "Lumi reports that the selected speech model is loaded and ready.");
|
||||
MarkStage(stages, 6, TestStageState.Running, "Checking the safe caption return path…");
|
||||
var caption = await WaitForTestSignalAsync(_captionSignal.Task, TimeSpan.FromSeconds(8), cancellationToken);
|
||||
if (!caption.Completed)
|
||||
{
|
||||
MarkStage(stages, 5, TestStageState.Blocked, caption.Failure ?? "No caption returned within 30 seconds. Check the loaded model and host diagnostics.");
|
||||
MarkStage(stages, 6, TestStageState.Blocked, caption.Failure ?? "The safe test caption did not return within 8 seconds.");
|
||||
return;
|
||||
}
|
||||
MarkStage(stages, 5, TestStageState.Passed, "Speech recognition returned a stable result.");
|
||||
MarkStage(stages, 6, TestStageState.Passed, "A caption returned over the secure connection.");
|
||||
MarkStage(stages, 6, TestStageState.Passed, "A safe test caption returned over the secure connection.");
|
||||
MarkStage(stages, 7, TestStageState.Passed, "The delivery adapter stayed in safe simulation mode.");
|
||||
MarkStage(stages, 8, TestStageState.Passed, "The exact outgoing caption was rendered locally, not sent to Twitch.");
|
||||
SetState(State with { Detail = "The safe end-to-end transcription test passed.", Health = TrayHealth.Ready });
|
||||
var fingerprint = CurrentPathFingerprint() ?? throw new InvalidOperationException("Lumi readiness changed while the check was running. Try the check again.");
|
||||
await _settings.SaveAsync(_settings.Current with { PathTestPassedAt = DateTimeOffset.UtcNow, PathTestFingerprint = fingerprint }, cancellationToken);
|
||||
RefreshPathReadiness("The voice-free full-path check passed. It stays valid until a relevant setup or model configuration changes.");
|
||||
await SendRuntimeStateAsync(cancellationToken);
|
||||
SetState(State with { Detail = "Everything is ready. The dedicated transcript test is available when you want to measure real speech.", Health = TrayHealth.Ready });
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (_socket is not null && State.Connected)
|
||||
try { await _socket.SendAsync("stop", new { reason = "test_complete" }, _socket.SessionId, CancellationToken.None); } catch { }
|
||||
_audioSignal = null;
|
||||
try
|
||||
{
|
||||
await _socket.SendAsync("stop", new { reason = "test_complete" }, _socket.SessionId, CancellationToken.None);
|
||||
if (_sessionStopSignal is not null) await Task.WhenAny(_sessionStopSignal.Task, Task.Delay(TimeSpan.FromSeconds(5)));
|
||||
}
|
||||
catch { }
|
||||
_captionSignal = null;
|
||||
_sessionStartSignal = null;
|
||||
_sessionStopSignal = null;
|
||||
_testFailure = null;
|
||||
SetState(State with { TestRunning = false, Detail = TestStages.Any(stage => stage.State == TestStageState.Blocked) ? "The test stopped at the first unavailable real boundary." : State.Detail });
|
||||
}
|
||||
@ -254,6 +266,7 @@ public sealed class CompanionRuntime : IAsyncDisposable
|
||||
Interlocked.Exchange(ref _benchmarkStopping, 0);
|
||||
Benchmark = EmptyBenchmark("Starting", DateTimeOffset.UtcNow);
|
||||
BenchmarkChanged?.Invoke(Benchmark);
|
||||
VoiceLevelChanged?.Invoke(-60);
|
||||
SetState(State with { BenchmarkRunning = true, BenchmarkDetail = "Starting server-hosted accuracy and latency measurement…", Health = TrayHealth.Operating });
|
||||
try
|
||||
{
|
||||
@ -293,6 +306,7 @@ public sealed class CompanionRuntime : IAsyncDisposable
|
||||
}
|
||||
finally
|
||||
{
|
||||
VoiceLevelChanged?.Invoke(-60);
|
||||
SetState(State with { BenchmarkRunning = false, BenchmarkDetail = Benchmark.Words.Count > 0 ? $"Test complete with {Benchmark.Words.Count} measured words or phrases." : "The test ended without a finalized transcript." });
|
||||
_sessionStartSignal = null;
|
||||
_testFailure = null;
|
||||
@ -388,6 +402,8 @@ public sealed class CompanionRuntime : IAsyncDisposable
|
||||
{
|
||||
var result = await _bridgeManager.InstallOrRepairAsync(cancellationToken);
|
||||
SetState(WithBridgeState(State with { Detail = "OBS integration installed. Start or restart OBS to connect it to Companion." }));
|
||||
RefreshPathReadiness();
|
||||
await SendRuntimeStateAsync(cancellationToken);
|
||||
await WriteLogAsync("obs_bridge_installed", $"Installed managed OBS integration {result.Version}.");
|
||||
}
|
||||
catch (Exception error)
|
||||
@ -404,6 +420,8 @@ public sealed class CompanionRuntime : IAsyncDisposable
|
||||
{
|
||||
await _bridgeManager.RemoveAsync();
|
||||
SetState(WithBridgeState(State with { ObsConnected = false, Detail = "OBS integration removed. Other Companion features remain installed." }));
|
||||
RefreshPathReadiness();
|
||||
await SendRuntimeStateAsync(CancellationToken.None);
|
||||
await WriteLogAsync("obs_bridge_removed", "Removed the managed OBS integration.");
|
||||
}
|
||||
catch (Exception error)
|
||||
@ -426,6 +444,8 @@ public sealed class CompanionRuntime : IAsyncDisposable
|
||||
await _settings.SaveAsync(_settings.Current with { PrimarySourceUuid = source.Uuid, PrimarySourceName = source.Name }, cancellationToken);
|
||||
await SyncBridgeSelectionAsync(cancellationToken);
|
||||
foreach (var item in ObsSources) await SendSourceUpdateAsync(item);
|
||||
RefreshPathReadiness();
|
||||
await SendRuntimeStateAsync(cancellationToken);
|
||||
await WriteLogAsync("source_selected", $"Selected OBS source {source.Name}.");
|
||||
}
|
||||
|
||||
@ -451,8 +471,18 @@ public sealed class CompanionRuntime : IAsyncDisposable
|
||||
|
||||
private Task OnServerMessageAsync(ServerEnvelope message)
|
||||
{
|
||||
if (message.Type == "status" && message.Payload.TryGetProperty("kind", out var statusKind) && statusKind.GetString() == "readiness")
|
||||
{
|
||||
_serverReady = ReadBoolean(message.Payload, "ready");
|
||||
_serverReadinessFingerprint = ReadString(message.Payload, "fingerprint");
|
||||
_serverReadinessDetail = ReadString(message.Payload, "detail") ?? "Lumi did not provide speech recognition readiness details.";
|
||||
RefreshPathReadiness();
|
||||
_ = SendRuntimeStateAsync(_lifetimeToken());
|
||||
}
|
||||
if (message.Type == "status" && message.Payload.TryGetProperty("kind", out var kind) && kind.GetString() == "session" &&
|
||||
message.Payload.TryGetProperty("state", out var state) && state.GetString() == "running")
|
||||
message.Payload.TryGetProperty("state", out var state))
|
||||
{
|
||||
if (state.GetString() == "running")
|
||||
{
|
||||
_sessionStartSignal?.TrySetResult(true);
|
||||
if (message.Payload.TryGetProperty("benchmark_id", out var benchmarkId) && benchmarkId.ValueKind == JsonValueKind.String)
|
||||
@ -461,6 +491,8 @@ public sealed class CompanionRuntime : IAsyncDisposable
|
||||
BenchmarkChanged?.Invoke(Benchmark);
|
||||
}
|
||||
}
|
||||
else if (state.GetString() == "idle") _sessionStopSignal?.TrySetResult(true);
|
||||
}
|
||||
if (message.Type == "caption")
|
||||
{
|
||||
var stableText = message.Payload.TryGetProperty("stable_text", out var stable) ? stable.GetString() : null;
|
||||
@ -499,9 +531,12 @@ public sealed class CompanionRuntime : IAsyncDisposable
|
||||
_obsBridge.ConnectionChanged += connected =>
|
||||
{
|
||||
var health = connected && State.Connected ? TrayHealth.Ready : State.Health;
|
||||
if (!connected) _bridgeSelectionAttached = null;
|
||||
SetState(State with { ObsBridgeInstalled = connected || State.ObsBridgeInstalled, ObsConnected = connected, Health = health, Detail = connected ? "OBS and Lumi are connected. Choose a microphone and run a safe test." : State.Detail });
|
||||
RefreshPathReadiness();
|
||||
_ = WriteLogAsync("obs_connection", connected ? "OBS bridge connected." : "OBS bridge disconnected.");
|
||||
if (connected) _ = SyncBridgeSelectionAsync();
|
||||
_ = SendRuntimeStateAsync(_lifetimeToken());
|
||||
};
|
||||
_obsBridge.Start();
|
||||
}
|
||||
@ -514,8 +549,8 @@ public sealed class CompanionRuntime : IAsyncDisposable
|
||||
var streaming = ReadBoolean(message, "streaming");
|
||||
var recording = ReadBoolean(message, "recording");
|
||||
SetState(State with { ObsConnected = true, ObsStreaming = streaming, ObsRecording = recording, Health = streaming ? TrayHealth.Operating : TrayHealth.Ready, Detail = streaming ? "OBS is live and companion services are active." : "OBS is connected and ready." });
|
||||
if (_socket is not null && State.Connected)
|
||||
await _socket.SendAsync("obs_state", new { streaming, recording, version = ReadString(message, "version"), auto_start = _settings.Current.StartWithObs }, _socket.SessionId, _lifetimeToken());
|
||||
RefreshPathReadiness();
|
||||
await SendRuntimeStateAsync(_lifetimeToken(), ReadString(message, "version"));
|
||||
}
|
||||
else if (type == "source_list" && message.TryGetProperty("sources", out var sources) && sources.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
@ -525,8 +560,10 @@ public sealed class CompanionRuntime : IAsyncDisposable
|
||||
ReadBoolean(source, "program_active") || ReadBoolean(source, "active"),
|
||||
ReadBoolean(source, "source_missing") || ReadBoolean(source, "missing")))
|
||||
.Where(source => Guid.TryParse(source.Uuid, out _)).ToArray();
|
||||
_bridgeSelectionAttached = null;
|
||||
ObsSourcesChanged?.Invoke(ObsSources);
|
||||
foreach (var source in ObsSources) await SendSourceUpdateAsync(source);
|
||||
RefreshPathReadiness();
|
||||
_ = SyncBridgeSelectionAsync(_lifetimeToken());
|
||||
}
|
||||
else if (type == "source_state")
|
||||
@ -537,6 +574,7 @@ public sealed class CompanionRuntime : IAsyncDisposable
|
||||
ObsSources = ObsSources.Where(item => item.Uuid != source.Uuid).Append(source).OrderBy(item => item.Name).ToArray();
|
||||
ObsSourcesChanged?.Invoke(ObsSources);
|
||||
await SendSourceUpdateAsync(source);
|
||||
RefreshPathReadiness();
|
||||
}
|
||||
}
|
||||
else if (type == "selection_state")
|
||||
@ -547,9 +585,14 @@ public sealed class CompanionRuntime : IAsyncDisposable
|
||||
|
||||
private Task OnObsAudioAsync(ReadOnlyMemory<byte> frame)
|
||||
{
|
||||
if (HasVoice(frame.Span))
|
||||
var level = MeasureDbfs(frame.Span);
|
||||
if (State.BenchmarkRunning && Environment.TickCount64 - Interlocked.Read(ref _lastVoiceMeterAt) >= 50)
|
||||
{
|
||||
Interlocked.Exchange(ref _lastVoiceMeterAt, Environment.TickCount64);
|
||||
VoiceLevelChanged?.Invoke(level);
|
||||
}
|
||||
if (level >= -38.5)
|
||||
{
|
||||
_audioSignal?.TrySetResult(true);
|
||||
if (State.BenchmarkRunning) _benchmarkLastVoiceAt = DateTimeOffset.UtcNow;
|
||||
}
|
||||
if (_socket is not null && State.Connected && !_socket.QueueBridgeAudio(frame))
|
||||
@ -575,7 +618,7 @@ public sealed class CompanionRuntime : IAsyncDisposable
|
||||
|
||||
private async Task<bool> SyncBridgeSelectionAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (_obsBridge is null || string.IsNullOrWhiteSpace(_settings.Current.PrimarySourceUuid)) return false;
|
||||
if (_obsBridge is null || string.IsNullOrWhiteSpace(_settings.Current.PrimarySourceUuid)) { _bridgeSelectionAttached = false; RefreshPathReadiness(); return false; }
|
||||
await _bridgeSelectionGate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
@ -589,11 +632,20 @@ public sealed class CompanionRuntime : IAsyncDisposable
|
||||
source_uuids = new[] { sourceUuid },
|
||||
primary_source_uuid = sourceUuid
|
||||
}, cancellationToken);
|
||||
if (!sent) return false;
|
||||
var attached = false;
|
||||
if (sent)
|
||||
{
|
||||
var completed = await Task.WhenAny(signal.Task, Task.Delay(TimeSpan.FromSeconds(2), cancellationToken));
|
||||
if (completed != signal.Task) { cancellationToken.ThrowIfCancellationRequested(); return true; }
|
||||
if (completed != signal.Task) { cancellationToken.ThrowIfCancellationRequested(); attached = true; }
|
||||
else
|
||||
{
|
||||
var result = await signal.Task;
|
||||
return result.Attached && string.Equals(result.SourceUuid, sourceUuid, StringComparison.OrdinalIgnoreCase);
|
||||
attached = result.Attached && string.Equals(result.SourceUuid, sourceUuid, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
}
|
||||
_bridgeSelectionAttached = attached;
|
||||
RefreshPathReadiness();
|
||||
return attached;
|
||||
}
|
||||
finally
|
||||
{
|
||||
@ -685,9 +737,62 @@ public sealed class CompanionRuntime : IAsyncDisposable
|
||||
return values[lower] + (values[upper] - values[lower]) * (position - lower);
|
||||
}
|
||||
|
||||
private static bool HasVoice(ReadOnlySpan<byte> frame)
|
||||
private async Task SendRuntimeStateAsync(CancellationToken cancellationToken, string? obsVersion = null)
|
||||
{
|
||||
if (frame.Length <= ProtocolV1.AudioHeaderBytes || (frame[5] & 1) == 0 || (frame[5] & 2) != 0) return false;
|
||||
if (_socket is null || !State.Connected) return;
|
||||
var bridge = _bridgeManager.Inspect();
|
||||
var fingerprint = CurrentPathFingerprint();
|
||||
var pathValid = _serverReady && fingerprint is not null && _settings.Current.PathTestPassedAt.HasValue &&
|
||||
string.Equals(_settings.Current.PathTestFingerprint, fingerprint, StringComparison.Ordinal);
|
||||
await _socket.SendAsync("obs_state", new
|
||||
{
|
||||
streaming = State.ObsStreaming,
|
||||
recording = State.ObsRecording,
|
||||
version = obsVersion,
|
||||
auto_start = _settings.Current.StartWithObs,
|
||||
bridge_installed = bridge.Valid,
|
||||
bridge_connected = State.ObsConnected,
|
||||
bridge_version = bridge.Version,
|
||||
path_test_valid = pathValid,
|
||||
path_test_at = pathValid ? _settings.Current.PathTestPassedAt?.ToUnixTimeMilliseconds() : null
|
||||
}, _socket.SessionId, cancellationToken);
|
||||
}
|
||||
|
||||
private string? CurrentPathFingerprint()
|
||||
{
|
||||
var bridge = _bridgeManager.Inspect();
|
||||
if (!_serverReady || string.IsNullOrWhiteSpace(_serverReadinessFingerprint) || !bridge.Valid ||
|
||||
string.IsNullOrWhiteSpace(State.Host) || string.IsNullOrWhiteSpace(_settings.Current.PrimarySourceUuid)) return null;
|
||||
if (State.ObsConnected && _bridgeSelectionAttached == false) return null;
|
||||
if (State.ObsConnected && ObsSources.FirstOrDefault(source => source.Uuid == _settings.Current.PrimarySourceUuid) is not { Missing: false, Active: true }) return null;
|
||||
var value = string.Join("|", PathValidationContract, State.Host, _settings.Current.PrimarySourceUuid, bridge.Version, _serverReadinessFingerprint);
|
||||
return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value))).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private void RefreshPathReadiness(string? validDetail = null)
|
||||
{
|
||||
var current = CurrentPathFingerprint();
|
||||
var valid = current is not null && _settings.Current.PathTestPassedAt.HasValue &&
|
||||
string.Equals(_settings.Current.PathTestFingerprint, current, StringComparison.Ordinal);
|
||||
var detail = valid
|
||||
? validDetail ?? $"Passed {_settings.Current.PathTestPassedAt!.Value.LocalDateTime:g}; no relevant setup or model changes detected."
|
||||
: !_serverReady ? _serverReadinessDetail
|
||||
: !_bridgeManager.Inspect().Valid ? "Install or repair the managed OBS integration."
|
||||
: string.IsNullOrWhiteSpace(_settings.Current.PrimarySourceUuid) ? "Choose the primary OBS microphone first."
|
||||
: State.ObsConnected && _bridgeSelectionAttached == false ? "Companion could not attach the saved OBS microphone. Re-select it or restart OBS."
|
||||
: State.ObsConnected && ObsSources.FirstOrDefault(source => source.Uuid == _settings.Current.PrimarySourceUuid) is not { Missing: false, Active: true } ? "The saved microphone is not currently available in the active OBS Program scene."
|
||||
: _settings.Current.PathTestPassedAt.HasValue ? "A relevant source, model, or integration setting changed. Run the short voice-free check when convenient."
|
||||
: "Run this short check once. It does not require speaking and remains valid until a relevant configuration changes.";
|
||||
var health = State.ObsStreaming ? TrayHealth.Operating : valid && State.Connected && State.ObsConnected ? TrayHealth.Ready : State.Connected ? TrayHealth.Degraded : State.Health;
|
||||
var overview = valid && State.Connected && State.ObsConnected && !State.ObsStreaming
|
||||
? "Everything is ready. Lumi will let you know if a relevant connection or configuration needs attention."
|
||||
: State.Detail;
|
||||
SetState(State with { PathTestValid = valid, PathTestDetail = detail, Health = health, Detail = overview });
|
||||
}
|
||||
|
||||
private static double MeasureDbfs(ReadOnlySpan<byte> frame)
|
||||
{
|
||||
if (frame.Length <= ProtocolV1.AudioHeaderBytes || (frame[5] & 1) == 0 || (frame[5] & 2) != 0) return -60;
|
||||
var samples = frame[ProtocolV1.AudioHeaderBytes..];
|
||||
double squares = 0;
|
||||
var count = samples.Length / 2;
|
||||
@ -696,7 +801,9 @@ public sealed class CompanionRuntime : IAsyncDisposable
|
||||
var value = BinaryPrimitives.ReadInt16LittleEndian(samples.Slice(index * 2, 2)) / 32768.0;
|
||||
squares += value * value;
|
||||
}
|
||||
return count > 0 && Math.Sqrt(squares / count) >= 0.012;
|
||||
if (count == 0) return -60;
|
||||
var rms = Math.Sqrt(squares / count);
|
||||
return Math.Clamp(rms <= 0 ? -60 : 20 * Math.Log10(rms), -60, 0);
|
||||
}
|
||||
|
||||
private static BenchmarkSnapshot EmptyBenchmark(string status, DateTimeOffset? startedAt = null) =>
|
||||
@ -783,10 +890,10 @@ public sealed class CompanionRuntime : IAsyncDisposable
|
||||
new("OBS integration", "Waiting to check the managed bridge.", TestStageState.Waiting),
|
||||
new("OBS connection", "Waiting for OBS.", TestStageState.Waiting),
|
||||
new("Microphone", "Waiting for a selected source.", TestStageState.Waiting),
|
||||
new("Audio capture", "Waiting for live microphone audio.", TestStageState.Waiting),
|
||||
new("Audio source", "Waiting to validate the selected OBS source. Speaking is not required.", TestStageState.Waiting),
|
||||
new("Lumi connection", "Waiting to verify secure transport.", TestStageState.Waiting),
|
||||
new("Speech recognition", "Waiting for real server-hosted inference.", TestStageState.Waiting),
|
||||
new("Caption return", "Waiting for a stable caption.", TestStageState.Waiting),
|
||||
new("Speech recognition", "Waiting for the server-hosted model readiness report.", TestStageState.Waiting),
|
||||
new("Caption return", "Waiting for a safe test caption.", TestStageState.Waiting),
|
||||
new("Delivery adapter", "Waiting for safe simulation mode.", TestStageState.Waiting),
|
||||
new("Simulated output", "Nothing is sent to Twitch during this test.", TestStageState.Waiting)
|
||||
];
|
||||
|
||||
@ -8,7 +8,9 @@ public sealed record CompanionSettings(
|
||||
bool StartWithObs = true,
|
||||
bool AdvancedMode = false,
|
||||
string? PrimarySourceUuid = null,
|
||||
string? PrimarySourceName = null);
|
||||
string? PrimarySourceName = null,
|
||||
DateTimeOffset? PathTestPassedAt = null,
|
||||
string? PathTestFingerprint = null);
|
||||
|
||||
public sealed class CompanionSettingsStore
|
||||
{
|
||||
|
||||
@ -22,6 +22,8 @@ public sealed record CompanionState(
|
||||
string? AvailableVersion = null,
|
||||
string UpdateDetail = "Checking for updates…",
|
||||
string BenchmarkDetail = "Start a dedicated test, speak naturally, then end it when you have enough material.",
|
||||
bool PathTestValid = false,
|
||||
string PathTestDetail = "Run once after setup or a relevant configuration change.",
|
||||
bool ObsBridgeRepairNeeded = false,
|
||||
bool ObsBridgePackageAvailable = false,
|
||||
string ObsBridgeDetail = "Checking the managed OBS integration…")
|
||||
|
||||
@ -5,7 +5,7 @@
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<ApplicationManifest>app.manifest</ApplicationManifest>
|
||||
<Version>0.1.0-experimental.5</Version>
|
||||
<Version>0.1.0-experimental.6</Version>
|
||||
<AssemblyVersion>0.1.0.0</AssemblyVersion>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
@ -21,5 +21,8 @@
|
||||
<Content Include="components/obs-bridge/lumi-obs-bridge.dll" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" ExcludeFromSingleFile="true" />
|
||||
<Content Include="components/obs-bridge/manifest.json" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
|
||||
<Content Include="components/obs-bridge/en-US.ini" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
|
||||
<EmbeddedResource Include="components/obs-bridge/lumi-obs-bridge.dll" LogicalName="Lumi.Companion.ObsBridge.dll" />
|
||||
<EmbeddedResource Include="components/obs-bridge/manifest.json" LogicalName="Lumi.Companion.ObsBridge.manifest.json" />
|
||||
<EmbeddedResource Include="components/obs-bridge/en-US.ini" LogicalName="Lumi.Companion.ObsBridge.en-US.ini" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@ -128,7 +128,7 @@
|
||||
<StackPanel Spacing="6">
|
||||
<TextBlock Text="SAFE CHECK" Classes="eyebrow" />
|
||||
<TextBlock Text="Transcription test" Classes="pageTitle" />
|
||||
<TextBlock Text="Exercises the real capture, network, speech recognition, and return path without sending anything to Twitch." Classes="muted" FontSize="15" />
|
||||
<TextBlock Text="Checks the selected OBS source, secure connection, model readiness, and safe caption return path. You do not need to speak, and nothing is sent to Twitch." Classes="muted" FontSize="15" />
|
||||
</StackPanel>
|
||||
<Button x:Name="RunTestButton" Classes="primary" Content="Run full path test" HorizontalAlignment="Left" />
|
||||
<StackPanel x:Name="TestStagesPanel" Spacing="8" />
|
||||
@ -145,10 +145,13 @@
|
||||
<TextBlock Text="Dedicated transcription test" Classes="sectionTitle" />
|
||||
<TextBlock Text="Speak naturally for as long as needed. End the test yourself, or Lumi will finalize it after 10 seconds of silence. Nothing is sent to Twitch." Classes="muted" />
|
||||
</StackPanel>
|
||||
<StackPanel Orientation="Horizontal" Spacing="10">
|
||||
<Grid ColumnDefinitions="Auto,Auto,*,Auto" ColumnSpacing="10">
|
||||
<Button x:Name="StartBenchmarkButton" Classes="primary" Content="Start dedicated test" />
|
||||
<Button x:Name="StopBenchmarkButton" Classes="secondary" Content="End test" IsEnabled="False" />
|
||||
</StackPanel>
|
||||
<Button Grid.Column="1" x:Name="StopBenchmarkButton" Classes="secondary" Content="End test" IsEnabled="False" />
|
||||
<ProgressBar Grid.Column="2" x:Name="VoiceLevelMeter" Minimum="-60" Maximum="0" Value="-60" Height="12" MinWidth="220" VerticalAlignment="Center" Foreground="{DynamicResource LumiDangerBrush}" />
|
||||
<TextBlock Grid.Column="3" x:Name="VoiceLevelText" Text="-60 dBFS" Classes="muted" FontSize="11" VerticalAlignment="Center" MinWidth="58" />
|
||||
</Grid>
|
||||
<TextBlock Text="Input level: green from -30 to -12 dBFS; orange within 10 dB outside that range; red otherwise." Classes="muted" FontSize="11" />
|
||||
<TextBlock x:Name="BenchmarkStatusText" Text="Not started." Classes="muted" />
|
||||
<Border Classes="soft">
|
||||
<StackPanel Spacing="5">
|
||||
|
||||
@ -12,7 +12,6 @@ public partial class MainWindow : Window
|
||||
private readonly CompanionRuntime _runtime;
|
||||
private readonly CompanionSettingsStore _settings;
|
||||
private bool _allowExit;
|
||||
private bool _testPassed;
|
||||
private bool _renderingSources;
|
||||
|
||||
public MainWindow() : this(CreateDefaultServices()) { }
|
||||
@ -34,6 +33,7 @@ public partial class MainWindow : Window
|
||||
runtime.TestStagesChanged += stages => Dispatcher.UIThread.Post(() => RenderTestStages(stages));
|
||||
runtime.ObsSourcesChanged += sources => Dispatcher.UIThread.Post(() => RenderSources(sources));
|
||||
runtime.BenchmarkChanged += benchmark => Dispatcher.UIThread.Post(() => RenderBenchmark(benchmark));
|
||||
runtime.VoiceLevelChanged += level => Dispatcher.UIThread.Post(() => RenderVoiceLevel(level));
|
||||
runtime.CaptionReceived += (text, simulated) => Dispatcher.UIThread.Post(() =>
|
||||
{
|
||||
if (simulated) SimulatedCaption.Text = text;
|
||||
@ -170,8 +170,8 @@ public partial class MainWindow : Window
|
||||
PairingStepDetail.Text = state.Connected ? "Securely connected" : state.Paired ? "Paired, currently offline" : "Not paired";
|
||||
ObsStepSymbol.Text = state.ObsConnected ? "✓" : state.ObsBridgeInstalled ? "◐" : "○";
|
||||
ObsStepDetail.Text = state.ObsConnected ? "Connected to OBS" : state.ObsBridgeInstalled ? "Installed; waiting for OBS" : "Installation required";
|
||||
TestStepSymbol.Text = _testPassed ? "✓" : "○";
|
||||
TestStepDetail.Text = _testPassed ? "Passed" : "Not passed";
|
||||
TestStepSymbol.Text = state.PathTestValid ? "✓" : "○";
|
||||
TestStepDetail.Text = state.PathTestDetail;
|
||||
DeviceNameText.Text = state.DeviceName ?? "Not paired";
|
||||
HostText.Text = state.Host ?? "—";
|
||||
LastConnectedText.Text = state.LastConnectedAt?.ToString("g") ?? "Never";
|
||||
@ -214,9 +214,9 @@ public partial class MainWindow : Window
|
||||
}
|
||||
else
|
||||
{
|
||||
NextActionTitle.Text = "Run a safe transcription test";
|
||||
NextActionDetail.Text = "Check the real end-to-end path without sending captions to Twitch.";
|
||||
NextActionButton.Content = "Open test";
|
||||
NextActionTitle.Text = state.PathTestValid ? "Ready when you are" : "Run one short readiness check";
|
||||
NextActionDetail.Text = state.PathTestValid ? state.PathTestDetail : "No speaking is required. The result remains valid until a relevant setup or model setting changes.";
|
||||
NextActionButton.Content = state.PathTestValid ? "View status" : "Open voice-free check";
|
||||
}
|
||||
}
|
||||
|
||||
@ -255,9 +255,10 @@ public partial class MainWindow : Window
|
||||
Grid.SetColumn(detail, 2); row.Children.Add(detail);
|
||||
TestStagesPanel.Children.Add(new Border { Classes = { "soft" }, Child = row, Padding = new Thickness(14, 11) });
|
||||
}
|
||||
_testPassed = stages.Count > 0 && stages.All(stage => stage.State == TestStageState.Passed);
|
||||
TestStepSymbol.Text = _testPassed ? "✓" : "○";
|
||||
TestStepDetail.Text = _testPassed ? "Passed" : "Not passed";
|
||||
var currentPass = stages.Count > 0 && stages.All(stage => stage.State == TestStageState.Passed);
|
||||
var valid = currentPass || _runtime.State.PathTestValid;
|
||||
TestStepSymbol.Text = valid ? "✓" : "○";
|
||||
TestStepDetail.Text = currentPass ? "Passed just now; this result will be reused while the relevant setup stays unchanged." : _runtime.State.PathTestDetail;
|
||||
RunTestButton.Content = _runtime.State.TestRunning ? "Testing…" : "Run full path test";
|
||||
RunTestButton.IsEnabled = !_runtime.State.TestRunning && !_runtime.State.BenchmarkRunning;
|
||||
}
|
||||
@ -269,6 +270,16 @@ public partial class MainWindow : Window
|
||||
RenderMetric(ConfidenceStatsPanel, benchmark.Confidence, value => $"{value:P0}");
|
||||
}
|
||||
|
||||
private void RenderVoiceLevel(double level)
|
||||
{
|
||||
var clamped = Math.Clamp(level, -60, 0);
|
||||
VoiceLevelMeter.Value = clamped;
|
||||
VoiceLevelText.Text = $"{clamped:0} dBFS";
|
||||
var color = clamped >= -30 && clamped <= -12 ? "#23845B" :
|
||||
clamped >= -40 && clamped <= -2 ? "#E58B2B" : "#BD4D4D";
|
||||
VoiceLevelMeter.Foreground = new SolidColorBrush(Color.Parse(color));
|
||||
}
|
||||
|
||||
private static void RenderMetric(Panel panel, MetricStatistics metric, Func<double, string> formatter)
|
||||
{
|
||||
panel.Children.Clear();
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
using System.Diagnostics;
|
||||
using System.ComponentModel;
|
||||
using System.Reflection;
|
||||
using System.Security.Cryptography;
|
||||
using System.Security.Principal;
|
||||
using System.Text.Json;
|
||||
@ -10,23 +11,23 @@ namespace Lumi.Companion.App;
|
||||
|
||||
public sealed class ObsBridgeManager
|
||||
{
|
||||
private readonly string _componentRoot = Path.Combine(AppContext.BaseDirectory, "components", "obs-bridge");
|
||||
private readonly Lazy<ObsBridgePackage?> _package;
|
||||
private readonly string _installRoot = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), "obs-studio", "plugins", "lumi-obs-bridge");
|
||||
private string SourceDll => Path.Combine(_componentRoot, "lumi-obs-bridge.dll");
|
||||
private string SourceManifest => Path.Combine(_componentRoot, "manifest.json");
|
||||
private string InstalledDll => Path.Combine(_installRoot, "bin", "64bit", "lumi-obs-bridge.dll");
|
||||
private string InstalledManifest => Path.Combine(_installRoot, "lumi-managed.json");
|
||||
|
||||
public ObsBridgeManager() => _package = new Lazy<ObsBridgePackage?>(LoadPackage);
|
||||
|
||||
public ObsBridgeStatus Inspect()
|
||||
{
|
||||
var package = ReadManifest(SourceManifest);
|
||||
var package = _package.Value;
|
||||
var installed = ReadManifest(InstalledManifest);
|
||||
var packageAvailable = package is not null && File.Exists(SourceDll) && HashFile(SourceDll) == package.Sha256;
|
||||
var packageAvailable = package is not null;
|
||||
var fileInstalled = File.Exists(InstalledDll);
|
||||
var valid = packageAvailable && fileInstalled && HashFile(InstalledDll) == package!.Sha256 && installed?.Version == package.Version;
|
||||
return new ObsBridgeStatus(packageAvailable, fileInstalled, valid, package?.Version,
|
||||
valid ? $"OBS integration {package!.Version} is installed. Restart OBS if it was open during the last repair." :
|
||||
!packageAvailable ? "This Companion build does not contain a valid OBS integration package." :
|
||||
var valid = packageAvailable && fileInstalled && HashFile(InstalledDll) == package!.Manifest.Sha256 && installed?.Version == package.Manifest.Version;
|
||||
return new ObsBridgeStatus(packageAvailable, fileInstalled, valid, package?.Manifest.Version,
|
||||
valid ? $"OBS integration {package!.Manifest.Version} is installed. Restart OBS if it was open during the last repair." :
|
||||
!packageAvailable ? "The bundled OBS integration could not be verified. Use Check now to repair the Companion package." :
|
||||
fileInstalled ? "The OBS integration is outdated or damaged. Repair it while OBS is closed." : "The OBS integration is ready to install.");
|
||||
}
|
||||
|
||||
@ -47,14 +48,14 @@ public sealed class ObsBridgeManager
|
||||
|
||||
private async Task<ObsBridgeStatus> InstallDirectAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var manifest = ReadManifest(SourceManifest)!;
|
||||
var package = _package.Value ?? throw new InvalidDataException("The bundled OBS integration could not be verified.");
|
||||
var manifest = package.Manifest;
|
||||
var bin = Path.GetDirectoryName(InstalledDll)!;
|
||||
var locale = Path.Combine(_installRoot, "data", "locale");
|
||||
Directory.CreateDirectory(bin);
|
||||
Directory.CreateDirectory(locale);
|
||||
await CopyAtomicAsync(SourceDll, InstalledDll, cancellationToken);
|
||||
var sourceLocale = Path.Combine(_componentRoot, "en-US.ini");
|
||||
if (File.Exists(sourceLocale)) await CopyAtomicAsync(sourceLocale, Path.Combine(locale, "en-US.ini"), cancellationToken);
|
||||
await WriteAtomicAsync(package.Dll, InstalledDll, cancellationToken);
|
||||
if (package.Locale is not null) await WriteAtomicAsync(package.Locale, Path.Combine(locale, "en-US.ini"), cancellationToken);
|
||||
var marker = JsonSerializer.SerializeToUtf8Bytes(manifest, ProtocolV1.JsonOptions);
|
||||
var temporary = $"{InstalledManifest}.{Environment.ProcessId}.tmp";
|
||||
await File.WriteAllBytesAsync(temporary, marker, cancellationToken);
|
||||
@ -126,15 +127,46 @@ public sealed class ObsBridgeManager
|
||||
throw new InvalidOperationException("Close OBS before installing, repairing, or removing the managed integration. Companion will never modify a loaded OBS plugin.");
|
||||
}
|
||||
private static ObsBridgeManifest? ReadManifest(string path) { try { return JsonSerializer.Deserialize<ObsBridgeManifest>(File.ReadAllBytes(path), ProtocolV1.JsonOptions); } catch { return null; } }
|
||||
private ObsBridgePackage? LoadPackage()
|
||||
{
|
||||
var roots = new[]
|
||||
{
|
||||
Path.Combine(Path.GetDirectoryName(Environment.ProcessPath) ?? string.Empty, "components", "obs-bridge"),
|
||||
Path.Combine(AppContext.BaseDirectory, "components", "obs-bridge")
|
||||
}.Distinct(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var root in roots)
|
||||
{
|
||||
try
|
||||
{
|
||||
var manifestBytes = File.ReadAllBytes(Path.Combine(root, "manifest.json"));
|
||||
var dll = File.ReadAllBytes(Path.Combine(root, "lumi-obs-bridge.dll"));
|
||||
var manifest = JsonSerializer.Deserialize<ObsBridgeManifest>(manifestBytes, ProtocolV1.JsonOptions);
|
||||
if (manifest is not null && HashBytes(dll) == manifest.Sha256)
|
||||
return new ObsBridgePackage(manifest, dll, File.Exists(Path.Combine(root, "en-US.ini")) ? File.ReadAllBytes(Path.Combine(root, "en-US.ini")) : null);
|
||||
}
|
||||
catch { }
|
||||
}
|
||||
try
|
||||
{
|
||||
var assembly = Assembly.GetExecutingAssembly();
|
||||
var manifestBytes = ReadResource(assembly, "Lumi.Companion.ObsBridge.manifest.json");
|
||||
var dll = ReadResource(assembly, "Lumi.Companion.ObsBridge.dll");
|
||||
var manifest = JsonSerializer.Deserialize<ObsBridgeManifest>(manifestBytes, ProtocolV1.JsonOptions);
|
||||
return manifest is not null && HashBytes(dll) == manifest.Sha256
|
||||
? new ObsBridgePackage(manifest, dll, TryReadResource(assembly, "Lumi.Companion.ObsBridge.en-US.ini")) : null;
|
||||
}
|
||||
catch { return null; }
|
||||
}
|
||||
private static byte[] ReadResource(Assembly assembly, string name) { using var stream = assembly.GetManifestResourceStream(name) ?? throw new FileNotFoundException($"Embedded resource {name} is missing."); using var body = new MemoryStream(); stream.CopyTo(body); return body.ToArray(); }
|
||||
private static byte[]? TryReadResource(Assembly assembly, string name) { try { return ReadResource(assembly, name); } catch { return null; } }
|
||||
private static string HashFile(string path) { using var stream = File.OpenRead(path); return Convert.ToHexString(SHA256.HashData(stream)).ToLowerInvariant(); }
|
||||
private static async Task CopyAtomicAsync(string source, string target, CancellationToken cancellationToken)
|
||||
private static string HashBytes(byte[] value) => Convert.ToHexString(SHA256.HashData(value)).ToLowerInvariant();
|
||||
private static async Task WriteAtomicAsync(byte[] value, string target, CancellationToken cancellationToken)
|
||||
{
|
||||
var temporary = $"{target}.{Environment.ProcessId}.tmp";
|
||||
try
|
||||
{
|
||||
await using (var input = File.OpenRead(source))
|
||||
await using (var output = new FileStream(temporary, FileMode.Create, FileAccess.Write, FileShare.None, 81920, true))
|
||||
await input.CopyToAsync(output, cancellationToken);
|
||||
await File.WriteAllBytesAsync(temporary, value, cancellationToken);
|
||||
File.Move(temporary, target, true);
|
||||
}
|
||||
catch
|
||||
@ -150,3 +182,4 @@ public sealed record ObsBridgeManifest(
|
||||
[property: JsonPropertyName("sha256")] string Sha256,
|
||||
[property: JsonPropertyName("obs_minimum_version")] string ObsMinimumVersion);
|
||||
public sealed record ObsBridgeStatus(bool PackageAvailable, bool Installed, bool Valid, string? Version, string Detail);
|
||||
internal sealed record ObsBridgePackage(ObsBridgeManifest Manifest, byte[] Dll, byte[]? Locale);
|
||||
|
||||
@ -14,7 +14,7 @@ editable: false
|
||||
Server-hosted whisper.cpp transcription for Lumi Companion and OBS closed captions.
|
||||
## Metadata
|
||||
Plugin ID: lumi_transcription
|
||||
Version: 0.1.0-experimental.5
|
||||
Version: 0.1.0-experimental.6
|
||||
Default state: enabled
|
||||
## Web Routes
|
||||
- /plugins/lumi_transcription
|
||||
|
||||
@ -83,6 +83,22 @@ class DeviceStore {
|
||||
const changed = this.db.prepare("UPDATE transcription_devices SET capabilities_json = ? WHERE id = ? AND revoked_at IS NULL").run(JSON.stringify(allowed), deviceId).changes;
|
||||
return changed ? allowed : null;
|
||||
}
|
||||
updateRuntime(deviceId, input = {}) {
|
||||
const row = this.db.prepare("SELECT metadata_json FROM transcription_devices WHERE id = ? AND revoked_at IS NULL").get(deviceId);
|
||||
if (!row) return false;
|
||||
let metadata = {};
|
||||
try { metadata = JSON.parse(row.metadata_json || "{}"); } catch { }
|
||||
if (typeof input.bridge_installed === "boolean") metadata.bridge_installed = input.bridge_installed;
|
||||
if (typeof input.bridge_connected === "boolean") metadata.bridge_connected = input.bridge_connected;
|
||||
if (input.bridge_version !== undefined) metadata.bridge_version = clean(input.bridge_version, 32) || null;
|
||||
if (typeof input.path_test_valid === "boolean") metadata.path_test_valid = input.path_test_valid;
|
||||
if (Number.isFinite(Number(input.path_test_at))) metadata.path_test_at = Math.max(0, Number(input.path_test_at));
|
||||
if (input.companion_version !== undefined) metadata.companion_version = clean(input.companion_version, 32);
|
||||
if (input.companion_plugin_version !== undefined) metadata.companion_plugin_version = clean(input.companion_plugin_version, 32);
|
||||
metadata.runtime_seen_at = this.now();
|
||||
return this.db.prepare("UPDATE transcription_devices SET metadata_json = ?, last_connected_at = ? WHERE id = ? AND revoked_at IS NULL")
|
||||
.run(JSON.stringify(metadata), this.now(), deviceId).changes === 1;
|
||||
}
|
||||
pairingAllowsHttp(token, requestOrigin) {
|
||||
const row = this.db.prepare("SELECT host, activated_at, expires_at FROM transcription_pairing_tokens WHERE token_hash = ?").get(digest(token));
|
||||
return Boolean(row && !row.activated_at && row.expires_at >= this.now() && sameLoopbackOrigin(row.host, requestOrigin));
|
||||
|
||||
@ -28,6 +28,7 @@ class CompanionGateway {
|
||||
let lastPong = Date.now();
|
||||
let windowStarted = Date.now();
|
||||
let messagesInWindow = 0;
|
||||
let messageChain = Promise.resolve();
|
||||
const send = (type, payload, sessionId = session?.id || null) => {
|
||||
if (socket.readyState === WebSocket.OPEN) socket.send(JSON.stringify(envelope(type, payload, sessionId)));
|
||||
};
|
||||
@ -37,20 +38,23 @@ class CompanionGateway {
|
||||
send("status", { kind: "heartbeat", state: session?.state || "connecting" });
|
||||
}, 10000);
|
||||
helloTimer.unref?.(); heartbeat.unref?.();
|
||||
socket.on("message", async (data, isBinary) => {
|
||||
socket.on("message", (data, isBinary) => {
|
||||
const body = Buffer.from(data);
|
||||
messageChain = messageChain.then(async () => {
|
||||
try {
|
||||
if (Date.now() - windowStarted >= 1000) { windowStarted = Date.now(); messagesInWindow = 0; }
|
||||
messagesInWindow += 1;
|
||||
if (messagesInWindow > 300) throw coded("RATE_LIMIT", "Companion message rate exceeded its limit.");
|
||||
if (isBinary) {
|
||||
if (!helloComplete || !session) throw coded("HELLO_REQUIRED", "Complete the handshake before sending audio.");
|
||||
const result = await this.sessions.audio(session.id, parseAudioFrame(data));
|
||||
const result = await this.sessions.audio(session.id, parseAudioFrame(body));
|
||||
if (result.gap) send("metric", { kind: "sequence_gap", missing_frames: result.gap });
|
||||
return;
|
||||
}
|
||||
const message = parseEnvelope(data);
|
||||
const message = parseEnvelope(body);
|
||||
if (!helloComplete) {
|
||||
const hello = validateHello(message);
|
||||
this.devices.updateRuntime(device.id, { companion_version: hello.companion_version, companion_plugin_version: hello.plugin_version });
|
||||
const created = this.sessions.create(device, send, hello.resume_session_id);
|
||||
session = created.session;
|
||||
helloComplete = true;
|
||||
@ -71,6 +75,7 @@ class CompanionGateway {
|
||||
if (["INCOMPATIBLE_VERSION", "HELLO_REQUIRED", "RATE_LIMIT"].includes(error.code)) closeWith(socket, 4400, error.code);
|
||||
}
|
||||
});
|
||||
});
|
||||
socket.on("close", () => {
|
||||
clearTimeout(helloTimer); clearInterval(heartbeat);
|
||||
if (session) this.sessions.disconnect(session.id);
|
||||
@ -80,9 +85,24 @@ class CompanionGateway {
|
||||
}
|
||||
async structured(session, message, send, pong) {
|
||||
switch (message.type) {
|
||||
case "ping": pong(); send("pong", { received_id: message.id }); break;
|
||||
case "source_update": send("status", { kind: "source", source: this.sessions.updateSource(session.id, message.payload || {}) }); break;
|
||||
case "obs_state": send("status", { kind: "obs", ...(await this.sessions.updateObsState(session.id, message.payload || {})) }); break;
|
||||
case "ping":
|
||||
pong();
|
||||
send("pong", { received_id: message.id });
|
||||
send("status", { kind: "readiness", ...(await this.sessions.readiness()), server_plugin_version: require("../../plugin.json").version });
|
||||
break;
|
||||
case "source_update": {
|
||||
const source = this.sessions.updateSource(session.id, message.payload || {});
|
||||
this.devices.updateRuntime(session.deviceId, { bridge_installed: true, bridge_connected: true });
|
||||
send("status", { kind: "source", source });
|
||||
break;
|
||||
}
|
||||
case "obs_state": {
|
||||
const status = await this.sessions.updateObsState(session.id, message.payload || {});
|
||||
this.devices.updateRuntime(session.deviceId, status);
|
||||
send("status", { kind: "obs", ...status });
|
||||
break;
|
||||
}
|
||||
case "readiness": send("status", { kind: "readiness", ...(await this.sessions.readiness()), server_plugin_version: require("../../plugin.json").version }); break;
|
||||
case "start": send("status", { kind: "session", ...(await this.sessions.start(session.id, message.payload || {})) }); break;
|
||||
case "stop": send("status", { kind: "session", ...(await this.sessions.stop(session.id, cleanReason(message.payload?.reason))) }); break;
|
||||
case "ack": break;
|
||||
|
||||
@ -5,7 +5,7 @@ const AUDIO_MAGIC = Buffer.from("LACP");
|
||||
const AUDIO_HEADER_BYTES = 64;
|
||||
const MAX_JSON_BYTES = 64 * 1024;
|
||||
const MAX_AUDIO_PAYLOAD_BYTES = 6400;
|
||||
const CLIENT_TYPES = new Set(["hello", "ping", "source_update", "obs_state", "start", "stop", "ack"]);
|
||||
const CLIENT_TYPES = new Set(["hello", "ping", "source_update", "obs_state", "readiness", "start", "stop", "ack"]);
|
||||
|
||||
function parseEnvelope(input) {
|
||||
const bytes = Buffer.isBuffer(input) ? input : Buffer.from(String(input || ""), "utf8");
|
||||
|
||||
@ -10,6 +10,7 @@ class SessionCoordinator {
|
||||
this.now = options.now || Date.now;
|
||||
this.graceMs = options.graceMs || 30000;
|
||||
this.benchmarks = options.benchmarks || null;
|
||||
this.readinessContext = options.readinessContext || (() => ({}));
|
||||
this.sessions = new Map();
|
||||
this.stabilizer = new CaptionStabilizer({ now: this.now });
|
||||
this.provider.on?.("hypothesis", (event) => Promise.resolve(this.onHypothesis(event)).catch((error) => {
|
||||
@ -34,7 +35,7 @@ class SessionCoordinator {
|
||||
const session = {
|
||||
id: crypto.randomUUID(), deviceId: device.id, connected: true, send,
|
||||
state: "idle", mode: null, obs: { streaming: false, recording: false },
|
||||
tracks: new Map(), delivery: this.deliveryFactory(send), createdAt: this.now(), graceUntil: 0, graceTimer: null, benchmarkId: null
|
||||
tracks: new Map(), delivery: this.deliveryFactory(send), createdAt: this.now(), graceUntil: 0, graceTimer: null, benchmarkId: null, providerActive: false
|
||||
};
|
||||
this.sessions.set(session.id, session);
|
||||
this.log.append({ kind: "session", state: "created", session_id: session.id, device_id: device.id });
|
||||
@ -59,7 +60,14 @@ class SessionCoordinator {
|
||||
async updateObsState(sessionId, obs) {
|
||||
const session = this.require(sessionId);
|
||||
const wasStreaming = session.obs.streaming;
|
||||
session.obs = { streaming: Boolean(obs.streaming), recording: Boolean(obs.recording), version: text(obs.version, 32) || null };
|
||||
session.obs = {
|
||||
streaming: Boolean(obs.streaming), recording: Boolean(obs.recording), version: text(obs.version, 32) || null,
|
||||
bridge_installed: typeof obs.bridge_installed === "boolean" ? obs.bridge_installed : session.obs.bridge_installed,
|
||||
bridge_connected: typeof obs.bridge_connected === "boolean" ? obs.bridge_connected : session.obs.bridge_connected,
|
||||
bridge_version: obs.bridge_version === undefined ? session.obs.bridge_version : text(obs.bridge_version, 32) || null,
|
||||
path_test_valid: typeof obs.path_test_valid === "boolean" ? obs.path_test_valid : session.obs.path_test_valid,
|
||||
path_test_at: Number.isFinite(Number(obs.path_test_at)) ? Math.max(0, Number(obs.path_test_at)) : session.obs.path_test_at
|
||||
};
|
||||
if (!wasStreaming && session.obs.streaming && session.state === "idle" && obs.auto_start !== false) await this.start(sessionId, { mode: "live" });
|
||||
if (wasStreaming && !session.obs.streaming && session.state === "running" && session.mode === "live") this.beginGrace(session);
|
||||
if (!wasStreaming && session.obs.streaming && session.state === "grace") await this.resume(session);
|
||||
@ -67,6 +75,7 @@ class SessionCoordinator {
|
||||
}
|
||||
async start(sessionId, options = {}) {
|
||||
const session = this.require(sessionId);
|
||||
if (session.state !== "idle") throw Object.assign(new Error("Another transcription activity is still finishing. Try again in a moment."), { code: "SESSION_BUSY" });
|
||||
const mode = options.mode === "test" ? "test" : options.mode === "benchmark" ? "benchmark" : "live";
|
||||
if (mode === "live" && !session.obs.streaming) throw Object.assign(new Error("Live transcription requires an active OBS stream."), { code: "OBS_NOT_STREAMING" });
|
||||
const tracks = Array.from(session.tracks.values()).filter((track) => track.enabled && !track.source_missing);
|
||||
@ -78,21 +87,36 @@ class SessionCoordinator {
|
||||
`Worker state is ${providerHealth.state || "unavailable"}.`;
|
||||
throw Object.assign(new Error(`Speech recognition is not ready. ${reason}`), { code: "PROVIDER_UNAVAILABLE", details: providerHealth });
|
||||
}
|
||||
try {
|
||||
if (mode === "benchmark" && this.benchmarks) session.benchmarkId = this.benchmarks.start({ sessionId: session.id, deviceId: session.deviceId, source: serializeTrack(tracks[0]) });
|
||||
if (mode !== "test") {
|
||||
await this.provider.startSession({ id: session.id, mode });
|
||||
session.providerActive = true;
|
||||
for (const track of tracks) await this.provider.addTrack(session.id, serializeTrack(track));
|
||||
}
|
||||
await session.delivery.start({ testMode: mode !== "live" });
|
||||
session.mode = mode;
|
||||
session.state = "running";
|
||||
if (mode === "benchmark" && this.benchmarks) session.benchmarkId = this.benchmarks.start({ sessionId: session.id, deviceId: session.deviceId, source: serializeTrack(tracks[0]) });
|
||||
this.log.append({ kind: "session", state: "running", mode, session_id: session.id, tracks: tracks.map((track) => track.source_uuid) });
|
||||
if (mode === "test") await session.delivery.deliver(pathTestCaption(session, tracks[0], providerHealth, this.now()));
|
||||
return this.status(session.id);
|
||||
} catch (error) {
|
||||
if (session.providerActive) try { await this.provider.stopSession(session.id); } catch { }
|
||||
try { await session.delivery.stop(); } catch { }
|
||||
if (session.benchmarkId && this.benchmarks) this.benchmarks.finish(session.id, "failed");
|
||||
session.providerActive = false;
|
||||
session.benchmarkId = null;
|
||||
session.mode = null;
|
||||
session.state = "idle";
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
async stop(sessionId, reason = "requested") {
|
||||
const session = this.require(sessionId);
|
||||
clearTimeout(session.graceTimer);
|
||||
session.graceTimer = null;
|
||||
await session.delivery.stop();
|
||||
if (session.state !== "idle") await this.provider.stopSession(session.id);
|
||||
if (session.providerActive) await this.provider.stopSession(session.id);
|
||||
if (session.benchmarkId && this.benchmarks) {
|
||||
const benchmarkStatus = reason === "silence_timeout" ? "silence_timeout" : reason === "provider_failed" ? "failed" : reason === "disconnect" || reason === "plugin_shutdown" ? "aborted" : "completed";
|
||||
const result = this.benchmarks.finish(session.id, benchmarkStatus);
|
||||
@ -101,8 +125,10 @@ class SessionCoordinator {
|
||||
}
|
||||
session.state = "idle";
|
||||
session.mode = null;
|
||||
session.providerActive = false;
|
||||
for (const track of session.tracks.values()) track.buffer.clear();
|
||||
this.log.append({ kind: "session", state: "stopped", reason, session_id: session.id });
|
||||
if (!session.connected) this.expireLater(session);
|
||||
return this.status(session.id);
|
||||
}
|
||||
async audio(sessionId, frame) {
|
||||
@ -140,6 +166,19 @@ class SessionCoordinator {
|
||||
live_running: sessions.filter((session) => session.state === "running" && session.mode === "live").length
|
||||
};
|
||||
}
|
||||
async readiness() {
|
||||
const health = await this.provider.health();
|
||||
const fingerprint = crypto.createHash("sha256").update(JSON.stringify({
|
||||
contract: "voice-free-path-v1", provider: health.provider || "whisper_cpp_server",
|
||||
model: health.model || null, backend: health.backend || null, settings: this.readinessContext()
|
||||
})).digest("hex");
|
||||
return {
|
||||
ready: Boolean(health.healthy && health.model_ready), state: health.state,
|
||||
model: health.model || null, backend: health.backend || health.model?.backend || "whisper.cpp",
|
||||
fingerprint,
|
||||
detail: health.healthy && health.model_ready ? "The speech model is loaded and ready." : "Speech recognition needs attention in Lumi."
|
||||
};
|
||||
}
|
||||
async close() { for (const session of Array.from(this.sessions.values())) await this.stop(session.id, "plugin_shutdown"); this.sessions.clear(); }
|
||||
require(id) { const session = this.sessions.get(id); if (!session) throw new Error("Session was not found."); return session; }
|
||||
beginGrace(session) {
|
||||
@ -181,6 +220,7 @@ class SessionCoordinator {
|
||||
session.graceUntil = 0;
|
||||
session.state = "idle";
|
||||
session.mode = null;
|
||||
session.providerActive = false;
|
||||
for (const track of session.tracks.values()) track.buffer.clear();
|
||||
try { await session.delivery.stop(); } catch { }
|
||||
if (session.benchmarkId && this.benchmarks) {
|
||||
@ -202,6 +242,17 @@ function speakerLabel(session, track) { const enabled = Array.from(session.track
|
||||
function serializeTrack(track) { return { source_uuid: track.source_uuid, display_name: track.display_name, speaker_label: track.speaker_label, enabled: track.enabled, primary: track.primary, single_speaker: track.single_speaker, delivery_enabled: track.delivery_enabled, program_active: track.program_active, source_missing: track.source_missing, last_activity_at: track.last_activity_at, sequence: track.sequence?.metrics?.() }; }
|
||||
function text(value, max) { return String(value || "").trim().slice(0, max); }
|
||||
function isUuid(value) { return /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(String(value || "")); }
|
||||
function pathTestCaption(session, track, health, now) {
|
||||
return {
|
||||
session_id: session.id, source_uuid: track.source_uuid, speaker_label: null,
|
||||
caption_id: crypto.randomUUID(), revision: 1, final: true, new_utterance: true,
|
||||
stable_text: "Lumi transcription path is ready.", uncertain_text: "", display_seconds: 2,
|
||||
audio: { start_us: 0, end_us: 0 },
|
||||
latency: { capture_ms: 0, network_ms: 0, queue_ms: 0, inference_ms: 0, stabilization_ms: 0, total_ms: 0 },
|
||||
model: { id: health.model || "unknown", provider: health.provider || "whisper_cpp_server", backend: health.backend || "ready" },
|
||||
analysis: { transcript: "Lumi transcription path is ready.", words: [], emitted_at_ms: now }
|
||||
};
|
||||
}
|
||||
function benchmarkCompletion(result) {
|
||||
return {
|
||||
id: result.id, status: result.status, transcript: result.transcript, stats: result.stats,
|
||||
|
||||
@ -13,7 +13,7 @@ class BenchmarkStore {
|
||||
migrate() {
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS transcription_benchmark_tests (
|
||||
id TEXT PRIMARY KEY, session_id TEXT NOT NULL UNIQUE, device_id TEXT NOT NULL,
|
||||
id TEXT PRIMARY KEY, session_id TEXT NOT NULL, device_id TEXT NOT NULL,
|
||||
source_uuid TEXT NOT NULL, source_name TEXT NOT NULL, started_at INTEGER NOT NULL,
|
||||
ended_at INTEGER, status TEXT NOT NULL, model_id TEXT, backend TEXT
|
||||
);
|
||||
@ -24,7 +24,28 @@ class BenchmarkStore {
|
||||
model_id TEXT, backend TEXT,
|
||||
UNIQUE(test_id, caption_id, revision)
|
||||
);
|
||||
`);
|
||||
const schema = this.db.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'transcription_benchmark_tests'").get()?.sql || "";
|
||||
if (/session_id\s+TEXT\s+NOT\s+NULL\s+UNIQUE/i.test(schema)) {
|
||||
this.db.transaction(() => {
|
||||
this.db.exec(`
|
||||
ALTER TABLE transcription_benchmark_tests RENAME TO transcription_benchmark_tests_legacy;
|
||||
CREATE TABLE transcription_benchmark_tests (
|
||||
id TEXT PRIMARY KEY, session_id TEXT NOT NULL, device_id TEXT NOT NULL,
|
||||
source_uuid TEXT NOT NULL, source_name TEXT NOT NULL, started_at INTEGER NOT NULL,
|
||||
ended_at INTEGER, status TEXT NOT NULL, model_id TEXT, backend TEXT
|
||||
);
|
||||
INSERT INTO transcription_benchmark_tests
|
||||
(id, session_id, device_id, source_uuid, source_name, started_at, ended_at, status, model_id, backend)
|
||||
SELECT id, session_id, device_id, source_uuid, source_name, started_at, ended_at, status, model_id, backend
|
||||
FROM transcription_benchmark_tests_legacy;
|
||||
DROP TABLE transcription_benchmark_tests_legacy;
|
||||
`);
|
||||
})();
|
||||
}
|
||||
this.db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS transcription_benchmark_started_idx ON transcription_benchmark_tests(started_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS transcription_benchmark_session_idx ON transcription_benchmark_tests(session_id, started_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS transcription_benchmark_revision_test_idx ON transcription_benchmark_revisions(test_id, received_at);
|
||||
`);
|
||||
}
|
||||
@ -40,7 +61,7 @@ class BenchmarkStore {
|
||||
}
|
||||
|
||||
record(sessionId, event) {
|
||||
const test = this.db.prepare("SELECT id FROM transcription_benchmark_tests WHERE session_id = ? AND status = 'running'").get(sessionId);
|
||||
const test = this.db.prepare("SELECT id FROM transcription_benchmark_tests WHERE session_id = ? AND status = 'running' ORDER BY started_at DESC LIMIT 1").get(sessionId);
|
||||
if (!test) return false;
|
||||
const count = this.db.prepare("SELECT COUNT(*) AS count FROM transcription_benchmark_revisions WHERE test_id = ?").get(test.id).count;
|
||||
if (count >= 10000) return false;
|
||||
@ -58,13 +79,19 @@ class BenchmarkStore {
|
||||
|
||||
finish(sessionId, status = "completed") {
|
||||
const allowed = ["completed", "silence_timeout", "aborted", "failed"].includes(status) ? status : "completed";
|
||||
this.db.prepare("UPDATE transcription_benchmark_tests SET ended_at = ?, status = ? WHERE session_id = ? AND status = 'running'")
|
||||
.run(this.now(), allowed, sessionId);
|
||||
return this.bySession(sessionId);
|
||||
const test = this.db.prepare("SELECT id FROM transcription_benchmark_tests WHERE session_id = ? AND status = 'running' ORDER BY started_at DESC LIMIT 1").get(sessionId);
|
||||
if (!test) return this.bySession(sessionId);
|
||||
this.db.prepare("UPDATE transcription_benchmark_tests SET ended_at = ?, status = ? WHERE id = ?").run(this.now(), allowed, test.id);
|
||||
return this.byId(test.id);
|
||||
}
|
||||
|
||||
bySession(sessionId) {
|
||||
const row = this.db.prepare("SELECT * FROM transcription_benchmark_tests WHERE session_id = ?").get(sessionId);
|
||||
const row = this.db.prepare("SELECT * FROM transcription_benchmark_tests WHERE session_id = ? ORDER BY started_at DESC LIMIT 1").get(sessionId);
|
||||
return row ? this.snapshot(row) : null;
|
||||
}
|
||||
|
||||
byId(id) {
|
||||
const row = this.db.prepare("SELECT * FROM transcription_benchmark_tests WHERE id = ?").get(id);
|
||||
return row ? this.snapshot(row) : null;
|
||||
}
|
||||
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"version": "0.1.0-experimental.5",
|
||||
"version": "0.1.0-experimental.6",
|
||||
"signed": false,
|
||||
"release_notes": "Reliably reattaches the saved OBS source after startup, waits for finalized path-test captions, and adds a dedicated accuracy and latency benchmark with live statistics.",
|
||||
"release_notes": "Makes the full-path check voice-free and reusable, adds a live dBFS meter, fixes repeat benchmark sessions, reports accurate setup state to Lumi, and embeds the verified OBS repair payload.",
|
||||
"artifacts": [
|
||||
{
|
||||
"id": "windows-x64-self-contained",
|
||||
@ -10,9 +10,9 @@
|
||||
"architecture": "x64",
|
||||
"label": "Windows x64 self-contained",
|
||||
"filename": "Lumi.Companion-win-x64.zip",
|
||||
"url": "https://git.rolfsvaag.no/Rolfsvaag_Datateknikk/Lumi/releases/download/companion-v0.1.0-experimental.5/Lumi.Companion-win-x64.zip",
|
||||
"sha256": "5728700f5559f957853b5e49f7f0efc3352ba631ee57526cc0fd56b96b24b733",
|
||||
"bytes": 41676248,
|
||||
"url": "https://git.rolfsvaag.no/Rolfsvaag_Datateknikk/Lumi/releases/download/companion-v0.1.0-experimental.6/Lumi.Companion-win-x64.zip",
|
||||
"sha256": "bd487528208347ee469ee58eb1a01bab9ed0c6a4597c93241886fcfbc80c5815",
|
||||
"bytes": 41756978,
|
||||
"entrypoint": "Lumi.Companion.App.exe"
|
||||
}
|
||||
]
|
||||
|
||||
@ -48,7 +48,12 @@ module.exports = {
|
||||
provider,
|
||||
deliveryFactory: (send) => new CompanionCaptionDeliveryAdapter(send),
|
||||
log: diagnosticLog,
|
||||
benchmarks
|
||||
benchmarks,
|
||||
readinessContext: () => {
|
||||
const current = revisions.list();
|
||||
return Object.fromEntries(["selected_model_id", "fallback_order", "decode_interval_ms", "silence_finalize_ms", "rolling_context_ms", "caption_max_chars", "minimum_display_ms", "tracks"]
|
||||
.filter((key) => current[key]).map((key) => [key, current[key].revision]));
|
||||
}
|
||||
});
|
||||
const gateway = new CompanionGateway({ devices, sessions, log: diagnosticLog });
|
||||
const unregisterUpgrade = web.addUpgradeHandler("/plugins/lumi_transcription/live", (request, socket, head) => gateway.upgrade(request, socket, head));
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
{
|
||||
"id": "lumi_transcription",
|
||||
"name": "Lumi Transcription",
|
||||
"version": "0.1.0-experimental.5",
|
||||
"version": "0.1.0-experimental.6",
|
||||
"description": "Server-hosted whisper.cpp transcription for Lumi Companion and OBS closed captions.",
|
||||
"main": "index.js",
|
||||
"channel": "experimental",
|
||||
|
||||
@ -38,6 +38,7 @@ async function run() {
|
||||
verifyQueues();
|
||||
verifyStabilization();
|
||||
verifyBenchmarkRetention();
|
||||
await verifyBenchmarkStartRollback();
|
||||
await verifySessionLifecycle();
|
||||
await verifyProviderFailureFeedback();
|
||||
await verifyWorkerRestart();
|
||||
@ -105,6 +106,7 @@ function verifyLocalhostTransportPolicy() {
|
||||
}
|
||||
|
||||
function verifyCompanionVersionOrdering() {
|
||||
assert.equal(plugin.compareVersions("0.1.0-experimental.6", "0.1.0-experimental.5"), 1);
|
||||
assert.equal(plugin.compareVersions("0.1.0-experimental.5", "0.1.0-experimental.4"), 1);
|
||||
assert.equal(plugin.compareVersions("0.1.0-experimental.4", "0.1.0-experimental.3"), 1);
|
||||
assert.equal(plugin.compareVersions("0.1.0-experimental.3", "0.1.0-experimental.2"), 1);
|
||||
@ -186,10 +188,50 @@ function verifyBenchmarkRetention() {
|
||||
assert.equal(result.words.length, 2);
|
||||
assert.equal(result.stats.latency.average, 1050);
|
||||
assert.equal(metricStats([1, 2, 3]).median, 2);
|
||||
const secondId = store.start({ sessionId: "benchmark-session", deviceId: "device", source: { source_uuid: source, display_name: "Mic" } });
|
||||
assert.notEqual(secondId, id);
|
||||
store.finish("benchmark-session", "completed");
|
||||
assert.equal(store.list().length, 2);
|
||||
now += 3600001;
|
||||
assert.equal(store.cleanup(), 1);
|
||||
assert.equal(store.cleanup(), 2);
|
||||
assert.equal(store.list().length, 0);
|
||||
db.close();
|
||||
|
||||
const legacy = new Database(":memory:");
|
||||
legacy.exec(`CREATE TABLE transcription_benchmark_tests (
|
||||
id TEXT PRIMARY KEY, session_id TEXT NOT NULL UNIQUE, device_id TEXT NOT NULL,
|
||||
source_uuid TEXT NOT NULL, source_name TEXT NOT NULL, started_at INTEGER NOT NULL,
|
||||
ended_at INTEGER, status TEXT NOT NULL, model_id TEXT, backend TEXT
|
||||
)`);
|
||||
const migrated = new BenchmarkStore(legacy, { now: () => 1000 });
|
||||
const migratedSource = crypto.randomUUID();
|
||||
migrated.start({ sessionId: "reused", deviceId: "device", source: { source_uuid: migratedSource, display_name: "Mic" } });
|
||||
migrated.finish("reused", "completed");
|
||||
migrated.start({ sessionId: "reused", deviceId: "device", source: { source_uuid: migratedSource, display_name: "Mic" } });
|
||||
assert.equal(migrated.list().length, 2);
|
||||
legacy.close();
|
||||
}
|
||||
|
||||
async function verifyBenchmarkStartRollback() {
|
||||
class Provider extends EventEmitter {
|
||||
constructor() { super(); this.starts = 0; }
|
||||
async health() { return { healthy: true, model_ready: true, state: "running" }; }
|
||||
async startSession() { this.starts += 1; }
|
||||
async addTrack() {}
|
||||
async stopSession() {}
|
||||
}
|
||||
const provider = new Provider();
|
||||
const coordinator = new SessionCoordinator({
|
||||
provider,
|
||||
benchmarks: { start() { throw new Error("storage unavailable"); }, finish() {} },
|
||||
deliveryFactory: () => ({ start: async () => {}, stop: async () => {}, pause: async () => {}, resume: async () => {}, deliver: async () => ({ disposition: "simulated" }) })
|
||||
});
|
||||
const { session } = coordinator.create({ id: "device" }, () => {});
|
||||
coordinator.updateSource(session.id, { source_uuid: crypto.randomUUID(), display_name: "Mic", primary: true, program_active: true });
|
||||
await assert.rejects(coordinator.start(session.id, { mode: "benchmark" }), /storage unavailable/);
|
||||
assert.equal(provider.starts, 0);
|
||||
assert.equal(coordinator.status(session.id).state, "idle");
|
||||
await coordinator.close();
|
||||
}
|
||||
|
||||
async function verifySessionLifecycle() {
|
||||
@ -224,19 +266,21 @@ async function verifySessionLifecycle() {
|
||||
|
||||
async function verifyProviderFailureFeedback() {
|
||||
class Provider extends EventEmitter {
|
||||
constructor() { super(); this.starts = 0; }
|
||||
async health() { return { healthy: true, state: "running", model_ready: true }; }
|
||||
async startSession() {} async addTrack() {} async stopSession() {}
|
||||
async startSession() { this.starts += 1; } async addTrack() {} async stopSession() {}
|
||||
}
|
||||
const provider = new Provider();
|
||||
const sent = [];
|
||||
const coordinator = new SessionCoordinator({
|
||||
provider,
|
||||
deliveryFactory: () => ({ start: async () => {}, stop: async () => {}, pause: async () => {}, resume: async () => {} })
|
||||
deliveryFactory: () => ({ start: async () => {}, stop: async () => {}, pause: async () => {}, resume: async () => {}, deliver: async () => ({ disposition: "simulated" }) })
|
||||
});
|
||||
const { session } = coordinator.create({ id: "device" }, (type, payload) => sent.push({ type, payload }));
|
||||
const source = crypto.randomUUID();
|
||||
coordinator.updateSource(session.id, { source_uuid: source, display_name: "Mic", primary: true, program_active: true });
|
||||
await coordinator.start(session.id, { mode: "test" });
|
||||
assert.equal(provider.starts, 0);
|
||||
provider.emit("provider_error", Object.assign(new Error("worker exited with code 3"), { code: "WORKER_CRASHED" }));
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
assert.equal(coordinator.status(session.id).state, "idle");
|
||||
@ -288,6 +332,10 @@ async function verifyNativeWorkerBoundary() {
|
||||
assert.match(source, /session_stopped/);
|
||||
const bridge = fs.readFileSync(path.join(__dirname, "../../../companion/native/obs-bridge/src/plugin.cpp"), "utf8");
|
||||
assert.match(bridge, /selection_state/);
|
||||
const companionProject = fs.readFileSync(path.join(__dirname, "../../../companion/src/Lumi.Companion.App/Lumi.Companion.App.csproj"), "utf8");
|
||||
const bridgeManager = fs.readFileSync(path.join(__dirname, "../../../companion/src/Lumi.Companion.App/ObsBridgeManager.cs"), "utf8");
|
||||
assert.match(companionProject, /EmbeddedResource Include="components\/obs-bridge\/lumi-obs-bridge\.dll"/);
|
||||
assert.match(bridgeManager, /Lumi\.Companion\.ObsBridge\.dll/);
|
||||
assert.doesNotMatch(source, /ofstream|fwrite|WriteAllBytes/);
|
||||
}
|
||||
|
||||
@ -313,10 +361,11 @@ async function verifyAuthenticatedGateway() {
|
||||
headers: { Authorization: `LumiDevice ${credential.device_id}.${credential.device_secret}` }
|
||||
});
|
||||
await new Promise((resolve, reject) => { client.once("open", resolve); client.once("error", reject); });
|
||||
client.send(JSON.stringify(protocol.envelope("hello", { companion_version: "0.1.0", plugin_version: "0.1.0", capabilities: ["transcription.capture.v1"], audio: { codec: "pcm_s16le", sample_rate: 16000, channels: 1, bits: 16 } })));
|
||||
client.send(JSON.stringify(protocol.envelope("hello", { companion_version: "0.1.0-experimental.6", plugin_version: "0.1.0", capabilities: ["transcription.capture.v1"], audio: { codec: "pcm_s16le", sample_rate: 16000, channels: 1, bits: 16 } })));
|
||||
const response = await new Promise((resolve, reject) => { client.once("message", (data) => resolve(JSON.parse(String(data)))); client.once("error", reject); });
|
||||
assert.equal(response.type, "hello_ack");
|
||||
assert.equal(response.session_id, sessionId);
|
||||
assert.equal(devices.list().find((device) => device.id === credential.device_id).metadata.companion_version, "0.1.0-experimental.6");
|
||||
await new Promise((resolve) => { client.once("close", resolve); client.close(); });
|
||||
for (let attempt = 0; attempt < 20 && !disconnected; attempt += 1) await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
assert.equal(disconnected, true);
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
<% const displayCompanionVersion = (value) => { const match = /^(\d+\.\d+\.\d+)-experimental\.(\d+)$/i.exec(String(value || '')); return match ? `${match[1]} · Experimental release ${match[2]}` : String(value || 'Unknown version'); }; %>
|
||||
<section class="card transcription-overview" data-transcription-admin>
|
||||
<div class="section-header">
|
||||
<%- include("../../../src/web/views/partials/page-header", {
|
||||
@ -20,10 +21,10 @@
|
||||
<div><span class="eyebrow">First usable path</span><h2 id="setup-title">Setup progress</h2><p class="hint">Complete one clear step at a time. Existing server administration remains in Lumi; streaming-computer controls remain in Companion.</p></div>
|
||||
</div>
|
||||
<ol>
|
||||
<li class="<%= devices.some((device) => !device.revoked_at) ? 'is-complete' : 'is-current' %>"><strong>Download and pair Companion</strong><span>Create a short-lived, single-use package for the streaming computer.</span></li>
|
||||
<li><strong>Install the OBS integration</strong><span>Companion owns installation and repair; the bridge has no separate settings.</span></li>
|
||||
<li class="<%= devices.length ? 'is-complete' : 'is-current' %>"><strong>Download and pair Companion</strong><span>Create a short-lived, single-use package for the streaming computer.</span></li>
|
||||
<li class="<%= devices.some((device) => device.metadata?.bridge_installed) ? 'is-complete' : devices.length ? 'is-current' : '' %>"><strong>Install the OBS integration</strong><span><%= devices.some((device) => device.metadata?.bridge_installed) ? 'A paired Companion has verified its managed OBS bridge installation.' : 'Companion owns installation and repair; the bridge has no separate settings.' %></span></li>
|
||||
<li class="<%= providerHealth.model_ready ? 'is-complete' : '' %>"><strong>Load and benchmark a model</strong><span>Small English is the recommended starting point.</span></li>
|
||||
<li><strong>Select and test a microphone</strong><span>The real safe test must pass before live delivery is enabled.</span></li>
|
||||
<li class="<%= devices.some((device) => device.metadata?.path_test_valid) ? 'is-complete' : '' %>"><strong>Select and test a microphone</strong><span><%= devices.some((device) => device.metadata?.path_test_valid) ? 'The saved voice-free path check is still valid for the current configuration.' : 'Run the short voice-free path check once after setup or a relevant configuration change.' %></span></li>
|
||||
</ol>
|
||||
<div class="inline-actions">
|
||||
<button class="button" type="button" data-create-pairing>Download Companion</button>
|
||||
@ -58,7 +59,7 @@
|
||||
<% if (devices.length) { %>
|
||||
<ul class="transcription-device-list">
|
||||
<% devices.forEach((device) => { %>
|
||||
<li><div><strong><%= device.name %></strong><span class="hint">Last connected <%= new Date(device.last_connected_at).toLocaleString() %><% if (device.metadata?.companion_version) { %> · Companion <%= device.metadata.companion_version %><% } %></span></div><span class="badge success">Allowed</span></li>
|
||||
<li><div><strong><%= device.name %></strong><span class="hint">Last connected <%= new Date(device.last_connected_at).toLocaleString() %><% if (device.metadata?.companion_version) { %> · Companion <%= displayCompanionVersion(device.metadata.companion_version) %><% } %></span></div><span class="badge success">Allowed</span></li>
|
||||
<% }) %>
|
||||
</ul>
|
||||
<% } %>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user