Build Lumi Companion shell and whisper worker
This commit is contained in:
Franz Rolfsvaag
parent
29cd4a0365
commit
2f31390602
19
TODO.md
19
TODO.md
@ -9,20 +9,25 @@ core WebSocket-upgrade capability; single-use pairing and revocable devices;
|
||||
TLS-only device transport; bounded PCM/session queues; revision-safe settings;
|
||||
provider/delivery interfaces; worker supervision; caption stabilization; pinned
|
||||
whisper.cpp/model manifests; short-lived JSONL diagnostics; protocol schemas;
|
||||
and .NET/native companion boundaries with focused verification.
|
||||
and .NET/native companion boundaries with focused verification. The companion now
|
||||
has a Lumi-styled Avalonia 12 single-instance tray shell, guided real-boundary
|
||||
test states, DPAPI pairing, local preferences/autostart, and bounded diagnostics.
|
||||
A pinned native whisper.cpp worker builds on the CPU verification target and the
|
||||
server offers consent-gated model download/load controls.
|
||||
|
||||
Release-blocking work remains:
|
||||
|
||||
- Build the real streaming whisper.cpp worker, install it through the manifest,
|
||||
and benchmark `small.en`, quantized `small.en`, and `base.en` on the RTX 3060.
|
||||
- Complete the Avalonia tray application, signed installer, DPAPI migration UX,
|
||||
managed OBS bridge install/repair/uninstall, and single-instance/live-quit flow.
|
||||
- Package the native whisper.cpp worker as the Windows CUDA runtime, then benchmark
|
||||
`small.en`, quantized `small.en`, and `base.en` on the RTX 3060.
|
||||
- Complete the signed companion installer, DPAPI migration UX, and managed OBS
|
||||
bridge install/repair/uninstall service.
|
||||
- Complete native selected-source capture, nested Program-scene evaluation,
|
||||
resampling/IPC, bridge health, and source rename/missing recovery.
|
||||
- Run the OBS 31+/Twitch compatibility spike and prove toggleable closed captions;
|
||||
tune replacement/display duration from player behavior without open-caption fallback.
|
||||
- Add setup/test/conflict-resolution UI and live settings synchronization to the
|
||||
companion, then run the target-topology failure and performance matrix.
|
||||
- Complete the remaining live source/setup steps, conflict-resolution UI, and
|
||||
live settings synchronization, then run the target-topology failure and
|
||||
performance matrix.
|
||||
|
||||
## Remaining DesignMotionHQ UX work — experimental-ux checkpoint (2026-07-22)
|
||||
|
||||
|
||||
101
companion/Lumi.Companion.sln
Normal file
101
companion/Lumi.Companion.sln
Normal file
@ -0,0 +1,101 @@
|
||||
|
||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
# Visual Studio Version 17
|
||||
VisualStudioVersion = 17.0.31903.59
|
||||
MinimumVisualStudioVersion = 10.0.40219.1
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lumi.Companion.Protocol", "src\Lumi.Companion.Protocol\Lumi.Companion.Protocol.csproj", "{3703AF57-828D-4E2F-BFBA-A95833555A5A}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lumi.Companion.Core", "src\Lumi.Companion.Core\Lumi.Companion.Core.csproj", "{0BDAE25E-C487-456D-A465-41A0CA48AF30}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lumi.Companion.PluginHost", "src\Lumi.Companion.PluginHost\Lumi.Companion.PluginHost.csproj", "{64C2B7E5-9356-457A-BAF8-F18F67DAE4CF}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lumi.Companion.App", "src\Lumi.Companion.App\Lumi.Companion.App.csproj", "{2F0041FD-FD06-4312-BF1E-180F0201EB97}"
|
||||
EndProject
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "plugins", "plugins", "{07D57EEB-2F50-60C4-C011-FE4FA775C9A8}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lumi.Companion.Transcription", "plugins\Lumi.Companion.Transcription\Lumi.Companion.Transcription.csproj", "{49C19CA8-4669-4C33-B3A8-9CB934CD1171}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
Debug|x64 = Debug|x64
|
||||
Debug|x86 = Debug|x86
|
||||
Release|Any CPU = Release|Any CPU
|
||||
Release|x64 = Release|x64
|
||||
Release|x86 = Release|x86
|
||||
EndGlobalSection
|
||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||
{3703AF57-828D-4E2F-BFBA-A95833555A5A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{3703AF57-828D-4E2F-BFBA-A95833555A5A}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{3703AF57-828D-4E2F-BFBA-A95833555A5A}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{3703AF57-828D-4E2F-BFBA-A95833555A5A}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{3703AF57-828D-4E2F-BFBA-A95833555A5A}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{3703AF57-828D-4E2F-BFBA-A95833555A5A}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{3703AF57-828D-4E2F-BFBA-A95833555A5A}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{3703AF57-828D-4E2F-BFBA-A95833555A5A}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{3703AF57-828D-4E2F-BFBA-A95833555A5A}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{3703AF57-828D-4E2F-BFBA-A95833555A5A}.Release|x64.Build.0 = Release|Any CPU
|
||||
{3703AF57-828D-4E2F-BFBA-A95833555A5A}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{3703AF57-828D-4E2F-BFBA-A95833555A5A}.Release|x86.Build.0 = Release|Any CPU
|
||||
{0BDAE25E-C487-456D-A465-41A0CA48AF30}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{0BDAE25E-C487-456D-A465-41A0CA48AF30}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{0BDAE25E-C487-456D-A465-41A0CA48AF30}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{0BDAE25E-C487-456D-A465-41A0CA48AF30}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{0BDAE25E-C487-456D-A465-41A0CA48AF30}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{0BDAE25E-C487-456D-A465-41A0CA48AF30}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{0BDAE25E-C487-456D-A465-41A0CA48AF30}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{0BDAE25E-C487-456D-A465-41A0CA48AF30}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{0BDAE25E-C487-456D-A465-41A0CA48AF30}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{0BDAE25E-C487-456D-A465-41A0CA48AF30}.Release|x64.Build.0 = Release|Any CPU
|
||||
{0BDAE25E-C487-456D-A465-41A0CA48AF30}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{0BDAE25E-C487-456D-A465-41A0CA48AF30}.Release|x86.Build.0 = Release|Any CPU
|
||||
{64C2B7E5-9356-457A-BAF8-F18F67DAE4CF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{64C2B7E5-9356-457A-BAF8-F18F67DAE4CF}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{64C2B7E5-9356-457A-BAF8-F18F67DAE4CF}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{64C2B7E5-9356-457A-BAF8-F18F67DAE4CF}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{64C2B7E5-9356-457A-BAF8-F18F67DAE4CF}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{64C2B7E5-9356-457A-BAF8-F18F67DAE4CF}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{64C2B7E5-9356-457A-BAF8-F18F67DAE4CF}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{64C2B7E5-9356-457A-BAF8-F18F67DAE4CF}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{64C2B7E5-9356-457A-BAF8-F18F67DAE4CF}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{64C2B7E5-9356-457A-BAF8-F18F67DAE4CF}.Release|x64.Build.0 = Release|Any CPU
|
||||
{64C2B7E5-9356-457A-BAF8-F18F67DAE4CF}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{64C2B7E5-9356-457A-BAF8-F18F67DAE4CF}.Release|x86.Build.0 = Release|Any CPU
|
||||
{2F0041FD-FD06-4312-BF1E-180F0201EB97}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{2F0041FD-FD06-4312-BF1E-180F0201EB97}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{2F0041FD-FD06-4312-BF1E-180F0201EB97}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{2F0041FD-FD06-4312-BF1E-180F0201EB97}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{2F0041FD-FD06-4312-BF1E-180F0201EB97}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{2F0041FD-FD06-4312-BF1E-180F0201EB97}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{2F0041FD-FD06-4312-BF1E-180F0201EB97}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{2F0041FD-FD06-4312-BF1E-180F0201EB97}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{2F0041FD-FD06-4312-BF1E-180F0201EB97}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{2F0041FD-FD06-4312-BF1E-180F0201EB97}.Release|x64.Build.0 = Release|Any CPU
|
||||
{2F0041FD-FD06-4312-BF1E-180F0201EB97}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{2F0041FD-FD06-4312-BF1E-180F0201EB97}.Release|x86.Build.0 = Release|Any CPU
|
||||
{49C19CA8-4669-4C33-B3A8-9CB934CD1171}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{49C19CA8-4669-4C33-B3A8-9CB934CD1171}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{49C19CA8-4669-4C33-B3A8-9CB934CD1171}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{49C19CA8-4669-4C33-B3A8-9CB934CD1171}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{49C19CA8-4669-4C33-B3A8-9CB934CD1171}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{49C19CA8-4669-4C33-B3A8-9CB934CD1171}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{49C19CA8-4669-4C33-B3A8-9CB934CD1171}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{49C19CA8-4669-4C33-B3A8-9CB934CD1171}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{49C19CA8-4669-4C33-B3A8-9CB934CD1171}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{49C19CA8-4669-4C33-B3A8-9CB934CD1171}.Release|x64.Build.0 = Release|Any CPU
|
||||
{49C19CA8-4669-4C33-B3A8-9CB934CD1171}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{49C19CA8-4669-4C33-B3A8-9CB934CD1171}.Release|x86.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
EndGlobalSection
|
||||
GlobalSection(NestedProjects) = preSolution
|
||||
{3703AF57-828D-4E2F-BFBA-A95833555A5A} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
|
||||
{0BDAE25E-C487-456D-A465-41A0CA48AF30} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
|
||||
{64C2B7E5-9356-457A-BAF8-F18F67DAE4CF} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
|
||||
{2F0041FD-FD06-4312-BF1E-180F0201EB97} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
|
||||
{49C19CA8-4669-4C33-B3A8-9CB934CD1171} = {07D57EEB-2F50-60C4-C011-FE4FA775C9A8}
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
@ -1,9 +1,17 @@
|
||||
# Lumi Companion (experimental transcription milestone)
|
||||
|
||||
This directory is the single Lumi Companion product boundary. The current milestone supplies the versioned protocol client, Windows credential protection, bounded outbound transport, plugin-process supervision, and same-user OBS bridge IPC. The Avalonia tray shell, installer, and signed native bridge package remain target-machine work and are not represented as complete.
|
||||
This directory is the single Lumi Companion product boundary. The current milestone supplies the versioned protocol client, Windows credential protection, bounded outbound transport, plugin-process supervision, same-user OBS bridge IPC boundary, and an Avalonia tray shell. The shell is single-instance, closes to the tray, confirms quit during OBS streaming/recording, supports explicit pairing, and reports real setup/test blockers without presenting simulated success.
|
||||
|
||||
Build prerequisites: .NET 8 SDK on Windows x64. The native bridge additionally requires CMake, Visual Studio C++ tools, and the OBS 31+ SDK.
|
||||
Build prerequisites: the current .NET SDK with the .NET 8 targeting pack on Windows x64. The app targets .NET 8; .NET SDK 10 is recommended for Avalonia 12 source-generator compatibility. The native bridge additionally requires CMake, Visual Studio C++ tools, and the OBS 31+ SDK.
|
||||
|
||||
The app accepts a `.lumi-pairing.json` bootstrap package. It exchanges the embedded token once and stores the returned device credential with Windows DPAPI. Do not commit bootstrap packages, credentials, generated installers, build output, or logs.
|
||||
|
||||
The companion never performs ASR in this MVP. Audio is normalized to 16 kHz mono signed 16-bit PCM, held in bounded memory, and sent to the paired Lumi host over TLS WebSockets. The OBS bridge talks only to the companion over a same-user named pipe.
|
||||
|
||||
Build all managed projects from the repository root:
|
||||
|
||||
```powershell
|
||||
dotnet build companion/Lumi.Companion.sln -p:EnableWindowsTargeting=true
|
||||
```
|
||||
|
||||
The signed installer, managed OBS bridge installation/repair service, and a target-machine OBS/Twitch acceptance run are still required. Until those pass, the UI deliberately stops its test at the unavailable real boundary.
|
||||
|
||||
@ -12,13 +12,16 @@ public sealed class ObsBridgePipe : IAsyncDisposable
|
||||
private readonly Func<JsonElement, Task> _onMessage;
|
||||
private readonly Func<ReadOnlyMemory<byte>, Task> _onAudio;
|
||||
private readonly CancellationTokenSource _lifetime = new();
|
||||
private readonly SemaphoreSlim _writeLock = new(1, 1);
|
||||
private Task? _loop;
|
||||
private Stream? _connection;
|
||||
|
||||
public ObsBridgePipe(string userSidHash, Func<JsonElement, Task> onMessage, Func<ReadOnlyMemory<byte>, Task> onAudio)
|
||||
{
|
||||
_pipeName = $"Lumi.Companion.ObsBridge.v1.{userSidHash}";
|
||||
_onMessage = onMessage; _onAudio = onAudio;
|
||||
}
|
||||
public event Action<bool>? ConnectionChanged;
|
||||
public void Start() => _loop ??= Task.Run(() => ListenAsync(_lifetime.Token));
|
||||
private async Task ListenAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
@ -27,8 +30,11 @@ public sealed class ObsBridgePipe : IAsyncDisposable
|
||||
await using var pipe = new NamedPipeServerStream(_pipeName, PipeDirection.InOut, 1, PipeTransmissionMode.Byte,
|
||||
PipeOptions.Asynchronous | PipeOptions.CurrentUserOnly, 65536, 65536);
|
||||
await pipe.WaitForConnectionAsync(cancellationToken);
|
||||
_connection = pipe;
|
||||
ConnectionChanged?.Invoke(true);
|
||||
try { await ReadConnectionAsync(pipe, cancellationToken); }
|
||||
catch (Exception) when (!cancellationToken.IsCancellationRequested) { /* reconnect without taking down the shell */ }
|
||||
finally { _connection = null; ConnectionChanged?.Invoke(false); }
|
||||
}
|
||||
}
|
||||
private async Task ReadConnectionAsync(Stream stream, CancellationToken cancellationToken)
|
||||
@ -59,6 +65,20 @@ public sealed class ObsBridgePipe : IAsyncDisposable
|
||||
var prefix = new byte[4]; BinaryPrimitives.WriteUInt32LittleEndian(prefix, (uint)body.Length);
|
||||
await stream.WriteAsync(prefix, cancellationToken); await stream.WriteAsync(body, cancellationToken); await stream.FlushAsync(cancellationToken);
|
||||
}
|
||||
public async Task<bool> SendAsync(object message, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var connection = _connection;
|
||||
if (connection is null) return false;
|
||||
await _writeLock.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
if (!ReferenceEquals(connection, _connection)) return false;
|
||||
await WriteAsync(connection, message, cancellationToken);
|
||||
return true;
|
||||
}
|
||||
catch (IOException) { return false; }
|
||||
finally { _writeLock.Release(); }
|
||||
}
|
||||
private static async Task ReadExactlyAsync(Stream stream, Memory<byte> buffer, CancellationToken cancellationToken)
|
||||
{
|
||||
var read = 0;
|
||||
@ -69,5 +89,5 @@ public sealed class ObsBridgePipe : IAsyncDisposable
|
||||
read += count;
|
||||
}
|
||||
}
|
||||
public async ValueTask DisposeAsync() { _lifetime.Cancel(); if (_loop is not null) try { await _loop; } catch (OperationCanceledException) { } _lifetime.Dispose(); }
|
||||
public async ValueTask DisposeAsync() { _lifetime.Cancel(); if (_loop is not null) try { await _loop; } catch (OperationCanceledException) { } _writeLock.Dispose(); _lifetime.Dispose(); }
|
||||
}
|
||||
|
||||
94
companion/src/Lumi.Companion.App/App.axaml
Normal file
94
companion/src/Lumi.Companion.App/App.axaml
Normal file
@ -0,0 +1,94 @@
|
||||
<Application xmlns="https://github.com/avaloniaui"
|
||||
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
|
||||
x:Class="Lumi.Companion.App.App"
|
||||
RequestedThemeVariant="Light">
|
||||
<Application.Resources>
|
||||
<Color x:Key="LumiInk">#182026</Color>
|
||||
<Color x:Key="LumiMuted">#5A6872</Color>
|
||||
<Color x:Key="LumiSea">#176B75</Color>
|
||||
<Color x:Key="LumiSun">#E58B2B</Color>
|
||||
<Color x:Key="LumiSuccess">#23845B</Color>
|
||||
<Color x:Key="LumiWarning">#A96612</Color>
|
||||
<Color x:Key="LumiDanger">#BD4D4D</Color>
|
||||
<SolidColorBrush x:Key="LumiTextBrush" Color="{DynamicResource LumiInk}" />
|
||||
<SolidColorBrush x:Key="LumiMutedBrush" Color="{DynamicResource LumiMuted}" />
|
||||
<SolidColorBrush x:Key="LumiPrimaryBrush" Color="{DynamicResource LumiSea}" />
|
||||
<SolidColorBrush x:Key="LumiAccentBrush" Color="{DynamicResource LumiSun}" />
|
||||
<SolidColorBrush x:Key="LumiSuccessBrush" Color="{DynamicResource LumiSuccess}" />
|
||||
<SolidColorBrush x:Key="LumiWarningBrush" Color="{DynamicResource LumiWarning}" />
|
||||
<SolidColorBrush x:Key="LumiDangerBrush" Color="{DynamicResource LumiDanger}" />
|
||||
<SolidColorBrush x:Key="LumiSurfaceBrush" Color="#FFFFFF" />
|
||||
<SolidColorBrush x:Key="LumiSubtleBrush" Color="#F4F7F8" />
|
||||
<SolidColorBrush x:Key="LumiBorderBrush" Color="#D8E0E3" />
|
||||
</Application.Resources>
|
||||
<Application.Styles>
|
||||
<FluentTheme />
|
||||
<Style Selector="Window">
|
||||
<Setter Property="FontFamily" Value="Inter" />
|
||||
<Setter Property="Background" Value="{DynamicResource LumiSurfaceBrush}" />
|
||||
<Setter Property="Foreground" Value="{DynamicResource LumiTextBrush}" />
|
||||
</Style>
|
||||
<Style Selector="TextBlock.eyebrow">
|
||||
<Setter Property="FontSize" Value="11" />
|
||||
<Setter Property="FontWeight" Value="SemiBold" />
|
||||
<Setter Property="Foreground" Value="{DynamicResource LumiMutedBrush}" />
|
||||
</Style>
|
||||
<Style Selector="TextBlock.pageTitle">
|
||||
<Setter Property="FontSize" Value="32" />
|
||||
<Setter Property="FontWeight" Value="Bold" />
|
||||
<Setter Property="LineHeight" Value="36" />
|
||||
</Style>
|
||||
<Style Selector="TextBlock.sectionTitle">
|
||||
<Setter Property="FontSize" Value="18" />
|
||||
<Setter Property="FontWeight" Value="SemiBold" />
|
||||
</Style>
|
||||
<Style Selector="TextBlock.muted">
|
||||
<Setter Property="Foreground" Value="{DynamicResource LumiMutedBrush}" />
|
||||
<Setter Property="TextWrapping" Value="Wrap" />
|
||||
<Setter Property="LineHeight" Value="21" />
|
||||
</Style>
|
||||
<Style Selector="Border.card">
|
||||
<Setter Property="Background" Value="{DynamicResource LumiSurfaceBrush}" />
|
||||
<Setter Property="BorderBrush" Value="{DynamicResource LumiBorderBrush}" />
|
||||
<Setter Property="BorderThickness" Value="1" />
|
||||
<Setter Property="CornerRadius" Value="14" />
|
||||
<Setter Property="Padding" Value="20" />
|
||||
</Style>
|
||||
<Style Selector="Border.soft">
|
||||
<Setter Property="Background" Value="{DynamicResource LumiSubtleBrush}" />
|
||||
<Setter Property="CornerRadius" Value="12" />
|
||||
<Setter Property="Padding" Value="16" />
|
||||
</Style>
|
||||
<Style Selector="Button.primary">
|
||||
<Setter Property="Background" Value="{DynamicResource LumiPrimaryBrush}" />
|
||||
<Setter Property="Foreground" Value="White" />
|
||||
<Setter Property="CornerRadius" Value="9" />
|
||||
<Setter Property="Padding" Value="18,10" />
|
||||
<Setter Property="FontWeight" Value="SemiBold" />
|
||||
</Style>
|
||||
<Style Selector="Button.secondary">
|
||||
<Setter Property="Background" Value="Transparent" />
|
||||
<Setter Property="BorderBrush" Value="{DynamicResource LumiBorderBrush}" />
|
||||
<Setter Property="BorderThickness" Value="1" />
|
||||
<Setter Property="CornerRadius" Value="9" />
|
||||
<Setter Property="Padding" Value="16,9" />
|
||||
</Style>
|
||||
<Style Selector="Button.nav">
|
||||
<Setter Property="HorizontalContentAlignment" Value="Left" />
|
||||
<Setter Property="Background" Value="Transparent" />
|
||||
<Setter Property="BorderThickness" Value="0" />
|
||||
<Setter Property="CornerRadius" Value="9" />
|
||||
<Setter Property="Padding" Value="13,10" />
|
||||
<Setter Property="Margin" Value="0,2" />
|
||||
</Style>
|
||||
<Style Selector="Button.nav.selected">
|
||||
<Setter Property="Background" Value="#DDEEF0" />
|
||||
<Setter Property="Foreground" Value="{DynamicResource LumiPrimaryBrush}" />
|
||||
<Setter Property="FontWeight" Value="SemiBold" />
|
||||
</Style>
|
||||
<Style Selector="Button:focus-visible">
|
||||
<Setter Property="BorderBrush" Value="{DynamicResource LumiPrimaryBrush}" />
|
||||
<Setter Property="BorderThickness" Value="2" />
|
||||
</Style>
|
||||
</Application.Styles>
|
||||
</Application>
|
||||
86
companion/src/Lumi.Companion.App/App.axaml.cs
Normal file
86
companion/src/Lumi.Companion.App/App.axaml.cs
Normal file
@ -0,0 +1,86 @@
|
||||
using Avalonia;
|
||||
using Avalonia.Controls;
|
||||
using Avalonia.Controls.ApplicationLifetimes;
|
||||
using Avalonia.Markup.Xaml;
|
||||
using Avalonia.Threading;
|
||||
|
||||
namespace Lumi.Companion.App;
|
||||
|
||||
public partial class App : Application
|
||||
{
|
||||
private TrayIcon? _trayIcon;
|
||||
private MainWindow? _mainWindow;
|
||||
|
||||
public override void Initialize() => AvaloniaXamlLoader.Load(this);
|
||||
|
||||
public override void OnFrameworkInitializationCompleted()
|
||||
{
|
||||
if (ApplicationLifetime is IClassicDesktopStyleApplicationLifetime desktop)
|
||||
{
|
||||
desktop.ShutdownMode = ShutdownMode.OnExplicitShutdown;
|
||||
var paths = CompanionPaths.ForCurrentUser();
|
||||
var settings = new CompanionSettingsStore(paths.SettingsPath);
|
||||
var runtime = new CompanionRuntime(paths, settings);
|
||||
_mainWindow = new MainWindow(runtime, settings);
|
||||
desktop.MainWindow = _mainWindow;
|
||||
ConfigureTray(desktop, runtime);
|
||||
Program.InstanceCoordinator!.ActivationRequested += () => Dispatcher.UIThread.Post(ShowMainWindow);
|
||||
if (!Program.LaunchInBackground) _mainWindow.Show();
|
||||
_ = runtime.InitializeAsync();
|
||||
}
|
||||
base.OnFrameworkInitializationCompleted();
|
||||
}
|
||||
|
||||
private void ConfigureTray(IClassicDesktopStyleApplicationLifetime desktop, CompanionRuntime runtime)
|
||||
{
|
||||
var open = new NativeMenuItem("Open Lumi Companion");
|
||||
var test = new NativeMenuItem("Run transcription test");
|
||||
var health = new NativeMenuItem("Health: Starting") { IsEnabled = false };
|
||||
var web = new NativeMenuItem("Open Lumi WebUI");
|
||||
var quit = new NativeMenuItem("Quit");
|
||||
open.Click += (_, _) => ShowMainWindow();
|
||||
test.Click += async (_, _) =>
|
||||
{
|
||||
ShowMainWindow();
|
||||
_mainWindow?.ShowPage(CompanionPage.Test);
|
||||
await runtime.RunTestAsync();
|
||||
};
|
||||
web.Click += (_, _) => runtime.OpenLumiWebUi();
|
||||
quit.Click += async (_, _) =>
|
||||
{
|
||||
if (runtime.State.RequiresQuitConfirmation) ShowMainWindow();
|
||||
if (_mainWindow is null || !await _mainWindow.ConfirmQuitAsync()) return;
|
||||
_mainWindow.AllowExit();
|
||||
_trayIcon?.Dispose();
|
||||
await runtime.DisposeAsync();
|
||||
desktop.Shutdown();
|
||||
};
|
||||
|
||||
_trayIcon = new TrayIcon
|
||||
{
|
||||
Icon = LumiIconFactory.Create(TrayHealth.Ready),
|
||||
ToolTipText = "Lumi Companion — starting",
|
||||
IsVisible = true,
|
||||
Menu = new NativeMenu { Items = { open, test, web, health, new NativeMenuItemSeparator(), quit } }
|
||||
};
|
||||
_trayIcon.Clicked += (_, _) => ShowMainWindow();
|
||||
TrayIcon.SetIcons(this, new TrayIcons { _trayIcon });
|
||||
runtime.StateChanged += state => Dispatcher.UIThread.Post(() =>
|
||||
{
|
||||
if (_trayIcon is null) return;
|
||||
_trayIcon.Icon = LumiIconFactory.Create(state.Health);
|
||||
_trayIcon.ToolTipText = $"Lumi Companion — {state.Summary}";
|
||||
health.Header = $"Health: {state.Summary}";
|
||||
test.Header = state.TestRunning ? "Transcription test running…" : "Run transcription test";
|
||||
test.IsEnabled = !state.TestRunning;
|
||||
});
|
||||
}
|
||||
|
||||
private void ShowMainWindow()
|
||||
{
|
||||
if (_mainWindow is null) return;
|
||||
_mainWindow.Show();
|
||||
_mainWindow.WindowState = WindowState.Normal;
|
||||
_mainWindow.Activate();
|
||||
}
|
||||
}
|
||||
11
companion/src/Lumi.Companion.App/CompanionPaths.cs
Normal file
11
companion/src/Lumi.Companion.App/CompanionPaths.cs
Normal file
@ -0,0 +1,11 @@
|
||||
namespace Lumi.Companion.App;
|
||||
|
||||
public sealed record CompanionPaths(string Root, string SettingsPath, string LogsDirectory, string BridgeDirectory)
|
||||
{
|
||||
public static CompanionPaths ForCurrentUser()
|
||||
{
|
||||
var local = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
|
||||
var root = Path.Combine(local, "Lumi", "Companion");
|
||||
return new CompanionPaths(root, Path.Combine(root, "settings.json"), Path.Combine(root, "logs"), Path.Combine(root, "obs-bridge"));
|
||||
}
|
||||
}
|
||||
422
companion/src/Lumi.Companion.App/CompanionRuntime.cs
Normal file
422
companion/src/Lumi.Companion.App/CompanionRuntime.cs
Normal file
@ -0,0 +1,422 @@
|
||||
using System.Diagnostics;
|
||||
using System.Reflection;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Lumi.Companion.Core;
|
||||
using Lumi.Companion.Protocol;
|
||||
using Lumi.Companion.Transcription;
|
||||
using Microsoft.Win32;
|
||||
|
||||
namespace Lumi.Companion.App;
|
||||
|
||||
public sealed class CompanionRuntime : IAsyncDisposable
|
||||
{
|
||||
private const string RunKey = @"Software\Microsoft\Windows\CurrentVersion\Run";
|
||||
private readonly CompanionPaths _paths;
|
||||
private readonly CompanionSettingsStore _settings;
|
||||
private readonly SecureCredentialStore _credentials;
|
||||
private readonly HttpClient _http = new() { Timeout = TimeSpan.FromSeconds(15) };
|
||||
private CompanionSocket? _socket;
|
||||
private ObsBridgePipe? _obsBridge;
|
||||
private TaskCompletionSource<bool>? _audioSignal;
|
||||
private TaskCompletionSource<bool>? _captionSignal;
|
||||
private bool _disposed;
|
||||
|
||||
public CompanionRuntime(CompanionPaths paths, CompanionSettingsStore settings)
|
||||
{
|
||||
_paths = paths;
|
||||
_settings = settings;
|
||||
_credentials = new SecureCredentialStore(paths.Root);
|
||||
State = new CompanionState();
|
||||
TestStages = CreateInitialTestStages();
|
||||
ObsSources = [];
|
||||
}
|
||||
|
||||
public CompanionState State { get; private set; }
|
||||
public IReadOnlyList<TestStage> TestStages { get; private set; }
|
||||
public IReadOnlyList<ObsSource> ObsSources { get; private set; }
|
||||
public event Action<CompanionState>? StateChanged;
|
||||
public event Action<IReadOnlyList<TestStage>>? TestStagesChanged;
|
||||
public event Action<IReadOnlyList<ObsSource>>? ObsSourcesChanged;
|
||||
public event Action<string, bool>? CaptionReceived;
|
||||
public event Action<string>? LogAdded;
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
try { await InitializeCoreAsync(); }
|
||||
catch (Exception error)
|
||||
{
|
||||
SetState(State with { Health = TrayHealth.Failed, Detail = $"Companion startup could not finish. {Friendly(error)}" });
|
||||
await WriteLogAsync("startup_failed", error.Message);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task InitializeCoreAsync()
|
||||
{
|
||||
await _settings.LoadAsync();
|
||||
StartObsBridgeBoundary();
|
||||
ApplyAutoStart(_settings.Current.AutoStartWithWindows);
|
||||
DeviceCredential? credential;
|
||||
try { credential = _credentials.Load(); }
|
||||
catch (Exception error)
|
||||
{
|
||||
SetState(State with { Health = TrayHealth.Failed, Detail = "The saved device credential could not be opened. Pair this computer again." });
|
||||
await WriteLogAsync("credential_load_failed", error.Message);
|
||||
return;
|
||||
}
|
||||
if (credential is null)
|
||||
{
|
||||
SetState(State with { ObsBridgeInstalled = DetectBridgeInstallation(), Detail = "Download a pairing package from Lumi, then open it here." });
|
||||
return;
|
||||
}
|
||||
SetState(State with { Paired = true, DeviceName = Environment.MachineName, Host = credential.Host, ObsBridgeInstalled = DetectBridgeInstallation(), Detail = "Connecting securely to Lumi…" });
|
||||
await ConnectAsync(credential);
|
||||
}
|
||||
|
||||
public async Task PairAsync(string packagePath, CancellationToken cancellationToken = default)
|
||||
{
|
||||
SetState(State with { Health = TrayHealth.Degraded, Detail = "Validating the pairing package…" });
|
||||
var client = new PairingClient(_http);
|
||||
try
|
||||
{
|
||||
var credential = await client.PairAsync(packagePath, new
|
||||
{
|
||||
install_id = _credentials.GetOrCreateInstallId(),
|
||||
name = Environment.MachineName,
|
||||
companion_version = Version
|
||||
}, cancellationToken);
|
||||
_credentials.Save(credential);
|
||||
SetState(State with { Paired = true, DeviceName = Environment.MachineName, Host = credential.Host, Detail = "Pairing complete. Connecting to Lumi…" });
|
||||
await WriteLogAsync("paired", $"Paired {Environment.MachineName} with {credential.Host}.");
|
||||
await ConnectAsync(credential, cancellationToken);
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
SetState(State with { Health = TrayHealth.Failed, Detail = Friendly(error) });
|
||||
await WriteLogAsync("pairing_failed", error.Message);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task RetryConnectionAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var credential = _credentials.Load() ?? throw new InvalidOperationException("Pair this computer before reconnecting.");
|
||||
await ConnectAsync(credential, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task ConnectAsync(DeviceCredential credential, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (_socket is not null) await _socket.DisposeAsync();
|
||||
_socket = new CompanionSocket();
|
||||
_socket.MessageReceived += OnServerMessageAsync;
|
||||
_socket.Disconnected += error =>
|
||||
{
|
||||
if (_disposed) return;
|
||||
SetState(State with { Connected = false, Health = TrayHealth.Degraded, Detail = "The secure Lumi connection closed. Retry when the host is available." });
|
||||
_ = WriteLogAsync("disconnected", error?.Message ?? "Connection closed.");
|
||||
};
|
||||
SetState(State with { Connected = false, Detail = "Connecting securely to Lumi…" });
|
||||
try
|
||||
{
|
||||
await _socket.ConnectAsync(credential, Version, "0.1.0", null, cancellationToken);
|
||||
var bridgeInstalled = State.ObsBridgeInstalled || DetectBridgeInstallation();
|
||||
SetState(State with
|
||||
{
|
||||
Paired = true,
|
||||
Connected = true,
|
||||
ObsBridgeInstalled = bridgeInstalled,
|
||||
Health = bridgeInstalled ? TrayHealth.Ready : TrayHealth.Degraded,
|
||||
Detail = bridgeInstalled ? "Lumi is connected. Waiting for OBS." : "Lumi is connected. Install the managed OBS integration to continue setup.",
|
||||
Host = credential.Host,
|
||||
LastConnectedAt = DateTimeOffset.Now
|
||||
});
|
||||
foreach (var source in ObsSources) await SendSourceUpdateAsync(source);
|
||||
if (State.ObsConnected)
|
||||
await _socket.SendAsync("obs_state", new { streaming = State.ObsStreaming, recording = State.ObsRecording, auto_start = _settings.Current.StartWithObs }, _socket.SessionId, cancellationToken);
|
||||
await WriteLogAsync("connected", $"Connected to {credential.Host}.");
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
SetState(State with { Paired = true, Connected = false, Health = TrayHealth.Degraded, Detail = $"Lumi could not be reached. {Friendly(error)}" });
|
||||
await WriteLogAsync("connection_failed", error.Message);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task RunTestAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (State.TestRunning) return;
|
||||
SetState(State with { TestRunning = true, Detail = "Running the transcription path check…" });
|
||||
var stages = CreateInitialTestStages().ToArray();
|
||||
TestStages = stages;
|
||||
TestStagesChanged?.Invoke(TestStages);
|
||||
try
|
||||
{
|
||||
await EvaluateStageAsync(stages, 0, State.ObsBridgeInstalled, "The managed OBS integration is installed.", "Install or repair the OBS integration before testing.", cancellationToken);
|
||||
if (!State.ObsBridgeInstalled) return;
|
||||
await EvaluateStageAsync(stages, 1, State.ObsConnected, "OBS reported a healthy local connection.", "Open OBS 31 or newer and retry.", cancellationToken);
|
||||
if (!State.ObsConnected) return;
|
||||
var selected = ObsSources.FirstOrDefault(source => source.Uuid == _settings.Current.PrimarySourceUuid);
|
||||
await EvaluateStageAsync(stages, 2, selected is { Missing: false, Active: true }, "The selected microphone is available and active in Program.", selected is null ? "Choose a microphone on the Transcription page." : selected.Missing ? "The selected microphone is missing from OBS." : "Put the selected microphone in the active Program scene, then retry.", cancellationToken);
|
||||
if (selected is not { Missing: false, Active: true }) return;
|
||||
await EvaluateStageAsync(stages, 4, State.Connected, "The secure Lumi transport is ready.", "Reconnect to Lumi before testing.", cancellationToken);
|
||||
if (!State.Connected) return;
|
||||
_audioSignal = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
_captionSignal = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
await _socket!.SendAsync("start", new { mode = "test" }, _socket.SessionId, cancellationToken);
|
||||
MarkStage(stages, 3, TestStageState.Running, "Speak normally into the selected microphone…");
|
||||
if (!await WaitForAsync(_audioSignal.Task, TimeSpan.FromSeconds(10), cancellationToken))
|
||||
{
|
||||
MarkStage(stages, 3, TestStageState.Blocked, "No active microphone audio reached the companion within 10 seconds.");
|
||||
return;
|
||||
}
|
||||
MarkStage(stages, 3, TestStageState.Passed, "Live microphone audio reached the companion.");
|
||||
MarkStage(stages, 5, TestStageState.Running, "Waiting for server-hosted speech recognition…");
|
||||
if (!await WaitForAsync(_captionSignal.Task, TimeSpan.FromSeconds(30), cancellationToken))
|
||||
{
|
||||
MarkStage(stages, 5, TestStageState.Blocked, "No caption returned within 30 seconds. Check the loaded model and host diagnostics.");
|
||||
return;
|
||||
}
|
||||
MarkStage(stages, 5, TestStageState.Passed, "Speech recognition returned a stable result.");
|
||||
MarkStage(stages, 6, TestStageState.Passed, "A caption returned over the secure connection.");
|
||||
MarkStage(stages, 7, TestStageState.Passed, "The delivery adapter stayed in safe simulation mode.");
|
||||
MarkStage(stages, 8, TestStageState.Passed, "The exact outgoing caption was rendered locally, not sent to Twitch.");
|
||||
SetState(State with { Detail = "The safe end-to-end transcription test passed.", Health = TrayHealth.Ready });
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (_socket is not null && State.Connected)
|
||||
try { await _socket.SendAsync("stop", new { reason = "test_complete" }, _socket.SessionId, CancellationToken.None); } catch { }
|
||||
_audioSignal = null;
|
||||
_captionSignal = null;
|
||||
SetState(State with { TestRunning = false, Detail = TestStages.Any(stage => stage.State == TestStageState.Blocked) ? "The test stopped at the first unavailable real boundary." : State.Detail });
|
||||
}
|
||||
}
|
||||
|
||||
private async Task EvaluateStageAsync(TestStage[] stages, int index, bool passed, string success, string blocked, CancellationToken cancellationToken)
|
||||
{
|
||||
MarkStage(stages, index, TestStageState.Running, "Checking…");
|
||||
await Task.Delay(180, cancellationToken);
|
||||
MarkStage(stages, index, passed ? TestStageState.Passed : TestStageState.Blocked, passed ? success : blocked);
|
||||
}
|
||||
|
||||
private void MarkStage(TestStage[] stages, int index, TestStageState state, string detail)
|
||||
{
|
||||
stages[index] = stages[index] with { State = state, Detail = detail };
|
||||
TestStages = stages.ToArray();
|
||||
TestStagesChanged?.Invoke(TestStages);
|
||||
}
|
||||
|
||||
public async Task SaveSettingsAsync(CompanionSettings value, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await _settings.SaveAsync(value, cancellationToken);
|
||||
ApplyAutoStart(value.AutoStartWithWindows);
|
||||
await WriteLogAsync("settings_saved", "Local companion preferences updated.");
|
||||
}
|
||||
|
||||
public async Task SelectSourceAsync(ObsSource source, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await _settings.SaveAsync(_settings.Current with { PrimarySourceUuid = source.Uuid, PrimarySourceName = source.Name }, cancellationToken);
|
||||
foreach (var item in ObsSources) await SendSourceUpdateAsync(item);
|
||||
await WriteLogAsync("source_selected", $"Selected OBS source {source.Name}.");
|
||||
}
|
||||
|
||||
public void OpenLumiWebUi()
|
||||
{
|
||||
if (!Uri.TryCreate(State.Host, UriKind.Absolute, out var uri)) return;
|
||||
Process.Start(new ProcessStartInfo(uri.ToString()) { UseShellExecute = true });
|
||||
}
|
||||
|
||||
public void OpenLogsDirectory()
|
||||
{
|
||||
Directory.CreateDirectory(_paths.LogsDirectory);
|
||||
Process.Start(new ProcessStartInfo(_paths.LogsDirectory) { UseShellExecute = true });
|
||||
}
|
||||
|
||||
public async Task ForgetDeviceAsync()
|
||||
{
|
||||
if (_socket is not null) { await _socket.DisposeAsync(); _socket = null; }
|
||||
_credentials.Remove();
|
||||
SetState(new CompanionState(ObsBridgeInstalled: DetectBridgeInstallation(), Detail: "Device removed. Pair this computer to reconnect."));
|
||||
await WriteLogAsync("device_removed", "Saved device credential removed locally.");
|
||||
}
|
||||
|
||||
private Task OnServerMessageAsync(ServerEnvelope message)
|
||||
{
|
||||
if (message.Type == "caption")
|
||||
{
|
||||
var text = message.Payload.TryGetProperty("stable_text", out var stable) ? stable.GetString() : null;
|
||||
var simulated = message.Payload.TryGetProperty("delivery", out var delivery) && delivery.TryGetProperty("disposition", out var disposition) && disposition.GetString() == "simulated";
|
||||
if (!string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
if (simulated) _captionSignal?.TrySetResult(true);
|
||||
CaptionReceived?.Invoke(text, simulated);
|
||||
if (!simulated) _ = _obsBridge?.SendAsync(new { type = "caption", payload = message.Payload });
|
||||
}
|
||||
}
|
||||
if (message.Type == "error")
|
||||
{
|
||||
var serverMessage = message.Payload.TryGetProperty("message", out var value) ? value.GetString() : "Lumi reported an error.";
|
||||
SetState(State with { Health = TrayHealth.Degraded, Detail = serverMessage ?? "Lumi reported an error." });
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private void StartObsBridgeBoundary()
|
||||
{
|
||||
if (_obsBridge is not null) return;
|
||||
var userKey = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes($"{Environment.UserDomainName}\\{Environment.UserName}"))).Substring(0, 16);
|
||||
_obsBridge = new ObsBridgePipe(userKey, OnObsMessageAsync, OnObsAudioAsync);
|
||||
_obsBridge.ConnectionChanged += connected =>
|
||||
{
|
||||
var health = connected && State.Connected ? TrayHealth.Ready : State.Health;
|
||||
SetState(State with { ObsBridgeInstalled = connected || State.ObsBridgeInstalled, ObsConnected = connected, Health = health, Detail = connected ? "OBS and Lumi are connected. Choose a microphone and run a safe test." : State.Detail });
|
||||
_ = WriteLogAsync("obs_connection", connected ? "OBS bridge connected." : "OBS bridge disconnected.");
|
||||
};
|
||||
_obsBridge.Start();
|
||||
}
|
||||
|
||||
private async Task OnObsMessageAsync(JsonElement message)
|
||||
{
|
||||
var type = message.GetProperty("type").GetString();
|
||||
if (type == "obs_state")
|
||||
{
|
||||
var streaming = ReadBoolean(message, "streaming");
|
||||
var recording = ReadBoolean(message, "recording");
|
||||
SetState(State with { ObsConnected = true, ObsStreaming = streaming, ObsRecording = recording, Health = streaming ? TrayHealth.Operating : TrayHealth.Ready, Detail = streaming ? "OBS is live and companion services are active." : "OBS is connected and ready." });
|
||||
if (_socket is not null && State.Connected)
|
||||
await _socket.SendAsync("obs_state", new { streaming, recording, version = ReadString(message, "version"), auto_start = _settings.Current.StartWithObs }, _socket.SessionId, _lifetimeToken());
|
||||
}
|
||||
else if (type == "source_list" && message.TryGetProperty("sources", out var sources) && sources.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
ObsSources = sources.EnumerateArray().Select(source => new ObsSource(
|
||||
ReadString(source, "source_uuid") ?? ReadString(source, "uuid") ?? string.Empty,
|
||||
ReadString(source, "display_name") ?? ReadString(source, "name") ?? "OBS source",
|
||||
ReadBoolean(source, "program_active") || ReadBoolean(source, "active"),
|
||||
ReadBoolean(source, "source_missing") || ReadBoolean(source, "missing")))
|
||||
.Where(source => Guid.TryParse(source.Uuid, out _)).ToArray();
|
||||
ObsSourcesChanged?.Invoke(ObsSources);
|
||||
foreach (var source in ObsSources) await SendSourceUpdateAsync(source);
|
||||
}
|
||||
else if (type == "source_state")
|
||||
{
|
||||
var source = new ObsSource(ReadString(message, "source_uuid") ?? string.Empty, ReadString(message, "display_name") ?? "OBS source", ReadBoolean(message, "program_active"), ReadBoolean(message, "source_missing"));
|
||||
if (Guid.TryParse(source.Uuid, out _))
|
||||
{
|
||||
ObsSources = ObsSources.Where(item => item.Uuid != source.Uuid).Append(source).OrderBy(item => item.Name).ToArray();
|
||||
ObsSourcesChanged?.Invoke(ObsSources);
|
||||
await SendSourceUpdateAsync(source);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Task OnObsAudioAsync(ReadOnlyMemory<byte> frame)
|
||||
{
|
||||
_audioSignal?.TrySetResult(true);
|
||||
if (_socket is not null && State.Connected && !_socket.QueueBridgeAudio(frame))
|
||||
_ = WriteLogAsync("audio_dropped", "An obsolete audio frame was dropped before network delivery.");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private async Task SendSourceUpdateAsync(ObsSource source)
|
||||
{
|
||||
if (_socket is null || !State.Connected) return;
|
||||
await _socket.SendAsync("source_update", new
|
||||
{
|
||||
source_uuid = source.Uuid,
|
||||
display_name = source.Name,
|
||||
enabled = source.Uuid == _settings.Current.PrimarySourceUuid,
|
||||
primary = source.Uuid == _settings.Current.PrimarySourceUuid,
|
||||
program_active = source.Active,
|
||||
source_missing = source.Missing,
|
||||
single_speaker = true,
|
||||
delivery_enabled = true
|
||||
}, _socket.SessionId, _lifetimeToken());
|
||||
}
|
||||
|
||||
private static bool ReadBoolean(JsonElement value, string name) => value.TryGetProperty(name, out var property) && property.ValueKind == JsonValueKind.True;
|
||||
private static string? ReadString(JsonElement value, string name) => value.TryGetProperty(name, out var property) && property.ValueKind == JsonValueKind.String ? property.GetString() : null;
|
||||
private CancellationToken _lifetimeToken() => _disposed ? new CancellationToken(true) : CancellationToken.None;
|
||||
private static async Task<bool> WaitForAsync(Task task, TimeSpan timeout, CancellationToken cancellationToken)
|
||||
{
|
||||
using var timeoutSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
var delay = Task.Delay(timeout, timeoutSource.Token);
|
||||
if (await Task.WhenAny(task, delay) != task) return false;
|
||||
timeoutSource.Cancel();
|
||||
await task;
|
||||
return true;
|
||||
}
|
||||
|
||||
private bool DetectBridgeInstallation()
|
||||
{
|
||||
if (File.Exists(Path.Combine(_paths.BridgeDirectory, "lumi-obs-bridge.dll"))) return true;
|
||||
var obsData = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "obs-studio", "plugins", "lumi-obs-bridge", "bin", "64bit", "lumi-obs-bridge.dll");
|
||||
return File.Exists(obsData);
|
||||
}
|
||||
|
||||
private void ApplyAutoStart(bool enabled)
|
||||
{
|
||||
if (!OperatingSystem.IsWindows()) return;
|
||||
try
|
||||
{
|
||||
using var key = Registry.CurrentUser.CreateSubKey(RunKey);
|
||||
if (enabled) key.SetValue("Lumi Companion", $"\"{Environment.ProcessPath}\" --background");
|
||||
else key.DeleteValue("Lumi Companion", false);
|
||||
}
|
||||
catch (Exception error) { _ = WriteLogAsync("autostart_failed", error.Message); }
|
||||
}
|
||||
|
||||
private async Task WriteLogAsync(string kind, string message)
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(_paths.LogsDirectory);
|
||||
PruneLogs();
|
||||
var path = Path.Combine(_paths.LogsDirectory, $"companion-{DateTime.UtcNow:yyyy-MM-dd}.jsonl");
|
||||
var line = JsonSerializer.Serialize(new { timestamp = DateTimeOffset.UtcNow, kind, message }, ProtocolV1.JsonOptions);
|
||||
await File.AppendAllTextAsync(path, line + Environment.NewLine);
|
||||
LogAdded?.Invoke($"{DateTime.Now:HH:mm:ss} {message}");
|
||||
}
|
||||
catch { }
|
||||
}
|
||||
|
||||
private void PruneLogs()
|
||||
{
|
||||
var files = new DirectoryInfo(_paths.LogsDirectory).GetFiles("*.jsonl").OrderBy(file => file.CreationTimeUtc).ToList();
|
||||
foreach (var file in files.Where(file => file.CreationTimeUtc < DateTime.UtcNow.AddDays(-7))) file.Delete();
|
||||
const long cap = 256L * 1024 * 1024;
|
||||
var total = files.Where(file => file.Exists).Sum(file => file.Length);
|
||||
foreach (var file in files.Where(file => file.Exists)) { if (total <= cap) break; total -= file.Length; file.Delete(); }
|
||||
}
|
||||
|
||||
private void SetState(CompanionState state) { State = state; StateChanged?.Invoke(state); }
|
||||
private static string Friendly(Exception error) => error switch
|
||||
{
|
||||
InvalidDataException => error.Message,
|
||||
HttpRequestException => "Check the Lumi address, TLS certificate, and network connection.",
|
||||
TaskCanceledException => "The connection timed out. Check that Lumi is reachable.",
|
||||
_ => error.Message
|
||||
};
|
||||
private static TestStage[] CreateInitialTestStages() =>
|
||||
[
|
||||
new("OBS integration", "Waiting to check the managed bridge.", TestStageState.Waiting),
|
||||
new("OBS connection", "Waiting for OBS.", TestStageState.Waiting),
|
||||
new("Microphone", "Waiting for a selected source.", TestStageState.Waiting),
|
||||
new("Audio capture", "Waiting for live microphone audio.", TestStageState.Waiting),
|
||||
new("Lumi connection", "Waiting to verify secure transport.", TestStageState.Waiting),
|
||||
new("Speech recognition", "Waiting for real server-hosted inference.", TestStageState.Waiting),
|
||||
new("Caption return", "Waiting for a stable caption.", TestStageState.Waiting),
|
||||
new("Delivery adapter", "Waiting for safe simulation mode.", TestStageState.Waiting),
|
||||
new("Simulated output", "Nothing is sent to Twitch during this test.", TestStageState.Waiting)
|
||||
];
|
||||
public static string Version => Assembly.GetExecutingAssembly().GetName().Version?.ToString(3) ?? "0.1.0";
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_disposed) return;
|
||||
_disposed = true;
|
||||
if (_socket is not null) await _socket.DisposeAsync();
|
||||
if (_obsBridge is not null) await _obsBridge.DisposeAsync();
|
||||
_http.Dispose();
|
||||
}
|
||||
}
|
||||
54
companion/src/Lumi.Companion.App/CompanionSettingsStore.cs
Normal file
54
companion/src/Lumi.Companion.App/CompanionSettingsStore.cs
Normal file
@ -0,0 +1,54 @@
|
||||
using System.Text.Json;
|
||||
using Lumi.Companion.Protocol;
|
||||
|
||||
namespace Lumi.Companion.App;
|
||||
|
||||
public sealed record CompanionSettings(
|
||||
bool AutoStartWithWindows = false,
|
||||
bool StartWithObs = true,
|
||||
bool AdvancedMode = false,
|
||||
string? PrimarySourceUuid = null,
|
||||
string? PrimarySourceName = null);
|
||||
|
||||
public sealed class CompanionSettingsStore
|
||||
{
|
||||
private readonly string _path;
|
||||
private readonly SemaphoreSlim _gate = new(1, 1);
|
||||
|
||||
public CompanionSettingsStore(string path) => _path = path;
|
||||
public CompanionSettings Current { get; private set; } = new();
|
||||
public event Action<CompanionSettings>? Changed;
|
||||
|
||||
public async Task LoadAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (!File.Exists(_path)) return;
|
||||
try
|
||||
{
|
||||
Current = JsonSerializer.Deserialize<CompanionSettings>(await File.ReadAllBytesAsync(_path, cancellationToken), ProtocolV1.JsonOptions) ?? new();
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
Current = new();
|
||||
}
|
||||
Changed?.Invoke(Current);
|
||||
}
|
||||
|
||||
public async Task SaveAsync(CompanionSettings settings, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await _gate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(_path)!);
|
||||
var temporary = $"{_path}.{Environment.ProcessId}.tmp";
|
||||
try
|
||||
{
|
||||
await File.WriteAllBytesAsync(temporary, JsonSerializer.SerializeToUtf8Bytes(settings, ProtocolV1.JsonOptions), cancellationToken);
|
||||
File.Move(temporary, _path, true);
|
||||
}
|
||||
finally { File.Delete(temporary); }
|
||||
Current = settings;
|
||||
}
|
||||
finally { _gate.Release(); }
|
||||
Changed?.Invoke(settings);
|
||||
}
|
||||
}
|
||||
@ -1,10 +1,42 @@
|
||||
namespace Lumi.Companion.App;
|
||||
|
||||
public enum TrayHealth { Ready, Operating, Degraded, Failed }
|
||||
public sealed record CompanionState(bool ObsStreaming, bool ObsRecording, bool TestRunning, TrayHealth Health)
|
||||
public enum CompanionPage { Overview, Transcription, Test, Connection, Logs, Settings }
|
||||
public enum TestStageState { Waiting, Running, Passed, Blocked, Failed }
|
||||
|
||||
public sealed record CompanionState(
|
||||
bool Paired = false,
|
||||
bool Connected = false,
|
||||
bool ObsBridgeInstalled = false,
|
||||
bool ObsConnected = false,
|
||||
bool ObsStreaming = false,
|
||||
bool ObsRecording = false,
|
||||
bool TestRunning = false,
|
||||
TrayHealth Health = TrayHealth.Degraded,
|
||||
string Detail = "Pair Lumi Companion to get started.",
|
||||
string? DeviceName = null,
|
||||
string? Host = null,
|
||||
DateTimeOffset? LastConnectedAt = null)
|
||||
{
|
||||
public string Summary => Health switch
|
||||
{
|
||||
TrayHealth.Operating => "Transcription active",
|
||||
TrayHealth.Ready => "Ready",
|
||||
TrayHealth.Failed => "Needs attention",
|
||||
_ when !Paired => "Setup required",
|
||||
_ when !Connected => "Lumi offline",
|
||||
_ when !ObsBridgeInstalled => "OBS setup required",
|
||||
_ => "Partially ready"
|
||||
};
|
||||
|
||||
public bool RequiresQuitConfirmation => ObsStreaming || ObsRecording;
|
||||
public string QuitWarning => ObsStreaming
|
||||
? "OBS is streaming. Quitting Lumi Companion will stop transcription and closed captions, but it will not stop the OBS stream."
|
||||
: ObsRecording ? "OBS is recording. Quitting Lumi Companion will stop active companion features." : string.Empty;
|
||||
}
|
||||
|
||||
public sealed record TestStage(string Name, string Detail, TestStageState State);
|
||||
public sealed record ObsSource(string Uuid, string Name, bool Active, bool Missing)
|
||||
{
|
||||
public override string ToString() => Missing ? $"{Name} (missing)" : Active ? $"{Name} (active)" : Name;
|
||||
}
|
||||
|
||||
31
companion/src/Lumi.Companion.App/DecisionWindow.cs
Normal file
31
companion/src/Lumi.Companion.App/DecisionWindow.cs
Normal file
@ -0,0 +1,31 @@
|
||||
using Avalonia;
|
||||
using Avalonia.Controls;
|
||||
using Avalonia.Layout;
|
||||
|
||||
namespace Lumi.Companion.App;
|
||||
|
||||
internal sealed class DecisionWindow : Window
|
||||
{
|
||||
public DecisionWindow(string title, string message, string confirmLabel)
|
||||
{
|
||||
Title = title;
|
||||
Width = 470;
|
||||
SizeToContent = SizeToContent.Height;
|
||||
CanResize = false;
|
||||
WindowStartupLocation = WindowStartupLocation.CenterOwner;
|
||||
var cancel = new Button { Content = "Cancel", Classes = { "secondary" } };
|
||||
var confirm = new Button { Content = confirmLabel, Classes = { "primary" } };
|
||||
cancel.Click += (_, _) => Close(false);
|
||||
confirm.Click += (_, _) => Close(true);
|
||||
Content = new StackPanel
|
||||
{
|
||||
Margin = new Thickness(28), Spacing = 18,
|
||||
Children =
|
||||
{
|
||||
new TextBlock { Text = title, Classes = { "sectionTitle" } },
|
||||
new TextBlock { Text = message, Classes = { "muted" }, TextWrapping = Avalonia.Media.TextWrapping.Wrap },
|
||||
new StackPanel { Orientation = Orientation.Horizontal, HorizontalAlignment = HorizontalAlignment.Right, Spacing = 10, Children = { cancel, confirm } }
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
@ -1 +1,20 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk"><PropertyGroup><OutputType>Exe</OutputType><TargetFramework>net8.0-windows</TargetFramework><Nullable>enable</Nullable><ImplicitUsings>enable</ImplicitUsings></PropertyGroup><ItemGroup><ProjectReference Include="../Lumi.Companion.Core/Lumi.Companion.Core.csproj" /><ProjectReference Include="../Lumi.Companion.Protocol/Lumi.Companion.Protocol.csproj" /></ItemGroup></Project>
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<OutputType>WinExe</OutputType>
|
||||
<TargetFramework>net8.0-windows</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<ApplicationManifest>app.manifest</ApplicationManifest>
|
||||
<Version>0.1.0-experimental.1</Version>
|
||||
<AssemblyVersion>0.1.0.0</AssemblyVersion>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../Lumi.Companion.Core/Lumi.Companion.Core.csproj" />
|
||||
<ProjectReference Include="../Lumi.Companion.Protocol/Lumi.Companion.Protocol.csproj" />
|
||||
<ProjectReference Include="../../plugins/Lumi.Companion.Transcription/Lumi.Companion.Transcription.csproj" />
|
||||
<PackageReference Include="Avalonia" Version="12.1.0" />
|
||||
<PackageReference Include="Avalonia.Desktop" Version="12.1.0" />
|
||||
<PackageReference Include="Avalonia.Fonts.Inter" Version="12.1.0" />
|
||||
<PackageReference Include="Avalonia.Themes.Fluent" Version="12.1.0" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
52
companion/src/Lumi.Companion.App/LumiIconFactory.cs
Normal file
52
companion/src/Lumi.Companion.App/LumiIconFactory.cs
Normal file
@ -0,0 +1,52 @@
|
||||
using Avalonia.Controls;
|
||||
|
||||
namespace Lumi.Companion.App;
|
||||
|
||||
internal static class LumiIconFactory
|
||||
{
|
||||
public static WindowIcon Create(TrayHealth health)
|
||||
{
|
||||
const int size = 32;
|
||||
var color = health switch
|
||||
{
|
||||
TrayHealth.Operating => (R: (byte)35, G: (byte)132, B: (byte)91),
|
||||
TrayHealth.Degraded => (R: (byte)169, G: (byte)102, B: (byte)18),
|
||||
TrayHealth.Failed => (R: (byte)189, G: (byte)77, B: (byte)77),
|
||||
_ => (R: (byte)79, G: (byte)182, B: (byte)194)
|
||||
};
|
||||
var pixels = new byte[size * size * 4];
|
||||
for (var y = 0; y < size; y++)
|
||||
for (var x = 0; x < size; x++)
|
||||
{
|
||||
var target = ((size - 1 - y) * size + x) * 4;
|
||||
var rounded = InsideRoundedSquare(x, y, size, 7);
|
||||
var diamond = Math.Abs(x - 15.5) / 11.5 + Math.Abs(y - 15.5) / 11.5 <= 1;
|
||||
pixels[target] = diamond && rounded ? (byte)255 : color.B;
|
||||
pixels[target + 1] = diamond && rounded ? (byte)255 : color.G;
|
||||
pixels[target + 2] = diamond && rounded ? (byte)255 : color.R;
|
||||
pixels[target + 3] = rounded ? (byte)255 : (byte)0;
|
||||
}
|
||||
|
||||
using var stream = new MemoryStream();
|
||||
using var writer = new BinaryWriter(stream, System.Text.Encoding.UTF8, leaveOpen: true);
|
||||
writer.Write((ushort)0); writer.Write((ushort)1); writer.Write((ushort)1);
|
||||
writer.Write((byte)size); writer.Write((byte)size); writer.Write((byte)0); writer.Write((byte)0);
|
||||
writer.Write((ushort)1); writer.Write((ushort)32);
|
||||
var imageBytes = 40 + pixels.Length + size * 4;
|
||||
writer.Write(imageBytes); writer.Write(22);
|
||||
writer.Write(40); writer.Write(size); writer.Write(size * 2); writer.Write((ushort)1); writer.Write((ushort)32);
|
||||
writer.Write(0); writer.Write(pixels.Length); writer.Write(0); writer.Write(0); writer.Write(0); writer.Write(0);
|
||||
writer.Write(pixels); writer.Write(new byte[size * 4]);
|
||||
stream.Position = 0;
|
||||
return new WindowIcon(stream);
|
||||
}
|
||||
|
||||
private static bool InsideRoundedSquare(int x, int y, int size, int radius)
|
||||
{
|
||||
if (x >= radius && x < size - radius || y >= radius && y < size - radius) return true;
|
||||
var cx = x < radius ? radius : size - radius - 1;
|
||||
var cy = y < radius ? radius : size - radius - 1;
|
||||
var dx = x - cx; var dy = y - cy;
|
||||
return dx * dx + dy * dy <= radius * radius;
|
||||
}
|
||||
}
|
||||
211
companion/src/Lumi.Companion.App/MainWindow.axaml
Normal file
211
companion/src/Lumi.Companion.App/MainWindow.axaml
Normal file
@ -0,0 +1,211 @@
|
||||
<Window xmlns="https://github.com/avaloniaui"
|
||||
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
|
||||
x:Class="Lumi.Companion.App.MainWindow"
|
||||
Title="Lumi Companion"
|
||||
Width="1120" Height="760" MinWidth="900" MinHeight="620"
|
||||
WindowStartupLocation="CenterScreen">
|
||||
<Grid ColumnDefinitions="248,*">
|
||||
<Border Grid.Column="0" Background="#F4F7F8" Padding="22,24">
|
||||
<Grid RowDefinitions="Auto,Auto,*,Auto">
|
||||
<StackPanel Orientation="Horizontal" Spacing="11">
|
||||
<Border Width="34" Height="34" CornerRadius="9" Background="#28B7C8">
|
||||
<Path Data="M 17,7 L 27,17 L 17,27 L 7,17 Z" Fill="White" Stretch="None" />
|
||||
</Border>
|
||||
<StackPanel VerticalAlignment="Center" Spacing="0">
|
||||
<TextBlock Text="LUMI" Classes="eyebrow" />
|
||||
<TextBlock Text="Companion" FontWeight="SemiBold" FontSize="16" />
|
||||
</StackPanel>
|
||||
</StackPanel>
|
||||
|
||||
<Border Grid.Row="1" Margin="0,25,0,20" Padding="11,9" CornerRadius="9" Background="#E6EEF0">
|
||||
<Grid ColumnDefinitions="Auto,*" ColumnSpacing="9">
|
||||
<Border x:Name="StatusMark" Width="10" Height="10" CornerRadius="5" Background="#A96612" VerticalAlignment="Center" />
|
||||
<StackPanel Grid.Column="1" Spacing="1">
|
||||
<TextBlock x:Name="StatusLabel" Text="Setup required" FontSize="13" FontWeight="SemiBold" />
|
||||
<TextBlock x:Name="StatusSymbol" Text="○ Incomplete" FontSize="11" Foreground="{DynamicResource LumiMutedBrush}" />
|
||||
</StackPanel>
|
||||
</Grid>
|
||||
</Border>
|
||||
|
||||
<StackPanel Grid.Row="2" x:Name="Navigation">
|
||||
<Button x:Name="OverviewNav" Classes="nav selected" Content="Overview" Tag="Overview" />
|
||||
<Button x:Name="TranscriptionNav" Classes="nav" Content="Transcription" Tag="Transcription" />
|
||||
<Button x:Name="TestNav" Classes="nav" Content="Test" Tag="Test" />
|
||||
<Button x:Name="ConnectionNav" Classes="nav" Content="Connection & device" Tag="Connection" />
|
||||
<Button x:Name="LogsNav" Classes="nav" Content="Logs & diagnostics" Tag="Logs" />
|
||||
<Button x:Name="SettingsNav" Classes="nav" Content="Settings" Tag="Settings" />
|
||||
</StackPanel>
|
||||
|
||||
<TextBlock Grid.Row="3" Text="Runs quietly in the notification area" Classes="muted" FontSize="11" />
|
||||
</Grid>
|
||||
</Border>
|
||||
|
||||
<Grid Grid.Column="1">
|
||||
<ScrollViewer x:Name="OverviewPage">
|
||||
<StackPanel Margin="44,38,52,48" MaxWidth="760" HorizontalAlignment="Left" Spacing="24">
|
||||
<StackPanel Spacing="6">
|
||||
<TextBlock Text="AT A GLANCE" Classes="eyebrow" />
|
||||
<TextBlock Text="Overview" Classes="pageTitle" />
|
||||
<TextBlock x:Name="OverviewDetail" Text="Checking what is ready…" Classes="muted" FontSize="15" />
|
||||
</StackPanel>
|
||||
|
||||
<Border Classes="soft" Padding="22">
|
||||
<Grid ColumnDefinitions="*,Auto" ColumnSpacing="24">
|
||||
<StackPanel Spacing="6">
|
||||
<TextBlock x:Name="NextActionTitle" Text="Pair this computer" Classes="sectionTitle" />
|
||||
<TextBlock x:Name="NextActionDetail" Text="Use a one-time pairing package from Lumi to connect this streaming computer." Classes="muted" MaxWidth="520" />
|
||||
</StackPanel>
|
||||
<Button Grid.Column="1" x:Name="NextActionButton" Classes="primary" Content="Choose pairing package" VerticalAlignment="Center" />
|
||||
</Grid>
|
||||
</Border>
|
||||
|
||||
<StackPanel Spacing="12">
|
||||
<TextBlock Text="Setup progress" Classes="sectionTitle" />
|
||||
<Grid ColumnDefinitions="*,*,*" ColumnSpacing="12">
|
||||
<Border Classes="card">
|
||||
<StackPanel Spacing="7">
|
||||
<TextBlock x:Name="PairingStepSymbol" Text="○" FontSize="20" />
|
||||
<TextBlock Text="Lumi connection" FontWeight="SemiBold" />
|
||||
<TextBlock x:Name="PairingStepDetail" Text="Not paired" Classes="muted" FontSize="12" />
|
||||
</StackPanel>
|
||||
</Border>
|
||||
<Border Grid.Column="1" Classes="card">
|
||||
<StackPanel Spacing="7">
|
||||
<TextBlock x:Name="ObsStepSymbol" Text="○" FontSize="20" />
|
||||
<TextBlock Text="OBS integration" FontWeight="SemiBold" />
|
||||
<TextBlock x:Name="ObsStepDetail" Text="Not installed" Classes="muted" FontSize="12" />
|
||||
</StackPanel>
|
||||
</Border>
|
||||
<Border Grid.Column="2" Classes="card">
|
||||
<StackPanel Spacing="7">
|
||||
<TextBlock x:Name="TestStepSymbol" Text="○" FontSize="20" />
|
||||
<TextBlock Text="Full path test" FontWeight="SemiBold" />
|
||||
<TextBlock x:Name="TestStepDetail" Text="Not run" Classes="muted" FontSize="12" />
|
||||
</StackPanel>
|
||||
</Border>
|
||||
</Grid>
|
||||
</StackPanel>
|
||||
|
||||
<Border Classes="soft">
|
||||
<TextBlock Text="Lumi Companion never runs speech recognition on this streaming computer. Audio stays in bounded memory and is sent to your paired Lumi host." Classes="muted" />
|
||||
</Border>
|
||||
</StackPanel>
|
||||
</ScrollViewer>
|
||||
|
||||
<ScrollViewer x:Name="TranscriptionPage" IsVisible="False">
|
||||
<StackPanel Margin="44,38,52,48" MaxWidth="760" HorizontalAlignment="Left" Spacing="25">
|
||||
<StackPanel Spacing="6">
|
||||
<TextBlock Text="CAPTURE" Classes="eyebrow" />
|
||||
<TextBlock Text="Transcription" Classes="pageTitle" />
|
||||
<TextBlock Text="Choose the OBS microphone Lumi should caption. Source identity remains stable if you rename it." Classes="muted" FontSize="15" />
|
||||
</StackPanel>
|
||||
<StackPanel Spacing="8">
|
||||
<TextBlock Text="Primary microphone" FontWeight="SemiBold" />
|
||||
<ComboBox x:Name="SourcePicker" IsEnabled="False" PlaceholderText="Open OBS after installing the managed integration" MinWidth="420" HorizontalAlignment="Left" />
|
||||
<TextBlock x:Name="SourceHelp" Text="No OBS source list is available yet." Classes="muted" />
|
||||
</StackPanel>
|
||||
<Border Classes="soft">
|
||||
<StackPanel Spacing="7">
|
||||
<TextBlock Text="Optional second microphone" FontWeight="SemiBold" />
|
||||
<TextBlock Text="The protocol supports more than one source, but source overlap policy remains disabled until the primary path is validated." Classes="muted" />
|
||||
</StackPanel>
|
||||
</Border>
|
||||
</StackPanel>
|
||||
</ScrollViewer>
|
||||
|
||||
<ScrollViewer x:Name="TestPage" IsVisible="False">
|
||||
<StackPanel Margin="44,38,52,48" MaxWidth="760" HorizontalAlignment="Left" Spacing="24">
|
||||
<StackPanel Spacing="6">
|
||||
<TextBlock Text="SAFE CHECK" Classes="eyebrow" />
|
||||
<TextBlock Text="Transcription test" Classes="pageTitle" />
|
||||
<TextBlock Text="Exercises the real capture, network, speech recognition, and return path without sending anything to Twitch." Classes="muted" FontSize="15" />
|
||||
</StackPanel>
|
||||
<Button x:Name="RunTestButton" Classes="primary" Content="Run full path test" HorizontalAlignment="Left" />
|
||||
<StackPanel x:Name="TestStagesPanel" Spacing="8" />
|
||||
<Border Classes="soft">
|
||||
<StackPanel Spacing="5">
|
||||
<TextBlock Text="Simulated outgoing caption" FontWeight="SemiBold" />
|
||||
<TextBlock x:Name="SimulatedCaption" Text="A real caption will appear here only after every upstream stage passes." Classes="muted" FontStyle="Italic" />
|
||||
</StackPanel>
|
||||
</Border>
|
||||
</StackPanel>
|
||||
</ScrollViewer>
|
||||
|
||||
<ScrollViewer x:Name="ConnectionPage" IsVisible="False">
|
||||
<StackPanel Margin="44,38,52,48" MaxWidth="760" HorizontalAlignment="Left" Spacing="24">
|
||||
<StackPanel Spacing="6">
|
||||
<TextBlock Text="DEVICE" Classes="eyebrow" />
|
||||
<TextBlock Text="Connection & device" Classes="pageTitle" />
|
||||
<TextBlock Text="The device credential is protected for your Windows account and never shared with OBS." Classes="muted" FontSize="15" />
|
||||
</StackPanel>
|
||||
<Grid ColumnDefinitions="150,*" RowDefinitions="Auto,Auto,Auto" RowSpacing="12">
|
||||
<TextBlock Text="Computer" Classes="muted" /><TextBlock Grid.Column="1" x:Name="DeviceNameText" Text="Not paired" FontWeight="SemiBold" />
|
||||
<TextBlock Grid.Row="1" Text="Lumi host" Classes="muted" /><TextBlock Grid.Row="1" Grid.Column="1" x:Name="HostText" Text="—" />
|
||||
<TextBlock Grid.Row="2" Text="Last connected" Classes="muted" /><TextBlock Grid.Row="2" Grid.Column="1" x:Name="LastConnectedText" Text="Never" />
|
||||
</Grid>
|
||||
<StackPanel Orientation="Horizontal" Spacing="10">
|
||||
<Button x:Name="ReconnectButton" Classes="secondary" Content="Retry connection" />
|
||||
<Button x:Name="PairButton" Classes="secondary" Content="Choose pairing package" />
|
||||
<Button x:Name="OpenWebButton" Classes="secondary" Content="Open Lumi WebUI" />
|
||||
</StackPanel>
|
||||
<Border Classes="soft">
|
||||
<StackPanel Spacing="8">
|
||||
<TextBlock Text="Remove this device" FontWeight="SemiBold" />
|
||||
<TextBlock Text="Removes the local credential. You can also revoke this computer from Lumi." Classes="muted" />
|
||||
<Button x:Name="ForgetButton" Classes="secondary" Content="Forget this device" HorizontalAlignment="Left" />
|
||||
</StackPanel>
|
||||
</Border>
|
||||
</StackPanel>
|
||||
</ScrollViewer>
|
||||
|
||||
<ScrollViewer x:Name="LogsPage" IsVisible="False">
|
||||
<StackPanel Margin="44,38,52,48" MaxWidth="760" HorizontalAlignment="Left" Spacing="24">
|
||||
<StackPanel Spacing="6">
|
||||
<TextBlock Text="DIAGNOSTICS" Classes="eyebrow" />
|
||||
<TextBlock Text="Logs & diagnostics" Classes="pageTitle" />
|
||||
<TextBlock Text="Short-lived diagnostic logs help explain connection and delivery failures. Raw audio is never written to disk." Classes="muted" FontSize="15" />
|
||||
</StackPanel>
|
||||
<Button x:Name="OpenLogsButton" Classes="secondary" Content="Open logs folder" HorizontalAlignment="Left" />
|
||||
<Border Background="#182026" CornerRadius="12" Padding="16" MinHeight="230">
|
||||
<StackPanel x:Name="RecentLogsPanel" Spacing="6">
|
||||
<TextBlock Text="Waiting for companion events…" Foreground="#CFDADD" FontFamily="Consolas" FontSize="12" />
|
||||
</StackPanel>
|
||||
</Border>
|
||||
<TextBlock Text="Logs are retained for 7 days and capped at 256 MiB. Caption text may appear in diagnostics during testing or troubleshooting." Classes="muted" FontSize="12" />
|
||||
</StackPanel>
|
||||
</ScrollViewer>
|
||||
|
||||
<ScrollViewer x:Name="SettingsPage" IsVisible="False">
|
||||
<StackPanel Margin="44,38,52,48" MaxWidth="760" HorizontalAlignment="Left" Spacing="25">
|
||||
<StackPanel Spacing="6">
|
||||
<TextBlock Text="PREFERENCES" Classes="eyebrow" />
|
||||
<TextBlock Text="Settings" Classes="pageTitle" />
|
||||
<TextBlock Text="Only controls useful on the streaming computer live here. Model and server administration stay in Lumi." Classes="muted" FontSize="15" />
|
||||
</StackPanel>
|
||||
<StackPanel Spacing="16">
|
||||
<Grid ColumnDefinitions="*,Auto" ColumnSpacing="24">
|
||||
<StackPanel Spacing="3"><TextBlock Text="Start with Windows" FontWeight="SemiBold" /><TextBlock Text="Keep Lumi Companion ready in the notification area after sign-in." Classes="muted" /></StackPanel>
|
||||
<ToggleSwitch Grid.Column="1" x:Name="AutoStartToggle" OffContent="Off" OnContent="On" />
|
||||
</Grid>
|
||||
<Grid ColumnDefinitions="*,Auto" ColumnSpacing="24">
|
||||
<StackPanel Spacing="3"><TextBlock Text="Start companion features with OBS" FontWeight="SemiBold" /><TextBlock Text="Prepare selected features when OBS becomes available." Classes="muted" /></StackPanel>
|
||||
<ToggleSwitch Grid.Column="1" x:Name="StartWithObsToggle" OffContent="Off" OnContent="On" />
|
||||
</Grid>
|
||||
<Grid ColumnDefinitions="*,Auto" ColumnSpacing="24">
|
||||
<StackPanel Spacing="3"><TextBlock Text="Advanced mode" FontWeight="SemiBold" /><TextBlock Text="Reveal protocol and component details useful for troubleshooting." Classes="muted" /></StackPanel>
|
||||
<ToggleSwitch Grid.Column="1" x:Name="AdvancedToggle" OffContent="Off" OnContent="On" />
|
||||
</Grid>
|
||||
</StackPanel>
|
||||
<Border x:Name="AdvancedPanel" Classes="soft" IsVisible="False">
|
||||
<StackPanel Spacing="6">
|
||||
<TextBlock Text="Advanced details" FontWeight="SemiBold" />
|
||||
<TextBlock Text="Protocol v1 · bounded in-memory audio · secure WebSocket transport · server-hosted inference" Classes="muted" />
|
||||
</StackPanel>
|
||||
</Border>
|
||||
<Button x:Name="SaveSettingsButton" Classes="primary" Content="Save preferences" HorizontalAlignment="Left" />
|
||||
<TextBlock x:Name="SettingsFeedback" Classes="muted" />
|
||||
</StackPanel>
|
||||
</ScrollViewer>
|
||||
</Grid>
|
||||
</Grid>
|
||||
</Window>
|
||||
253
companion/src/Lumi.Companion.App/MainWindow.axaml.cs
Normal file
253
companion/src/Lumi.Companion.App/MainWindow.axaml.cs
Normal file
@ -0,0 +1,253 @@
|
||||
using Avalonia;
|
||||
using Avalonia.Controls;
|
||||
using Avalonia.Interactivity;
|
||||
using Avalonia.Media;
|
||||
using Avalonia.Platform.Storage;
|
||||
using Avalonia.Threading;
|
||||
|
||||
namespace Lumi.Companion.App;
|
||||
|
||||
public partial class MainWindow : Window
|
||||
{
|
||||
private readonly CompanionRuntime _runtime;
|
||||
private readonly CompanionSettingsStore _settings;
|
||||
private bool _allowExit;
|
||||
private bool _testPassed;
|
||||
private bool _renderingSources;
|
||||
|
||||
public MainWindow() : this(CreateDefaultServices()) { }
|
||||
|
||||
private MainWindow((CompanionRuntime Runtime, CompanionSettingsStore Settings) services)
|
||||
: this(services.Runtime, services.Settings) { }
|
||||
|
||||
public MainWindow(CompanionRuntime runtime, CompanionSettingsStore settings)
|
||||
{
|
||||
_runtime = runtime;
|
||||
_settings = settings;
|
||||
InitializeComponent();
|
||||
WireActions();
|
||||
RenderState(runtime.State);
|
||||
RenderTestStages(runtime.TestStages);
|
||||
Closing += OnClosing;
|
||||
runtime.StateChanged += state => Dispatcher.UIThread.Post(() => RenderState(state));
|
||||
runtime.TestStagesChanged += stages => Dispatcher.UIThread.Post(() => RenderTestStages(stages));
|
||||
runtime.ObsSourcesChanged += sources => Dispatcher.UIThread.Post(() => RenderSources(sources));
|
||||
runtime.CaptionReceived += (text, simulated) => Dispatcher.UIThread.Post(() =>
|
||||
{
|
||||
if (simulated) SimulatedCaption.Text = text;
|
||||
});
|
||||
runtime.LogAdded += line => Dispatcher.UIThread.Post(() => AddLog(line));
|
||||
settings.Changed += value => Dispatcher.UIThread.Post(() => RenderSettings(value));
|
||||
}
|
||||
|
||||
private void WireActions()
|
||||
{
|
||||
foreach (var button in Navigation.Children.OfType<Button>()) button.Click += OnNavigate;
|
||||
NextActionButton.Click += OnNextAction;
|
||||
PairButton.Click += async (_, _) => await ChoosePairingPackageAsync();
|
||||
ReconnectButton.Click += async (_, _) => await RunUiActionAsync(() => _runtime.RetryConnectionAsync(), ReconnectButton);
|
||||
OpenWebButton.Click += (_, _) => _runtime.OpenLumiWebUi();
|
||||
OpenLogsButton.Click += (_, _) => _runtime.OpenLogsDirectory();
|
||||
RunTestButton.Click += async (_, _) => await RunUiActionAsync(() => _runtime.RunTestAsync(), RunTestButton);
|
||||
ForgetButton.Click += async (_, _) => await ForgetDeviceAsync();
|
||||
SaveSettingsButton.Click += async (_, _) => await SaveSettingsAsync();
|
||||
AdvancedToggle.IsCheckedChanged += (_, _) => AdvancedPanel.IsVisible = AdvancedToggle.IsChecked == true;
|
||||
SourcePicker.SelectionChanged += async (_, _) =>
|
||||
{
|
||||
if (!_renderingSources && SourcePicker.SelectedItem is ObsSource source) await _runtime.SelectSourceAsync(source);
|
||||
};
|
||||
}
|
||||
|
||||
private void OnNavigate(object? sender, RoutedEventArgs args)
|
||||
{
|
||||
if (sender is Button { Tag: string tag } && Enum.TryParse<CompanionPage>(tag, out var page)) ShowPage(page);
|
||||
}
|
||||
|
||||
public void ShowPage(CompanionPage page)
|
||||
{
|
||||
var pages = new Dictionary<CompanionPage, Control>
|
||||
{
|
||||
[CompanionPage.Overview] = OverviewPage, [CompanionPage.Transcription] = TranscriptionPage,
|
||||
[CompanionPage.Test] = TestPage, [CompanionPage.Connection] = ConnectionPage,
|
||||
[CompanionPage.Logs] = LogsPage, [CompanionPage.Settings] = SettingsPage
|
||||
};
|
||||
foreach (var item in pages) item.Value.IsVisible = item.Key == page;
|
||||
foreach (var button in Navigation.Children.OfType<Button>())
|
||||
{
|
||||
button.Classes.Set("selected", string.Equals(button.Tag as string, page.ToString(), StringComparison.Ordinal));
|
||||
}
|
||||
}
|
||||
|
||||
private async void OnNextAction(object? sender, RoutedEventArgs args)
|
||||
{
|
||||
if (!_runtime.State.Paired) { await ChoosePairingPackageAsync(); return; }
|
||||
if (!_runtime.State.Connected) { await RunUiActionAsync(() => _runtime.RetryConnectionAsync(), NextActionButton); return; }
|
||||
if (!_runtime.State.ObsBridgeInstalled) { ShowPage(CompanionPage.Connection); return; }
|
||||
ShowPage(CompanionPage.Test);
|
||||
}
|
||||
|
||||
private async Task ChoosePairingPackageAsync()
|
||||
{
|
||||
var files = await StorageProvider.OpenFilePickerAsync(new FilePickerOpenOptions
|
||||
{
|
||||
Title = "Choose a Lumi pairing package",
|
||||
AllowMultiple = false,
|
||||
FileTypeFilter = [new FilePickerFileType("Lumi pairing package") { Patterns = ["*.lumi-pairing.json", "*.json"] }]
|
||||
});
|
||||
var path = files.FirstOrDefault()?.TryGetLocalPath();
|
||||
if (path is null) return;
|
||||
await RunUiActionAsync(() => _runtime.PairAsync(path), PairButton);
|
||||
}
|
||||
|
||||
private async Task RunUiActionAsync(Func<Task> action, Button button)
|
||||
{
|
||||
button.IsEnabled = false;
|
||||
try { await action(); }
|
||||
catch (Exception error) { AddLog($"{DateTime.Now:HH:mm:ss} {error.Message}"); }
|
||||
finally { button.IsEnabled = true; }
|
||||
}
|
||||
|
||||
private async Task SaveSettingsAsync()
|
||||
{
|
||||
SaveSettingsButton.IsEnabled = false;
|
||||
SettingsFeedback.Text = "Saving…";
|
||||
try
|
||||
{
|
||||
await _runtime.SaveSettingsAsync(_settings.Current with
|
||||
{
|
||||
AutoStartWithWindows = AutoStartToggle.IsChecked == true,
|
||||
StartWithObs = StartWithObsToggle.IsChecked == true,
|
||||
AdvancedMode = AdvancedToggle.IsChecked == true
|
||||
});
|
||||
SettingsFeedback.Text = "Preferences saved.";
|
||||
}
|
||||
catch (Exception error) { SettingsFeedback.Text = $"Could not save preferences: {error.Message}"; }
|
||||
finally { SaveSettingsButton.IsEnabled = true; }
|
||||
}
|
||||
|
||||
private async Task ForgetDeviceAsync()
|
||||
{
|
||||
var dialog = new DecisionWindow("Forget this device?", "This removes the protected credential from this computer. Pairing can be restored with a new package.", "Forget device");
|
||||
if (!await dialog.ShowDialog<bool>(this)) return;
|
||||
await _runtime.ForgetDeviceAsync();
|
||||
}
|
||||
|
||||
private void RenderState(CompanionState state)
|
||||
{
|
||||
StatusLabel.Text = state.Summary;
|
||||
StatusSymbol.Text = state.Health switch { TrayHealth.Ready => "✓ Ready", TrayHealth.Operating => "▶ Active", TrayHealth.Failed => "! Action required", _ => "△ Incomplete" };
|
||||
StatusMark.Background = new SolidColorBrush(Color.Parse(state.Health switch { TrayHealth.Ready => "#176B75", TrayHealth.Operating => "#23845B", TrayHealth.Failed => "#BD4D4D", _ => "#A96612" }));
|
||||
OverviewDetail.Text = state.Detail;
|
||||
PairingStepSymbol.Text = state.Paired ? "✓" : "○";
|
||||
PairingStepDetail.Text = state.Connected ? "Securely connected" : state.Paired ? "Paired, currently offline" : "Not paired";
|
||||
ObsStepSymbol.Text = state.ObsConnected ? "✓" : state.ObsBridgeInstalled ? "◐" : "○";
|
||||
ObsStepDetail.Text = state.ObsConnected ? "Connected to OBS" : state.ObsBridgeInstalled ? "Installed; waiting for OBS" : "Installation required";
|
||||
TestStepSymbol.Text = _testPassed ? "✓" : "○";
|
||||
TestStepDetail.Text = _testPassed ? "Passed" : "Not passed";
|
||||
DeviceNameText.Text = state.DeviceName ?? "Not paired";
|
||||
HostText.Text = state.Host ?? "—";
|
||||
LastConnectedText.Text = state.LastConnectedAt?.ToString("g") ?? "Never";
|
||||
OpenWebButton.IsEnabled = state.Host is not null;
|
||||
ReconnectButton.IsEnabled = state.Paired && !state.Connected;
|
||||
ForgetButton.IsEnabled = state.Paired;
|
||||
RunTestButton.Content = state.TestRunning ? "Testing…" : "Run full path test";
|
||||
RunTestButton.IsEnabled = !state.TestRunning;
|
||||
|
||||
if (!state.Paired)
|
||||
{
|
||||
NextActionTitle.Text = "Pair this computer";
|
||||
NextActionDetail.Text = "Use a one-time pairing package from Lumi to connect this streaming computer.";
|
||||
NextActionButton.Content = "Choose pairing package";
|
||||
}
|
||||
else if (!state.Connected)
|
||||
{
|
||||
NextActionTitle.Text = "Reconnect to Lumi";
|
||||
NextActionDetail.Text = "Your device is paired, but Lumi cannot currently be reached.";
|
||||
NextActionButton.Content = "Retry connection";
|
||||
}
|
||||
else if (!state.ObsBridgeInstalled)
|
||||
{
|
||||
NextActionTitle.Text = "Install the OBS integration";
|
||||
NextActionDetail.Text = "The native bridge package is not installed yet. The signed installer and repair service remain required.";
|
||||
NextActionButton.Content = "View connection details";
|
||||
}
|
||||
else
|
||||
{
|
||||
NextActionTitle.Text = "Run a safe transcription test";
|
||||
NextActionDetail.Text = "Check the real end-to-end path without sending captions to Twitch.";
|
||||
NextActionButton.Content = "Open test";
|
||||
}
|
||||
}
|
||||
|
||||
private void RenderSettings(CompanionSettings settings)
|
||||
{
|
||||
AutoStartToggle.IsChecked = settings.AutoStartWithWindows;
|
||||
StartWithObsToggle.IsChecked = settings.StartWithObs;
|
||||
AdvancedToggle.IsChecked = settings.AdvancedMode;
|
||||
AdvancedPanel.IsVisible = settings.AdvancedMode;
|
||||
if (settings.PrimarySourceUuid is not null)
|
||||
SourcePicker.SelectedItem = _runtime.ObsSources.FirstOrDefault(source => source.Uuid == settings.PrimarySourceUuid);
|
||||
}
|
||||
|
||||
|
||||
private void RenderSources(IReadOnlyList<ObsSource> sources)
|
||||
{
|
||||
_renderingSources = true;
|
||||
SourcePicker.ItemsSource = sources;
|
||||
SourcePicker.IsEnabled = sources.Count > 0;
|
||||
SourcePicker.SelectedItem = sources.FirstOrDefault(source => source.Uuid == _settings.Current.PrimarySourceUuid);
|
||||
SourceHelp.Text = sources.Count == 0 ? "OBS is connected, but no audio sources were reported." : "Choose the source that carries the primary speaker.";
|
||||
_renderingSources = false;
|
||||
}
|
||||
|
||||
private void RenderTestStages(IReadOnlyList<TestStage> stages)
|
||||
{
|
||||
TestStagesPanel.Children.Clear();
|
||||
foreach (var stage in stages)
|
||||
{
|
||||
var symbol = stage.State switch { TestStageState.Passed => "✓", TestStageState.Running => "…", TestStageState.Blocked => "△", TestStageState.Failed => "!", _ => "○" };
|
||||
var row = new Grid { ColumnDefinitions = ColumnDefinitions.Parse("28,180,*"), ColumnSpacing = 10 };
|
||||
row.Children.Add(new TextBlock { Text = symbol, FontSize = 17, VerticalAlignment = Avalonia.Layout.VerticalAlignment.Center });
|
||||
var name = new TextBlock { Text = stage.Name, FontWeight = FontWeight.SemiBold, VerticalAlignment = Avalonia.Layout.VerticalAlignment.Center };
|
||||
Grid.SetColumn(name, 1); row.Children.Add(name);
|
||||
var detail = new TextBlock { Text = stage.Detail, Foreground = new SolidColorBrush(Color.Parse("#5A6872")), TextWrapping = TextWrapping.Wrap };
|
||||
Grid.SetColumn(detail, 2); row.Children.Add(detail);
|
||||
TestStagesPanel.Children.Add(new Border { Classes = { "soft" }, Child = row, Padding = new Thickness(14, 11) });
|
||||
}
|
||||
_testPassed = stages.Count > 0 && stages.All(stage => stage.State == TestStageState.Passed);
|
||||
TestStepSymbol.Text = _testPassed ? "✓" : "○";
|
||||
TestStepDetail.Text = _testPassed ? "Passed" : "Not passed";
|
||||
RunTestButton.Content = _runtime.State.TestRunning ? "Testing…" : "Run full path test";
|
||||
RunTestButton.IsEnabled = !_runtime.State.TestRunning;
|
||||
}
|
||||
|
||||
private void AddLog(string line)
|
||||
{
|
||||
if (RecentLogsPanel.Children.Count == 1 && RecentLogsPanel.Children[0] is TextBlock text && text.Text?.StartsWith("Waiting") == true) RecentLogsPanel.Children.Clear();
|
||||
RecentLogsPanel.Children.Insert(0, new TextBlock { Text = line, Foreground = new SolidColorBrush(Color.Parse("#CFDADD")), FontFamily = new FontFamily("Consolas"), FontSize = 12, TextWrapping = TextWrapping.Wrap });
|
||||
while (RecentLogsPanel.Children.Count > 20) RecentLogsPanel.Children.RemoveAt(RecentLogsPanel.Children.Count - 1);
|
||||
}
|
||||
|
||||
private void OnClosing(object? sender, WindowClosingEventArgs args)
|
||||
{
|
||||
if (_allowExit) return;
|
||||
args.Cancel = true;
|
||||
Hide();
|
||||
}
|
||||
|
||||
public async Task<bool> ConfirmQuitAsync()
|
||||
{
|
||||
if (!_runtime.State.RequiresQuitConfirmation) return true;
|
||||
var dialog = new DecisionWindow("Quit while OBS is active?", _runtime.State.QuitWarning, "Quit Companion");
|
||||
return await dialog.ShowDialog<bool>(this);
|
||||
}
|
||||
|
||||
public void AllowExit() => _allowExit = true;
|
||||
|
||||
private static (CompanionRuntime, CompanionSettingsStore) CreateDefaultServices()
|
||||
{
|
||||
var paths = CompanionPaths.ForCurrentUser();
|
||||
var settings = new CompanionSettingsStore(paths.SettingsPath);
|
||||
return (new CompanionRuntime(paths, settings), settings);
|
||||
}
|
||||
}
|
||||
@ -1,33 +1,42 @@
|
||||
using Lumi.Companion.Core;
|
||||
using Avalonia;
|
||||
|
||||
if (!OperatingSystem.IsWindows())
|
||||
{
|
||||
Console.Error.WriteLine("The first Lumi Companion build supports Windows x64 only.");
|
||||
return 2;
|
||||
}
|
||||
namespace Lumi.Companion.App;
|
||||
|
||||
var appData = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "Lumi", "Companion");
|
||||
var credentialStore = new SecureCredentialStore(appData);
|
||||
if (args is ["pair", var packagePath])
|
||||
internal static class Program
|
||||
{
|
||||
using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(30) };
|
||||
var client = new PairingClient(http);
|
||||
var credential = await client.PairAsync(packagePath, new
|
||||
internal static SingleInstanceCoordinator? InstanceCoordinator { get; private set; }
|
||||
internal static bool LaunchInBackground { get; private set; }
|
||||
|
||||
[STAThread]
|
||||
public static int Main(string[] args)
|
||||
{
|
||||
install_id = credentialStore.GetOrCreateInstallId(), name = Environment.MachineName,
|
||||
companion_version = "0.1.0-experimental.1", os = Environment.OSVersion.VersionString,
|
||||
architecture = System.Runtime.InteropServices.RuntimeInformation.OSArchitecture.ToString()
|
||||
}, CancellationToken.None);
|
||||
credentialStore.Save(credential);
|
||||
Console.WriteLine("Lumi Companion paired successfully. The device credential is protected for the current Windows user.");
|
||||
return 0;
|
||||
}
|
||||
if (!OperatingSystem.IsWindows())
|
||||
{
|
||||
Console.Error.WriteLine("The first Lumi Companion release supports Windows x64 only.");
|
||||
return 2;
|
||||
}
|
||||
|
||||
if (credentialStore.Load() is null)
|
||||
{
|
||||
Console.WriteLine("Lumi Companion is not paired. Run: Lumi.Companion.App pair <package.lumi-pairing.json>");
|
||||
return 1;
|
||||
}
|
||||
LaunchInBackground = args.Contains("--background", StringComparer.OrdinalIgnoreCase);
|
||||
InstanceCoordinator = new SingleInstanceCoordinator();
|
||||
if (!InstanceCoordinator.IsPrimary)
|
||||
{
|
||||
InstanceCoordinator.SignalPrimary();
|
||||
InstanceCoordinator.Dispose();
|
||||
return 0;
|
||||
}
|
||||
|
||||
Console.WriteLine("Lumi Companion protocol foundation is installed. The Avalonia tray UI and managed OBS bridge installer are not included in this milestone.");
|
||||
return 0;
|
||||
try
|
||||
{
|
||||
return BuildAvaloniaApp().StartWithClassicDesktopLifetime(args);
|
||||
}
|
||||
finally
|
||||
{
|
||||
InstanceCoordinator.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
public static AppBuilder BuildAvaloniaApp() => AppBuilder.Configure<App>()
|
||||
.UsePlatformDetect()
|
||||
.WithInterFont()
|
||||
.LogToTrace();
|
||||
}
|
||||
|
||||
@ -0,0 +1,62 @@
|
||||
using System.IO.Pipes;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
|
||||
namespace Lumi.Companion.App;
|
||||
|
||||
internal sealed class SingleInstanceCoordinator : IDisposable
|
||||
{
|
||||
private readonly Mutex _mutex;
|
||||
private readonly CancellationTokenSource _lifetime = new();
|
||||
private readonly string _pipeName;
|
||||
private Task? _listener;
|
||||
|
||||
public SingleInstanceCoordinator()
|
||||
{
|
||||
var userKey = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes($"{Environment.UserDomainName}\\{Environment.UserName}"))).Substring(0, 16);
|
||||
_pipeName = $"Lumi.Companion.Activate.v1.{userKey}";
|
||||
_mutex = new Mutex(true, $"Local\\Lumi.Companion.v1.{userKey}", out var created);
|
||||
IsPrimary = created;
|
||||
if (created) _listener = Task.Run(() => ListenAsync(_lifetime.Token));
|
||||
}
|
||||
|
||||
public bool IsPrimary { get; }
|
||||
public event Action? ActivationRequested;
|
||||
|
||||
public void SignalPrimary()
|
||||
{
|
||||
try
|
||||
{
|
||||
using var pipe = new NamedPipeClientStream(".", _pipeName, PipeDirection.Out, PipeOptions.Asynchronous);
|
||||
pipe.Connect(1000);
|
||||
pipe.WriteByte(1);
|
||||
pipe.Flush();
|
||||
}
|
||||
catch { }
|
||||
}
|
||||
|
||||
private async Task ListenAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var pipe = new NamedPipeServerStream(_pipeName, PipeDirection.In, 1, PipeTransmissionMode.Byte,
|
||||
PipeOptions.Asynchronous | PipeOptions.CurrentUserOnly, 1, 1);
|
||||
await pipe.WaitForConnectionAsync(cancellationToken);
|
||||
if (pipe.ReadByte() == 1) ActivationRequested?.Invoke();
|
||||
}
|
||||
catch (OperationCanceledException) { break; }
|
||||
catch when (!cancellationToken.IsCancellationRequested) { await Task.Delay(200, cancellationToken); }
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_lifetime.Cancel();
|
||||
try { _listener?.Wait(500); } catch { }
|
||||
if (IsPrimary) try { _mutex.ReleaseMutex(); } catch { }
|
||||
_mutex.Dispose();
|
||||
_lifetime.Dispose();
|
||||
}
|
||||
}
|
||||
15
companion/src/Lumi.Companion.App/app.manifest
Normal file
15
companion/src/Lumi.Companion.App/app.manifest
Normal file
@ -0,0 +1,15 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<assembly manifestVersion="1.0" xmlns="urn:schemas-microsoft-com:asm.v1">
|
||||
<assemblyIdentity version="1.0.0.0" name="Lumi.Companion.App" />
|
||||
<compatibility xmlns="urn:schemas-microsoft-com:compatibility.v1">
|
||||
<application>
|
||||
<supportedOS Id="{8e0f7a12-bfb3-4fe8-b9a5-48fd50a15a9a}" />
|
||||
</application>
|
||||
</compatibility>
|
||||
<application xmlns="urn:schemas-microsoft-com:asm.v3">
|
||||
<windowsSettings>
|
||||
<dpiAware xmlns="http://schemas.microsoft.com/SMI/2005/WindowsSettings">true/pm</dpiAware>
|
||||
<longPathAware xmlns="http://schemas.microsoft.com/SMI/2016/WindowsSettings">true</longPathAware>
|
||||
</windowsSettings>
|
||||
</application>
|
||||
</assembly>
|
||||
@ -10,10 +10,12 @@ public sealed class CompanionSocket : IAsyncDisposable
|
||||
private ClientWebSocket? _socket;
|
||||
private CancellationTokenSource? _lifetime;
|
||||
private readonly SemaphoreSlim _sendLock = new(1, 1);
|
||||
private readonly List<Task> _backgroundTasks = [];
|
||||
private long _droppedFrames;
|
||||
public long DroppedFrames => Interlocked.Read(ref _droppedFrames);
|
||||
public Guid SessionId { get; private set; }
|
||||
public event Func<ServerEnvelope, Task>? MessageReceived;
|
||||
public event Action<Exception?>? Disconnected;
|
||||
|
||||
public async Task ConnectAsync(DeviceCredential credential, string companionVersion, string pluginVersion, string? obsVersion, CancellationToken cancellationToken)
|
||||
{
|
||||
@ -29,18 +31,31 @@ public sealed class CompanionSocket : IAsyncDisposable
|
||||
throw new InvalidDataException("Lumi returned an incompatible companion handshake.");
|
||||
SessionId = acknowledgement.SessionId.Value;
|
||||
_lifetime = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
_ = Task.Run(() => SendAudioAsync(_lifetime.Token), _lifetime.Token);
|
||||
_ = Task.Run(() => ReceiveAsync(_lifetime.Token), _lifetime.Token);
|
||||
_backgroundTasks.Add(Task.Run(() => RunGuardedAsync(SendAudioAsync, _lifetime.Token), _lifetime.Token));
|
||||
_backgroundTasks.Add(Task.Run(() => RunGuardedAsync(ReceiveAsync, _lifetime.Token), _lifetime.Token));
|
||||
_backgroundTasks.Add(Task.Run(() => RunGuardedAsync(SendHeartbeatsAsync, _lifetime.Token), _lifetime.Token));
|
||||
}
|
||||
|
||||
public bool QueueAudio(AudioFrame frame)
|
||||
{
|
||||
var encoded = ProtocolV1.EncodeAudio(frame);
|
||||
return QueueEncodedAudio(ProtocolV1.EncodeAudio(frame));
|
||||
}
|
||||
public bool QueueEncodedAudio(ReadOnlyMemory<byte> encoded)
|
||||
{
|
||||
if (encoded.Length < ProtocolV1.AudioHeaderBytes || encoded.Length > ProtocolV1.AudioHeaderBytes + ProtocolV1.MaxAudioPayloadBytes ||
|
||||
!encoded.Span[..4].SequenceEqual("LACP"u8)) return false;
|
||||
if (_audio.Writer.TryWrite(encoded)) return true;
|
||||
_audio.Reader.TryRead(out _);
|
||||
Interlocked.Increment(ref _droppedFrames);
|
||||
return _audio.Writer.TryWrite(encoded);
|
||||
}
|
||||
public bool QueueBridgeAudio(ReadOnlyMemory<byte> encoded)
|
||||
{
|
||||
if (encoded.Length < ProtocolV1.AudioHeaderBytes || !encoded.Span[..4].SequenceEqual("LACP"u8)) return false;
|
||||
var normalized = encoded.ToArray();
|
||||
SessionId.TryWriteBytes(normalized.AsSpan(20, 16), bigEndian: true, out _);
|
||||
return QueueEncodedAudio(normalized);
|
||||
}
|
||||
public async Task SendAsync(string type, object payload, Guid? sessionId, CancellationToken cancellationToken)
|
||||
{
|
||||
if (_socket?.State != WebSocketState.Open) throw new InvalidOperationException("Lumi is not connected.");
|
||||
@ -68,6 +83,22 @@ public sealed class CompanionSocket : IAsyncDisposable
|
||||
if (MessageReceived is { } handler) await handler(message);
|
||||
}
|
||||
}
|
||||
private async Task SendHeartbeatsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(8));
|
||||
while (await timer.WaitForNextTickAsync(cancellationToken))
|
||||
await SendAsync("ping", new { }, SessionId, cancellationToken);
|
||||
}
|
||||
private async Task RunGuardedAsync(Func<CancellationToken, Task> operation, CancellationToken cancellationToken)
|
||||
{
|
||||
try { await operation(cancellationToken); }
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { }
|
||||
catch (Exception error)
|
||||
{
|
||||
_lifetime?.Cancel();
|
||||
Disconnected?.Invoke(error);
|
||||
}
|
||||
}
|
||||
private static async Task<ServerEnvelope> ReceiveMessageAsync(ClientWebSocket socket, CancellationToken cancellationToken)
|
||||
{
|
||||
using var body = new MemoryStream();
|
||||
@ -87,7 +118,12 @@ public sealed class CompanionSocket : IAsyncDisposable
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
_lifetime?.Cancel();
|
||||
if (_socket?.State == WebSocketState.Open) await _socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "companion_exit", CancellationToken.None);
|
||||
if (_socket?.State == WebSocketState.Open)
|
||||
{
|
||||
try { await _socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "companion_exit", CancellationToken.None); }
|
||||
catch (WebSocketException) { }
|
||||
}
|
||||
try { await Task.WhenAll(_backgroundTasks); } catch (OperationCanceledException) { }
|
||||
_socket?.Dispose(); _lifetime?.Dispose(); _sendLock.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
@ -13,7 +13,7 @@ OBS bridge boundary -> same-user companion IPC -> bounded companion queue
|
||||
-> WebSocket -> companion delivery adapter -> OBS native caption boundary
|
||||
```
|
||||
|
||||
The repository does not yet contain a validated whisper streaming worker or a complete signed OBS bridge installer. Accordingly, the admin page reports inference setup as required and this branch does not claim live Twitch acceptance.
|
||||
The repository contains a pinned native whisper.cpp rolling-window worker and a CPU build/protocol smoke test. It has not yet been benchmarked with a model on the target RTX 3060, packaged as the Windows CUDA runtime, or validated end to end. The repository also does not contain a complete signed OBS bridge installer. Accordingly, this branch does not claim live Twitch acceptance.
|
||||
|
||||
## Trust and privacy
|
||||
|
||||
@ -28,11 +28,11 @@ The repository does not yet contain a validated whisper streaming worker or a co
|
||||
|
||||
1. Serve Lumi through HTTPS and enable `lumi_transcription` under Admin > Plugins.
|
||||
2. Open Plugins > Transcription and create a pairing package. It is a bootstrap package for the generic companion, not a credential that should be shared or committed.
|
||||
3. On the Windows streaming computer, build the .NET 8 projects under `companion/` and run `Lumi.Companion.App pair <package>` for the current protocol milestone.
|
||||
3. On the Windows streaming computer, build `companion/Lumi.Companion.sln`, start `Lumi.Companion.App`, and choose the package from the Overview or Connection page.
|
||||
4. Install a pinned runtime/model only after explicit confirmation. `small.en` is recommended; `small.en-q5_1` and `base.en` are fallbacks. Every artifact is checksum-verified before install.
|
||||
5. Configure `LUMI_TRANSCRIPTION_WORKER` with the supervised streaming-worker executable once that worker is built for the target host.
|
||||
|
||||
The normal signed installer, Avalonia tray UI, managed bridge install/repair, source selector, and model benchmark wizard are not complete yet.
|
||||
The Avalonia tray UI and host-side model download/load controls now exist. The normal signed installer, managed bridge install/repair, live OBS source enumeration, and measured model benchmark wizard are not complete yet.
|
||||
|
||||
## Operation and recovery
|
||||
|
||||
|
||||
163
knowledge/plugins/lumi-transcription.md
Normal file
163
knowledge/plugins/lumi-transcription.md
Normal file
@ -0,0 +1,163 @@
|
||||
---
|
||||
id: plugin.lumi_transcription
|
||||
title: Lumi Transcription
|
||||
scope: plugins
|
||||
status: active
|
||||
priority: 10
|
||||
visibility: user
|
||||
category: Plugin
|
||||
tags: plugin, lumi_transcription
|
||||
generated: true
|
||||
editable: false
|
||||
---
|
||||
# Lumi Transcription
|
||||
Server-hosted whisper.cpp transcription for Lumi Companion and OBS closed captions.
|
||||
## Metadata
|
||||
Plugin ID: lumi_transcription
|
||||
Version: 0.1.0-experimental.1
|
||||
Default state: enabled
|
||||
## Web Routes
|
||||
- /plugins/lumi_transcription
|
||||
- GET /plugins/lumi_transcription
|
||||
- GET /plugins/lumi_transcription/api/status
|
||||
- POST /plugins/lumi_transcription/api/pairing-package
|
||||
- POST /plugins/lumi_transcription/api/pair
|
||||
- GET /plugins/lumi_transcription/api/devices
|
||||
- POST /plugins/lumi_transcription/api/devices/:id/revoke
|
||||
- POST /plugins/lumi_transcription/api/devices/:id/capabilities
|
||||
- GET /plugins/lumi_transcription/api/settings
|
||||
- PATCH /plugins/lumi_transcription/api/settings
|
||||
- POST /plugins/lumi_transcription/api/models/:id/download
|
||||
- POST /plugins/lumi_transcription/api/models/:id/load
|
||||
- POST /plugins/lumi_transcription/api/runtime/:id/install
|
||||
- GET /plugins/lumi_transcription/api/logs
|
||||
## Route Reference
|
||||
### MOUNT /plugins/lumi_transcription
|
||||
|
||||
- Purpose: Mounts the plugin router at this base WebUI path.
|
||||
- Inputs: No request parameters detected by static analysis.
|
||||
- Response format: Plugin router mount point.
|
||||
- Access: Access is controlled by the mount options and individual plugin routes.
|
||||
- Side effects: No direct route action; child routes handle requests.
|
||||
- Limits/notes: Mount metadata is inferred from static source scanning.
|
||||
|
||||
### GET /plugins/lumi_transcription
|
||||
|
||||
- Purpose: Renders or serves the lumi_transcription plugin page.
|
||||
- Inputs: No request parameters detected by static analysis.
|
||||
- Response format: HTML page rendered from an EJS view
|
||||
- Access: admin access expected
|
||||
- Side effects: Usually read-only.
|
||||
- Limits/notes: Generated from static route source analysis; confirm exact behavior in the handler before changing integrations.
|
||||
|
||||
### GET /plugins/lumi_transcription/api/status
|
||||
|
||||
- Purpose: Renders or serves the lumi_transcription plugin page for api status.
|
||||
- Inputs: No request parameters detected by static analysis.
|
||||
- Response format: JSON response
|
||||
- Access: admin access expected
|
||||
- Side effects: Usually read-only.
|
||||
- Limits/notes: Generated from static route source analysis; confirm exact behavior in the handler before changing integrations. API consumers should expect JSON unless the response format says otherwise.
|
||||
|
||||
### POST /plugins/lumi_transcription/api/pairing-package
|
||||
|
||||
- Purpose: Processes the lumi_transcription plugin action for api pairing package.
|
||||
- Inputs: No request parameters detected by static analysis.
|
||||
- Response format: plain or HTML response
|
||||
- Access: admin access expected; logged-in session required or used
|
||||
- Side effects: Action route; side effects were not detected statically.
|
||||
- Limits/notes: Generated from static route source analysis; confirm exact behavior in the handler before changing integrations. Errors are caught and usually returned as a flash message, JSON error, or error page. Most non-API POST routes are browser form submissions and usually redirect after completion. API consumers should expect JSON unless the response format says otherwise.
|
||||
|
||||
### POST /plugins/lumi_transcription/api/pair
|
||||
|
||||
- Purpose: Processes the lumi_transcription plugin action for api pair.
|
||||
- Inputs: body: full submitted body is passed to a helper; exact fields are defined by the matching form/service
|
||||
- Response format: Form/action response; exact format was not detected statically.
|
||||
- Access: No explicit access guard detected in the route handler; check surrounding router/mount middleware.
|
||||
- Side effects: Action route; side effects were not detected statically.
|
||||
- Limits/notes: Generated from static route source analysis; confirm exact behavior in the handler before changing integrations. Errors are caught and usually returned as a flash message, JSON error, or error page. Most non-API POST routes are browser form submissions and usually redirect after completion. API consumers should expect JSON unless the response format says otherwise.
|
||||
|
||||
### GET /plugins/lumi_transcription/api/devices
|
||||
|
||||
- Purpose: Renders or serves the lumi_transcription plugin page for api devices.
|
||||
- Inputs: No request parameters detected by static analysis.
|
||||
- Response format: JSON response
|
||||
- Access: admin access expected
|
||||
- Side effects: Usually read-only.
|
||||
- Limits/notes: Generated from static route source analysis; confirm exact behavior in the handler before changing integrations. API consumers should expect JSON unless the response format says otherwise.
|
||||
|
||||
### POST /plugins/lumi_transcription/api/devices/:id/revoke
|
||||
|
||||
- Purpose: Processes the lumi_transcription plugin action for api devices id revoke.
|
||||
- Inputs: path params: `id`
|
||||
- Response format: JSON response
|
||||
- Access: admin access expected
|
||||
- Side effects: Action route; side effects were not detected statically.
|
||||
- Limits/notes: Generated from static route source analysis; confirm exact behavior in the handler before changing integrations. Most non-API POST routes are browser form submissions and usually redirect after completion. API consumers should expect JSON unless the response format says otherwise.
|
||||
|
||||
### POST /plugins/lumi_transcription/api/devices/:id/capabilities
|
||||
|
||||
- Purpose: Processes the lumi_transcription plugin action for api devices id capabilities.
|
||||
- Inputs: path params: `id`; body: `capabilities`
|
||||
- Response format: JSON response
|
||||
- Access: admin access expected
|
||||
- Side effects: writes or mutates server-side state
|
||||
- Limits/notes: Generated from static route source analysis; confirm exact behavior in the handler before changing integrations. Most non-API POST routes are browser form submissions and usually redirect after completion. API consumers should expect JSON unless the response format says otherwise.
|
||||
|
||||
### GET /plugins/lumi_transcription/api/settings
|
||||
|
||||
- Purpose: Renders or serves the lumi_transcription plugin page for api settings.
|
||||
- Inputs: No request parameters detected by static analysis.
|
||||
- Response format: JSON response
|
||||
- Access: No explicit access guard detected in the route handler; check surrounding router/mount middleware.
|
||||
- Side effects: Usually read-only.
|
||||
- Limits/notes: Generated from static route source analysis; confirm exact behavior in the handler before changing integrations. API consumers should expect JSON unless the response format says otherwise.
|
||||
|
||||
### PATCH /plugins/lumi_transcription/api/settings
|
||||
|
||||
- Purpose: Processes the lumi_transcription plugin action for api settings.
|
||||
- Inputs: body: `changes`
|
||||
- Response format: Form/action response; exact format was not detected statically.
|
||||
- Access: No explicit access guard detected in the route handler; check surrounding router/mount middleware.
|
||||
- Side effects: No side effects detected statically.
|
||||
- Limits/notes: Generated from static route source analysis; confirm exact behavior in the handler before changing integrations. Errors are caught and usually returned as a flash message, JSON error, or error page. API consumers should expect JSON unless the response format says otherwise.
|
||||
|
||||
### POST /plugins/lumi_transcription/api/models/:id/download
|
||||
|
||||
- Purpose: Processes the lumi_transcription plugin action for api models id download.
|
||||
- Inputs: path params: `id`; body: `confirmed`
|
||||
- Response format: Form/action response; exact format was not detected statically.
|
||||
- Access: admin access expected
|
||||
- Side effects: Consumes submitted data; state mutation happens in called helpers if present.
|
||||
- Limits/notes: Generated from static route source analysis; confirm exact behavior in the handler before changing integrations. Errors are caught and usually returned as a flash message, JSON error, or error page. Most non-API POST routes are browser form submissions and usually redirect after completion. API consumers should expect JSON unless the response format says otherwise.
|
||||
|
||||
### POST /plugins/lumi_transcription/api/models/:id/load
|
||||
|
||||
- Purpose: Processes the lumi_transcription plugin action for api models id load.
|
||||
- Inputs: path params: `id`
|
||||
- Response format: JSON response
|
||||
- Access: admin access expected; logged-in session required or used
|
||||
- Side effects: Action route; side effects were not detected statically.
|
||||
- Limits/notes: Generated from static route source analysis; confirm exact behavior in the handler before changing integrations. Errors are caught and usually returned as a flash message, JSON error, or error page. Most non-API POST routes are browser form submissions and usually redirect after completion. API consumers should expect JSON unless the response format says otherwise.
|
||||
|
||||
### POST /plugins/lumi_transcription/api/runtime/:id/install
|
||||
|
||||
- Purpose: Processes the lumi_transcription plugin action for api runtime id install.
|
||||
- Inputs: path params: `id`; body: `confirmed`
|
||||
- Response format: Form/action response; exact format was not detected statically.
|
||||
- Access: admin access expected
|
||||
- Side effects: writes or mutates server-side state
|
||||
- Limits/notes: Generated from static route source analysis; confirm exact behavior in the handler before changing integrations. Errors are caught and usually returned as a flash message, JSON error, or error page. Most non-API POST routes are browser form submissions and usually redirect after completion. API consumers should expect JSON unless the response format says otherwise.
|
||||
|
||||
### GET /plugins/lumi_transcription/api/logs
|
||||
|
||||
- Purpose: Renders or serves the lumi_transcription plugin page for api logs.
|
||||
- Inputs: No request parameters detected by static analysis.
|
||||
- Response format: JSON response
|
||||
- Access: admin access expected
|
||||
- Side effects: Usually read-only.
|
||||
- Limits/notes: Generated from static route source analysis; confirm exact behavior in the handler before changing integrations. API consumers should expect JSON unless the response format says otherwise.
|
||||
## Commands
|
||||
- No plugin command triggers detected.
|
||||
## Source
|
||||
Plugin folder: plugins/lumi_transcription
|
||||
@ -108,7 +108,20 @@ class WhisperCppServerProvider extends TranscriptionProvider {
|
||||
supervisor.on("crash", (event) => this.emit("provider_error", Object.assign(new Error("Whisper worker crashed."), { details: event })));
|
||||
supervisor.on("error", (error) => this.emit("provider_error", error));
|
||||
}
|
||||
async loadModel(model) { this.worker.start(); this.model = model; this.worker.send({ type: "load_model", model }); return this.health(); }
|
||||
async loadModel(model) {
|
||||
this.worker.start();
|
||||
const loaded = new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => { cleanup(); reject(new Error("The whisper.cpp worker did not confirm model loading in time.")); }, 30000);
|
||||
const onLoaded = (event) => { cleanup(); resolve(event); };
|
||||
const onError = (event) => { cleanup(); reject(Object.assign(new Error(event.message || "The model could not be loaded."), { code: event.code })); };
|
||||
const cleanup = () => { clearTimeout(timer); this.off("model_loaded", onLoaded); this.off("model_error", onError); };
|
||||
this.once("model_loaded", onLoaded); this.once("model_error", onError);
|
||||
});
|
||||
this.worker.send({ type: "load_model", model });
|
||||
const result = await loaded;
|
||||
this.model = model;
|
||||
return { ...(await this.health()), loaded: result };
|
||||
}
|
||||
async benchmark(options = {}) { this.worker.send({ type: "benchmark", options }); return { accepted: true, model: this.model?.id || null }; }
|
||||
async startSession(session) { this.sessions.add(session.id); this.worker.send({ type: "start_session", session }); }
|
||||
async addTrack(sessionId, track) { this.worker.send({ type: "add_track", session_id: sessionId, track }); }
|
||||
|
||||
@ -0,0 +1,37 @@
|
||||
cmake_minimum_required(VERSION 3.24)
|
||||
project(lumi-whisper-worker VERSION 0.1.0 LANGUAGES CXX)
|
||||
|
||||
include(FetchContent)
|
||||
option(LUMI_WHISPER_CUDA "Build whisper.cpp with CUDA support" OFF)
|
||||
|
||||
set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
|
||||
set(WHISPER_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
|
||||
set(WHISPER_BUILD_TESTS OFF CACHE BOOL "" FORCE)
|
||||
set(WHISPER_BUILD_SERVER OFF CACHE BOOL "" FORCE)
|
||||
set(GGML_CUDA ${LUMI_WHISPER_CUDA} CACHE BOOL "" FORCE)
|
||||
|
||||
FetchContent_Declare(
|
||||
whisper
|
||||
GIT_REPOSITORY https://github.com/ggml-org/whisper.cpp.git
|
||||
GIT_TAG f049fff95a089aa9969deb009cdd4892b3e74916
|
||||
GIT_SHALLOW FALSE
|
||||
)
|
||||
FetchContent_Declare(
|
||||
json
|
||||
URL https://github.com/nlohmann/json/releases/download/v3.12.0/json.tar.xz
|
||||
URL_HASH SHA256=42f6e95cad6ec532fd372391373363b62a14af6d771056dbfc86160e6dfff7aa
|
||||
)
|
||||
FetchContent_MakeAvailable(whisper json)
|
||||
|
||||
add_executable(lumi-whisper-worker src/main.cpp)
|
||||
target_compile_features(lumi-whisper-worker PRIVATE cxx_std_20)
|
||||
target_link_libraries(lumi-whisper-worker PRIVATE whisper nlohmann_json::nlohmann_json)
|
||||
if(LUMI_WHISPER_CUDA)
|
||||
target_compile_definitions(lumi-whisper-worker PRIVATE LUMI_BACKEND_CUDA=1)
|
||||
endif()
|
||||
if(MSVC)
|
||||
target_compile_options(lumi-whisper-worker PRIVATE /W4 /EHsc)
|
||||
else()
|
||||
target_compile_options(lumi-whisper-worker PRIVATE -Wall -Wextra -Wpedantic)
|
||||
endif()
|
||||
install(TARGETS lumi-whisper-worker RUNTIME DESTINATION bin)
|
||||
@ -0,0 +1,12 @@
|
||||
# Lumi whisper.cpp worker
|
||||
|
||||
This is the dedicated native inference process used by `WhisperWorkerSupervisor`. It reads the existing length-prefixed metadata/PCM protocol from standard input and emits newline-delimited structured events on standard output. Audio remains in bounded memory and is never written to disk.
|
||||
|
||||
The build pins whisper.cpp to the commit recorded in `runtime_manifest.json` and nlohmann/json to v3.12.0. CPU is the portable verification target; the Lumi server release uses the CUDA option:
|
||||
|
||||
```powershell
|
||||
cmake -S . -B build -G Ninja -DLUMI_WHISPER_CUDA=ON -DCMAKE_BUILD_TYPE=Release
|
||||
cmake --build build
|
||||
```
|
||||
|
||||
The worker keeps at most six seconds per source, attempts an incremental decode around every 600 ms during speech, and finalizes after roughly 750 ms of measured silence. The server-side stabilizer remains authoritative for stable-prefix reconciliation and obsolete-revision suppression.
|
||||
@ -0,0 +1,219 @@
|
||||
#include "whisper.h"
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
using json = nlohmann::json;
|
||||
using clock_type = std::chrono::steady_clock;
|
||||
|
||||
namespace {
|
||||
constexpr std::size_t sample_rate = 16000;
|
||||
constexpr std::size_t max_samples = sample_rate * 6;
|
||||
constexpr std::size_t min_decode_samples = sample_rate / 2;
|
||||
constexpr auto decode_interval = std::chrono::milliseconds(600);
|
||||
constexpr std::uint64_t finalize_silence_us = 750000;
|
||||
|
||||
struct whisper_deleter { void operator()(whisper_context * value) const { if (value) whisper_free(value); } };
|
||||
using whisper_pointer = std::unique_ptr<whisper_context, whisper_deleter>;
|
||||
|
||||
struct track_state {
|
||||
std::vector<float> samples;
|
||||
std::uint64_t audio_start_us = 0;
|
||||
std::uint64_t audio_end_us = 0;
|
||||
std::uint64_t last_voice_us = 0;
|
||||
std::uint64_t last_decode_us = 0;
|
||||
bool voiced = false;
|
||||
bool new_utterance = true;
|
||||
};
|
||||
|
||||
whisper_pointer context;
|
||||
std::string model_id = "unknown";
|
||||
std::string backend = "cpu";
|
||||
std::unordered_map<std::string, track_state> tracks;
|
||||
|
||||
void emit(const json & value) {
|
||||
std::cout << value.dump() << '\n' << std::flush;
|
||||
}
|
||||
|
||||
void diagnostic(const std::string & state, const std::string & message) {
|
||||
emit({{"type", "diagnostic"}, {"state", state}, {"message", message}});
|
||||
}
|
||||
|
||||
bool read_exact(char * destination, std::size_t size) {
|
||||
std::cin.read(destination, static_cast<std::streamsize>(size));
|
||||
return static_cast<std::size_t>(std::cin.gcount()) == size;
|
||||
}
|
||||
|
||||
bool read_packet(json & metadata, std::vector<std::uint8_t> & pcm) {
|
||||
std::uint8_t header[8];
|
||||
if (!read_exact(reinterpret_cast<char *>(header), sizeof(header))) return false;
|
||||
auto read_u32 = [](const std::uint8_t * value) {
|
||||
return static_cast<std::uint32_t>(value[0]) | (static_cast<std::uint32_t>(value[1]) << 8U) |
|
||||
(static_cast<std::uint32_t>(value[2]) << 16U) | (static_cast<std::uint32_t>(value[3]) << 24U);
|
||||
};
|
||||
const auto json_size = read_u32(header);
|
||||
const auto pcm_size = read_u32(header + 4);
|
||||
if (json_size == 0 || json_size > 64 * 1024 || pcm_size > 5 * 32000 || pcm_size % 2 != 0) {
|
||||
diagnostic("packet_rejected", "Worker packet exceeded its framing limits.");
|
||||
return false;
|
||||
}
|
||||
std::string body(json_size, '\0');
|
||||
pcm.resize(pcm_size);
|
||||
if (!read_exact(body.data(), body.size()) || (pcm_size > 0 && !read_exact(reinterpret_cast<char *>(pcm.data()), pcm.size()))) return false;
|
||||
metadata = json::parse(body, nullptr, false);
|
||||
if (metadata.is_discarded() || !metadata.is_object()) {
|
||||
diagnostic("packet_rejected", "Worker metadata was not valid JSON.");
|
||||
return true;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string model_path(const json & model) {
|
||||
for (const char * key : {"path", "file", "filename"}) {
|
||||
if (model.contains(key) && model[key].is_string()) return model[key].get<std::string>();
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
void load_model(const json & packet) {
|
||||
const auto model = packet.value("model", json::object());
|
||||
const auto path = model_path(model);
|
||||
if (path.empty()) {
|
||||
emit({{"type", "model_error"}, {"code", "MODEL_PATH_REQUIRED"}, {"message", "The model packet did not include a local path."}});
|
||||
return;
|
||||
}
|
||||
auto parameters = whisper_context_default_params();
|
||||
context.reset(whisper_init_from_file_with_params(path.c_str(), parameters));
|
||||
if (!context) {
|
||||
emit({{"type", "model_error"}, {"code", "MODEL_LOAD_FAILED"}, {"message", "whisper.cpp could not load the selected model."}});
|
||||
return;
|
||||
}
|
||||
model_id = model.value("id", "unknown");
|
||||
#if defined(LUMI_BACKEND_CUDA)
|
||||
backend = "cuda";
|
||||
#else
|
||||
backend = "cpu";
|
||||
#endif
|
||||
emit({{"type", "model_loaded"}, {"model_id", model_id}, {"backend", backend}});
|
||||
}
|
||||
|
||||
std::string transcribe(const std::vector<float> & samples, double & inference_ms) {
|
||||
if (!context) return {};
|
||||
auto parameters = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);
|
||||
parameters.print_realtime = false;
|
||||
parameters.print_progress = false;
|
||||
parameters.print_timestamps = false;
|
||||
parameters.print_special = false;
|
||||
parameters.translate = false;
|
||||
parameters.no_context = true;
|
||||
parameters.single_segment = true;
|
||||
parameters.language = "en";
|
||||
parameters.n_threads = std::max(1U, std::min(4U, std::thread::hardware_concurrency()));
|
||||
const auto started = clock_type::now();
|
||||
const auto result = whisper_full(context.get(), parameters, samples.data(), static_cast<int>(samples.size()));
|
||||
inference_ms = std::chrono::duration<double, std::milli>(clock_type::now() - started).count();
|
||||
if (result != 0) return {};
|
||||
std::string text;
|
||||
const int segments = whisper_full_n_segments(context.get());
|
||||
for (int index = 0; index < segments; ++index) text += whisper_full_get_segment_text(context.get(), index);
|
||||
const auto first = text.find_first_not_of(" \t\r\n");
|
||||
const auto last = text.find_last_not_of(" \t\r\n");
|
||||
return first == std::string::npos ? std::string{} : text.substr(first, last - first + 1);
|
||||
}
|
||||
|
||||
void decode_track(const std::string & session_id, const std::string & track_id, track_state & track, bool final) {
|
||||
if (!context || track.samples.size() < min_decode_samples) return;
|
||||
double inference_ms = 0;
|
||||
const auto text = transcribe(track.samples, inference_ms);
|
||||
if (!text.empty()) {
|
||||
const bool incomplete = !final && std::isalnum(static_cast<unsigned char>(text.back())) != 0;
|
||||
emit({
|
||||
{"type", "hypothesis"}, {"session_id", session_id}, {"track_id", track_id}, {"text", text},
|
||||
{"final", final}, {"new_utterance", track.new_utterance}, {"incomplete_word", incomplete},
|
||||
{"audio_start_us", track.audio_start_us}, {"audio_end_us", track.audio_end_us},
|
||||
{"inference_ms", inference_ms}, {"model_id", model_id}, {"backend", backend}
|
||||
});
|
||||
track.new_utterance = false;
|
||||
}
|
||||
track.last_decode_us = track.audio_end_us;
|
||||
if (final) {
|
||||
track.samples.clear();
|
||||
track.voiced = false;
|
||||
track.new_utterance = true;
|
||||
track.audio_start_us = 0;
|
||||
}
|
||||
}
|
||||
|
||||
void audio(const json & packet, const std::vector<std::uint8_t> & pcm) {
|
||||
if (!context || pcm.empty()) return;
|
||||
const auto session_id = packet.value("session_id", "");
|
||||
const auto track_id = packet.value("track_id", "");
|
||||
if (session_id.empty() || track_id.empty()) return;
|
||||
auto & track = tracks[session_id + "\n" + track_id];
|
||||
const auto captured_us = packet.value("capture_timestamp_us", std::uint64_t{0});
|
||||
if (track.samples.empty()) track.audio_start_us = captured_us;
|
||||
const auto count = pcm.size() / 2;
|
||||
double squares = 0;
|
||||
for (std::size_t index = 0; index < count; ++index) {
|
||||
const auto sample = static_cast<std::int16_t>(static_cast<std::uint16_t>(pcm[index * 2]) |
|
||||
(static_cast<std::uint16_t>(pcm[index * 2 + 1]) << 8U));
|
||||
const float normalized = static_cast<float>(sample) / 32768.0F;
|
||||
track.samples.push_back(normalized);
|
||||
squares += static_cast<double>(normalized) * normalized;
|
||||
}
|
||||
if (track.samples.size() > max_samples) {
|
||||
const auto removed = track.samples.size() - max_samples;
|
||||
track.samples.erase(track.samples.begin(), track.samples.begin() + static_cast<std::ptrdiff_t>(removed));
|
||||
track.audio_start_us += static_cast<std::uint64_t>(removed * 1000000ULL / sample_rate);
|
||||
}
|
||||
const auto duration_us = static_cast<std::uint64_t>(count * 1000000ULL / sample_rate);
|
||||
track.audio_end_us = captured_us + duration_us;
|
||||
const auto rms = std::sqrt(squares / static_cast<double>(std::max<std::size_t>(1, count)));
|
||||
if (rms >= 0.012) { track.voiced = true; track.last_voice_us = track.audio_end_us; }
|
||||
const bool final = track.voiced && track.audio_end_us > track.last_voice_us && track.audio_end_us - track.last_voice_us >= finalize_silence_us;
|
||||
const bool decode_due = track.last_decode_us == 0 || track.audio_end_us - track.last_decode_us >= static_cast<std::uint64_t>(decode_interval.count()) * 1000ULL;
|
||||
if (final || (track.voiced && decode_due)) decode_track(session_id, track_id, track, final);
|
||||
}
|
||||
|
||||
void handle(const json & packet, const std::vector<std::uint8_t> & pcm, bool & running) {
|
||||
const auto type = packet.value("type", "");
|
||||
if (type == "load_model") load_model(packet);
|
||||
else if (type == "benchmark") emit({{"type", "benchmark"}, {"accepted", context != nullptr}, {"model_id", model_id}, {"backend", backend}});
|
||||
else if (type == "start_session") emit({{"type", "session_started"}, {"session_id", packet.value("session", json::object()).value("id", "")}});
|
||||
else if (type == "add_track") tracks.try_emplace(packet.value("session_id", "") + "\n" + packet.value("track", json::object()).value("source_uuid", ""));
|
||||
else if (type == "audio") audio(packet, pcm);
|
||||
else if (type == "remove_track") tracks.erase(packet.value("session_id", "") + "\n" + packet.value("track_id", ""));
|
||||
else if (type == "stop_session") {
|
||||
const auto prefix = packet.value("session_id", "") + "\n";
|
||||
std::erase_if(tracks, [&](const auto & item) { return item.first.starts_with(prefix); });
|
||||
}
|
||||
else if (type == "shutdown") running = false;
|
||||
else diagnostic("packet_rejected", "Unknown worker message type.");
|
||||
}
|
||||
}
|
||||
|
||||
int main() {
|
||||
std::ios::sync_with_stdio(false);
|
||||
whisper_log_set([](enum ggml_log_level, const char *, void *) {}, nullptr);
|
||||
emit({{"type", "ready"}, {"protocol_version", 1}, {"backend", backend}});
|
||||
bool running = true;
|
||||
while (running && std::cin.good()) {
|
||||
json packet;
|
||||
std::vector<std::uint8_t> pcm;
|
||||
if (!read_packet(packet, pcm)) break;
|
||||
if (!packet.is_discarded()) handle(packet, pcm, running);
|
||||
}
|
||||
tracks.clear();
|
||||
context.reset();
|
||||
return 0;
|
||||
}
|
||||
@ -44,6 +44,13 @@ module.exports = {
|
||||
const models = new ArtifactManager(dataPath("models"));
|
||||
const runtimeArchives = new ArtifactManager(dataPath("tmp"));
|
||||
const runtimes = new ArtifactManager(dataPath("runtime"));
|
||||
const selectedModelId = revisions.list().selected_model_id?.value;
|
||||
const selectedModel = modelManifest.models.find((entry) => entry.id === selectedModelId);
|
||||
if (supervisor.executable && selectedModel) {
|
||||
const selectedStatus = models.status(selectedModel);
|
||||
if (selectedStatus.valid) provider.loadModel({ ...selectedModel, path: selectedStatus.path })
|
||||
.catch((error) => diagnosticLog.append({ kind: "worker", state: "model_restore_failed", model_id: selectedModel.id, message: error.message }));
|
||||
}
|
||||
|
||||
const router = web.createRouter();
|
||||
router.use("/assets", express.static(path.join(__dirname, "public")));
|
||||
@ -101,6 +108,19 @@ module.exports = {
|
||||
try { res.status(201).json({ ok: true, status: await models.download(entry, { confirmed: req.body.confirmed === true }) }); }
|
||||
catch (error) { res.status(400).json({ ok: false, error: error.message }); }
|
||||
});
|
||||
router.post("/api/models/:id/load", requireAdmin, async (req, res) => {
|
||||
const entry = modelManifest.models.find((candidate) => candidate.id === req.params.id);
|
||||
if (!entry) return res.status(404).json({ ok: false, error: "Model was not found." });
|
||||
const status = models.status(entry);
|
||||
if (!status.valid) return res.status(409).json({ ok: false, error: "Download and verify this model before loading it." });
|
||||
try {
|
||||
const providerStatus = await provider.loadModel({ ...entry, path: status.path });
|
||||
const current = revisions.list().selected_model_id;
|
||||
revisions.apply([{ key: "selected_model_id", value: entry.id, base_revision: current?.revision || 0 }], req.session.user.id);
|
||||
res.json({ ok: true, status: providerStatus });
|
||||
}
|
||||
catch (error) { res.status(500).json({ ok: false, error: error.message }); }
|
||||
});
|
||||
router.post("/api/runtime/:id/install", requireAdmin, async (req, res) => {
|
||||
const entry = runtimeManifest.artifacts.find((candidate) => candidate.id === req.params.id);
|
||||
if (!entry) return res.status(404).json({ ok: false, error: "Runtime was not found." });
|
||||
|
||||
@ -1 +1,2 @@
|
||||
.transcription-shell{--accent:#85e3c4;--warn:#f5c56b;max-width:1120px;margin:0 auto;padding:clamp(1rem,3vw,2.5rem);display:grid;gap:1.35rem}.transcription-shell section{padding:1.35rem 0;border-top:1px solid color-mix(in srgb,currentColor 16%,transparent)}.transcription-hero{display:flex;justify-content:space-between;align-items:flex-start;gap:2rem;padding:1rem 0 2rem}.transcription-hero h1{font-size:clamp(2rem,5vw,4.25rem);line-height:.95;margin:.25rem 0 1rem}.transcription-hero p:not(.eyebrow){max-width:67ch;opacity:.78}.eyebrow{text-transform:uppercase;letter-spacing:.13em;font-size:.72rem;font-weight:750;opacity:.65;margin:0}.health-pill{display:inline-flex;align-items:center;gap:.55rem;padding:.55rem .8rem;border-radius:999px;white-space:nowrap;background:color-mix(in srgb,var(--warn) 16%,transparent)}.health-pill span{width:.65rem;height:.65rem;border-radius:50%;background:var(--warn)}.health-pill.is-ready{background:color-mix(in srgb,var(--accent) 16%,transparent)}.health-pill.is-ready span{background:var(--accent)}.section-heading{display:flex;justify-content:space-between;align-items:end;gap:1rem;margin-bottom:1rem}.section-heading h2{margin:.2rem 0 0;font-size:1.35rem}.setup-path ol{display:grid;grid-template-columns:repeat(4,1fr);list-style:none;padding:0;margin:0 0 1.25rem;counter-reset:steps}.setup-path li{counter-increment:steps;padding:0 1rem 1rem 2.25rem;position:relative;opacity:.62}.setup-path li:before{content:counter(steps);position:absolute;left:0;top:-.2rem;width:1.6rem;height:1.6rem;border:1px solid currentColor;border-radius:50%;display:grid;place-items:center;font-size:.75rem}.setup-path li.is-current{opacity:1}.setup-path li.is-current:before{background:var(--accent);color:#10251f;border-color:var(--accent)}.setup-path li span,.plain-list span{display:block;font-size:.88rem;opacity:.7;margin-top:.25rem}.primary-action{border:0;border-radius:.6rem;background:var(--accent);color:#10251f;font-weight:750;padding:.7rem 1rem;cursor:pointer}.primary-action:disabled{opacity:.55;cursor:wait}.inline-status{display:inline;margin-left:.8rem}.transcription-grid{display:grid;grid-template-columns:1fr 1fr;gap:2.5rem}.health-list{margin:0}.health-list div{display:flex;justify-content:space-between;gap:1rem;padding:.55rem 0}.health-list dt{opacity:.65}.notice,.privacy-note,.empty-state{padding:1rem;border-radius:.65rem;background:color-mix(in srgb,currentColor 6%,transparent);font-size:.9rem}.plain-list{list-style:none;padding:0;margin:0}.plain-list li{display:flex;justify-content:space-between;gap:1rem;padding:.7rem 0}.plain-list code{font-size:.72rem;opacity:.6}.model-row{display:grid;gap:.5rem}.model-row article{display:flex;justify-content:space-between;align-items:center;gap:1rem;padding:.9rem 0}.model-row h3,.model-row p{margin:0}.model-row p{font-size:.88rem;opacity:.7;margin-top:.2rem}.state-label{font-size:.78rem;font-weight:700;white-space:nowrap}@media(max-width:760px){.transcription-hero,.transcription-grid{display:grid;grid-template-columns:1fr}.setup-path ol{grid-template-columns:1fr}.health-pill{justify-self:start}.plain-list li{display:block}.plain-list code{display:block;margin-top:.5rem;overflow-wrap:anywhere}}
|
||||
.model-actions{display:flex;align-items:center;justify-content:flex-end;gap:.65rem}.model-actions button{border:1px solid color-mix(in srgb,currentColor 24%,transparent);border-radius:.55rem;background:transparent;color:inherit;padding:.5rem .7rem;cursor:pointer}.model-actions button:disabled{opacity:.55;cursor:wait}@media(max-width:760px){.model-row article{align-items:flex-start}.model-actions{align-items:flex-end;flex-direction:column}}
|
||||
|
||||
@ -19,4 +19,26 @@
|
||||
} catch (error) { status.textContent = error.message; }
|
||||
finally { button.disabled = false; }
|
||||
});
|
||||
|
||||
root.addEventListener("click", async (event) => {
|
||||
const download = event.target.closest("[data-download-model]");
|
||||
const load = event.target.closest("[data-load-model]");
|
||||
if (!download && !load) return;
|
||||
const modelId = download?.dataset.downloadModel || load?.dataset.loadModel;
|
||||
if (download && !window.confirm(`Download ${download.dataset.modelLabel} (${download.dataset.modelSize} MiB) to the Lumi host?`)) return;
|
||||
const action = download ? "download" : "load";
|
||||
const target = download || load;
|
||||
target.disabled = true;
|
||||
status.textContent = download ? "Downloading and verifying the model…" : "Loading the model into whisper.cpp…";
|
||||
try {
|
||||
const response = await fetch(`/plugins/lumi_transcription/api/models/${encodeURIComponent(modelId)}/${action}`, {
|
||||
method: "POST", headers: { "Content-Type": "application/json", Accept: "application/json" },
|
||||
body: download ? JSON.stringify({ confirmed: true }) : "{}"
|
||||
});
|
||||
const result = await response.json();
|
||||
if (!response.ok) throw new Error(result.error || `Model ${action} failed.`);
|
||||
status.textContent = download ? "Model verified. You can load it now." : "Model loaded. Run the host benchmark before live use.";
|
||||
setTimeout(() => window.location.reload(), 700);
|
||||
} catch (error) { status.textContent = error.message; target.disabled = false; }
|
||||
});
|
||||
})();
|
||||
|
||||
@ -3,6 +3,13 @@
|
||||
"provider": "whisper.cpp",
|
||||
"tested_version": "v1.9.1",
|
||||
"tested_commit": "f049fff95a089aa9969deb009cdd4892b3e74916",
|
||||
"worker": {
|
||||
"source": "backend/transcription/worker-native",
|
||||
"executable": "lumi-whisper-worker.exe",
|
||||
"protocol_version": 1,
|
||||
"cuda_cmake_option": "LUMI_WHISPER_CUDA",
|
||||
"whisper_commit": "f049fff95a089aa9969deb009cdd4892b3e74916"
|
||||
},
|
||||
"artifacts": [
|
||||
{
|
||||
"id": "windows-x64-cuda-12.4",
|
||||
|
||||
@ -18,7 +18,7 @@ const { ArtifactManager, sha256File } = require("../backend/models/artifact_mana
|
||||
const { BoundedQueue, SequenceTracker, RollingPcmBuffer } = require("../backend/sessions/bounded_queue");
|
||||
const { SessionCoordinator } = require("../backend/sessions/session_coordinator");
|
||||
const { CaptionStabilizer, LatestCaptionGate } = require("../backend/transcription/stabilizer");
|
||||
const { WhisperWorkerSupervisor } = require("../backend/transcription/provider");
|
||||
const { WhisperWorkerSupervisor, WhisperCppServerProvider } = require("../backend/transcription/provider");
|
||||
const plugin = require("../index");
|
||||
const { createWebUpgradeRegistry } = require("../../../src/services/web-upgrades");
|
||||
|
||||
@ -32,10 +32,11 @@ async function run() {
|
||||
verifyStabilization();
|
||||
await verifySessionLifecycle();
|
||||
await verifyWorkerRestart();
|
||||
await verifyNativeWorkerBoundary();
|
||||
await verifyAuthenticatedGateway();
|
||||
verifyArtifactsAndLogs(temp);
|
||||
await verifyPluginIsolation();
|
||||
console.log("Lumi transcription verification passed: protocol, pairing, revocation, revisions, queues, stabilization, lifecycle, worker recovery, artifacts, logs, and plugin isolation.");
|
||||
console.log("Lumi transcription verification passed: protocol, pairing, revocation, revisions, queues, stabilization, lifecycle, native worker boundary, worker recovery, artifacts, logs, and plugin isolation.");
|
||||
} finally { fs.rmSync(temp, { recursive: true, force: true }); }
|
||||
}
|
||||
|
||||
@ -169,6 +170,26 @@ async function verifyWorkerRestart() {
|
||||
supervisor.stopping = true; children[1].emit("exit", 0, null);
|
||||
}
|
||||
|
||||
async function verifyNativeWorkerBoundary() {
|
||||
class Worker extends EventEmitter {
|
||||
start() { this.started = true; }
|
||||
send(message) { if (message.type === "load_model") setImmediate(() => this.emit("message", { type: "model_loaded", model_id: message.model.id, backend: "cpu" })); return true; }
|
||||
health() { return { healthy: true, state: "running" }; }
|
||||
async stop() {}
|
||||
}
|
||||
const worker = new Worker();
|
||||
const provider = new WhisperCppServerProvider(worker);
|
||||
const health = await provider.loadModel({ id: "base.en", path: "verified-model.bin" });
|
||||
assert.equal(worker.started, true);
|
||||
assert.equal(health.model, "base.en");
|
||||
const sourceRoot = path.join(__dirname, "../backend/transcription/worker-native");
|
||||
const cmake = fs.readFileSync(path.join(sourceRoot, "CMakeLists.txt"), "utf8");
|
||||
const source = fs.readFileSync(path.join(sourceRoot, "src/main.cpp"), "utf8");
|
||||
assert.match(cmake, /f049fff95a089aa9969deb009cdd4892b3e74916/);
|
||||
assert.match(source, /max_samples = sample_rate \* 6/);
|
||||
assert.doesNotMatch(source, /ofstream|fwrite|WriteAllBytes/);
|
||||
}
|
||||
|
||||
async function verifyAuthenticatedGateway() {
|
||||
const db = new Database(":memory:");
|
||||
const devices = new DeviceStore(db);
|
||||
|
||||
@ -50,7 +50,14 @@
|
||||
<div class="section-heading"><div><p class="eyebrow">Curated choices</p><h2 id="models-title">Speech models</h2></div></div>
|
||||
<div class="model-row">
|
||||
<% models.forEach((model) => { %>
|
||||
<article><div><h3><%= model.label %></h3><p><%= model.recommended ? 'Recommended starting point' : 'Fallback option' %> · <%= Math.round(model.bytes / 1048576) %> MiB</p></div><span class="state-label"><%= model.status.valid ? 'Verified' : model.status.installed ? 'Checksum failed' : 'Not installed' %></span></article>
|
||||
<article>
|
||||
<div><h3><%= model.label %></h3><p><%= model.recommended ? 'Recommended starting point' : 'Fallback option' %> · <%= Math.round(model.bytes / 1048576) %> MiB</p></div>
|
||||
<div class="model-actions">
|
||||
<span class="state-label"><%= providerHealth.model === model.id ? 'Loaded' : model.status.valid ? 'Verified' : model.status.installed ? 'Checksum failed' : 'Not installed' %></span>
|
||||
<% if (!model.status.valid) { %><button type="button" data-download-model="<%= model.id %>" data-model-label="<%= model.label %>" data-model-size="<%= Math.round(model.bytes / 1048576) %>">Download</button><% } %>
|
||||
<% if (model.status.valid && providerHealth.model !== model.id) { %><button type="button" data-load-model="<%= model.id %>">Load model</button><% } %>
|
||||
</div>
|
||||
</article>
|
||||
<% }) %>
|
||||
</div>
|
||||
<p class="privacy-note"><strong>Privacy:</strong> raw audio is held only in bounded memory and is never written to disk by default. Diagnostic caption text is retained for seven days unless disabled.</p>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user