diff --git a/TODO.md b/TODO.md index 2b4e9c2..c49a0cf 100644 --- a/TODO.md +++ b/TODO.md @@ -6,14 +6,19 @@ This file tracks larger Lumi work that cannot safely be completed in one pass. K Foundation implemented locally: independent `lumi_transcription` plugin; generic core WebSocket-upgrade capability; single-use pairing and revocable devices; -TLS-only device transport; bounded PCM/session queues; revision-safe settings; +HTTPS/WSS device transport with exact localhost-origin HTTP allowed only for +locally generated pairing packages; bounded PCM/session queues; revision-safe settings; provider/delivery interfaces; worker supervision; caption stabilization; pinned whisper.cpp/model manifests; short-lived JSONL diagnostics; protocol schemas; and .NET/native companion boundaries with focused verification. The companion now has a Lumi-styled Avalonia 12 single-instance tray shell, guided real-boundary test states, DPAPI pairing, local preferences/autostart, and bounded diagnostics. A pinned native whisper.cpp worker builds on the CPU verification target and the -server offers consent-gated model download/load controls. +server offers consent-gated model download/load controls. The Transcription +settings now render inside the shared Lumi shell, and Admin shows a compact +Companion health/download section beneath its shortcut cards. A checksum-pinned +self-contained Windows bundle includes a one-time pairing file and pairs after +extraction without a separate import workflow. Release-blocking work remains: diff --git a/companion/README.md b/companion/README.md index 4859bb5..6e54963 100644 --- a/companion/README.md +++ b/companion/README.md @@ -4,9 +4,9 @@ This directory is the single Lumi Companion product boundary. The current milest Build prerequisites: the current .NET SDK with the .NET 8 targeting pack on Windows x64. The app targets .NET 8; .NET SDK 10 is recommended for Avalonia 12 source-generator compatibility. The native bridge additionally requires CMake, Visual Studio C++ tools, and the OBS 31+ SDK. -The app accepts a `.lumi-pairing.json` bootstrap package. It exchanges the embedded token once and stores the returned device credential with Windows DPAPI. Do not commit bootstrap packages, credentials, generated installers, build output, or logs. +The app accepts a `.lumi-pairing.json` bootstrap package. A paired ZIP downloaded from Lumi includes one beside the executable; the app detects it automatically, exchanges the embedded token once, stores the returned device credential with Windows DPAPI, and removes the pairing file after success. Do not commit bootstrap packages, credentials, generated installers, build output, or logs. -The companion never performs ASR in this MVP. Audio is normalized to 16 kHz mono signed 16-bit PCM, held in bounded memory, and sent to the paired Lumi host over TLS WebSockets. The OBS bridge talks only to the companion over a same-user named pipe. +The companion never performs ASR in this MVP. Audio is normalized to 16 kHz mono signed 16-bit PCM, held in bounded memory, and sent to the paired Lumi host over TLS WebSockets. HTTP/WS works only for a package generated from the exact matching loopback Lumi origin. The OBS bridge talks only to the companion over a same-user named pipe. Build all managed projects from the repository root: @@ -15,3 +15,5 @@ dotnet build companion/Lumi.Companion.sln -p:EnableWindowsTargeting=true ``` The signed installer, managed OBS bridge installation/repair service, and a target-machine OBS/Twitch acceptance run are still required. Until those pass, the UI deliberately stops its test at the unavailable real boundary. + +The Admin **Download Companion** action currently distributes a checksum-pinned, self-contained Windows x64 ZIP. It is intentionally marked experimental and is not code-signed yet. diff --git a/companion/src/Lumi.Companion.App/CompanionRuntime.cs b/companion/src/Lumi.Companion.App/CompanionRuntime.cs index b6df2de..ac30dae 100644 --- a/companion/src/Lumi.Companion.App/CompanionRuntime.cs +++ b/companion/src/Lumi.Companion.App/CompanionRuntime.cs @@ -67,6 +67,13 @@ public sealed class CompanionRuntime : IAsyncDisposable } if (credential is null) { + var bundledPairing = FindBundledPairingPackage(); + if (bundledPairing is not null) + { + await PairAsync(bundledPairing); + try { File.Delete(bundledPairing); } catch { } + return; + } SetState(State with { ObsBridgeInstalled = DetectBridgeInstallation(), Detail = "Download a pairing package from Lumi, then open it here." }); return; } @@ -354,6 +361,15 @@ public sealed class CompanionRuntime : IAsyncDisposable return File.Exists(obsData); } + private static string? FindBundledPairingPackage() + { + try + { + return Directory.EnumerateFiles(AppContext.BaseDirectory, "*.lumi-pairing.json", SearchOption.TopDirectoryOnly).Take(2).SingleOrDefault(); + } + catch { return null; } + } + private void ApplyAutoStart(bool enabled) { if (!OperatingSystem.IsWindows()) return; diff --git a/companion/src/Lumi.Companion.App/Lumi.Companion.App.csproj b/companion/src/Lumi.Companion.App/Lumi.Companion.App.csproj index cc6274e..a3bc841 100644 --- a/companion/src/Lumi.Companion.App/Lumi.Companion.App.csproj +++ b/companion/src/Lumi.Companion.App/Lumi.Companion.App.csproj @@ -5,7 +5,7 @@ enable enable app.manifest - 0.1.0-experimental.1 + 0.1.0-experimental.2 0.1.0.0 diff --git a/companion/src/Lumi.Companion.Core/CompanionSocket.cs b/companion/src/Lumi.Companion.Core/CompanionSocket.cs index 94a862b..eab1b80 100644 --- a/companion/src/Lumi.Companion.Core/CompanionSocket.cs +++ b/companion/src/Lumi.Companion.Core/CompanionSocket.cs @@ -22,6 +22,8 @@ public sealed class CompanionSocket : IAsyncDisposable _socket = new ClientWebSocket(); _socket.Options.SetRequestHeader("Authorization", $"LumiDevice {credential.DeviceId}.{credential.DeviceSecret}"); var host = new Uri(credential.Host); + if (host.Scheme != "https" && !(host.Scheme == "http" && host.IsLoopback)) + throw new InvalidDataException("Companion transport requires HTTPS unless this credential was issued for localhost."); var uri = new UriBuilder(host) { Scheme = host.Scheme == "https" ? "wss" : "ws", Path = "/plugins/lumi_transcription/live" }.Uri; await _socket.ConnectAsync(uri, cancellationToken); var hello = ProtocolV1.EncodeEnvelope("hello", new { companion_version = companionVersion, plugin_version = pluginVersion, obs_version = obsVersion, capabilities = credential.Capabilities, audio = new { codec = "pcm_s16le", sample_rate = 16000, channels = 1, bits = 16 } }); diff --git a/companion/src/Lumi.Companion.Core/PairingClient.cs b/companion/src/Lumi.Companion.Core/PairingClient.cs index 5709184..ed68c06 100644 --- a/companion/src/Lumi.Companion.Core/PairingClient.cs +++ b/companion/src/Lumi.Companion.Core/PairingClient.cs @@ -15,9 +15,11 @@ public sealed class PairingClient(HttpClient http) if (DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() >= bootstrap.ExpiresAt) throw new InvalidDataException("Pairing package has expired. Download a new package from Lumi."); var exchangeUri = new Uri(bootstrap.ExchangeUrl); - var insecureDev = Environment.GetEnvironmentVariable("LUMI_COMPANION_DEV_ALLOW_INSECURE") == "1" && exchangeUri.IsLoopback; - if (!exchangeUri.Scheme.Equals("https", StringComparison.OrdinalIgnoreCase) && !insecureDev) - throw new InvalidDataException("Pairing requires an HTTPS Lumi host."); + var hostUri = new Uri(bootstrap.Host); + var secure = exchangeUri.Scheme.Equals("https", StringComparison.OrdinalIgnoreCase) && hostUri.Scheme.Equals("https", StringComparison.OrdinalIgnoreCase); + var localHttp = exchangeUri.Scheme.Equals("http", StringComparison.OrdinalIgnoreCase) && hostUri.Scheme.Equals("http", StringComparison.OrdinalIgnoreCase) && exchangeUri.IsLoopback && hostUri.IsLoopback; + if ((!secure && !localHttp) || exchangeUri.GetLeftPart(UriPartial.Authority) != hostUri.GetLeftPart(UriPartial.Authority)) + throw new InvalidDataException("Pairing requires HTTPS unless the package explicitly uses one matching localhost origin."); using var response = await http.PostAsJsonAsync(bootstrap.ExchangeUrl, new { token = bootstrap.Token, device }, ProtocolV1.JsonOptions, cancellationToken); var body = await response.Content.ReadAsStringAsync(cancellationToken); if (!response.IsSuccessStatusCode) throw new InvalidOperationException($"Pairing failed: {body}"); diff --git a/docs/lumi-companion-transcription.md b/docs/lumi-companion-transcription.md index 415b632..869b843 100644 --- a/docs/lumi-companion-transcription.md +++ b/docs/lumi-companion-transcription.md @@ -19,20 +19,20 @@ The repository contains a pinned native whisper.cpp rolling-window worker and a - Pairing packages contain a cryptographically random credential that expires after 15 minutes and can be activated once. Lumi stores only its SHA-256 digest. - Activation returns a revocable device secret once. The Windows companion stores it with current-user DPAPI. -- Device HTTP and WebSocket credentials require HTTPS/WSS. Insecure transport is available only from localhost when `LUMI_COMPANION_DEV_ALLOW_INSECURE=1` is explicitly set. +- Device HTTP and WebSocket credentials require HTTPS/WSS. HTTP/WS is accepted only when an administrator downloads the package through an explicit loopback URL and Companion returns to that exact loopback origin, including its port. There is no environment-variable bypass. - The OBS bridge never receives the Lumi credential and never connects to Lumi directly. - Audio frames are capped at 200 ms and recovery buffers at five seconds. Old or excess frames are dropped; capture paths never block for inference. - Raw audio is never logged or written to disk. JSON Lines diagnostics default to seven days and 256 MiB. Caption text can be disabled in diagnostics. ## Pairing and installation milestone -1. Serve Lumi through HTTPS and enable `lumi_transcription` under Admin > Plugins. -2. Open Plugins > Transcription and create a pairing package. It is a bootstrap package for the generic companion, not a credential that should be shared or committed. -3. On the Windows streaming computer, build `companion/Lumi.Companion.sln`, start `Lumi.Companion.App`, and choose the package from the Overview or Connection page. +1. Serve Lumi through HTTPS and enable `lumi_transcription` under Admin > Plugins. Local development may use HTTP only from the exact `localhost` or loopback URL used to download Companion. +2. Use **Download Companion** in Admin or Plugins > Transcription. Lumi downloads and checksum-verifies the pinned Windows artifact, adds a short-lived single-use pairing file, and returns a private ZIP. +3. On the Windows streaming computer, extract the complete ZIP and start `Lumi.Companion.App.exe` within 15 minutes. Companion discovers the adjacent pairing file, exchanges it once, stores the credential with current-user DPAPI, and removes the pairing file after success. 4. Install a pinned runtime/model only after explicit confirmation. `small.en` is recommended; `small.en-q5_1` and `base.en` are fallbacks. Every artifact is checksum-verified before install. 5. Configure `LUMI_TRANSCRIPTION_WORKER` with the supervised streaming-worker executable once that worker is built for the target host. -The Avalonia tray UI and host-side model download/load controls now exist. The normal signed installer, managed bridge install/repair, live OBS source enumeration, and measured model benchmark wizard are not complete yet. +The Avalonia tray UI, self-contained paired ZIP, shared Lumi settings shell, Admin summary, and host-side model download/load controls now exist. The ZIP is not code-signed, so Windows may warn. The normal signed installer, managed bridge install/repair, live OBS source enumeration, and measured model benchmark wizard are not complete yet. ## Operation and recovery @@ -42,15 +42,15 @@ Device and capability revocation take effect on the next authenticated request/c ## Diagnostics -Use the Transcription admin page for provider, model, device, and log health. Recovery errors distinguish missing setup, unavailable inference, source inactivity, protocol incompatibility, and revoked access. The target-machine test must additionally record latency, resource, OBS missed-frame, audio-dropout, queue-drop, and network metrics using the acceptance template in `companion/docs/performance-acceptance-template.md`. +Use the compact Lumi Companion section on Admin for connection, device, inference, session, and package health. Open the shared-shell Transcription settings page for provider, model, device, and log details. Recovery errors distinguish missing setup, unavailable inference, source inactivity, protocol incompatibility, and revoked access. The target-machine test must additionally record latency, resource, OBS missed-frame, audio-dropout, queue-drop, and network metrics using the acceptance template in `companion/docs/performance-acceptance-template.md`. ## Known limitations -- The .NET SDK and OBS SDK are not available in the development environment used for this milestone, so those projects have not been compiled here. +- The managed .NET solution and self-contained Windows x64 application have been compiled in the development environment; the OBS SDK integration still requires its target toolchain and acceptance host. - No real whisper.cpp streaming worker has been integrated or benchmarked. - The bridge skeleton does not yet run its named-pipe worker or selected-source audio callback. - Twitch toggleable caption behavior has not been tested; native API presence is not acceptance evidence. -- The companion shell is a protocol/bootstrap executable, not yet the Avalonia tray application. -- Installer signing, bridge repair, auto-start, source discovery/nested Program-scene evaluation, benchmark UX, and conflict-resolution UI remain pending. +- The downloadable bundle is an unsigned experimental self-contained ZIP, not the final signed installer. +- Bridge repair, source discovery/nested Program-scene evaluation, benchmark UX, and conflict-resolution UI remain pending. See `docs/adr/0001-companion-transcription-boundaries.md`, `protocol/companion-protocol-v1.md`, and `companion/docs/obs-native-caption-compatibility-spike.md`. diff --git a/knowledge/plugins/lumi-transcription.md b/knowledge/plugins/lumi-transcription.md index 047edc7..ce12138 100644 --- a/knowledge/plugins/lumi-transcription.md +++ b/knowledge/plugins/lumi-transcription.md @@ -14,13 +14,14 @@ editable: false Server-hosted whisper.cpp transcription for Lumi Companion and OBS closed captions. ## Metadata Plugin ID: lumi_transcription -Version: 0.1.0-experimental.1 +Version: 0.1.0-experimental.2 Default state: enabled ## Web Routes - /plugins/lumi_transcription - GET /plugins/lumi_transcription - GET /plugins/lumi_transcription/api/status - POST /plugins/lumi_transcription/api/pairing-package +- POST /plugins/lumi_transcription/api/companion/download - POST /plugins/lumi_transcription/api/pair - GET /plugins/lumi_transcription/api/devices - POST /plugins/lumi_transcription/api/devices/:id/revoke @@ -45,7 +46,7 @@ Default state: enabled - Purpose: Renders or serves the lumi_transcription plugin page. - Inputs: No request parameters detected by static analysis. -- Response format: HTML page rendered from an EJS view +- Response format: plain or HTML response - Access: admin access expected - Side effects: Usually read-only. - Limits/notes: Generated from static route source analysis; confirm exact behavior in the handler before changing integrations. @@ -68,6 +69,15 @@ Default state: enabled - Side effects: Action route; side effects were not detected statically. - Limits/notes: Generated from static route source analysis; confirm exact behavior in the handler before changing integrations. Errors are caught and usually returned as a flash message, JSON error, or error page. Most non-API POST routes are browser form submissions and usually redirect after completion. API consumers should expect JSON unless the response format says otherwise. +### POST /plugins/lumi_transcription/api/companion/download + +- Purpose: Processes the lumi_transcription plugin action for api companion download. +- Inputs: No request parameters detected by static analysis. +- Response format: plain or HTML response +- Access: admin access expected; logged-in session required or used +- Side effects: Action route; side effects were not detected statically. +- Limits/notes: Generated from static route source analysis; confirm exact behavior in the handler before changing integrations. Errors are caught and usually returned as a flash message, JSON error, or error page. Most non-API POST routes are browser form submissions and usually redirect after completion. API consumers should expect JSON unless the response format says otherwise. + ### POST /plugins/lumi_transcription/api/pair - Purpose: Processes the lumi_transcription plugin action for api pair. diff --git a/plugins/lumi_transcription/backend/companion/device_store.js b/plugins/lumi_transcription/backend/companion/device_store.js index 411c5cb..97c5a2a 100644 --- a/plugins/lumi_transcription/backend/companion/device_store.js +++ b/plugins/lumi_transcription/backend/companion/device_store.js @@ -7,7 +7,6 @@ class DeviceStore { this.db = db; this.now = options.now || Date.now; this.randomBytes = options.randomBytes || crypto.randomBytes; - this.allowInsecure = options.allowInsecure === true; this.migrate(); } @@ -20,10 +19,12 @@ class DeviceStore { CREATE TABLE IF NOT EXISTS transcription_devices ( id TEXT PRIMARY KEY, install_id TEXT, name TEXT NOT NULL, lumi_user_id TEXT NOT NULL, credential_hash TEXT NOT NULL, capabilities_json TEXT NOT NULL, metadata_json TEXT NOT NULL, - first_connected_at INTEGER NOT NULL, last_connected_at INTEGER NOT NULL, revoked_at INTEGER + first_connected_at INTEGER NOT NULL, last_connected_at INTEGER NOT NULL, revoked_at INTEGER, + pairing_host TEXT ); CREATE INDEX IF NOT EXISTS transcription_devices_user_idx ON transcription_devices(lumi_user_id); `); + ensureColumn(this.db, "transcription_devices", "pairing_host", "TEXT"); } issuePairing({ userId, host, ttlMs = 15 * 60 * 1000 }) { @@ -32,8 +33,8 @@ class DeviceStore { const token = tokenValue(this.randomBytes(32)); const now = this.now(); this.db.prepare("INSERT INTO transcription_pairing_tokens (id, token_hash, lumi_user_id, host, expires_at, activated_at, created_at) VALUES (?, ?, ?, ?, ?, NULL, ?)") - .run(id, digest(token), String(userId), normalizeHost(host, this.allowInsecure), now + ttlMs, now); - return { pairing_id: id, token, host: normalizeHost(host, this.allowInsecure), expires_at: now + ttlMs, protocol_version: 1 }; + .run(id, digest(token), String(userId), normalizeHost(host), now + ttlMs, now); + return { pairing_id: id, token, host: normalizeHost(host), expires_at: now + ttlMs, protocol_version: 1 }; } exchange({ token, device = {} }) { @@ -51,8 +52,8 @@ class DeviceStore { const transaction = this.db.transaction(() => { const consumed = this.db.prepare("UPDATE transcription_pairing_tokens SET activated_at = ? WHERE id = ? AND activated_at IS NULL").run(now, row.id); if (consumed.changes !== 1) { const error = new Error("This pairing package was already activated. Download a new companion package."); error.code = "PAIRING_ALREADY_USED"; throw error; } - this.db.prepare("INSERT INTO transcription_devices (id, install_id, name, lumi_user_id, credential_hash, capabilities_json, metadata_json, first_connected_at, last_connected_at, revoked_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL)") - .run(deviceId, clean(device.install_id, 128) || null, clean(device.name, 160) || "Lumi Companion", row.lumi_user_id, digest(secret), JSON.stringify(capabilities), JSON.stringify(safeMetadata(device)), now, now); + this.db.prepare("INSERT INTO transcription_devices (id, install_id, name, lumi_user_id, credential_hash, capabilities_json, metadata_json, first_connected_at, last_connected_at, revoked_at, pairing_host) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?)") + .run(deviceId, clean(device.install_id, 128) || null, clean(device.name, 160) || "Lumi Companion", row.lumi_user_id, digest(secret), JSON.stringify(capabilities), JSON.stringify(safeMetadata(device)), now, now, row.host); }); transaction(); return { device_id: deviceId, device_secret: secret, host: row.host, capabilities, protocol_version: 1 }; @@ -76,15 +77,23 @@ class DeviceStore { const changed = this.db.prepare("UPDATE transcription_devices SET capabilities_json = ? WHERE id = ? AND revoked_at IS NULL").run(JSON.stringify(allowed), deviceId).changes; return changed ? allowed : null; } + pairingAllowsHttp(token, requestOrigin) { + const row = this.db.prepare("SELECT host, activated_at, expires_at FROM transcription_pairing_tokens WHERE token_hash = ?").get(digest(token)); + return Boolean(row && !row.activated_at && row.expires_at >= this.now() && sameLoopbackOrigin(row.host, requestOrigin)); + } } function digest(value) { return crypto.createHash("sha256").update(String(value || ""), "utf8").digest("hex"); } function safeEqual(left, right) { const a = Buffer.from(String(left)); const b = Buffer.from(String(right)); return a.length === b.length && crypto.timingSafeEqual(a, b); } function tokenValue(buffer) { return Buffer.from(buffer).toString("base64url"); } -function normalizeHost(value, allowInsecure = false) { const url = new URL(String(value)); if (url.protocol !== "https:" && !(allowInsecure && url.protocol === "http:" && ["localhost", "127.0.0.1", "::1"].includes(url.hostname))) throw new Error("Lumi Companion requires HTTPS. Insecure HTTP is allowed only for explicit localhost development."); return url.origin; } +function normalizeHost(value) { const url = new URL(String(value)); if (url.protocol !== "https:" && !isLoopbackHttpOrigin(url)) throw new Error("Lumi Companion requires HTTPS. HTTP is allowed only when the pairing URL explicitly uses localhost or a loopback address."); return url.origin; } +function isLoopbackHttpOrigin(value) { try { const url = value instanceof URL ? value : new URL(String(value)); return url.protocol === "http:" && ["localhost", "127.0.0.1", "::1", "[::1]"].includes(url.hostname); } catch { return false; } } +function sameLoopbackOrigin(left, right) { try { const a = new URL(String(left)); const b = new URL(String(right)); return isLoopbackHttpOrigin(a) && isLoopbackHttpOrigin(b) && a.origin === b.origin; } catch { return false; } } +function insecureDeviceAllowed(device, requestOrigin, remoteAddress) { return ["127.0.0.1", "::1", "::ffff:127.0.0.1"].includes(String(remoteAddress || "")) && sameLoopbackOrigin(device?.pairing_host, requestOrigin); } +function ensureColumn(db, table, column, type) { if (!db.prepare(`PRAGMA table_info(${table})`).all().some((entry) => entry.name === column)) db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${type}`); } function clean(value, max) { return String(value || "").trim().slice(0, max); } function parseArray(value) { try { const parsed = JSON.parse(value); return Array.isArray(parsed) ? parsed : []; } catch { return []; } } function safeMetadata(device) { return { companion_version: clean(device.companion_version, 32), obs_version: clean(device.obs_version, 32), os: clean(device.os, 120), architecture: clean(device.architecture, 32), hardware: clean(device.hardware, 500) }; } -function serialize(row, capabilities) { return { id: row.id, install_id: row.install_id, name: row.name, lumi_user_id: row.lumi_user_id, capabilities, metadata: JSON.parse(row.metadata_json || "{}"), first_connected_at: row.first_connected_at, last_connected_at: row.last_connected_at, revoked_at: row.revoked_at }; } +function serialize(row, capabilities) { return { id: row.id, install_id: row.install_id, name: row.name, lumi_user_id: row.lumi_user_id, capabilities, metadata: JSON.parse(row.metadata_json || "{}"), first_connected_at: row.first_connected_at, last_connected_at: row.last_connected_at, revoked_at: row.revoked_at, pairing_host: row.pairing_host || null }; } -module.exports = { DeviceStore, DEFAULT_CAPABILITIES, digest }; +module.exports = { DeviceStore, DEFAULT_CAPABILITIES, digest, normalizeHost, isLoopbackHttpOrigin, sameLoopbackOrigin, insecureDeviceAllowed }; diff --git a/plugins/lumi_transcription/backend/companion/gateway.js b/plugins/lumi_transcription/backend/companion/gateway.js index 832bc3d..00d12db 100644 --- a/plugins/lumi_transcription/backend/companion/gateway.js +++ b/plugins/lumi_transcription/backend/companion/gateway.js @@ -1,22 +1,23 @@ const { WebSocketServer, WebSocket } = require("ws"); const { MAX_JSON_BYTES, MAX_AUDIO_PAYLOAD_BYTES, AUDIO_HEADER_BYTES, parseEnvelope, validateHello, parseAudioFrame, envelope } = require("./protocol"); +const { insecureDeviceAllowed } = require("./device_store"); class CompanionGateway { constructor(options) { this.devices = options.devices; this.sessions = options.sessions; this.log = options.log || { append() {} }; - this.allowInsecure = options.allowInsecure === true; this.wss = new WebSocketServer({ noServer: true, maxPayload: Math.max(MAX_JSON_BYTES, AUDIO_HEADER_BYTES + MAX_AUDIO_PAYLOAD_BYTES), perMessageDeflate: false, clientTracking: true }); this.wss.on("connection", (socket, request, device) => this.connection(socket, request, device)); } upgrade(request, socket, head) { const forwardedProto = String(request.headers["x-forwarded-proto"] || "").split(",")[0].trim(); - const secure = Boolean(request.socket.encrypted) || forwardedProto === "https"; - const local = ["127.0.0.1", "::1", "::ffff:127.0.0.1"].includes(request.socket.remoteAddress); - if (!secure && !(this.allowInsecure && local)) return reject(socket, 426, "tls_required"); + const proxyIsLocal = ["127.0.0.1", "::1", "::ffff:127.0.0.1"].includes(request.socket.remoteAddress); + const secure = Boolean(request.socket.encrypted) || (proxyIsLocal && forwardedProto === "https"); const auth = this.devices.authenticate(request.headers.authorization, "transcription.capture.v1"); if (!auth.allowed) return reject(socket, auth.reason === "capability_revoked" ? 403 : 401, auth.reason); + const requestOrigin = `http://${request.headers.host || "invalid"}`; + if (!secure && !insecureDeviceAllowed(auth.device, requestOrigin, request.socket.remoteAddress)) return reject(socket, 426, "tls_required"); const origin = request.headers.origin; if (origin && !sameHostOrigin(origin, request.headers.host)) return reject(socket, 403, "origin_rejected"); this.wss.handleUpgrade(request, socket, head, (ws) => this.wss.emit("connection", ws, request, auth.device)); diff --git a/plugins/lumi_transcription/backend/companion/package_service.js b/plugins/lumi_transcription/backend/companion/package_service.js new file mode 100644 index 0000000..b2387fb --- /dev/null +++ b/plugins/lumi_transcription/backend/companion/package_service.js @@ -0,0 +1,61 @@ +const fs = require("fs"); +const path = require("path"); +const AdmZip = require("adm-zip"); +const { ArtifactManager } = require("../models/artifact_manager"); + +class CompanionPackageService { + constructor(root, manifest, options = {}) { + this.root = root; + this.manifest = manifest; + this.artifacts = new ArtifactManager(root, { fetch: options.fetch }); + fs.mkdirSync(root, { recursive: true }); + } + + entry() { return this.manifest?.artifacts?.find((entry) => entry.platform === "win32" && entry.architecture === "x64") || null; } + + status() { + const entry = this.entry(); + if (!entry) return { available: false, installed: false, valid: false, reason: "No Windows Companion artifact is configured." }; + const artifact = this.artifacts.status(entry); + return { available: true, version: this.manifest.version, artifact: entry.id, ...artifact }; + } + + async build(pairing) { + const entry = this.entry(); + if (!entry) throw new Error("No Windows Companion artifact is configured."); + let status = this.artifacts.status(entry); + if (!status.valid) status = await this.artifacts.download(entry, { confirmed: true }); + const source = new AdmZip(status.path); + const output = new AdmZip(); + let expandedBytes = 0; + for (const item of source.getEntries()) { + const name = safeArchivePath(item.entryName); + if (!name || item.isDirectory) continue; + const body = item.getData(); + expandedBytes += body.length; + if (expandedBytes > 300 * 1024 * 1024) throw new Error("The Companion artifact exceeds its expanded size limit."); + output.addFile(name, body); + } + if (!output.getEntry(entry.entrypoint)) throw new Error("The Companion artifact is missing its expected application entrypoint."); + const pairingName = `lumi-companion-${pairing.pairing_id}.lumi-pairing.json`; + output.addFile(pairingName, Buffer.from(`${JSON.stringify(pairing.bootstrap, null, 2)}\n`, "utf8")); + output.addFile("START-HERE.txt", Buffer.from([ + "Lumi Companion — experimental transcription MVP", "", + "1. Extract every file in this ZIP to a folder on the Windows streaming computer.", + `2. Start ${entry.entrypoint} within 15 minutes.`, + "3. Companion finds the adjacent one-time pairing package automatically and removes it after successful pairing.", "", + "Do not share this ZIP. Its pairing package works once and expires after 15 minutes.", + "Windows may warn because this experimental build is not code-signed yet.", "" + ].join("\r\n"), "utf8")); + return { buffer: output.toBuffer(), filename: `Lumi-Companion-${this.manifest.version}-paired.zip`, pairingName }; + } +} + +function safeArchivePath(value) { + const portable = String(value || "").replace(/\\/g, "/"); + const normalized = path.posix.normalize(portable).replace(/^\/+/, ""); + if (!normalized || normalized === ".." || normalized.startsWith("../") || /^[A-Za-z]:/.test(normalized)) throw new Error("The Companion artifact contains an unsafe path."); + return normalized; +} + +module.exports = { CompanionPackageService, safeArchivePath }; diff --git a/plugins/lumi_transcription/backend/sessions/session_coordinator.js b/plugins/lumi_transcription/backend/sessions/session_coordinator.js index a53cce9..3044428 100644 --- a/plugins/lumi_transcription/backend/sessions/session_coordinator.js +++ b/plugins/lumi_transcription/backend/sessions/session_coordinator.js @@ -111,6 +111,16 @@ class SessionCoordinator { const session = this.require(sessionId); return { session_id: session.id, state: session.state, mode: session.mode, obs: { ...session.obs }, grace_until: session.graceUntil || null, tracks: Array.from(session.tracks.values()).map(serializeTrack) }; } + summary() { + const sessions = Array.from(this.sessions.values()); + return { + total: sessions.length, + connected: sessions.filter((session) => session.connected).length, + running: sessions.filter((session) => session.state === "running").length, + test_running: sessions.filter((session) => session.state === "running" && session.mode === "test").length, + live_running: sessions.filter((session) => session.state === "running" && session.mode === "live").length + }; + } async close() { for (const session of Array.from(this.sessions.values())) await this.stop(session.id, "plugin_shutdown"); this.sessions.clear(); } require(id) { const session = this.sessions.get(id); if (!session) throw new Error("Session was not found."); return session; } beginGrace(session) { diff --git a/plugins/lumi_transcription/companion_manifest.json b/plugins/lumi_transcription/companion_manifest.json new file mode 100644 index 0000000..7101362 --- /dev/null +++ b/plugins/lumi_transcription/companion_manifest.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "version": "0.1.0-experimental.2", + "artifacts": [ + { + "id": "windows-x64-self-contained", + "platform": "win32", + "architecture": "x64", + "label": "Windows x64 self-contained", + "filename": "Lumi.Companion-win-x64.zip", + "url": "https://git.rolfsvaag.no/Rolfsvaag_Datateknikk/Lumi/releases/download/companion-v0.1.0-experimental.2/Lumi.Companion-win-x64.zip", + "sha256": "2884522cb368fa990ff6475c99324a41a9a7ce4871029078808f2f9a2902c6e5", + "bytes": 41568887, + "entrypoint": "Lumi.Companion.App.exe" + } + ] +} diff --git a/plugins/lumi_transcription/index.js b/plugins/lumi_transcription/index.js index e85e115..e8909c5 100644 --- a/plugins/lumi_transcription/index.js +++ b/plugins/lumi_transcription/index.js @@ -1,8 +1,11 @@ const express = require("express"); +const ejs = require("ejs"); const fs = require("fs"); const path = require("path"); const { DeviceStore } = require("./backend/companion/device_store"); +const { insecureDeviceAllowed } = require("./backend/companion/device_store"); const { CompanionGateway } = require("./backend/companion/gateway"); +const { CompanionPackageService } = require("./backend/companion/package_service"); const { RevisionStore } = require("./backend/config/revision_store"); const { CompanionCaptionDeliveryAdapter } = require("./backend/delivery/caption_delivery"); const { JsonlDiagnosticLog } = require("./backend/logs/jsonl_log"); @@ -12,6 +15,7 @@ const { WhisperWorkerSupervisor, WhisperCppServerProvider } = require("./backend const { ensureDataDirs, dataPath } = require("./backend/paths"); const modelManifest = require("./models_manifest.json"); const runtimeManifest = require("./runtime_manifest.json"); +const companionManifest = require("./companion_manifest.json"); const manifest = require("./plugin.json"); const PLUGIN_ID = "lumi_transcription"; @@ -20,8 +24,7 @@ module.exports = { id: PLUGIN_ID, init({ web, db, logger }) { ensureDataDirs(); - const allowInsecure = process.env.LUMI_COMPANION_DEV_ALLOW_INSECURE === "1"; - const devices = new DeviceStore(db, { allowInsecure }); + const devices = new DeviceStore(db); const revisions = new RevisionStore(db); const diagnosticLog = new JsonlDiagnosticLog(dataPath("logs")); diagnosticLog.cleanup(); @@ -39,11 +42,12 @@ module.exports = { deliveryFactory: (send) => new CompanionCaptionDeliveryAdapter(send), log: diagnosticLog }); - const gateway = new CompanionGateway({ devices, sessions, log: diagnosticLog, allowInsecure }); + const gateway = new CompanionGateway({ devices, sessions, log: diagnosticLog }); const unregisterUpgrade = web.addUpgradeHandler("/plugins/lumi_transcription/live", (request, socket, head) => gateway.upgrade(request, socket, head)); const models = new ArtifactManager(dataPath("models")); const runtimeArchives = new ArtifactManager(dataPath("tmp")); const runtimes = new ArtifactManager(dataPath("runtime")); + const companionPackages = new CompanionPackageService(dataPath("companion"), companionManifest); const selectedModelId = revisions.list().selected_model_id?.value; const selectedModel = modelManifest.models.find((entry) => entry.id === selectedModelId); if (supervisor.executable && selectedModel) { @@ -55,8 +59,13 @@ module.exports = { const router = web.createRouter(); router.use("/assets", express.static(path.join(__dirname, "public"))); router.get("/", requireAdmin, async (_req, res) => { - res.render(path.join(__dirname, "views", "settings.ejs"), { + const locals = { + ...res.locals, title: "Lumi Transcription", + pageWidth: "wide", + pageId: "lumi-transcription", + extraStyles: [`/plugins/${PLUGIN_ID}/assets/transcription.css?v=${manifest.version}`], + extraScripts: [`/plugins/${PLUGIN_ID}/assets/transcription.js?v=${manifest.version}`], pluginVersion: manifest.version, providerHealth: await provider.health(), devices: devices.list(), @@ -64,7 +73,8 @@ module.exports = { models: modelManifest.models.map((entry) => ({ ...entry, status: models.status(entry) })), runtimeManifest, logs: diagnosticLog.files() - }); + }; + res.send(await renderLumiPage(locals)); }); router.get("/api/status", requireAdmin, async (_req, res) => res.json({ ok: true, plugin: { id: PLUGIN_ID, version: manifest.version }, protocol_version: 1, @@ -82,7 +92,18 @@ module.exports = { res.send(`${JSON.stringify(bootstrap, null, 2)}\n`); } catch (error) { res.status(400).json({ ok: false, error: error.message }); } }); - router.post("/api/pair", requireSecureRequest(allowInsecure), (req, res) => { + router.post("/api/companion/download", requireAdmin, async (req, res) => { + try { + const host = requestHost(req); + const pairing = devices.issuePairing({ userId: req.session.user.id, host }); + const bootstrap = { format: "lumi-companion-bootstrap-v1", ...pairing, exchange_url: `${host}/plugins/${PLUGIN_ID}/api/pair` }; + const bundle = await companionPackages.build({ ...pairing, bootstrap }); + res.set("Cache-Control", "no-store"); + res.attachment(bundle.filename); + res.send(bundle.buffer); + } catch (error) { res.status(503).json({ ok: false, error: error.message }); } + }); + router.post("/api/pair", requirePairingTransport(devices), (req, res) => { try { res.set("Cache-Control", "no-store"); res.status(201).json({ ok: true, ...devices.exchange(req.body || {}) }); } catch (error) { res.status(error.code === "PAIRING_ALREADY_USED" ? 409 : 400).json({ ok: false, code: error.code, error: error.message }); } }); @@ -93,8 +114,8 @@ module.exports = { if (!capabilities) return res.status(404).json({ ok: false, error: "Device was not found or is revoked." }); res.json({ ok: true, capabilities }); }); - router.get("/api/settings", requireSettingsAccess(devices, allowInsecure), (_req, res) => res.json({ fields: revisions.list() })); - router.patch("/api/settings", requireSettingsAccess(devices, allowInsecure), (req, res) => { + router.get("/api/settings", requireSettingsAccess(devices), (_req, res) => res.json({ fields: revisions.list() })); + router.patch("/api/settings", requireSettingsAccess(devices), (req, res) => { try { const actor = req.session?.user?.id || req.lumiDevice?.id; const result = revisions.apply(req.body.changes, actor); @@ -134,7 +155,12 @@ module.exports = { web.mount(`/plugins/${PLUGIN_ID}`, router, { label: "Transcription", role: "admin", section: "plugins" }); global.lumiFrameworks = global.lumiFrameworks || {}; - global.lumiFrameworks.transcription = { version: manifest.version, protocol_version: 1, health: () => provider.health() }; + global.lumiFrameworks.transcription = { + version: manifest.version, + protocol_version: 1, + health: () => provider.health(), + dashboardSummary: () => buildDashboardSummary({ provider, devices, sessions, companionPackages }) + }; return async () => { clearInterval(cleanupTimer); @@ -149,6 +175,45 @@ module.exports = { }; function requireAdmin(req, res, next) { if (req.session?.user?.isAdmin) return next(); return res.status(403).json({ error: "Administrator access is required." }); } -function requireSettingsAccess(devices, allowInsecure) { return (req, res, next) => { if (req.session?.user?.isAdmin) return next(); const forwarded = String(req.headers["x-forwarded-proto"] || "").split(",")[0].trim(); const local = ["127.0.0.1", "::1", "::ffff:127.0.0.1"].includes(req.socket.remoteAddress); if (!req.secure && forwarded !== "https" && !(allowInsecure && local)) return res.status(426).json({ error: "Device settings synchronization requires HTTPS." }); const auth = devices.authenticate(req.headers.authorization, "transcription.settings.v1"); if (!auth.allowed) return res.status(401).json({ error: "A paired device credential is required." }); req.lumiDevice = auth.device; next(); }; } -function requestHost(req) { const forwarded = String(req.headers["x-forwarded-proto"] || "").split(",")[0].trim(); const protocol = forwarded === "https" ? "https" : req.protocol; return `${protocol}://${req.get("host")}`; } -function requireSecureRequest(allowInsecure) { return (req, res, next) => { const forwarded = String(req.headers["x-forwarded-proto"] || "").split(",")[0].trim(); const local = ["127.0.0.1", "::1", "::ffff:127.0.0.1"].includes(req.socket.remoteAddress); if (req.secure || forwarded === "https" || (allowInsecure && local)) return next(); return res.status(426).json({ error: "Lumi Companion pairing requires HTTPS." }); }; } +async function buildDashboardSummary({ provider, devices, sessions, companionPackages }) { + const inference = await provider.health(); + const activeDevices = devices.list().filter((device) => !device.revoked_at); + const connections = sessions.summary(); + const packageStatus = companionPackages.status(); + const issues = []; + if (!inference.healthy) issues.push("Server-hosted speech recognition is not ready."); + if (!inference.model) issues.push("No speech model is loaded."); + if (!activeDevices.length) issues.push("No Companion device is paired."); + else if (!connections.connected) issues.push("No paired Companion is currently connected."); + if (!packageStatus.available) issues.push("The Windows Companion download is not configured."); + const tone = inference.state === "failed" ? "danger" : issues.length ? "warning" : "success"; + return { + id: "lumi-companion", eyebrow: "Streaming computer", title: "Lumi Companion", + description: "Download the paired Windows Companion and review the live transcription path at a glance.", + status: { tone, label: issues.length ? `${issues.length} issue${issues.length === 1 ? "" : "s"} discovered` : "Everything healthy" }, + metrics: [ + { label: "Connection", value: connections.connected ? "Active" : "Offline" }, + { label: "Paired devices", value: activeDevices.length }, + { label: "Inference", value: inference.healthy ? "Healthy" : "Needs setup" }, + { label: "Active sessions", value: connections.running } + ], + issues, + actions: [ + { label: "Download Companion", href: `/plugins/${PLUGIN_ID}/api/companion/download`, method: "post", primary: true, disabled: !packageStatus.available, disabledReason: packageStatus.reason }, + { label: "Open transcription settings", href: `/plugins/${PLUGIN_ID}`, method: "get" } + ] + }; +} +async function renderLumiPage(locals) { + const coreViews = path.join(__dirname, "..", "..", "src", "web", "views"); + const page = path.join(__dirname, "views", "settings.ejs"); + const [top, body, bottom] = await Promise.all([ + ejs.renderFile(path.join(coreViews, "partials", "layout-top.ejs"), locals), + ejs.renderFile(page, locals), + ejs.renderFile(path.join(coreViews, "partials", "layout-bottom.ejs"), locals) + ]); + return `${top}${body}${bottom}`; +} +function requireSettingsAccess(devices) { return (req, res, next) => { if (req.session?.user?.isAdmin) return next(); const auth = devices.authenticate(req.headers.authorization, "transcription.settings.v1"); if (!auth.allowed) return res.status(401).json({ error: "A paired device credential is required." }); if (!req.secure && !insecureDeviceAllowed(auth.device, requestHost(req), req.socket.remoteAddress)) return res.status(426).json({ error: "Device settings synchronization requires HTTPS unless this device was paired from the same localhost origin." }); req.lumiDevice = auth.device; next(); }; } +function requestHost(req) { return `${req.protocol === "https" ? "https" : "http"}://${req.get("host")}`; } +function requirePairingTransport(devices) { return (req, res, next) => { if (req.secure) return next(); const local = ["127.0.0.1", "::1", "::ffff:127.0.0.1"].includes(req.socket.remoteAddress); if (local && devices.pairingAllowsHttp(req.body?.token, requestHost(req))) return next(); return res.status(426).json({ error: "Lumi Companion pairing requires HTTPS unless the package was generated from this exact localhost URL." }); }; } diff --git a/plugins/lumi_transcription/plugin.json b/plugins/lumi_transcription/plugin.json index 4f3e895..1546c24 100644 --- a/plugins/lumi_transcription/plugin.json +++ b/plugins/lumi_transcription/plugin.json @@ -1,7 +1,7 @@ { "id": "lumi_transcription", "name": "Lumi Transcription", - "version": "0.1.0-experimental.1", + "version": "0.1.0-experimental.2", "description": "Server-hosted whisper.cpp transcription for Lumi Companion and OBS closed captions.", "main": "index.js", "channel": "experimental", diff --git a/plugins/lumi_transcription/public/transcription.css b/plugins/lumi_transcription/public/transcription.css index a45575d..df77f5d 100644 --- a/plugins/lumi_transcription/public/transcription.css +++ b/plugins/lumi_transcription/public/transcription.css @@ -1,2 +1 @@ -.transcription-shell{--accent:#85e3c4;--warn:#f5c56b;max-width:1120px;margin:0 auto;padding:clamp(1rem,3vw,2.5rem);display:grid;gap:1.35rem}.transcription-shell section{padding:1.35rem 0;border-top:1px solid color-mix(in srgb,currentColor 16%,transparent)}.transcription-hero{display:flex;justify-content:space-between;align-items:flex-start;gap:2rem;padding:1rem 0 2rem}.transcription-hero h1{font-size:clamp(2rem,5vw,4.25rem);line-height:.95;margin:.25rem 0 1rem}.transcription-hero p:not(.eyebrow){max-width:67ch;opacity:.78}.eyebrow{text-transform:uppercase;letter-spacing:.13em;font-size:.72rem;font-weight:750;opacity:.65;margin:0}.health-pill{display:inline-flex;align-items:center;gap:.55rem;padding:.55rem .8rem;border-radius:999px;white-space:nowrap;background:color-mix(in srgb,var(--warn) 16%,transparent)}.health-pill span{width:.65rem;height:.65rem;border-radius:50%;background:var(--warn)}.health-pill.is-ready{background:color-mix(in srgb,var(--accent) 16%,transparent)}.health-pill.is-ready span{background:var(--accent)}.section-heading{display:flex;justify-content:space-between;align-items:end;gap:1rem;margin-bottom:1rem}.section-heading h2{margin:.2rem 0 0;font-size:1.35rem}.setup-path ol{display:grid;grid-template-columns:repeat(4,1fr);list-style:none;padding:0;margin:0 0 1.25rem;counter-reset:steps}.setup-path li{counter-increment:steps;padding:0 1rem 1rem 2.25rem;position:relative;opacity:.62}.setup-path li:before{content:counter(steps);position:absolute;left:0;top:-.2rem;width:1.6rem;height:1.6rem;border:1px solid currentColor;border-radius:50%;display:grid;place-items:center;font-size:.75rem}.setup-path li.is-current{opacity:1}.setup-path li.is-current:before{background:var(--accent);color:#10251f;border-color:var(--accent)}.setup-path li span,.plain-list span{display:block;font-size:.88rem;opacity:.7;margin-top:.25rem}.primary-action{border:0;border-radius:.6rem;background:var(--accent);color:#10251f;font-weight:750;padding:.7rem 1rem;cursor:pointer}.primary-action:disabled{opacity:.55;cursor:wait}.inline-status{display:inline;margin-left:.8rem}.transcription-grid{display:grid;grid-template-columns:1fr 1fr;gap:2.5rem}.health-list{margin:0}.health-list div{display:flex;justify-content:space-between;gap:1rem;padding:.55rem 0}.health-list dt{opacity:.65}.notice,.privacy-note,.empty-state{padding:1rem;border-radius:.65rem;background:color-mix(in srgb,currentColor 6%,transparent);font-size:.9rem}.plain-list{list-style:none;padding:0;margin:0}.plain-list li{display:flex;justify-content:space-between;gap:1rem;padding:.7rem 0}.plain-list code{font-size:.72rem;opacity:.6}.model-row{display:grid;gap:.5rem}.model-row article{display:flex;justify-content:space-between;align-items:center;gap:1rem;padding:.9rem 0}.model-row h3,.model-row p{margin:0}.model-row p{font-size:.88rem;opacity:.7;margin-top:.2rem}.state-label{font-size:.78rem;font-weight:700;white-space:nowrap}@media(max-width:760px){.transcription-hero,.transcription-grid{display:grid;grid-template-columns:1fr}.setup-path ol{grid-template-columns:1fr}.health-pill{justify-self:start}.plain-list li{display:block}.plain-list code{display:block;margin-top:.5rem;overflow-wrap:anywhere}} -.model-actions{display:flex;align-items:center;justify-content:flex-end;gap:.65rem}.model-actions button{border:1px solid color-mix(in srgb,currentColor 24%,transparent);border-radius:.55rem;background:transparent;color:inherit;padding:.5rem .7rem;cursor:pointer}.model-actions button:disabled{opacity:.55;cursor:wait}@media(max-width:760px){.model-row article{align-items:flex-start}.model-actions{align-items:flex-end;flex-direction:column}} +.transcription-overview > .section-header{align-items:flex-start}.transcription-overview .page-header{margin:0}.transcription-overview .page-header h1{font-size:clamp(2rem,5vw,3.7rem)}.setup-path ol{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));list-style:none;padding:0;margin:0 0 var(--lumi-space-5);counter-reset:steps;gap:var(--lumi-space-3)}.setup-path li{counter-increment:steps;padding:var(--lumi-space-4);border-radius:var(--lumi-radius-md);background:var(--lumi-surface-subtle);min-width:0}.setup-path li::before{content:counter(steps);display:grid;place-items:center;width:1.8rem;height:1.8rem;margin-bottom:var(--lumi-space-3);border:1px solid var(--lumi-border);border-radius:50%;font-weight:700}.setup-path li.is-current::before{background:var(--lumi-primary);border-color:var(--lumi-primary);color:var(--lumi-button-text)}.setup-path li.is-complete::before{content:"✓";background:var(--lumi-success-bg);border-color:var(--lumi-success);color:var(--lumi-success)}.setup-path li span{display:block;margin-top:var(--lumi-space-2);color:var(--lumi-text-muted);font-size:.9rem}.transcription-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:var(--lumi-space-5)}.transcription-metrics{grid-template-columns:repeat(2,minmax(0,1fr));margin-bottom:var(--lumi-space-4)}.transcription-device-list{list-style:none;padding:0;margin:0;display:grid;gap:var(--lumi-space-2)}.transcription-device-list li,.model-row article{display:flex;align-items:center;justify-content:space-between;gap:var(--lumi-space-4);padding:var(--lumi-space-3) 0;border-bottom:1px solid var(--lumi-border)}.transcription-device-list li:last-child,.model-row article:last-child{border-bottom:0}.transcription-device-list .hint{display:block;margin-top:var(--lumi-space-1)}.model-row h3,.model-row p{margin:0}.model-actions{display:flex;align-items:center;justify-content:flex-end;gap:var(--lumi-space-2);flex-wrap:wrap}.empty-state{padding:var(--lumi-space-5);border-radius:var(--lumi-radius-md);background:var(--lumi-surface-subtle)}.empty-state p{margin-bottom:0;color:var(--lumi-text-muted)}@media(max-width:860px){.setup-path ol,.transcription-grid{grid-template-columns:1fr}.transcription-overview > .section-header{align-items:stretch}.transcription-metrics{grid-template-columns:1fr}}@media(max-width:560px){.model-row article,.transcription-device-list li{align-items:flex-start;flex-direction:column}.model-actions{justify-content:flex-start}} diff --git a/plugins/lumi_transcription/public/transcription.js b/plugins/lumi_transcription/public/transcription.js index 352204d..7edfb8b 100644 --- a/plugins/lumi_transcription/public/transcription.js +++ b/plugins/lumi_transcription/public/transcription.js @@ -1,5 +1,5 @@ (() => { - const root = document.querySelector("[data-transcription-admin]"); + const root = document.querySelector('[data-lumi-page="lumi-transcription"]'); if (!root) return; const button = root.querySelector("[data-create-pairing]"); const status = root.querySelector("[data-status]"); @@ -7,15 +7,15 @@ button.disabled = true; status.textContent = "Creating a one-time pairing package…"; try { - const response = await fetch("/plugins/lumi_transcription/api/pairing-package", { method: "POST", headers: { Accept: "application/json" } }); + const response = await fetch("/plugins/lumi_transcription/api/companion/download", { method: "POST", headers: { Accept: "application/json" } }); if (!response.ok) throw new Error((await response.json()).error || "Pairing package could not be created."); const blob = await response.blob(); const disposition = response.headers.get("content-disposition") || ""; - const filename = /filename="?([^";]+)"?/i.exec(disposition)?.[1] || "lumi-companion.lumi-pairing.json"; + const filename = /filename="?([^";]+)"?/i.exec(disposition)?.[1] || "Lumi-Companion-paired.zip"; const link = document.createElement("a"); link.href = URL.createObjectURL(blob); link.download = filename; link.click(); setTimeout(() => URL.revokeObjectURL(link.href), 1000); - status.textContent = "Pairing package created. It expires in 15 minutes and works once."; + status.textContent = "Companion package created. Extract and start it within 15 minutes; the included pairing key works once."; } catch (error) { status.textContent = error.message; } finally { button.disabled = false; } }); diff --git a/plugins/lumi_transcription/stats.js b/plugins/lumi_transcription/stats.js new file mode 100644 index 0000000..9aaf6ad --- /dev/null +++ b/plugins/lumi_transcription/stats.js @@ -0,0 +1,7 @@ +async function getAdminDashboardStats() { + const api = global.lumiFrameworks?.transcription; + if (!api?.dashboardSummary) throw new Error("The transcription plugin is not ready."); + return api.dashboardSummary(); +} + +module.exports = { getAdminDashboardStats }; diff --git a/plugins/lumi_transcription/stats.json b/plugins/lumi_transcription/stats.json new file mode 100644 index 0000000..3c379ca --- /dev/null +++ b/plugins/lumi_transcription/stats.json @@ -0,0 +1,9 @@ +{ + "pluginId": "lumi_transcription", + "pluginName": "Lumi Companion", + "provider": "stats.js", + "adminDashboard": { + "title": "Lumi Companion", + "description": "Streaming-computer connection and server-hosted transcription health." + } +} diff --git a/plugins/lumi_transcription/tests/verify.js b/plugins/lumi_transcription/tests/verify.js index b5cd634..623127c 100644 --- a/plugins/lumi_transcription/tests/verify.js +++ b/plugins/lumi_transcription/tests/verify.js @@ -10,6 +10,8 @@ const Database = require("better-sqlite3"); const express = require("express"); const { WebSocket } = require("ws"); const { DeviceStore } = require("../backend/companion/device_store"); +const { normalizeHost } = require("../backend/companion/device_store"); +const { CompanionPackageService, safeArchivePath } = require("../backend/companion/package_service"); const { CompanionGateway } = require("../backend/companion/gateway"); const protocol = require("../backend/companion/protocol"); const { RevisionStore } = require("../backend/config/revision_store"); @@ -27,6 +29,7 @@ async function run() { try { verifyProtocol(); verifyPairingAndRevocation(); + verifyLocalhostTransportPolicy(); verifyRevisions(); verifyQueues(); verifyStabilization(); @@ -35,8 +38,9 @@ async function run() { await verifyNativeWorkerBoundary(); await verifyAuthenticatedGateway(); verifyArtifactsAndLogs(temp); + await verifyCompanionPackage(temp); await verifyPluginIsolation(); - console.log("Lumi transcription verification passed: protocol, pairing, revocation, revisions, queues, stabilization, lifecycle, native worker boundary, worker recovery, artifacts, logs, and plugin isolation."); + console.log("Lumi transcription verification passed: protocol, pairing, localhost-only HTTP, revocation, revisions, queues, stabilization, lifecycle, native worker boundary, paired Companion package, worker recovery, artifacts, logs, dashboard summary, and plugin isolation."); } finally { fs.rmSync(temp, { recursive: true, force: true }); } } @@ -75,6 +79,21 @@ function verifyPairingAndRevocation() { db.close(); } +function verifyLocalhostTransportPolicy() { + assert.equal(normalizeHost("http://localhost:3000/path"), "http://localhost:3000"); + assert.equal(normalizeHost("http://127.0.0.1:3000"), "http://127.0.0.1:3000"); + assert.throws(() => normalizeHost("http://lumi.example"), /localhost|HTTPS/i); + const db = new Database(":memory:"); + const store = new DeviceStore(db); + const local = store.issuePairing({ userId: "admin", host: "http://localhost:3000" }); + assert.equal(store.pairingAllowsHttp(local.token, "http://localhost:3000"), true); + assert.equal(store.pairingAllowsHttp(local.token, "http://127.0.0.1:3000"), false); + assert.equal(store.pairingAllowsHttp(local.token, "http://localhost:3001"), false); + store.exchange({ token: local.token, device: {} }); + assert.equal(store.pairingAllowsHttp(local.token, "http://localhost:3000"), false); + db.close(); +} + function verifyRevisions() { const db = new Database(":memory:"); const store = new RevisionStore(db, { now: () => 1234 }); @@ -193,8 +212,6 @@ async function verifyNativeWorkerBoundary() { async function verifyAuthenticatedGateway() { const db = new Database(":memory:"); const devices = new DeviceStore(db); - const pairing = devices.issuePairing({ userId: "admin", host: "https://lumi.example" }); - const credential = devices.exchange({ token: pairing.token, device: { name: "Stream PC" } }); const sessionId = crypto.randomUUID(); let disconnected = false; const sessions = { @@ -202,12 +219,14 @@ async function verifyAuthenticatedGateway() { disconnect: () => { disconnected = true; }, audio: async () => ({ accepted: true }), updateSource: () => ({}), updateObsState: async () => ({}), start: async () => ({}), stop: async () => ({}) }; - const gateway = new CompanionGateway({ devices, sessions, allowInsecure: true }); + const gateway = new CompanionGateway({ devices, sessions }); const registry = createWebUpgradeRegistry(); registry.add("/plugins/lumi_transcription/live", (request, socket, head) => gateway.upgrade(request, socket, head)); const server = http.createServer((_req, res) => res.end("ok")); registry.attach(server); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const pairing = devices.issuePairing({ userId: "admin", host: `http://127.0.0.1:${server.address().port}` }); + const credential = devices.exchange({ token: pairing.token, device: { name: "Stream PC" } }); const client = new WebSocket(`ws://127.0.0.1:${server.address().port}/plugins/lumi_transcription/live`, { headers: { Authorization: `LumiDevice ${credential.device_id}.${credential.device_secret}` } }); @@ -219,6 +238,17 @@ async function verifyAuthenticatedGateway() { await new Promise((resolve) => { client.once("close", resolve); client.close(); }); for (let attempt = 0; attempt < 20 && !disconnected; attempt += 1) await new Promise((resolve) => setTimeout(resolve, 5)); assert.equal(disconnected, true); + const productionPairing = devices.issuePairing({ userId: "admin", host: "https://lumi.example" }); + const productionCredential = devices.exchange({ token: productionPairing.token, device: { name: "Production Stream PC" } }); + const insecureProductionClient = new WebSocket(`ws://127.0.0.1:${server.address().port}/plugins/lumi_transcription/live`, { + headers: { Authorization: `LumiDevice ${productionCredential.device_id}.${productionCredential.device_secret}` } + }); + const rejectionStatus = await new Promise((resolve, reject) => { + insecureProductionClient.once("unexpected-response", (_request, response) => { response.resume(); resolve(response.statusCode); }); + insecureProductionClient.once("open", () => reject(new Error("A production credential connected over insecure WebSocket."))); + insecureProductionClient.once("error", (error) => { if (!String(error.message).includes("Unexpected server response")) reject(error); }); + }); + assert.equal(rejectionStatus, 426); await gateway.close(); registry.close(); await new Promise((resolve) => server.close(resolve)); db.close(); @@ -238,6 +268,26 @@ function verifyArtifactsAndLogs(temp) { assert.equal(fs.readdirSync(logsRoot).some((name) => name.endsWith(".wav") || name.endsWith(".pcm")), false); } +async function verifyCompanionPackage(temp) { + const root = path.join(temp, "companion-package"); + fs.mkdirSync(root, { recursive: true }); + const sourcePath = path.join(root, "companion.zip"); + const zip = new (require("adm-zip"))(); + zip.addFile("Lumi.Companion.App.exe", Buffer.from("portable-app")); + zip.writeZip(sourcePath); + const manifest = { version: "test", artifacts: [{ + id: "windows-x64", platform: "win32", architecture: "x64", filename: "companion.zip", + entrypoint: "Lumi.Companion.App.exe", url: "https://example.invalid/companion.zip", sha256: sha256File(sourcePath) + }] }; + const service = new CompanionPackageService(root, manifest); + const bundle = await service.build({ pairing_id: "pairing", bootstrap: { format: "lumi-companion-bootstrap-v1", token: "single-use" } }); + const output = new (require("adm-zip"))(bundle.buffer); + assert.equal(output.readAsText("Lumi.Companion.App.exe"), "portable-app"); + assert.match(output.readAsText("lumi-companion-pairing.lumi-pairing.json"), /single-use/); + assert.match(output.readAsText("START-HERE.txt"), /works once/i); + assert.throws(() => safeArchivePath("../escape.exe"), /unsafe path/i); +} + async function verifyPluginIsolation() { const db = new Database(":memory:"); const mounts = []; @@ -253,6 +303,9 @@ async function verifyPluginIsolation() { }); assert.deepEqual(mounts, ["/plugins/lumi_transcription"]); assert.equal(typeof global.lumiFrameworks.transcription.health, "function"); + const dashboard = await global.lumiFrameworks.transcription.dashboardSummary(); + assert.equal(dashboard.title, "Lumi Companion"); + assert.equal(dashboard.metrics.find((entry) => entry.label === "Connection").value, "Offline"); await cleanup(); assert.equal(upgradeRemoved, true); assert.equal(global.lumiFrameworks.transcription, undefined); diff --git a/plugins/lumi_transcription/views/settings.ejs b/plugins/lumi_transcription/views/settings.ejs index c7e5851..0abaf9c 100644 --- a/plugins/lumi_transcription/views/settings.ejs +++ b/plugins/lumi_transcription/views/settings.ejs @@ -1,66 +1,75 @@ - -
-
-
-

Experimental companion

-

Lumi Transcription

-

Server-hosted speech recognition for selected OBS sources. Live Twitch delivery remains blocked until the native OBS caption compatibility test passes on the target setup.

-
- - <%= providerHealth.healthy ? 'Inference ready' : 'Inference setup required' %> +
+
+ <%- include("../../../src/web/views/partials/page-header", { + eyebrow: "Experimental companion", + pageTitle: "Lumi Transcription", + description: "Server-hosted speech recognition for selected OBS sources, managed through one Lumi Companion application." + }) %> + + <%= providerHealth.healthy ? 'Inference ready' : 'Setup required' %> -
- -
-

First usable path

Setup progress

-
    -
  1. Pair a companionCreate a single-use package for the streaming computer.
    2. -
  2. Install the OBS bridgeManaged by Lumi Companion; no separate bridge settings.
    3. -
  3. Install and benchmark a modelSmall English is recommended. Nothing downloads without confirmation.
    4. -
  4. Select and test a microphoneTest mode must pass before live delivery is enabled.
    5. -
- -

-
- -
-
-

Lumi host

Inference

-
-
Provider
whisper.cpp <%= runtimeManifest.tested_version %>
-
Worker
<%= providerHealth.state %>
-
Selected model
<%= providerHealth.model || 'Not loaded' %>
-
Active sessions
<%= providerHealth.sessions || 0 %>
-
-

Lumi AI may already occupy most GPU memory. The MVP warns and benchmarks; it does not unload Lumi AI models automatically.

-
- -
-

Access

Paired devices

<%= devices.length %>
- <% if (!devices.length) { %>

No companion is paired yet.

<% } %> -
    - <% devices.forEach((device) => { %> -
  • <%= device.name %><%= device.revoked_at ? 'Revoked' : 'Last connected ' + new Date(device.last_connected_at).toLocaleString() %>
    <%= device.id %>
    • - <% }) %> -
-
+
+ Live Twitch delivery is not accepted yet +

The native OBS 31+ caption path still needs target-machine validation. Lumi will not present a local simulated caption as live Twitch success.

+
+ -
-

Curated choices

Speech models

-
- <% models.forEach((model) => { %> -
-

<%= model.label %>

<%= model.recommended ? 'Recommended starting point' : 'Fallback option' %> · <%= Math.round(model.bytes / 1048576) %> MiB

-
- <%= providerHealth.model === model.id ? 'Loaded' : model.status.valid ? 'Verified' : model.status.installed ? 'Checksum failed' : 'Not installed' %> - <% if (!model.status.valid) { %><% } %> - <% if (model.status.valid && providerHealth.model !== model.id) { %><% } %> -
-
- <% }) %> +
+
+
First usable path

Setup progress

Complete one clear step at a time. Existing server administration remains in Lumi; streaming-computer controls remain in Companion.

+
+
    +
  1. Download and pair CompanionCreate a short-lived, single-use package for the streaming computer.
    2. +
  2. Install the OBS integrationCompanion owns installation and repair; the bridge has no separate settings.
    3. +
  3. Load and benchmark a modelSmall English is the recommended starting point.
    4. +
  4. Select and test a microphoneThe real safe test must pass before live delivery is enabled.
    5. +
+
+ + Review speech models +
+

+
+ +
+
+
Lumi host

Inference health

Speech recognition runs on the Lumi server, never on the streaming computer.

+
+
Providerwhisper.cpp <%= runtimeManifest.tested_version %>
+
Worker<%= providerHealth.state %>
+
Selected model<%= providerHealth.model || 'Not loaded' %>
+
Active sessions<%= providerHealth.sessions || 0 %>
-

Privacy: raw audio is held only in bounded memory and is never written to disk by default. Diagnostic caption text is retained for seven days unless disabled.

+
Shared GPU awareness

Lumi warns and benchmarks when Lumi AI already occupies GPU memory. It never unloads AI models automatically.

-
- + +
+
Access

Paired devices

Credentials are revocable and scoped to Companion capabilities.

<%= devices.filter((device) => !device.revoked_at).length %> active
+ <% if (!devices.length) { %>
No Companion is paired

Use Download Companion to create a package that expires after 15 minutes and works once.

<% } %> + <% if (devices.length) { %> + + <% } %> +
+ + +
+
Curated choices

Speech models

Downloads require confirmation and checksum verification before a model can load.

+
+ <% models.forEach((model) => { %> +
+

<%= model.label %>

<%= model.recommended ? 'Recommended starting point' : 'Fallback option' %> · <%= Math.round(model.bytes / 1048576) %> MiB

+
+ <%= providerHealth.model === model.id ? 'Loaded' : model.status.valid ? 'Verified' : model.status.installed ? 'Checksum failed' : 'Not installed' %> + <% if (!model.status.valid) { %><% } %> + <% if (model.status.valid && providerHealth.model !== model.id) { %><% } %> +
+
+ <% }) %> +
+
Audio privacy

Raw audio is held only in bounded memory and is never written to disk by default. Diagnostic caption text is retained for seven days unless disabled.

+
diff --git a/src/services/plugin-stats.js b/src/services/plugin-stats.js index 9ddab35..57b3006 100644 --- a/src/services/plugin-stats.js +++ b/src/services/plugin-stats.js @@ -95,6 +95,57 @@ function getPluginProfileStats(userId) { .filter(Boolean); } +async function getAdminDashboardSections() { + const sections = await Promise.all(loadStatProviders().map(async ({ plugin, manifest, provider }) => { + if (!manifest.adminDashboard || typeof provider.getAdminDashboardStats !== "function") return null; + try { + const result = await provider.getAdminDashboardStats({ db, plugin, manifest }); + if (!result || typeof result !== "object") return null; + return { + id: String(result.id || manifest.pluginId || plugin.id).slice(0, 120), + eyebrow: String(result.eyebrow || "Companion service").slice(0, 80), + title: String(result.title || manifest.adminDashboard.title || plugin.name).slice(0, 160), + description: String(result.description || manifest.adminDashboard.description || "").slice(0, 500), + status: normalizeDashboardStatus(result.status), + metrics: normalizeDashboardMetrics(result.metrics), + issues: Array.isArray(result.issues) ? result.issues.map((issue) => String(issue).slice(0, 240)).slice(0, 6) : [], + actions: normalizeDashboardActions(result.actions) + }; + } catch (error) { + console.error(`Failed to load ${plugin.id} admin dashboard stats`, error); + return { + id: plugin.id, + eyebrow: "Companion service", + title: manifest.adminDashboard.title || plugin.name, + description: "This plugin's dashboard summary could not be loaded. Its normal settings remain available.", + status: { label: "Summary unavailable", tone: "danger" }, + metrics: [], issues: ["Open the plugin page or diagnostics for details."], + actions: normalizeDashboardActions([{ label: "Open settings", href: `/plugins/${encodeURIComponent(plugin.id)}`, method: "get" }]) + }; + } + })); + return sections.filter(Boolean); +} + +function normalizeDashboardStatus(value) { + const tone = ["success", "warning", "danger", "info", "muted"].includes(value?.tone) ? value.tone : "muted"; + return { label: String(value?.label || "Status unavailable").slice(0, 120), tone }; +} + +function normalizeDashboardMetrics(values) { + if (!Array.isArray(values)) return []; + return values.map((entry) => ({ label: String(entry?.label || "Status").slice(0, 80), value: String(entry?.value ?? "—").slice(0, 120) })).slice(0, 6); +} + +function normalizeDashboardActions(values) { + if (!Array.isArray(values)) return []; + return values.filter((entry) => /^\/(?!\/)/.test(String(entry?.href || ""))).map((entry) => ({ + label: String(entry.label || "Open").slice(0, 80), href: String(entry.href).slice(0, 300), + method: entry.method === "post" ? "post" : "get", primary: entry.primary === true, + disabled: entry.disabled === true, disabledReason: String(entry.disabledReason || "").slice(0, 200) + })).slice(0, 4); +} + function getPluginLeaderboards(limit = 10) { return loadStatProviders() .map((entry) => buildLeaderboardSection({ ...entry, limit })) @@ -103,5 +154,6 @@ function getPluginLeaderboards(limit = 10) { module.exports = { getPluginProfileStats, - getPluginLeaderboards + getPluginLeaderboards, + getAdminDashboardSections }; diff --git a/src/web/server.js b/src/web/server.js index 960e284..0fa88a4 100644 --- a/src/web/server.js +++ b/src/web/server.js @@ -48,7 +48,7 @@ const { exchangeYouTubeCode, fetchYouTubeChannel } = require("../services/auth"); -const { getPluginProfileStats } = require("../services/plugin-stats"); +const { getPluginProfileStats, getAdminDashboardSections } = require("../services/plugin-stats"); const { getLeaderboardSections, getTopCommandOptions @@ -5372,9 +5372,10 @@ function createWebServer({ loadPlugins, discordClient, commandRouter }) { }); }); - app.get("/admin", requireRole("admin"), (req, res) => { + app.get("/admin", requireRole("admin"), async (req, res) => { res.render("admin-dashboard", { - title: "Admin dashboard" + title: "Admin dashboard", + dashboardSections: await getAdminDashboardSections() }); }); diff --git a/src/web/views/admin-dashboard.ejs b/src/web/views/admin-dashboard.ejs index 0f2e6a0..695a6da 100644 --- a/src/web/views/admin-dashboard.ejs +++ b/src/web/views/admin-dashboard.ejs @@ -48,6 +48,30 @@ +<% (typeof dashboardSections !== "undefined" && Array.isArray(dashboardSections) ? dashboardSections : []).forEach((section) => { %> +
+
+
<%= section.eyebrow %>

<%= section.title %>

<%= section.description %>

+ <%= section.status.label %> +
+ <% if (section.metrics.length) { %> +
+ <% section.metrics.forEach((metric) => { %>
<%= metric.label %><%= metric.value %>
<% }) %> +
+ <% } %> + <% if (section.issues.length) { %> +
<%= section.issues.length %> issue<%= section.issues.length === 1 ? '' : 's' %> discovered
    <% section.issues.forEach((issue) => { %>
  • <%= issue %>
    • <% }) %>
+ <% } else { %> +
Everything healthy

No Companion issues are currently reported.

+ <% } %> +
+ <% section.actions.forEach((action) => { %> + <% if (action.method === "post") { %>
+ <% } else { %> title="<%= action.disabledReason %>"><%= action.label %><% } %> + <% }) %> +
+
+<% }) %>
diff --git a/src/web/views/partials/layout-bottom.ejs b/src/web/views/partials/layout-bottom.ejs index ee80ab3..bfcdf29 100644 --- a/src/web/views/partials/layout-bottom.ejs +++ b/src/web/views/partials/layout-bottom.ejs @@ -166,5 +166,8 @@ + <% (typeof extraScripts !== "undefined" && Array.isArray(extraScripts) ? extraScripts : []).forEach((script) => { %> + + <% }) %> diff --git a/src/web/views/partials/layout-top.ejs b/src/web/views/partials/layout-top.ejs index 8927c0a..40e9577 100644 --- a/src/web/views/partials/layout-top.ejs +++ b/src/web/views/partials/layout-top.ejs @@ -9,6 +9,9 @@ + <% (typeof extraStyles !== "undefined" && Array.isArray(extraStyles) ? extraStyles : []).forEach((stylesheet) => { %> + + <% }) %> <%- include("theme-vars", { theme }) %>