From 7760ed0a3f61252d44a7f6eec2144da6b0a54540 Mon Sep 17 00:00:00 2001 From: Franz Rolfsvaag Date: Thu, 23 Jul 2026 09:12:28 +0200 Subject: [PATCH] Accelerate server transcription and finalize tests --- TODO.md | 13 +- companion/installer/Lumi.Companion.iss | 2 +- companion/scripts/publish-companion.ps1 | 2 +- .../Lumi.Companion.App/CompanionRuntime.cs | 19 +- .../Lumi.Companion.App.csproj | 2 +- knowledge/plugins/lumi-transcription.md | 2 +- .../backend/models/artifact_manager.js | 34 +++- plugins/lumi_transcription/backend/paths.js | 20 +- .../backend/sessions/session_coordinator.js | 17 +- .../backend/tests/benchmark_store.js | 36 +++- .../backend/transcription/provider.js | 2 +- .../worker-native/CMakeLists.txt | 12 ++ .../transcription/worker-native/src/main.cpp | 28 ++- .../companion_manifest.json | 16 +- plugins/lumi_transcription/index.js | 26 ++- plugins/lumi_transcription/plugin.json | 2 +- .../public/transcription.js | 19 ++ .../lumi_transcription/runtime_manifest.json | 21 +- .../scripts/benchmark-worker.js | 189 ++++++++++++++++++ .../scripts/build-worker.ps1 | 66 +++++- plugins/lumi_transcription/tests/verify.js | 77 ++++++- plugins/lumi_transcription/views/settings.ejs | 8 + 22 files changed, 562 insertions(+), 51 deletions(-) create mode 100644 plugins/lumi_transcription/scripts/benchmark-worker.js diff --git a/TODO.md b/TODO.md index 008598d..beb2806 100644 --- a/TODO.md +++ b/TODO.md @@ -63,10 +63,19 @@ Experimental.10 deduplicates OBS source inventories and unchanged source-state notifications at both ends, removing the high-volume source-update feedback seen with large scene collections while retaining the non-disconnecting safety budget. +Experimental.11 removes server-side inference backlog, filters whisper.cpp +control tokens, preserves first-visible word latency while retaining finalized +confidence, and makes manual test completion wait for the final worker result. +The portable CUDA 12.4 worker targets RTX 3060 and newer NVIDIA GPUs, activates +from the Lumi WebUI without a host SDK dependency, and retains a CPU +compatibility package. On the RTX 3080 Ti acceptance phrase, `small.en` returned +the exact 19-word transcript at 300 ms average, 106 ms median, and 1,007 ms maximum +first-visible latency. + Release-blocking work remains: -- Package the native whisper.cpp worker as the Windows CUDA runtime, then benchmark - `small.en`, quantized `small.en`, and `base.en` on the RTX 3060. +- Repeat the `small.en` latency gate and benchmark quantized `small.en` and + `base.en` on the production RTX 3060. - Sign the companion installer and add a rollback policy after the experimental updater has target-machine acceptance evidence. - Complete target-machine bridge acceptance and source rename/missing recovery. diff --git a/companion/installer/Lumi.Companion.iss b/companion/installer/Lumi.Companion.iss index 16cd263..f73d2f1 100644 --- a/companion/installer/Lumi.Companion.iss +++ b/companion/installer/Lumi.Companion.iss @@ -1,5 +1,5 @@ #ifndef AppVersion - #define AppVersion "0.1.0-experimental.10" + #define AppVersion "0.1.0-experimental.11" #endif #ifndef SourceRoot #error SourceRoot must point at the self-contained Companion publish directory. diff --git a/companion/scripts/publish-companion.ps1 b/companion/scripts/publish-companion.ps1 index 8bd7a04..a74f37d 100644 --- a/companion/scripts/publish-companion.ps1 +++ b/companion/scripts/publish-companion.ps1 @@ -1,5 +1,5 @@ param( - [string]$Version = "0.1.0-experimental.10", + [string]$Version = "0.1.0-experimental.11", [string]$BridgeVersion = "0.1.0-experimental.5" ) diff --git a/companion/src/Lumi.Companion.App/CompanionRuntime.cs b/companion/src/Lumi.Companion.App/CompanionRuntime.cs index 256bb8d..b088df1 100644 --- a/companion/src/Lumi.Companion.App/CompanionRuntime.cs +++ b/companion/src/Lumi.Companion.App/CompanionRuntime.cs @@ -303,18 +303,29 @@ public sealed class CompanionRuntime : IAsyncDisposable if (!State.BenchmarkRunning || _socket is null || Interlocked.Exchange(ref _benchmarkStopping, 1) != 0) return; _benchmarkLifetime?.Cancel(); SetState(State with { BenchmarkDetail = reason == "silence_timeout" ? "Ten seconds of silence detected. Finalizing the test…" : "Finalizing the test…" }); + var finalized = false; try { await _socket.SendAsync("stop", new { reason }, _socket.SessionId, cancellationToken); if (_benchmarkCompleteSignal is not null) - await Task.WhenAny(_benchmarkCompleteSignal.Task, Task.Delay(TimeSpan.FromSeconds(7), cancellationToken)); + { + var completed = await Task.WhenAny(_benchmarkCompleteSignal.Task, Task.Delay(TimeSpan.FromSeconds(20), cancellationToken)); + if (completed != _benchmarkCompleteSignal.Task) + throw new TimeoutException("Lumi did not finalize the transcription test in time. The test was ended safely, but its final confidence result is unavailable."); + finalized = Benchmark.Words.Count == 0 || Benchmark.Confidence.Count > 0; + if (!finalized) + throw new InvalidOperationException("Lumi returned the test transcript without finalized confidence. The incomplete result was kept for troubleshooting."); + } } finally { VoiceLevelChanged?.Invoke(-60); - SetState(State with { BenchmarkRunning = false, BenchmarkDetail = Benchmark.Words.Count > 0 ? $"Test complete with {Benchmark.Words.Count} measured words or phrases." : "The test ended without a finalized transcript." }); + SetState(State with { BenchmarkRunning = false, BenchmarkDetail = finalized + ? (Benchmark.Words.Count > 0 ? $"Test complete with {Benchmark.Words.Count} measured words or phrases." : "The test ended without recognized speech.") + : "The test ended, but Lumi did not return finalized confidence results." }); _sessionStartSignal = null; _testFailure = null; + _benchmarkCompleteSignal = null; Interlocked.Exchange(ref _benchmarkStopping, 0); } } @@ -605,7 +616,9 @@ public sealed class CompanionRuntime : IAsyncDisposable { if (State.BenchmarkRunning) _benchmarkLastVoiceAt = DateTimeOffset.UtcNow; } - var audioNeeded = State.TestRunning || State.BenchmarkRunning || State.ObsStreaming; + var audioNeeded = State.TestRunning || (State.BenchmarkRunning + ? Volatile.Read(ref _benchmarkStopping) == 0 + : State.ObsStreaming); if (audioNeeded && _socket is not null && State.Connected && !_socket.QueueBridgeAudio(frame)) _ = WriteLogAsync("audio_dropped", "An obsolete audio frame was dropped before network delivery."); return Task.CompletedTask; diff --git a/companion/src/Lumi.Companion.App/Lumi.Companion.App.csproj b/companion/src/Lumi.Companion.App/Lumi.Companion.App.csproj index f0567ec..1e0263d 100644 --- a/companion/src/Lumi.Companion.App/Lumi.Companion.App.csproj +++ b/companion/src/Lumi.Companion.App/Lumi.Companion.App.csproj @@ -5,7 +5,7 @@ enable enable app.manifest - 0.1.0-experimental.10 + 0.1.0-experimental.11 0.1.0.0 diff --git a/knowledge/plugins/lumi-transcription.md b/knowledge/plugins/lumi-transcription.md index b689fce..128edf4 100644 --- a/knowledge/plugins/lumi-transcription.md +++ b/knowledge/plugins/lumi-transcription.md @@ -14,7 +14,7 @@ editable: false Server-hosted whisper.cpp transcription for Lumi Companion and OBS closed captions. ## Metadata Plugin ID: lumi_transcription -Version: 0.1.0-experimental.10 +Version: 0.1.0-experimental.11 Default state: enabled ## Web Routes - /plugins/lumi_transcription diff --git a/plugins/lumi_transcription/backend/models/artifact_manager.js b/plugins/lumi_transcription/backend/models/artifact_manager.js index 9489da1..9ee2b3c 100644 --- a/plugins/lumi_transcription/backend/models/artifact_manager.js +++ b/plugins/lumi_transcription/backend/models/artifact_manager.js @@ -47,8 +47,7 @@ class ArtifactManager { } zip.extractAllTo(staged, true); for (const expected of entry.expected_paths || []) if (!findBasename(staged, expected)) throw new Error(`Runtime is missing ${expected}.`); - fs.rmSync(target, { recursive: true, force: true }); - fs.renameSync(staged, target); + replaceDirectory(staged, target); return { installed: true, path: target, backend: entry.backend, version: entry.id }; } } @@ -56,5 +55,36 @@ class ArtifactManager { function validateManifestEntry(entry) { if (!entry?.id || !/^https:\/\//.test(entry.url || "") || !/^[a-f0-9]{64}$/.test(entry.sha256 || "")) throw new Error("Artifact manifest entry is invalid."); } function sha256File(target) { const hash = crypto.createHash("sha256"); const fd = fs.openSync(target, "r"); const buffer = Buffer.alloc(1024 * 1024); try { let read; while ((read = fs.readSync(fd, buffer, 0, buffer.length, null)) > 0) hash.update(buffer.subarray(0, read)); } finally { fs.closeSync(fd); } return hash.digest("hex"); } function findBasename(root, basename) { const pending = [root]; while (pending.length) { const current = pending.pop(); for (const entry of fs.readdirSync(current, { withFileTypes: true })) { const target = path.join(current, entry.name); if (entry.isDirectory()) pending.push(target); else if (entry.name === basename) return target; } } return null; } +function replaceDirectory(staged, target) { + const backup = `${target}.${process.pid}.backup`; + fs.rmSync(backup, { recursive: true, force: true }); + const hadTarget = fs.existsSync(target); + let backupDisposable = false; + try { + if (hadTarget) moveDirectory(target, backup); + moveDirectory(staged, target); + backupDisposable = true; + fs.rmSync(backup, { recursive: true, force: true }); + } catch (error) { + fs.rmSync(target, { recursive: true, force: true }); + if (hadTarget && fs.existsSync(backup)) { + moveDirectory(backup, target); + backupDisposable = true; + } else if (!hadTarget) backupDisposable = true; + throw error; + } finally { + fs.rmSync(staged, { recursive: true, force: true }); + if (backupDisposable) fs.rmSync(backup, { recursive: true, force: true }); + } +} +function moveDirectory(source, target) { + try { fs.renameSync(source, target); } + catch (error) { + if (!["EPERM", "EACCES", "EXDEV"].includes(error.code)) throw error; + fs.mkdirSync(target, { recursive: true }); + fs.cpSync(source, target, { recursive: true, force: true }); + fs.rmSync(source, { recursive: true, force: true }); + } +} module.exports = { ArtifactManager, sha256File, validateManifestEntry }; diff --git a/plugins/lumi_transcription/backend/paths.js b/plugins/lumi_transcription/backend/paths.js index ce63c07..d571ba6 100644 --- a/plugins/lumi_transcription/backend/paths.js +++ b/plugins/lumi_transcription/backend/paths.js @@ -4,15 +4,33 @@ const ROOT = path.resolve(__dirname, ".."); const DATA = path.join(ROOT, "data"); function ensureDataDirs() { for (const name of ["logs", "models", "runtime", "tmp"]) fs.mkdirSync(path.join(DATA, name), { recursive: true }); } function dataPath(...parts) { const target = path.resolve(DATA, ...parts); if (target !== DATA && !target.startsWith(`${DATA}${path.sep}`)) throw new Error("Path escapes transcription plugin data."); return target; } +function activeWorkerExecutable() { + try { + const requested = fs.readFileSync(dataPath("runtime", "active-worker.txt"), "utf8").trim(); + const target = path.resolve(requested); + const runtimeRoot = dataPath("runtime"); + return target.startsWith(`${runtimeRoot}${path.sep}`) && fs.statSync(target).isFile() ? target : ""; + } catch { return ""; } +} +function setActiveWorkerExecutable(target) { + const executable = path.resolve(String(target || "")); + const runtimeRoot = dataPath("runtime"); + if (!executable.startsWith(`${runtimeRoot}${path.sep}`) || !fs.existsSync(executable) || !fs.statSync(executable).isFile()) + throw new Error("Active transcription worker must be an installed runtime file."); + fs.writeFileSync(dataPath("runtime", "active-worker.txt"), `${executable}\n`, { encoding: "utf8", mode: 0o600 }); +} function resolveWorkerExecutable(override = "") { const requested = String(override || "").trim(); if (requested) return fs.existsSync(requested) ? path.resolve(requested) : requested; const executable = process.platform === "win32" ? "lumi-whisper-worker.exe" : "lumi-whisper-worker"; const candidates = [ + activeWorkerExecutable(), + dataPath("runtime", process.platform === "win32" ? "windows-x64-cuda-12.4" : `${process.platform}-${process.arch}-cuda`, "bin", executable), dataPath("runtime", "worker", process.platform === "win32" ? "windows-x64-cuda" : `${process.platform}-${process.arch}-cuda`, "bin", executable), + dataPath("runtime", process.platform === "win32" ? "windows-x64-cpu" : `${process.platform}-${process.arch}`, "bin", executable), dataPath("runtime", "worker", process.platform === "win32" ? "windows-x64-cpu" : `${process.platform}-${process.arch}`, "bin", executable), path.join(ROOT, "backend", "transcription", "worker-native", "build", executable) ]; return candidates.find((candidate) => fs.existsSync(candidate) && fs.statSync(candidate).isFile()) || ""; } -module.exports = { ROOT, DATA, ensureDataDirs, dataPath, resolveWorkerExecutable }; +module.exports = { ROOT, DATA, ensureDataDirs, dataPath, resolveWorkerExecutable, setActiveWorkerExecutable }; diff --git a/plugins/lumi_transcription/backend/sessions/session_coordinator.js b/plugins/lumi_transcription/backend/sessions/session_coordinator.js index 2b25114..e37a4d6 100644 --- a/plugins/lumi_transcription/backend/sessions/session_coordinator.js +++ b/plugins/lumi_transcription/backend/sessions/session_coordinator.js @@ -115,8 +115,8 @@ class SessionCoordinator { const session = this.require(sessionId); clearTimeout(session.graceTimer); session.graceTimer = null; - await session.delivery.stop(); if (session.providerActive) await this.provider.stopSession(session.id); + await session.delivery.stop(); if (session.benchmarkId && this.benchmarks) { const benchmarkStatus = reason === "silence_timeout" ? "silence_timeout" : reason === "provider_failed" ? "failed" : reason === "disconnect" || reason === "plugin_shutdown" ? "aborted" : "completed"; const result = this.benchmarks.finish(session.id, benchmarkStatus); @@ -196,10 +196,12 @@ class SessionCoordinator { const session = this.sessions.get(raw.session_id); const track = session?.tracks.get(raw.track_id || raw.source_uuid); if (!session || !track || session.state !== "running") return; + const hypothesisText = cleanAnalysisText(raw.text); + if (!hypothesisText) return; track.lastSpeechAt = this.now(); const primary = Array.from(session.tracks.values()).find((candidate) => candidate.primary); if (primary && track !== primary && this.now() - primary.lastSpeechAt < 800) return; - const stable = this.stabilizer.update(track.source_uuid, raw.text, { final: raw.final, newUtterance: raw.new_utterance, trailingIncomplete: raw.incomplete_word }); + const stable = this.stabilizer.update(track.source_uuid, hypothesisText, { final: raw.final, newUtterance: raw.new_utterance, trailingIncomplete: raw.incomplete_word }); const event = { session_id: session.id, source_uuid: track.source_uuid, speaker_label: speakerLabel(session, track), ...stable, @@ -207,7 +209,13 @@ class SessionCoordinator { latency: { capture_ms: raw.capture_ms || 0, network_ms: raw.network_ms || 0, queue_ms: raw.queue_ms || 0, inference_ms: raw.inference_ms || 0, stabilization_ms: stable.stabilization_ms, total_ms: raw.total_ms || 0 }, model: { id: raw.model_id || "unknown", provider: "whisper.cpp", backend: raw.backend || "unknown" } }; - event.analysis = { transcript: String(raw.text || "").slice(0, 8000), words: Array.isArray(raw.words) ? raw.words.slice(0, 500) : [], emitted_at_ms: raw.emitted_at_ms || this.now() }; + event.analysis = { + transcript: hypothesisText.slice(0, 8000), + words: Array.isArray(raw.words) ? raw.words.slice(0, 500) + .map((word) => ({ ...word, text: cleanAnalysisText(word?.text) })) + .filter((word) => word.text) : [], + emitted_at_ms: raw.emitted_at_ms || this.now() + }; if (session.mode === "benchmark") this.benchmarks?.record(session.id, event); const result = await session.delivery.deliver(event); this.log.append({ kind: "caption", session_id: session.id, source_uuid: track.source_uuid, caption_text: session.mode === "benchmark" ? undefined : event.stable_text, uncertain_text: session.mode === "benchmark" ? undefined : event.uncertain_text, revision: event.revision, final: event.final, delivery: result.disposition, latency: event.latency, model: event.model, benchmark_id: session.benchmarkId }); @@ -241,6 +249,9 @@ class SessionCoordinator { function speakerLabel(session, track) { const enabled = Array.from(session.tracks.values()).filter((candidate) => candidate.enabled); return enabled.length === 1 && track.single_speaker ? null : track.speaker_label || track.display_name; } function serializeTrack(track) { return { source_uuid: track.source_uuid, display_name: track.display_name, speaker_label: track.speaker_label, enabled: track.enabled, primary: track.primary, single_speaker: track.single_speaker, delivery_enabled: track.delivery_enabled, program_active: track.program_active, source_missing: track.source_missing, last_activity_at: track.last_activity_at, sequence: track.sequence?.metrics?.() }; } function text(value, max) { return String(value || "").trim().slice(0, max); } +function cleanAnalysisText(value) { + return String(value || "").replace(/\[_(?:BEG_|TT_\d+)\]/gi, "").replace(/\[BLANK_AUDIO\]/gi, "").trim(); +} function isUuid(value) { return /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(String(value || "")); } function pathTestCaption(session, track, health, now) { return { diff --git a/plugins/lumi_transcription/backend/tests/benchmark_store.js b/plugins/lumi_transcription/backend/tests/benchmark_store.js index c7cd818..8d7354a 100644 --- a/plugins/lumi_transcription/backend/tests/benchmark_store.js +++ b/plugins/lumi_transcription/backend/tests/benchmark_store.js @@ -110,7 +110,8 @@ class BenchmarkStore { const selected = Array.from(captions.values()).sort((a, b) => a.received_at - b.received_at); const words = []; for (const revision of selected) { - for (const word of parseWords(revision.words_json)) words.push({ ...word, confidence: revision.final ? word.confidence : null, final: Boolean(revision.final) }); + const history = revisions.filter((candidate) => candidate.caption_id === revision.caption_id); + for (const word of mergeRevisionWords(revision, history)) words.push({ ...word, confidence: revision.final ? word.confidence : null, final: Boolean(revision.final) }); } const latency = metricStats(words.map((word) => word.latency_ms)); const confidence = metricStats(words.map((word) => word.confidence).filter(Number.isFinite)); @@ -119,7 +120,7 @@ class BenchmarkStore { source_uuid: row.source_uuid, source_name: row.source_name, started_at: row.started_at, ended_at: row.ended_at, duration_ms: Math.max(0, (row.ended_at || this.now()) - row.started_at), status: row.status, model_id: row.model_id, backend: row.backend, - transcript: selected.map((revision) => revision.text).filter(Boolean).join(" "), words, + transcript: selected.map((revision) => cleanTranscript(revision.text)).filter(Boolean).join(" "), words, stats: { latency, confidence }, revisions: revisions.length }; } @@ -136,13 +137,42 @@ class BenchmarkStore { function safeWord(word) { return { - text: String(word?.text || "").slice(0, 120), + text: cleanWordText(word?.text).slice(0, 120), latency_ms: finite(word?.latency_ms), confidence: clamp(word?.confidence, 0, 1), audio_start_ms: finite(word?.audio_start_ms), audio_end_ms: finite(word?.audio_end_ms), captured_at_ms: finite(word?.captured_at_ms) }; } function parseWords(value) { try { const parsed = JSON.parse(value); return Array.isArray(parsed) ? parsed.map(safeWord).filter((word) => word.text) : []; } catch { return []; } } +function mergeRevisionWords(selected, revisions) { + const words = withOccurrences(parseWords(selected.words_json)); + const candidates = revisions.flatMap((revision) => withOccurrences(parseWords(revision.words_json))); + return words.map((word) => { + const matches = candidates.filter((candidate) => candidate.key === word.key && candidate.occurrence === word.occurrence && + Math.abs(candidate.captured_at_ms - word.captured_at_ms) <= 1500); + const { key: _key, occurrence: _occurrence, ...plain } = word; + return matches.length ? { ...plain, latency_ms: Math.min(...matches.map((candidate) => candidate.latency_ms)) } : plain; + }); +} +function withOccurrences(words) { + const seen = new Map(); + return words.map((word) => { + const key = comparableWord(word.text); + const occurrence = seen.get(key) || 0; + seen.set(key, occurrence + 1); + return { ...word, key, occurrence }; + }); +} +function cleanWordText(value) { + return String(value || "") + .replace(/\[_(?:BEG_|TT_\d+)\]/gi, "") + .replace(/\[BLANK_AUDIO\]/gi, "") + .trim(); +} +function cleanTranscript(value) { + return cleanWordText(value).replace(/\s+/g, " ").trim(); +} +function comparableWord(value) { return cleanWordText(value).toLocaleLowerCase().replace(/[^\p{L}\p{N}]+/gu, ""); } function finite(value) { const number = Number(value); return Number.isFinite(number) ? Math.max(0, number) : 0; } function clamp(value, min, max) { return Math.min(max, Math.max(min, finite(value))); } function metricStats(input) { diff --git a/plugins/lumi_transcription/backend/transcription/provider.js b/plugins/lumi_transcription/backend/transcription/provider.js index ed4fe94..7fe5235 100644 --- a/plugins/lumi_transcription/backend/transcription/provider.js +++ b/plugins/lumi_transcription/backend/transcription/provider.js @@ -191,7 +191,7 @@ class WhisperCppServerProvider extends TranscriptionProvider { async stopSession(sessionId) { this.sessions.delete(sessionId); const stopped = new Promise((resolve) => { - const timer = setTimeout(() => { cleanup(); resolve(false); }, 5000); + const timer = setTimeout(() => { cleanup(); resolve(false); }, 15000); const onStopped = (event) => { if (event.session_id !== sessionId) return; cleanup(); resolve(true); }; const cleanup = () => { clearTimeout(timer); this.off("session_stopped", onStopped); }; this.on("session_stopped", onStopped); diff --git a/plugins/lumi_transcription/backend/transcription/worker-native/CMakeLists.txt b/plugins/lumi_transcription/backend/transcription/worker-native/CMakeLists.txt index 6374811..e90d38f 100644 --- a/plugins/lumi_transcription/backend/transcription/worker-native/CMakeLists.txt +++ b/plugins/lumi_transcription/backend/transcription/worker-native/CMakeLists.txt @@ -9,6 +9,18 @@ set(WHISPER_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) set(WHISPER_BUILD_TESTS OFF CACHE BOOL "" FORCE) set(WHISPER_BUILD_SERVER OFF CACHE BOOL "" FORCE) set(GGML_CUDA ${LUMI_WHISPER_CUDA} CACHE BOOL "" FORCE) +# Build a portable AVX2 host path instead of inheriting the build machine's +# AVX-512 features. The GPU package must run on ordinary RTX 3060-era hosts. +set(GGML_NATIVE OFF CACHE BOOL "" FORCE) +set(GGML_AVX ON CACHE BOOL "" FORCE) +set(GGML_AVX2 ON CACHE BOOL "" FORCE) +set(GGML_AVX512 OFF CACHE BOOL "" FORCE) +if(LUMI_WHISPER_CUDA) + # The RTX 3060 production baseline and RTX 3080 Ti development host are + # both Ampere (SM 8.6). Include PTX for forward compatibility as well. + set(LUMI_CUDA_ARCHITECTURES "86-real;86-virtual" CACHE STRING "CUDA architectures included in the portable worker") + set(CMAKE_CUDA_ARCHITECTURES "${LUMI_CUDA_ARCHITECTURES}" CACHE STRING "" FORCE) +endif() FetchContent_Declare( whisper diff --git a/plugins/lumi_transcription/backend/transcription/worker-native/src/main.cpp b/plugins/lumi_transcription/backend/transcription/worker-native/src/main.cpp index f3a3f27..ff135aa 100644 --- a/plugins/lumi_transcription/backend/transcription/worker-native/src/main.cpp +++ b/plugins/lumi_transcription/backend/transcription/worker-native/src/main.cpp @@ -23,7 +23,8 @@ using clock_type = std::chrono::steady_clock; namespace { constexpr std::size_t sample_rate = 16000; -constexpr std::size_t max_samples = sample_rate * 6; +constexpr std::size_t max_samples = sample_rate * 15; +constexpr std::size_t preroll_samples = sample_rate * 2 / 5; constexpr std::size_t min_decode_samples = sample_rate / 2; constexpr auto decode_interval = std::chrono::milliseconds(600); constexpr std::uint64_t finalize_silence_us = 750000; @@ -37,7 +38,7 @@ struct track_state { std::uint64_t audio_end_us = 0; std::uint64_t captured_at_ms = 0; std::uint64_t last_voice_us = 0; - std::uint64_t last_decode_us = 0; + clock_type::time_point last_decode_at{}; bool voiced = false; bool new_utterance = true; }; @@ -138,6 +139,9 @@ transcription_result transcribe(const std::vector & samples, double & inf parameters.single_segment = true; parameters.token_timestamps = true; parameters.split_on_word = true; + parameters.max_tokens = 96; + parameters.suppress_blank = true; + parameters.suppress_nst = true; parameters.language = "en"; parameters.n_threads = std::max(1U, std::min(4U, std::thread::hardware_concurrency())); const auto started = clock_type::now(); @@ -152,13 +156,14 @@ transcription_result transcribe(const std::vector & samples, double & inf for (int index = 0; index < tokens; ++index) { const char * value = whisper_full_get_token_text(context.get(), segment, index); if (!value || !*value) continue; + const auto data = whisper_full_get_token_data(context.get(), segment, index); + if (data.id >= whisper_token_eot(context.get())) continue; std::string raw(value); if (raw.starts_with("<|")) continue; const auto first = raw.find_first_not_of(" \t\r\n"); if (first == std::string::npos) continue; const auto last = raw.find_last_not_of(" \t\r\n"); const bool begins_word = first > 0 || output.words.empty(); - const auto data = whisper_full_get_token_data(context.get(), segment, index); if (begins_word) output.words.push_back({}); auto & word = output.words.back(); word.text += raw.substr(first, last - first + 1); @@ -203,13 +208,14 @@ void decode_track(const std::string & session_id, const std::string & track_id, }); track.new_utterance = false; } - track.last_decode_us = track.audio_end_us; + track.last_decode_at = clock_type::now(); if (final) { track.samples.clear(); track.voiced = false; track.new_utterance = true; track.audio_start_us = 0; track.captured_at_ms = 0; + track.last_decode_at = {}; } } @@ -236,10 +242,20 @@ void audio(const json & packet, const std::vector & pcm) { const auto duration_us = static_cast(count * 1000000ULL / sample_rate); track.audio_end_us = captured_us + duration_us; const auto rms = std::sqrt(squares / static_cast(std::max(1, count))); - if (rms >= 0.012) { track.voiced = true; track.last_voice_us = track.audio_end_us; } + if (rms >= 0.012) { + track.voiced = true; + track.last_voice_us = track.audio_end_us; + } else if (!track.voiced && track.samples.size() > preroll_samples) { + const auto removed = track.samples.size() - preroll_samples; + track.samples.erase(track.samples.begin(), track.samples.begin() + static_cast(removed)); + const auto removed_us = static_cast(removed * 1000000ULL / sample_rate); + track.audio_start_us += removed_us; + track.captured_at_ms += removed_us / 1000ULL; + } const bool final = track.voiced && (track.samples.size() >= max_samples || (track.audio_end_us > track.last_voice_us && track.audio_end_us - track.last_voice_us >= finalize_silence_us)); - const bool decode_due = track.last_decode_us == 0 || track.audio_end_us - track.last_decode_us >= static_cast(decode_interval.count()) * 1000ULL; + const auto now = clock_type::now(); + const bool decode_due = track.last_decode_at == clock_type::time_point{} || now - track.last_decode_at >= decode_interval; if (final || (track.voiced && decode_due)) decode_track(session_id, track_id, track, final); } diff --git a/plugins/lumi_transcription/companion_manifest.json b/plugins/lumi_transcription/companion_manifest.json index fcc2970..0ab6d2a 100644 --- a/plugins/lumi_transcription/companion_manifest.json +++ b/plugins/lumi_transcription/companion_manifest.json @@ -1,17 +1,17 @@ { "schema_version": 1, - "version": "0.1.0-experimental.10", + "version": "0.1.0-experimental.11", "signed": false, - "release_notes": "Prevents OBS capture bursts from rate-limiting the Companion connection, sends steady 20 ms audio frames, and removes duplicate source-state traffic from large scene collections.", + "release_notes": "Manual transcription tests now wait for finalized confidence results, stop sending obsolete audio while finalizing, and clearly report incomplete server finalization.", "installer": { "id": "windows-x64-installer", "platform": "win32", "architecture": "x64", "label": "Windows x64 per-user installer", "filename": "Lumi.Companion-Setup.exe", - "url": "https://git.rolfsvaag.no/Rolfsvaag_Datateknikk/Lumi/releases/download/companion-v0.1.0-experimental.10/Lumi.Companion-Setup.exe", - "sha256": "8207620c1fa2a3f6371c7f5fad40f9a467146a3c44068ed6ae655ee701d98e6a", - "bytes": 32031916 + "url": "https://git.rolfsvaag.no/Rolfsvaag_Datateknikk/Lumi/releases/download/companion-v0.1.0-experimental.11/Lumi.Companion-Setup.exe", + "sha256": "17ac5b059b3d358c6dc11caeea3ba96c962edc063db25d06c9f2a0e91e4d2e12", + "bytes": 32034881 }, "artifacts": [ { @@ -20,9 +20,9 @@ "architecture": "x64", "label": "Windows x64 self-contained", "filename": "Lumi.Companion-win-x64.zip", - "url": "https://git.rolfsvaag.no/Rolfsvaag_Datateknikk/Lumi/releases/download/companion-v0.1.0-experimental.10/Lumi.Companion-win-x64.zip", - "sha256": "f5a00c9bb778790caec596da475c33b3f7c8e6e67921da75e1e8e3b463bee624", - "bytes": 41757850, + "url": "https://git.rolfsvaag.no/Rolfsvaag_Datateknikk/Lumi/releases/download/companion-v0.1.0-experimental.11/Lumi.Companion-win-x64.zip", + "sha256": "a981d798ee123f982342a4b90b65084bf9da85ca2955cccb069abd0184fc38be", + "bytes": 41757483, "entrypoint": "Lumi.Companion.App.exe" } ] diff --git a/plugins/lumi_transcription/index.js b/plugins/lumi_transcription/index.js index a41a9d3..137308e 100644 --- a/plugins/lumi_transcription/index.js +++ b/plugins/lumi_transcription/index.js @@ -13,7 +13,7 @@ const { ArtifactManager } = require("./backend/models/artifact_manager"); const { SessionCoordinator } = require("./backend/sessions/session_coordinator"); const { WhisperWorkerSupervisor, WhisperCppServerProvider } = require("./backend/transcription/provider"); const { BenchmarkStore } = require("./backend/tests/benchmark_store"); -const { ensureDataDirs, dataPath, resolveWorkerExecutable } = require("./backend/paths"); +const { ensureDataDirs, dataPath, resolveWorkerExecutable, setActiveWorkerExecutable } = require("./backend/paths"); const modelManifest = require("./models_manifest.json"); const runtimeManifest = require("./runtime_manifest.json"); const manifest = require("./plugin.json"); @@ -185,9 +185,31 @@ module.exports = { const entry = runtimeManifest.artifacts.find((candidate) => candidate.id === req.params.id); if (!entry) return res.status(404).json({ ok: false, error: "Runtime was not found." }); try { + if ((await provider.health()).sessions > 0) return res.status(409).json({ ok: false, error: "End active transcription sessions before changing the inference runtime." }); const filename = `${entry.id}.zip`; const archive = await runtimeArchives.download({ ...entry, filename }, { confirmed: req.body.confirmed === true }); - res.status(201).json({ ok: true, status: runtimes.installZip(entry, archive.path) }); + const selected = modelManifest.models.find((candidate) => candidate.id === revisions.list().selected_model_id?.value); + const selectedStatus = selected ? models.status(selected) : null; + const previousExecutable = supervisor.executable; + let activated; + try { + await provider.stop(); + const installed = runtimes.installZip(entry, archive.path); + const executable = path.join(installed.path, "bin", process.platform === "win32" ? "lumi-whisper-worker.exe" : "lumi-whisper-worker"); + if (!fs.existsSync(executable)) throw new Error("Installed runtime did not contain the Lumi transcription worker."); + supervisor.executable = executable; + activated = selected && selectedStatus?.valid + ? await provider.loadModel({ ...selected, path: selectedStatus.path }) + : await provider.health(); + setActiveWorkerExecutable(executable); + res.status(201).json({ ok: true, status: installed, activated }); + } catch (activationError) { + await provider.stop().catch(() => {}); + supervisor.executable = previousExecutable; + if (previousExecutable && selected && selectedStatus?.valid) + await provider.loadModel({ ...selected, path: selectedStatus.path }).catch(() => {}); + throw activationError; + } } catch (error) { res.status(400).json({ ok: false, error: error.message }); } }); router.get("/api/logs", requireAdmin, (_req, res) => res.json({ files: diagnosticLog.files().map(({ path: _path, ...entry }) => entry) })); diff --git a/plugins/lumi_transcription/plugin.json b/plugins/lumi_transcription/plugin.json index 9ee6be0..7507d63 100644 --- a/plugins/lumi_transcription/plugin.json +++ b/plugins/lumi_transcription/plugin.json @@ -1,7 +1,7 @@ { "id": "lumi_transcription", "name": "Lumi Transcription", - "version": "0.1.0-experimental.10", + "version": "0.1.0-experimental.11", "description": "Server-hosted whisper.cpp transcription for Lumi Companion and OBS closed captions.", "main": "index.js", "channel": "experimental", diff --git a/plugins/lumi_transcription/public/transcription.js b/plugins/lumi_transcription/public/transcription.js index 886a514..15ffe9a 100644 --- a/plugins/lumi_transcription/public/transcription.js +++ b/plugins/lumi_transcription/public/transcription.js @@ -90,6 +90,25 @@ } }); + root.addEventListener("click", async (event) => { + const runtime = event.target.closest("[data-install-runtime]"); + if (!runtime) return; + const size = Number(runtime.dataset.runtimeSize) || 0; + if (!window.confirm(`Download and activate ${runtime.dataset.runtimeLabel}${size ? ` (${size} MiB)` : ""} on the Lumi host? Active transcription tests must be stopped first.`)) return; + runtime.disabled = true; + status.textContent = "Downloading, verifying, and activating the inference runtime…"; + try { + const response = await fetch(`/plugins/lumi_transcription/api/runtime/${encodeURIComponent(runtime.dataset.installRuntime)}/install`, { + method: "POST", headers: { "Content-Type": "application/json", Accept: "application/json" }, + body: JSON.stringify({ confirmed: true }) + }); + const result = await response.json(); + if (!response.ok) throw new Error(result.error || "The inference runtime could not be installed."); + status.textContent = "Inference runtime verified and activated."; + setTimeout(() => window.location.reload(), 700); + } catch (error) { status.textContent = error.message; runtime.disabled = false; } + }); + root.addEventListener("click", async (event) => { const download = event.target.closest("[data-download-model]"); const load = event.target.closest("[data-load-model]"); diff --git a/plugins/lumi_transcription/runtime_manifest.json b/plugins/lumi_transcription/runtime_manifest.json index 57875c9..f7efaea 100644 --- a/plugins/lumi_transcription/runtime_manifest.json +++ b/plugins/lumi_transcription/runtime_manifest.json @@ -17,9 +17,12 @@ "architecture": "x64", "backend": "cuda", "cuda": "12.4", - "url": "https://github.com/ggml-org/whisper.cpp/releases/download/v1.9.1/whisper-cublas-12.4.0-bin-x64.zip", - "sha256": "106a2030eff8998e4ef320fe72e263a78449e9040386ee27c41ea80b001b601b", - "expected_paths": ["whisper-cli.exe"], + "label": "NVIDIA GPU runtime (RTX 3060 and newer)", + "url": "https://git.rolfsvaag.no/Rolfsvaag_Datateknikk/Lumi/releases/download/companion-v0.1.0-experimental.11/lumi-whisper-worker-windows-x64-cuda.zip", + "sha256": "ab8f5e7ac1d8cfc6453574d69a4b61e5ab2f892d27707486c614b13c3a5debef", + "bytes": 469478354, + "expected_paths": ["lumi-whisper-worker.exe", "cudart64_12.dll", "cublas64_12.dll", "cublasLt64_12.dll"], + "minimum_gpu": "NVIDIA RTX 3060", "tested": true }, { @@ -27,11 +30,13 @@ "platform": "win32", "architecture": "x64", "backend": "cpu", - "url": "https://github.com/ggml-org/whisper.cpp/releases/download/v1.9.1/whisper-bin-x64.zip", - "sha256": "7d8be46ecd31828e1eb7a2ecdd0d6b314feafd82163038ab6092594b0a063539", - "expected_paths": ["whisper-cli.exe"], - "tested": false, - "note": "CPU fallback remains disabled until the host benchmark meets the latency target." + "label": "CPU compatibility fallback", + "url": "https://git.rolfsvaag.no/Rolfsvaag_Datateknikk/Lumi/releases/download/companion-v0.1.0-experimental.11/lumi-whisper-worker-windows-x64-cpu.zip", + "sha256": "044f06396765ce64a32c4fb3701def8f1c27ebab71dde95f0e4c1b7d32134c6b", + "bytes": 631098, + "expected_paths": ["lumi-whisper-worker.exe"], + "tested": true, + "note": "Compatibility fallback only; the NVIDIA runtime is required for Lumi's low-latency target." } ] } diff --git a/plugins/lumi_transcription/scripts/benchmark-worker.js b/plugins/lumi_transcription/scripts/benchmark-worker.js new file mode 100644 index 0000000..9cc5676 --- /dev/null +++ b/plugins/lumi_transcription/scripts/benchmark-worker.js @@ -0,0 +1,189 @@ +const fs = require("fs"); +const path = require("path"); +const { spawn } = require("child_process"); + +const options = parseArguments(process.argv.slice(2)); +if (!options.worker || !options.audio || !options.model) { + console.error("Usage: node benchmark-worker.js --worker --audio <16-kHz-mono.wav> --model [--max-latency-ms 2160] [--expect-transcript \"...\"]"); + process.exit(2); +} + +run().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); + +async function run() { + const pcm = readPcmWave(options.audio); + const child = spawn(options.worker, [], { + stdio: ["pipe", "pipe", "pipe"], + windowsHide: true, + env: process.env + }); + const messages = []; + let stdout = ""; + let stderr = ""; + child.stderr.on("data", (chunk) => { stderr = `${stderr}${chunk}`.slice(-8000); }); + child.stdout.on("data", (chunk) => { + stdout += chunk; + let newline; + while ((newline = stdout.indexOf("\n")) >= 0) { + const line = stdout.slice(0, newline).trim(); + stdout = stdout.slice(newline + 1); + if (!line) continue; + try { messages.push(JSON.parse(line)); } catch { } + } + }); + const exited = new Promise((resolve) => child.once("exit", (code) => resolve(code))); + await waitFor(messages, (message) => message.type === "ready", 20000, exited, stderr); + send(child, { type: "load_model", model: { id: "benchmark-model", path: path.resolve(options.model) } }); + const loaded = await waitFor(messages, (message) => ["model_loaded", "model_error"].includes(message.type), 30000, exited, stderr); + if (loaded.type !== "model_loaded") throw new Error(loaded.message || "Model loading failed."); + + const sessionId = "benchmark-session"; + const trackId = "benchmark-track"; + send(child, { type: "start_session", session: { id: sessionId, mode: "benchmark" } }); + send(child, { type: "add_track", session_id: sessionId, track: { source_uuid: trackId } }); + const frameBytes = 640; + const started = Date.now(); + let sequence = 0; + for (let offset = 0; offset < pcm.length; offset += frameBytes) { + const dueAt = started + sequence * 20; + const wait = dueAt - Date.now(); + if (wait > 0) await delay(wait); + const frame = Buffer.alloc(frameBytes); + pcm.copy(frame, 0, offset, Math.min(pcm.length, offset + frameBytes)); + const capturedAt = Date.now(); + send(child, { + type: "audio", session_id: sessionId, track_id: trackId, sequence, + capture_timestamp_us: capturedAt * 1000, captured_at: capturedAt + }, frame); + sequence += 1; + } + for (let index = 0; index < 50; index += 1) { + const dueAt = started + sequence * 20; + const wait = dueAt - Date.now(); + if (wait > 0) await delay(wait); + const capturedAt = Date.now(); + send(child, { + type: "audio", session_id: sessionId, track_id: trackId, sequence, + capture_timestamp_us: capturedAt * 1000, captured_at: capturedAt + }, Buffer.alloc(frameBytes)); + sequence += 1; + } + await waitFor(messages, (message) => message.type === "hypothesis" && message.final, 15000, exited, stderr); + send(child, { type: "stop_session", session_id: sessionId }); + await waitFor(messages, (message) => message.type === "session_stopped" && message.session_id === sessionId, 5000, exited, stderr); + send(child, { type: "shutdown" }); + await Promise.race([exited, delay(3000)]); + + const hypotheses = messages.filter((message) => message.type === "hypothesis"); + const selected = hypotheses.findLast((message) => message.final) || hypotheses.at(-1); + if (!selected) throw new Error("The worker did not produce a hypothesis."); + const words = mergeFirstSeen(selected.words || [], hypotheses); + const latencies = words.map((word) => word.latency_ms).filter(Number.isFinite); + const inference = hypotheses.map((message) => message.inference_ms).filter(Number.isFinite); + const result = { + backend: loaded.backend, + audio_ms: Math.round(pcm.length / 32), + revisions: hypotheses.length, + transcript: selected.text, + word_count: words.length, + latency_ms: statistics(latencies), + inference_ms: statistics(inference), + control_tokens_present: words.some((word) => /\[_(?:BEG_|TT_\d+)\]/i.test(word.text)), + transcript_matches_expected: options.expectTranscript === undefined ? null + : comparableTranscript(selected.text) === comparableTranscript(options.expectTranscript) + }; + console.log(JSON.stringify(result, null, 2)); + if (result.control_tokens_present) throw new Error("Whisper control tokens leaked into word analysis."); + if (result.transcript_matches_expected === false) + throw new Error(`Transcript did not match the expected acceptance phrase: ${selected.text}`); + if (Number.isFinite(options.maxLatencyMs) && result.latency_ms.max > options.maxLatencyMs) + throw new Error(`Maximum first-seen word latency ${result.latency_ms.max} ms exceeds ${options.maxLatencyMs} ms.`); +} + +function mergeFirstSeen(selected, hypotheses) { + const chosen = withOccurrences(selected); + const candidates = hypotheses.flatMap((hypothesis) => withOccurrences(Array.isArray(hypothesis.words) ? hypothesis.words : [])); + return chosen.map((word) => { + const matches = candidates.filter((candidate) => candidate.key === word.key && candidate.occurrence === word.occurrence && + Math.abs(Number(candidate.captured_at_ms) - Number(word.captured_at_ms)) <= 1500); + const { key: _key, occurrence: _occurrence, ...plain } = word; + return matches.length ? { ...plain, latency_ms: Math.min(...matches.map((candidate) => Number(candidate.latency_ms))) } : plain; + }).filter((word) => comparable(word.text)); +} +function withOccurrences(words) { + const seen = new Map(); + return words.map((word) => { + const key = comparable(word.text); + const occurrence = seen.get(key) || 0; + seen.set(key, occurrence + 1); + return { ...word, key, occurrence }; + }); +} + +function comparable(value) { return String(value || "").toLocaleLowerCase().replace(/[^\p{L}\p{N}]+/gu, ""); } +function comparableTranscript(value) { return String(value || "").toLocaleLowerCase().match(/[\p{L}\p{N}]+/gu)?.join(" ") || ""; } +function statistics(values) { + if (!values.length) return { count: 0, min: null, average: null, median: null, p99: null, max: null }; + const sorted = [...values].sort((left, right) => left - right); + return { + count: sorted.length, min: sorted[0], + average: sorted.reduce((sum, value) => sum + value, 0) / sorted.length, + median: percentile(sorted, .5), p99: percentile(sorted, .99), max: sorted.at(-1) + }; +} +function percentile(values, ratio) { + const position = (values.length - 1) * ratio; + const lower = Math.floor(position); const upper = Math.ceil(position); + return values[lower] + (values[upper] - values[lower]) * (position - lower); +} +function send(child, metadata, pcm = Buffer.alloc(0)) { + const json = Buffer.from(JSON.stringify(metadata)); + const header = Buffer.alloc(8); + header.writeUInt32LE(json.length, 0); + header.writeUInt32LE(pcm.length, 4); + child.stdin.write(Buffer.concat([header, json, pcm])); +} +async function waitFor(messages, predicate, timeoutMs, exited, stderr) { + const started = Date.now(); + while (Date.now() - started < timeoutMs) { + const match = messages.find(predicate); + if (match) return match; + const exit = await Promise.race([exited, delay(20).then(() => null)]); + if (exit !== null) throw new Error(`Worker exited with code ${exit}.${stderr ? ` ${stderr.trim()}` : ""}`); + } + throw new Error(`Worker response timed out after ${timeoutMs} ms.${stderr ? ` ${stderr.trim()}` : ""}`); +} +function readPcmWave(filename) { + const body = fs.readFileSync(filename); + if (body.toString("ascii", 0, 4) !== "RIFF" || body.toString("ascii", 8, 12) !== "WAVE") throw new Error("Audio input is not a WAV file."); + let offset = 12; + let format = null; + let pcm = null; + while (offset + 8 <= body.length) { + const type = body.toString("ascii", offset, offset + 4); + const size = body.readUInt32LE(offset + 4); + const start = offset + 8; + if (type === "fmt ") format = { + codec: body.readUInt16LE(start), channels: body.readUInt16LE(start + 2), + sampleRate: body.readUInt32LE(start + 4), bits: body.readUInt16LE(start + 14) + }; + if (type === "data") pcm = body.subarray(start, start + size); + offset = start + size + (size % 2); + } + if (!format || format.codec !== 1 || format.channels !== 1 || format.sampleRate !== 16000 || format.bits !== 16 || !pcm) + throw new Error("Audio input must be 16-kHz mono PCM16."); + return pcm; +} +function parseArguments(args) { + const result = {}; + for (let index = 0; index < args.length; index += 2) { + const key = args[index]?.replace(/^--/, "").replace(/-([a-z])/g, (_match, letter) => letter.toUpperCase()); + if (key) result[key] = args[index + 1]; + } + result.maxLatencyMs = result.maxLatencyMs === undefined ? null : Number(result.maxLatencyMs); + return result; +} +function delay(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } diff --git a/plugins/lumi_transcription/scripts/build-worker.ps1 b/plugins/lumi_transcription/scripts/build-worker.ps1 index f602a78..ef4bc49 100644 --- a/plugins/lumi_transcription/scripts/build-worker.ps1 +++ b/plugins/lumi_transcription/scripts/build-worker.ps1 @@ -1,5 +1,8 @@ param( - [switch]$Cuda + [switch]$Cuda, + [string]$CudaToolkitRoot = "", + [switch]$Clean, + [switch]$Package ) $ErrorActionPreference = "Stop" @@ -21,17 +24,55 @@ function Find-Tool([string]$name) { $cmake = Find-Tool "cmake.exe" $ninja = Find-Tool "ninja.exe" +$cudaRoot = if ($Cuda -and $CudaToolkitRoot) { + Get-Item -LiteralPath $CudaToolkitRoot -ErrorAction Stop +} elseif ($Cuda) { + Get-ChildItem "$env:ProgramFiles\NVIDIA GPU Computing Toolkit\CUDA\v*" -Directory -ErrorAction SilentlyContinue | + Where-Object { Test-Path (Join-Path $_.FullName "bin\nvcc.exe") } | + Sort-Object FullName -Descending | + Select-Object -First 1 +} else { $null } +if ($Cuda -and !$cudaRoot) { throw "The CUDA Toolkit was not found. Install it before building the CUDA worker." } +if ($Cuda -and !(Test-Path (Join-Path $cudaRoot.FullName "bin\nvcc.exe"))) { + throw "The selected CUDA Toolkit does not contain bin\\nvcc.exe." +} +$cudaToolRoot = $null +if ($cudaRoot) { + $cudaToolRoot = Join-Path $env:TEMP "lumi-cuda-sdk" + if (Test-Path $cudaToolRoot) { Remove-Item $cudaToolRoot -Force -Recurse } + New-Item -ItemType Junction -Path $cudaToolRoot -Target $cudaRoot.FullName | Out-Null +} $vcvars = Get-ChildItem "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2022\*\VC\Auxiliary\Build\vcvars64.bat" -ErrorAction SilentlyContinue | Select-Object -First 1 if (!$vcvars) { throw "Visual Studio 2022 C++ Build Tools were not found." } +$windowsSdkBin = Get-ChildItem "${env:ProgramFiles(x86)}\Windows Kits\10\bin\*\x64" -Directory -ErrorAction SilentlyContinue | + Where-Object { (Test-Path (Join-Path $_.FullName "rc.exe")) -and (Test-Path (Join-Path $_.FullName "mt.exe")) } | + Sort-Object FullName -Descending | + Select-Object -First 1 +if (!$windowsSdkBin) { throw "The Windows SDK resource and manifest tools were not found." } +$resourceCompiler = (Join-Path $windowsSdkBin.FullName "rc.exe").Replace("\", "/") +$manifestTool = (Join-Path $windowsSdkBin.FullName "mt.exe").Replace("\", "/") +$windowsSdkVersion = $windowsSdkBin.Parent.Name +$windowsSdkRoot = Join-Path "${env:ProgramFiles(x86)}" "Windows Kits\10" +$windowsSdkInclude = Join-Path $windowsSdkRoot "Include\$windowsSdkVersion" +$windowsSdkLib = Join-Path $windowsSdkRoot "Lib\$windowsSdkVersion" +if ($Clean -and (Test-Path $buildRoot)) { Remove-Item $buildRoot -Force -Recurse } New-Item -ItemType Directory -Force -Path $buildRoot, $targetRoot | Out-Null $cudaValue = if ($Cuda) { "ON" } else { "OFF" } +$cudaArgument = if ($cudaRoot) { + '-DCUDAToolkit_ROOT="{0}" -DCMAKE_CUDA_COMPILER="{1}"' -f $cudaToolRoot.Replace("\", "/"), (Join-Path $cudaToolRoot "bin\nvcc.exe").Replace("\", "/") +} else { "" } $buildScript = Join-Path $buildRoot "build-worker.cmd" @( "@echo off", ('call "{0}" >nul' -f $vcvars.FullName), - ('"{0}" -S "{1}" -B "{2}" -G Ninja -DLUMI_WHISPER_CUDA={3} -DCMAKE_BUILD_TYPE=Release -DCMAKE_MAKE_PROGRAM="{4}"' -f $cmake, $sourceRoot, $buildRoot, $cudaValue, $ninja), + 'set "PATHEXT=.COM;.EXE;.BAT;.CMD"', + ('set "PATH={0};%PATH%"' -f $windowsSdkBin.FullName), + $(if ($cudaRoot) { 'set "PATH={0};%PATH%"' -f (Join-Path $cudaToolRoot "bin") } else { "rem CUDA is disabled" }), + ('set "INCLUDE=%INCLUDE%;{0};{1};{2};{3}"' -f (Join-Path $windowsSdkInclude "ucrt"), (Join-Path $windowsSdkInclude "shared"), (Join-Path $windowsSdkInclude "um"), (Join-Path $windowsSdkInclude "winrt")), + ('set "LIB=%LIB%;{0};{1}"' -f (Join-Path $windowsSdkLib "ucrt\x64"), (Join-Path $windowsSdkLib "um\x64")), + ('"{0}" -S "{1}" -B "{2}" -G Ninja -DLUMI_WHISPER_CUDA={3} -DCMAKE_BUILD_TYPE=Release -DCMAKE_MAKE_PROGRAM="{4}" -DCMAKE_RC_COMPILER="{5}" -DCMAKE_MT="{6}" {7}' -f $cmake, $sourceRoot, $buildRoot, $cudaValue, $ninja, $resourceCompiler, $manifestTool, $cudaArgument), "if errorlevel 1 exit /b %errorlevel%", ('"{0}" --build "{1}" --config Release' -f $cmake, $buildRoot), "exit /b %errorlevel%" @@ -43,4 +84,25 @@ if ($process.ExitCode -ne 0) { throw "Worker build failed with exit code $($proc $worker = Join-Path $buildRoot "lumi-whisper-worker.exe" if (!(Test-Path $worker)) { throw "The build completed without producing lumi-whisper-worker.exe." } Copy-Item -Force $worker (Join-Path $targetRoot "lumi-whisper-worker.exe") +if ($Cuda) { + Get-ChildItem $targetRoot -Filter "*.dll" -ErrorAction SilentlyContinue | Remove-Item -Force + foreach ($pattern in @("cudart64_*.dll", "cublas64_*.dll", "cublasLt64_*.dll")) { + $dependency = Get-ChildItem (Join-Path $cudaRoot.FullName "bin") -Filter $pattern -File | + Sort-Object Name | + Select-Object -First 1 + if (!$dependency) { throw "The selected CUDA Toolkit is missing $pattern." } + Copy-Item -Force $dependency.FullName $targetRoot + } +} Write-Host "Installed Lumi whisper worker ($backend) in $targetRoot" +if ($Package) { + $archive = Join-Path $pluginRoot "data\tmp\lumi-whisper-worker-windows-x64-$backend.zip" + New-Item -ItemType Directory -Force -Path (Split-Path -Parent $archive) | Out-Null + Remove-Item $archive -Force -ErrorAction SilentlyContinue + Compress-Archive -Path $targetRoot -DestinationPath $archive -CompressionLevel Optimal + $file = Get-Item $archive + $sha = (Get-FileHash $archive -Algorithm SHA256).Hash.ToLowerInvariant() + Write-Host "Runtime package: $archive" + Write-Host "Runtime bytes: $($file.Length)" + Write-Host "Runtime SHA256: $sha" +} diff --git a/plugins/lumi_transcription/tests/verify.js b/plugins/lumi_transcription/tests/verify.js index 63ba710..c3c382e 100644 --- a/plugins/lumi_transcription/tests/verify.js +++ b/plugins/lumi_transcription/tests/verify.js @@ -40,6 +40,7 @@ async function run() { verifyBenchmarkRetention(); verifyAdminDeviceRevocationUx(); await verifyBenchmarkStartRollback(); + await verifyManualBenchmarkFinalization(); await verifySessionLifecycle(); await verifyProviderFailureFeedback(); await verifyWorkerRestart(); @@ -192,16 +193,26 @@ function verifyBenchmarkRetention() { const source = crypto.randomUUID(); const id = store.start({ sessionId: "benchmark-session", deviceId: "device", source: { source_uuid: source, display_name: "Mic" } }); store.record("benchmark-session", { - caption_id: "caption", revision: 1, final: true, stable_text: "hello world", + caption_id: "caption", revision: 1, final: false, stable_text: "hello world", analysis: { transcript: "hello world", words: [ - { text: "hello", latency_ms: 700, confidence: .96, audio_start_ms: 0, audio_end_ms: 300 }, - { text: "world", latency_ms: 1400, confidence: .82, audio_start_ms: 300, audio_end_ms: 700 } + { text: "[_BEG_]", latency_ms: 400, confidence: .2, captured_at_ms: 1000 }, + { text: "hello", latency_ms: 500, confidence: .7, audio_start_ms: 0, audio_end_ms: 300, captured_at_ms: 1000 }, + { text: "world[_TT_300]", latency_ms: 900, confidence: .6, audio_start_ms: 300, audio_end_ms: 700, captured_at_ms: 1300 } + ] }, latency: { inference_ms: 400 }, model: { id: "small.en", backend: "cpu" } + }); + store.record("benchmark-session", { + caption_id: "caption", revision: 2, final: true, stable_text: "hello world", + analysis: { transcript: "hello world", words: [ + { text: "hello", latency_ms: 700, confidence: .96, audio_start_ms: 0, audio_end_ms: 300, captured_at_ms: 1000 }, + { text: "world", latency_ms: 1400, confidence: .82, audio_start_ms: 300, audio_end_ms: 700, captured_at_ms: 1300 } ] }, latency: { inference_ms: 500 }, model: { id: "small.en", backend: "cpu" } }); const result = store.finish("benchmark-session", "completed"); assert.equal(result.id, id); assert.equal(result.words.length, 2); - assert.equal(result.stats.latency.average, 1050); + assert.deepEqual(result.words.map((word) => word.text), ["hello", "world"]); + assert.equal(result.stats.latency.average, 700); + assert.ok(Math.abs(result.stats.confidence.average - .89) < 1e-9); assert.equal(metricStats([1, 2, 3]).median, 2); const secondId = store.start({ sessionId: "benchmark-session", deviceId: "device", source: { source_uuid: source, display_name: "Mic" } }); assert.notEqual(secondId, id); @@ -227,6 +238,47 @@ function verifyBenchmarkRetention() { legacy.close(); } +async function verifyManualBenchmarkFinalization() { + class Provider extends EventEmitter { + async health() { return { healthy: true, model_ready: true, state: "running" }; } + async startSession() {} + async addTrack() {} + async stopSession(sessionId) { + this.emit("hypothesis", { + session_id: sessionId, track_id: source, text: "manual ending", final: true, new_utterance: true, + words: [{ text: "manual", latency_ms: 600, confidence: .94, captured_at_ms: 1000 }], + model_id: "small.en", backend: "cpu" + }); + await new Promise((resolve) => setImmediate(resolve)); + } + } + const provider = new Provider(); + const source = crypto.randomUUID(); + let deliveryStopped = false; + let finalRecorded = false; + const sent = []; + const benchmarks = { + start: () => "benchmark-id", + record: (_sessionId, event) => { finalRecorded = event.final && !deliveryStopped; }, + finish: () => ({ id: "benchmark-id", status: "completed", transcript: "manual ending", stats: { latency: metricStats([600]), confidence: metricStats([.94]) }, words: [{ text: "manual", latency_ms: 600, confidence: .94, final: true }] }) + }; + const coordinator = new SessionCoordinator({ + provider, benchmarks, + deliveryFactory: () => ({ + start: async () => {}, stop: async () => { deliveryStopped = true; }, pause: async () => {}, resume: async () => {}, + deliver: async () => ({ disposition: "simulated" }) + }) + }); + const { session } = coordinator.create({ id: "device" }, (type, payload) => sent.push({ type, payload })); + coordinator.updateSource(session.id, { source_uuid: source, display_name: "Mic", primary: true, program_active: true }); + await coordinator.start(session.id, { mode: "benchmark" }); + await coordinator.stop(session.id, "benchmark_complete"); + assert.equal(finalRecorded, true); + assert.equal(sent.at(-1).type, "benchmark_complete"); + assert.equal(sent.at(-1).payload.stats.confidence.average, .94); + await coordinator.close(); +} + async function verifyBenchmarkStartRollback() { class Provider extends EventEmitter { constructor() { super(); this.starts = 0; } @@ -341,9 +393,11 @@ async function verifyNativeWorkerBoundary() { const cmake = fs.readFileSync(path.join(sourceRoot, "CMakeLists.txt"), "utf8"); const source = fs.readFileSync(path.join(sourceRoot, "src/main.cpp"), "utf8"); assert.match(cmake, /f049fff95a089aa9969deb009cdd4892b3e74916/); - assert.match(source, /max_samples = sample_rate \* 6/); + assert.match(source, /max_samples = sample_rate \* 15/); assert.match(source, /_setmode\(_fileno\(stdin\), _O_BINARY\)/); assert.match(source, /token_timestamps = true/); + assert.match(source, /last_decode_at/); + assert.match(source, /data\.id >= whisper_token_eot/); assert.match(source, /session_stopped/); const bridge = fs.readFileSync(path.join(__dirname, "../../../companion/native/obs-bridge/src/plugin.cpp"), "utf8"); assert.match(bridge, /selection_state/); @@ -415,6 +469,19 @@ function verifyArtifactsAndLogs(temp) { const file = path.join(artifactRoot, "model.bin"); fs.writeFileSync(file, "verified"); const entry = { id: "model", filename: "model.bin", url: "https://example.invalid/model.bin", sha256: sha256File(file) }; assert.equal(new ArtifactManager(artifactRoot).status(entry).valid, true); + const runtimeZip = path.join(temp, "runtime.zip"); + const archive = new (require("adm-zip"))(); + archive.addFile("bin/lumi-whisper-worker.exe", Buffer.from("portable-worker")); + archive.writeZip(runtimeZip); + const runtimeEntry = { + id: "windows-x64-cpu", url: "https://example.invalid/runtime.zip", sha256: sha256File(runtimeZip), + expected_paths: ["lumi-whisper-worker.exe"], backend: "cpu" + }; + const runtimeRoot = path.join(temp, "runtime"); + const manager = new ArtifactManager(runtimeRoot); + manager.installZip(runtimeEntry, runtimeZip); + manager.installZip(runtimeEntry, runtimeZip); + assert.equal(fs.readFileSync(path.join(runtimeRoot, runtimeEntry.id, "bin", "lumi-whisper-worker.exe"), "utf8"), "portable-worker"); assert.equal(sanitize({ pcm: Buffer.alloc(10), device_secret: "secret", stable_text: "hello" }, false).pcm, "[redacted]"); const logsRoot = path.join(temp, "logs"); const logs = new JsonlDiagnosticLog(logsRoot, { retentionDays: 1, maxBytes: 100 }); diff --git a/plugins/lumi_transcription/views/settings.ejs b/plugins/lumi_transcription/views/settings.ejs index 466fecb..fe19dd0 100644 --- a/plugins/lumi_transcription/views/settings.ejs +++ b/plugins/lumi_transcription/views/settings.ejs @@ -45,6 +45,14 @@ <% if (!providerHealth.healthy) { const workerReason = providerHealth.last_error?.message || (providerHealth.last_exit ? `Last exit code: ${providerHealth.last_exit.code ?? 'unknown'}` : null) || providerHealth.last_diagnostic?.message || 'No worker failure detail has been reported yet.'; %>
Speech recognition needs attention

<%= workerReason %> Reload the verified model below, then rerun the Companion test.

<% } %> +
+ <% runtimeManifest.artifacts.forEach((artifact) => { %> + + <% }) %> +
+

The low-latency package targets <%= runtimeManifest.artifacts.find((artifact) => artifact.backend === 'cuda')?.minimum_gpu || 'a supported NVIDIA GPU' %> and newer. Runtime changes activate immediately when no transcription session is active.

Shared GPU awareness

Lumi warns and benchmarks when Lumi AI already occupies GPU memory. It never unloads AI models automatically.