From 134d3efa74f085d9a031887de16bfbb2e91976cc Mon Sep 17 00:00:00 2001 From: Franz Rolfsvaag Date: Wed, 22 Jul 2026 11:01:49 +0200 Subject: [PATCH] Add Lumi transcription server and protocol foundation --- .gitignore | 5 + package.json | 3 +- .../backend/companion/device_store.js | 90 +++++++ .../backend/companion/gateway.js | 102 ++++++++ .../backend/companion/protocol.js | 110 ++++++++ .../backend/config/revision_store.js | 52 ++++ .../backend/delivery/caption_delivery.js | 28 ++ .../backend/logs/jsonl_log.js | 55 ++++ .../backend/models/artifact_manager.js | 59 +++++ plugins/lumi_transcription/backend/paths.js | 7 + .../backend/sessions/bounded_queue.js | 77 ++++++ .../backend/sessions/session_coordinator.js | 152 +++++++++++ .../backend/transcription/provider.js | 122 +++++++++ .../backend/transcription/stabilizer.js | 67 +++++ plugins/lumi_transcription/data/.gitkeep | 1 + plugins/lumi_transcription/index.js | 134 ++++++++++ .../lumi_transcription/models_manifest.json | 33 +++ plugins/lumi_transcription/plugin.json | 10 + .../public/transcription.css | 1 + .../public/transcription.js | 22 ++ .../lumi_transcription/runtime_manifest.json | 30 +++ plugins/lumi_transcription/tests/verify.js | 241 ++++++++++++++++++ plugins/lumi_transcription/views/settings.ejs | 59 +++++ protocol/companion-protocol-v1.md | 50 ++++ protocol/schemas/caption.schema.json | 22 ++ protocol/schemas/envelope.schema.json | 15 ++ protocol/schemas/hello.schema.json | 35 +++ scripts/verify-all.js | 1 + src/main.js | 2 + src/services/web-upgrades.js | 62 +++++ src/web/server.js | 4 + 31 files changed, 1650 insertions(+), 1 deletion(-) create mode 100644 plugins/lumi_transcription/backend/companion/device_store.js create mode 100644 plugins/lumi_transcription/backend/companion/gateway.js create mode 100644 plugins/lumi_transcription/backend/companion/protocol.js create mode 100644 plugins/lumi_transcription/backend/config/revision_store.js create mode 100644 plugins/lumi_transcription/backend/delivery/caption_delivery.js create mode 100644 plugins/lumi_transcription/backend/logs/jsonl_log.js create mode 100644 plugins/lumi_transcription/backend/models/artifact_manager.js create mode 100644 plugins/lumi_transcription/backend/paths.js create mode 100644 plugins/lumi_transcription/backend/sessions/bounded_queue.js create mode 100644 plugins/lumi_transcription/backend/sessions/session_coordinator.js create mode 100644 plugins/lumi_transcription/backend/transcription/provider.js create mode 100644 plugins/lumi_transcription/backend/transcription/stabilizer.js create mode 100644 plugins/lumi_transcription/data/.gitkeep create mode 100644 plugins/lumi_transcription/index.js create mode 100644 plugins/lumi_transcription/models_manifest.json create mode 100644 plugins/lumi_transcription/plugin.json create mode 100644 plugins/lumi_transcription/public/transcription.css create mode 100644 plugins/lumi_transcription/public/transcription.js create mode 100644 plugins/lumi_transcription/runtime_manifest.json create mode 100644 plugins/lumi_transcription/tests/verify.js create mode 100644 plugins/lumi_transcription/views/settings.ejs create mode 100644 protocol/companion-protocol-v1.md create mode 100644 protocol/schemas/caption.schema.json create mode 100644 protocol/schemas/envelope.schema.json create mode 100644 protocol/schemas/hello.schema.json create mode 100644 src/services/web-upgrades.js diff --git a/.gitignore b/.gitignore index 68a7e6d..cfb2a77 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,11 @@ plugins/*/data/** *.sqlite-* npm-debug.log /dist/ +companion/**/bin/ +companion/**/obj/ +companion/installer/output/ +companion/**/logs/ +*.lumi-pairing.json security-audit-*.json security-audit-*.md taskfile.txt diff --git a/package.json b/package.json index 8847d07..89cc00d 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,8 @@ "test:ui:update": "playwright test --update-snapshots", "verify:webui": "node scripts/verify-webui.js && node scripts/verify-destructive-actions.js", "benchmark:okf": "node scripts/benchmark-okf-search.js", - "verify:content": "node scripts/verify-content-library.js" + "verify:content": "node scripts/verify-content-library.js", + "verify:transcription": "node plugins/lumi_transcription/tests/verify.js" }, "engines": { "node": ">=18" diff --git a/plugins/lumi_transcription/backend/companion/device_store.js b/plugins/lumi_transcription/backend/companion/device_store.js new file mode 100644 index 0000000..411c5cb --- /dev/null +++ b/plugins/lumi_transcription/backend/companion/device_store.js @@ -0,0 +1,90 @@ +const crypto = require("crypto"); + +const DEFAULT_CAPABILITIES = Object.freeze(["transcription.capture.v1", "transcription.settings.v1", "obs.caption.native.v1"]); + +class DeviceStore { + constructor(db, options = {}) { + this.db = db; + this.now = options.now || Date.now; + this.randomBytes = options.randomBytes || crypto.randomBytes; + this.allowInsecure = options.allowInsecure === true; + this.migrate(); + } + + migrate() { + this.db.exec(` + CREATE TABLE IF NOT EXISTS transcription_pairing_tokens ( + id TEXT PRIMARY KEY, token_hash TEXT NOT NULL UNIQUE, lumi_user_id TEXT NOT NULL, + host TEXT NOT NULL, expires_at INTEGER NOT NULL, activated_at INTEGER, created_at INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS transcription_devices ( + id TEXT PRIMARY KEY, install_id TEXT, name TEXT NOT NULL, lumi_user_id TEXT NOT NULL, + credential_hash TEXT NOT NULL, capabilities_json TEXT NOT NULL, metadata_json TEXT NOT NULL, + first_connected_at INTEGER NOT NULL, last_connected_at INTEGER NOT NULL, revoked_at INTEGER + ); + CREATE INDEX IF NOT EXISTS transcription_devices_user_idx ON transcription_devices(lumi_user_id); + `); + } + + issuePairing({ userId, host, ttlMs = 15 * 60 * 1000 }) { + if (!userId || !host) throw new Error("A Lumi user and host are required."); + const id = crypto.randomUUID(); + const token = tokenValue(this.randomBytes(32)); + const now = this.now(); + this.db.prepare("INSERT INTO transcription_pairing_tokens (id, token_hash, lumi_user_id, host, expires_at, activated_at, created_at) VALUES (?, ?, ?, ?, ?, NULL, ?)") + .run(id, digest(token), String(userId), normalizeHost(host, this.allowInsecure), now + ttlMs, now); + return { pairing_id: id, token, host: normalizeHost(host, this.allowInsecure), expires_at: now + ttlMs, protocol_version: 1 }; + } + + exchange({ token, device = {} }) { + const hash = digest(token); + const row = this.db.prepare("SELECT * FROM transcription_pairing_tokens WHERE token_hash = ?").get(hash); + if (!row || row.activated_at || row.expires_at < this.now()) { + const error = new Error(row?.activated_at ? "This pairing package was already activated. Download a new companion package." : "This pairing package is invalid or expired. Download a new companion package."); + error.code = row?.activated_at ? "PAIRING_ALREADY_USED" : "PAIRING_INVALID"; + throw error; + } + const deviceId = crypto.randomUUID(); + const secret = tokenValue(this.randomBytes(32)); + const now = this.now(); + const capabilities = DEFAULT_CAPABILITIES.slice(); + const transaction = this.db.transaction(() => { + const consumed = this.db.prepare("UPDATE transcription_pairing_tokens SET activated_at = ? WHERE id = ? AND activated_at IS NULL").run(now, row.id); + if (consumed.changes !== 1) { const error = new Error("This pairing package was already activated. Download a new companion package."); error.code = "PAIRING_ALREADY_USED"; throw error; } + this.db.prepare("INSERT INTO transcription_devices (id, install_id, name, lumi_user_id, credential_hash, capabilities_json, metadata_json, first_connected_at, last_connected_at, revoked_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL)") + .run(deviceId, clean(device.install_id, 128) || null, clean(device.name, 160) || "Lumi Companion", row.lumi_user_id, digest(secret), JSON.stringify(capabilities), JSON.stringify(safeMetadata(device)), now, now); + }); + transaction(); + return { device_id: deviceId, device_secret: secret, host: row.host, capabilities, protocol_version: 1 }; + } + + authenticate(header, requiredCapability = null) { + const match = /^LumiDevice\s+([0-9a-f-]{36})\.([A-Za-z0-9_-]{40,})$/i.exec(String(header || "")); + if (!match) return { allowed: false, reason: "missing_credentials" }; + const row = this.db.prepare("SELECT * FROM transcription_devices WHERE id = ?").get(match[1]); + if (!row || row.revoked_at || !safeEqual(row.credential_hash, digest(match[2]))) return { allowed: false, reason: row?.revoked_at ? "device_revoked" : "invalid_credentials" }; + const capabilities = parseArray(row.capabilities_json); + if (requiredCapability && !capabilities.includes(requiredCapability)) return { allowed: false, reason: "capability_revoked" }; + this.db.prepare("UPDATE transcription_devices SET last_connected_at = ? WHERE id = ?").run(this.now(), row.id); + return { allowed: true, device: serialize(row, capabilities) }; + } + + list() { return this.db.prepare("SELECT * FROM transcription_devices ORDER BY last_connected_at DESC").all().map((row) => serialize(row, parseArray(row.capabilities_json))); } + revoke(deviceId) { return this.db.prepare("UPDATE transcription_devices SET revoked_at = ? WHERE id = ? AND revoked_at IS NULL").run(this.now(), deviceId).changes === 1; } + setCapabilities(deviceId, capabilities) { + const allowed = DEFAULT_CAPABILITIES.filter((capability) => new Set(capabilities || []).has(capability)); + const changed = this.db.prepare("UPDATE transcription_devices SET capabilities_json = ? WHERE id = ? AND revoked_at IS NULL").run(JSON.stringify(allowed), deviceId).changes; + return changed ? allowed : null; + } +} + +function digest(value) { return crypto.createHash("sha256").update(String(value || ""), "utf8").digest("hex"); } +function safeEqual(left, right) { const a = Buffer.from(String(left)); const b = Buffer.from(String(right)); return a.length === b.length && crypto.timingSafeEqual(a, b); } +function tokenValue(buffer) { return Buffer.from(buffer).toString("base64url"); } +function normalizeHost(value, allowInsecure = false) { const url = new URL(String(value)); if (url.protocol !== "https:" && !(allowInsecure && url.protocol === "http:" && ["localhost", "127.0.0.1", "::1"].includes(url.hostname))) throw new Error("Lumi Companion requires HTTPS. Insecure HTTP is allowed only for explicit localhost development."); return url.origin; } +function clean(value, max) { return String(value || "").trim().slice(0, max); } +function parseArray(value) { try { const parsed = JSON.parse(value); return Array.isArray(parsed) ? parsed : []; } catch { return []; } } +function safeMetadata(device) { return { companion_version: clean(device.companion_version, 32), obs_version: clean(device.obs_version, 32), os: clean(device.os, 120), architecture: clean(device.architecture, 32), hardware: clean(device.hardware, 500) }; } +function serialize(row, capabilities) { return { id: row.id, install_id: row.install_id, name: row.name, lumi_user_id: row.lumi_user_id, capabilities, metadata: JSON.parse(row.metadata_json || "{}"), first_connected_at: row.first_connected_at, last_connected_at: row.last_connected_at, revoked_at: row.revoked_at }; } + +module.exports = { DeviceStore, DEFAULT_CAPABILITIES, digest }; diff --git a/plugins/lumi_transcription/backend/companion/gateway.js b/plugins/lumi_transcription/backend/companion/gateway.js new file mode 100644 index 0000000..832bc3d --- /dev/null +++ b/plugins/lumi_transcription/backend/companion/gateway.js @@ -0,0 +1,102 @@ +const { WebSocketServer, WebSocket } = require("ws"); +const { MAX_JSON_BYTES, MAX_AUDIO_PAYLOAD_BYTES, AUDIO_HEADER_BYTES, parseEnvelope, validateHello, parseAudioFrame, envelope } = require("./protocol"); + +class CompanionGateway { + constructor(options) { + this.devices = options.devices; + this.sessions = options.sessions; + this.log = options.log || { append() {} }; + this.allowInsecure = options.allowInsecure === true; + this.wss = new WebSocketServer({ noServer: true, maxPayload: Math.max(MAX_JSON_BYTES, AUDIO_HEADER_BYTES + MAX_AUDIO_PAYLOAD_BYTES), perMessageDeflate: false, clientTracking: true }); + this.wss.on("connection", (socket, request, device) => this.connection(socket, request, device)); + } + upgrade(request, socket, head) { + const forwardedProto = String(request.headers["x-forwarded-proto"] || "").split(",")[0].trim(); + const secure = Boolean(request.socket.encrypted) || forwardedProto === "https"; + const local = ["127.0.0.1", "::1", "::ffff:127.0.0.1"].includes(request.socket.remoteAddress); + if (!secure && !(this.allowInsecure && local)) return reject(socket, 426, "tls_required"); + const auth = this.devices.authenticate(request.headers.authorization, "transcription.capture.v1"); + if (!auth.allowed) return reject(socket, auth.reason === "capability_revoked" ? 403 : 401, auth.reason); + const origin = request.headers.origin; + if (origin && !sameHostOrigin(origin, request.headers.host)) return reject(socket, 403, "origin_rejected"); + this.wss.handleUpgrade(request, socket, head, (ws) => this.wss.emit("connection", ws, request, auth.device)); + } + connection(socket, _request, device) { + let session = null; + let helloComplete = false; + let lastPong = Date.now(); + let windowStarted = Date.now(); + let messagesInWindow = 0; + const send = (type, payload, sessionId = session?.id || null) => { + if (socket.readyState === WebSocket.OPEN) socket.send(JSON.stringify(envelope(type, payload, sessionId))); + }; + const helloTimer = setTimeout(() => closeWith(socket, 4408, "hello_timeout"), 5000); + const heartbeat = setInterval(() => { + if (Date.now() - lastPong > 30000) return closeWith(socket, 4408, "heartbeat_timeout"); + send("status", { kind: "heartbeat", state: session?.state || "connecting" }); + }, 10000); + helloTimer.unref?.(); heartbeat.unref?.(); + socket.on("message", async (data, isBinary) => { + try { + if (Date.now() - windowStarted >= 1000) { windowStarted = Date.now(); messagesInWindow = 0; } + messagesInWindow += 1; + if (messagesInWindow > 300) throw coded("RATE_LIMIT", "Companion message rate exceeded its limit."); + if (isBinary) { + if (!helloComplete || !session) throw coded("HELLO_REQUIRED", "Complete the handshake before sending audio."); + const result = await this.sessions.audio(session.id, parseAudioFrame(data)); + if (result.gap) send("metric", { kind: "sequence_gap", missing_frames: result.gap }); + return; + } + const message = parseEnvelope(data); + if (!helloComplete) { + const hello = validateHello(message); + const created = this.sessions.create(device, send, hello.resume_session_id); + session = created.session; + helloComplete = true; + clearTimeout(helloTimer); + send("hello_ack", { + protocol_version: 1, resumed: created.resumed, + server_plugin_version: require("../../plugin.json").version, + capabilities: ["transcription.server.v1", "settings.revision.v1", "pcm_s16le"], + heartbeat_ms: 10000, recovery_window_ms: 5000 + }); + this.log.append({ kind: "connection", state: "authenticated", device_id: device.id, session_id: session.id }); + return; + } + await this.structured(session, message, send, () => { lastPong = Date.now(); }); + } catch (error) { + send("error", { code: error.code || "INVALID_MESSAGE", message: error.message, recoverable: !["INCOMPATIBLE_VERSION", "HELLO_REQUIRED"].includes(error.code) }); + this.log.append({ kind: "protocol_error", device_id: device.id, session_id: session?.id, code: error.code || "INVALID_MESSAGE", message: error.message }); + if (["INCOMPATIBLE_VERSION", "HELLO_REQUIRED", "RATE_LIMIT"].includes(error.code)) closeWith(socket, 4400, error.code); + } + }); + socket.on("close", () => { + clearTimeout(helloTimer); clearInterval(heartbeat); + if (session) this.sessions.disconnect(session.id); + this.log.append({ kind: "connection", state: "closed", device_id: device.id, session_id: session?.id }); + }); + socket.on("error", (error) => this.log.append({ kind: "connection", state: "error", device_id: device.id, session_id: session?.id, message: error.message })); + } + async structured(session, message, send, pong) { + switch (message.type) { + case "ping": pong(); send("pong", { received_id: message.id }); break; + case "source_update": send("status", { kind: "source", source: this.sessions.updateSource(session.id, message.payload || {}) }); break; + case "obs_state": send("status", { kind: "obs", ...(await this.sessions.updateObsState(session.id, message.payload || {})) }); break; + case "start": send("status", { kind: "session", ...(await this.sessions.start(session.id, message.payload || {})) }); break; + case "stop": send("status", { kind: "session", ...(await this.sessions.stop(session.id)) }); break; + case "ack": break; + default: throw coded("UNEXPECTED_MESSAGE", `Message ${message.type} is not valid after the handshake.`); + } + } + async close() { + for (const client of this.wss.clients) closeWith(client, 1001, "plugin_shutdown"); + await new Promise((resolve) => this.wss.close(resolve)); + } +} + +function reject(socket, status, reason) { const labels = { 401: "Unauthorized", 403: "Forbidden", 426: "Upgrade Required" }; socket.write(`HTTP/1.1 ${status} ${labels[status] || "Rejected"}\r\nConnection: close\r\nContent-Type: application/json\r\n\r\n${JSON.stringify({ error: reason })}`); socket.destroy(); } +function closeWith(socket, code, reason) { if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) socket.close(code, String(reason).slice(0, 120)); } +function sameHostOrigin(origin, host) { try { return new URL(origin).host === host; } catch { return false; } } +function coded(code, message) { return Object.assign(new Error(message), { code }); } + +module.exports = { CompanionGateway, sameHostOrigin }; diff --git a/plugins/lumi_transcription/backend/companion/protocol.js b/plugins/lumi_transcription/backend/companion/protocol.js new file mode 100644 index 0000000..1c1bc65 --- /dev/null +++ b/plugins/lumi_transcription/backend/companion/protocol.js @@ -0,0 +1,110 @@ +const crypto = require("crypto"); + +const PROTOCOL_VERSION = 1; +const AUDIO_MAGIC = Buffer.from("LACP"); +const AUDIO_HEADER_BYTES = 64; +const MAX_JSON_BYTES = 64 * 1024; +const MAX_AUDIO_PAYLOAD_BYTES = 6400; +const CLIENT_TYPES = new Set(["hello", "ping", "source_update", "obs_state", "start", "stop", "ack"]); + +function parseEnvelope(input) { + const bytes = Buffer.isBuffer(input) ? input : Buffer.from(String(input || ""), "utf8"); + if (!bytes.length || bytes.length > MAX_JSON_BYTES) throw protocolError("MESSAGE_SIZE", "Structured message size is invalid."); + let value; + try { value = JSON.parse(bytes.toString("utf8")); } + catch { throw protocolError("INVALID_JSON", "Structured message is not valid JSON."); } + if (!value || Array.isArray(value) || typeof value !== "object") throw protocolError("INVALID_ENVELOPE", "Message envelope must be an object."); + if (value.version !== PROTOCOL_VERSION) throw protocolError("INCOMPATIBLE_VERSION", `Protocol version ${value.version} is unsupported.`); + if (!CLIENT_TYPES.has(value.type)) throw protocolError("UNKNOWN_TYPE", "Message type is not allowed."); + if (!isUuid(value.id)) throw protocolError("INVALID_ID", "Message id must be a UUID."); + if (!value.sent_at || !Number.isFinite(Date.parse(value.sent_at))) throw protocolError("INVALID_TIME", "Message sent_at must be an ISO timestamp."); + if (value.session_id != null && !isUuid(value.session_id)) throw protocolError("INVALID_SESSION", "Session id must be a UUID."); + if (value.payload != null && (Array.isArray(value.payload) || typeof value.payload !== "object")) throw protocolError("INVALID_PAYLOAD", "Message payload must be an object."); + return value; +} + +function validateHello(envelope) { + if (envelope.type !== "hello") throw protocolError("HELLO_REQUIRED", "The first message must be hello."); + const payload = envelope.payload || {}; + if (!shortVersion(payload.companion_version) || !shortVersion(payload.plugin_version)) throw protocolError("INVALID_HELLO", "Companion and plugin versions are required."); + if (!Array.isArray(payload.capabilities) || payload.capabilities.length > 32) throw protocolError("INVALID_CAPABILITIES", "Capabilities are invalid."); + if (!payload.capabilities.includes("transcription.capture.v1")) throw protocolError("MISSING_CAPABILITY", "The transcription capture capability is required."); + const audio = payload.audio || {}; + if (audio.codec !== "pcm_s16le" || audio.sample_rate !== 16000 || audio.channels !== 1 || audio.bits !== 16) { + throw protocolError("UNSUPPORTED_AUDIO", "Protocol v1 requires 16 kHz mono signed 16-bit PCM."); + } + if (payload.resume_session_id != null && !isUuid(payload.resume_session_id)) throw protocolError("INVALID_RESUME", "Resume session id is invalid."); + return payload; +} + +function parseAudioFrame(input) { + const frame = Buffer.from(input || []); + if (frame.length < AUDIO_HEADER_BYTES || frame.length > AUDIO_HEADER_BYTES + MAX_AUDIO_PAYLOAD_BYTES) { + throw protocolError("AUDIO_SIZE", "Audio frame size is invalid."); + } + if (!frame.subarray(0, 4).equals(AUDIO_MAGIC)) throw protocolError("AUDIO_MAGIC", "Audio frame magic is invalid."); + const version = frame.readUInt8(4); + const flags = frame.readUInt8(5); + const headerBytes = frame.readUInt16LE(6); + const payloadBytes = frame.readUInt32LE(60); + if (version !== PROTOCOL_VERSION || headerBytes !== AUDIO_HEADER_BYTES || frame.length !== headerBytes + payloadBytes) { + throw protocolError("AUDIO_HEADER", "Audio frame header is invalid."); + } + if (payloadBytes > MAX_AUDIO_PAYLOAD_BYTES || payloadBytes % 2 !== 0) throw protocolError("AUDIO_PAYLOAD", "PCM payload length is invalid."); + if (frame.readUInt32LE(52) !== 16000 || frame.readUInt16LE(56) !== 1 || frame.readUInt16LE(58) !== 16) { + throw protocolError("UNSUPPORTED_AUDIO", "Audio format is unsupported."); + } + return { + version, + active: Boolean(flags & 1), + muted: Boolean(flags & 2), + sequence: frame.readUInt32LE(8), + capture_timestamp_us: Number(frame.readBigUInt64LE(12)), + session_id: bytesToUuid(frame.subarray(20, 36)), + source_uuid: bytesToUuid(frame.subarray(36, 52)), + sample_rate: 16000, + channels: 1, + bits: 16, + pcm: frame.subarray(AUDIO_HEADER_BYTES) + }; +} + +function encodeAudioFrame(value) { + const pcm = Buffer.from(value.pcm || []); + if (pcm.length > MAX_AUDIO_PAYLOAD_BYTES || pcm.length % 2) throw protocolError("AUDIO_PAYLOAD", "PCM payload length is invalid."); + const frame = Buffer.alloc(AUDIO_HEADER_BYTES + pcm.length); + AUDIO_MAGIC.copy(frame, 0); + frame.writeUInt8(PROTOCOL_VERSION, 4); + frame.writeUInt8((value.active === false ? 0 : 1) | (value.muted ? 2 : 0), 5); + frame.writeUInt16LE(AUDIO_HEADER_BYTES, 6); + frame.writeUInt32LE(Number(value.sequence) >>> 0, 8); + frame.writeBigUInt64LE(BigInt(value.capture_timestamp_us || 0), 12); + uuidToBytes(value.session_id).copy(frame, 20); + uuidToBytes(value.source_uuid).copy(frame, 36); + frame.writeUInt32LE(16000, 52); + frame.writeUInt16LE(1, 56); + frame.writeUInt16LE(16, 58); + frame.writeUInt32LE(pcm.length, 60); + pcm.copy(frame, AUDIO_HEADER_BYTES); + return frame; +} + +function envelope(type, payload = {}, sessionId = null) { + return { version: 1, type, id: crypto.randomUUID(), sent_at: new Date().toISOString(), session_id: sessionId, payload }; +} + +function protocolError(code, message) { + const error = new Error(message); + error.code = code; + return error; +} + +function isUuid(value) { + return /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(String(value || "")); +} + +function shortVersion(value) { return typeof value === "string" && value.length > 0 && value.length <= 32; } +function uuidToBytes(value) { if (!isUuid(value)) throw protocolError("INVALID_UUID", "UUID is invalid."); return Buffer.from(value.replace(/-/g, ""), "hex"); } +function bytesToUuid(value) { const hex = Buffer.from(value).toString("hex"); return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; } + +module.exports = { PROTOCOL_VERSION, AUDIO_HEADER_BYTES, MAX_JSON_BYTES, MAX_AUDIO_PAYLOAD_BYTES, parseEnvelope, validateHello, parseAudioFrame, encodeAudioFrame, envelope, protocolError, isUuid }; diff --git a/plugins/lumi_transcription/backend/config/revision_store.js b/plugins/lumi_transcription/backend/config/revision_store.js new file mode 100644 index 0000000..7823090 --- /dev/null +++ b/plugins/lumi_transcription/backend/config/revision_store.js @@ -0,0 +1,52 @@ +const ALLOWED_KEYS = new Set([ + "selected_model_id", "fallback_order", "decode_interval_ms", "silence_finalize_ms", + "rolling_context_ms", "caption_max_chars", "minimum_display_ms", "auto_start_stream", + "offline_authorization_severity", "diagnostic_caption_text", "tracks" +]); + +class RevisionStore { + constructor(db, options = {}) { this.db = db; this.now = options.now || Date.now; this.migrate(); } + migrate() { + this.db.exec(`CREATE TABLE IF NOT EXISTS transcription_settings ( + key TEXT PRIMARY KEY, value_json TEXT NOT NULL, revision INTEGER NOT NULL, + actor_id TEXT NOT NULL, updated_at INTEGER NOT NULL + );`); + } + list() { + return Object.fromEntries(this.db.prepare("SELECT * FROM transcription_settings ORDER BY key").all().map((row) => [row.key, decode(row)])); + } + apply(changes, actorId) { + if (!Array.isArray(changes) || !changes.length || changes.length > 50) throw new Error("One to fifty field changes are required."); + const normalized = changes.map(validateChange); + const applied = []; + const conflicts = []; + this.db.transaction(() => { + for (const change of normalized) { + const current = this.db.prepare("SELECT * FROM transcription_settings WHERE key = ?").get(change.key); + const currentRevision = current?.revision || 0; + if (change.base_revision !== currentRevision) { + conflicts.push({ key: change.key, local_value: change.value, local_base_revision: change.base_revision, server: current ? decode(current) : { value: null, revision: 0, actor_id: null, updated_at: null } }); + continue; + } + const revision = currentRevision + 1; + const updatedAt = this.now(); + this.db.prepare("INSERT INTO transcription_settings (key, value_json, revision, actor_id, updated_at) VALUES (?, ?, ?, ?, ?) ON CONFLICT(key) DO UPDATE SET value_json = excluded.value_json, revision = excluded.revision, actor_id = excluded.actor_id, updated_at = excluded.updated_at") + .run(change.key, JSON.stringify(change.value), revision, String(actorId || "unknown"), updatedAt); + applied.push({ key: change.key, value: change.value, revision, actor_id: String(actorId || "unknown"), updated_at: updatedAt }); + } + })(); + return { applied, conflicts, current: this.list() }; + } +} + +function validateChange(change) { + if (!change || !ALLOWED_KEYS.has(change.key)) throw new Error(`Setting key ${change?.key || "(missing)"} is not allowed.`); + const baseRevision = Number(change.base_revision); + if (!Number.isInteger(baseRevision) || baseRevision < 0) throw new Error("A non-negative base revision is required for every changed field."); + const encoded = JSON.stringify(change.value); + if (encoded == null || Buffer.byteLength(encoded) > 64 * 1024) throw new Error("Setting value is too large."); + return { key: change.key, value: change.value, base_revision: baseRevision }; +} +function decode(row) { return { value: JSON.parse(row.value_json), revision: row.revision, actor_id: row.actor_id, updated_at: row.updated_at }; } + +module.exports = { RevisionStore, ALLOWED_KEYS }; diff --git a/plugins/lumi_transcription/backend/delivery/caption_delivery.js b/plugins/lumi_transcription/backend/delivery/caption_delivery.js new file mode 100644 index 0000000..356782c --- /dev/null +++ b/plugins/lumi_transcription/backend/delivery/caption_delivery.js @@ -0,0 +1,28 @@ +const { LatestCaptionGate } = require("../transcription/stabilizer"); + +class CaptionDeliveryAdapter { + async test() { throw new Error("Caption delivery test is not implemented."); } + async start() { throw new Error("Caption delivery start is not implemented."); } + async deliver() { throw new Error("Caption delivery is not implemented."); } + async pause() {} + async resume() {} + async stop() {} + async health() { return { healthy: false, state: "unavailable" }; } +} + +class CompanionCaptionDeliveryAdapter extends CaptionDeliveryAdapter { + constructor(send) { super(); this.send = send; this.gate = new LatestCaptionGate(); this.state = "idle"; this.testMode = false; } + async test() { return { supported: true, mode: "simulated", note: "The companion simulates the exact outgoing native-caption stream without sending it to Twitch." }; } + async start(options = {}) { this.testMode = Boolean(options.testMode); this.state = "running"; return this.health(); } + async deliver(event) { + if (this.state !== "running" || !this.gate.accept(event)) return { disposition: "obsolete_or_paused" }; + this.send("caption", { ...event, delivery: { disposition: this.testMode ? "simulated" : "forwarded_to_obs" } }, event.session_id); + return { disposition: this.testMode ? "simulated" : "forwarded_to_obs" }; + } + async pause() { this.state = "paused"; } + async resume() { this.state = "running"; } + async stop() { this.state = "idle"; } + async health() { return { healthy: this.state !== "idle", state: this.state, adapter: "companion_obs_native", test_mode: this.testMode }; } +} + +module.exports = { CaptionDeliveryAdapter, CompanionCaptionDeliveryAdapter }; diff --git a/plugins/lumi_transcription/backend/logs/jsonl_log.js b/plugins/lumi_transcription/backend/logs/jsonl_log.js new file mode 100644 index 0000000..5af2024 --- /dev/null +++ b/plugins/lumi_transcription/backend/logs/jsonl_log.js @@ -0,0 +1,55 @@ +const fs = require("fs"); +const path = require("path"); + +class JsonlDiagnosticLog { + constructor(directory, options = {}) { + this.directory = directory; + this.retentionMs = (options.retentionDays || 7) * 86400000; + this.maxBytes = options.maxBytes || 256 * 1024 * 1024; + this.includeCaptionText = options.includeCaptionText !== false; + fs.mkdirSync(directory, { recursive: true }); + } + append(entry) { + const safe = sanitize({ timestamp: new Date().toISOString(), ...entry }, this.includeCaptionText); + fs.appendFileSync(this.fileForToday(), `${JSON.stringify(safe)}\n`, { encoding: "utf8", mode: 0o600 }); + } + cleanup(now = Date.now()) { + const files = this.files(); + let removed = 0; + for (const file of files) { + if (now - file.mtimeMs > this.retentionMs) { fs.rmSync(file.path, { force: true }); removed += 1; } + } + const retained = this.files(); + let total = retained.reduce((sum, file) => sum + file.size, 0); + for (const file of retained) { + if (total <= this.maxBytes) break; + fs.rmSync(file.path, { force: true }); + total -= file.size; + removed += 1; + } + return { removed, bytes: total }; + } + files() { + return fs.readdirSync(this.directory).filter((name) => /^transcription-\d{4}-\d{2}-\d{2}\.jsonl$/.test(name)).map((name) => { + const target = path.join(this.directory, name); + const stat = fs.statSync(target); + return { path: target, name, size: stat.size, mtimeMs: stat.mtimeMs }; + }).sort((a, b) => a.mtimeMs - b.mtimeMs); + } + fileForToday() { return path.join(this.directory, `transcription-${new Date().toISOString().slice(0, 10)}.jsonl`); } +} + +function sanitize(value, includeCaptionText) { + if (Buffer.isBuffer(value)) return "[binary omitted]"; + if (Array.isArray(value)) return value.map((entry) => sanitize(entry, includeCaptionText)); + if (!value || typeof value !== "object") return value; + const result = {}; + for (const [key, child] of Object.entries(value)) { + if (/audio|pcm|credential|secret|token/i.test(key)) result[key] = "[redacted]"; + else if (!includeCaptionText && /(?:stable|uncertain|caption)_text/i.test(key)) result[key] = "[caption text disabled]"; + else result[key] = sanitize(child, includeCaptionText); + } + return result; +} + +module.exports = { JsonlDiagnosticLog, sanitize }; diff --git a/plugins/lumi_transcription/backend/models/artifact_manager.js b/plugins/lumi_transcription/backend/models/artifact_manager.js new file mode 100644 index 0000000..ef54844 --- /dev/null +++ b/plugins/lumi_transcription/backend/models/artifact_manager.js @@ -0,0 +1,59 @@ +const crypto = require("crypto"); +const fs = require("fs"); +const path = require("path"); +const AdmZip = require("adm-zip"); + +class ArtifactManager { + constructor(root, options = {}) { this.root = root; this.fetch = options.fetch || global.fetch; fs.mkdirSync(root, { recursive: true }); } + status(entry) { + const target = path.join(this.root, entry.filename || entry.id); + if (!fs.existsSync(target)) return { installed: false, valid: false, path: target }; + return { installed: true, valid: sha256File(target) === entry.sha256, path: target, bytes: fs.statSync(target).size }; + } + async download(entry, options = {}) { + validateManifestEntry(entry); + if (options.confirmed !== true) throw new Error("Download requires explicit setup confirmation."); + const filename = entry.filename || path.basename(new URL(entry.url).pathname); + const target = path.join(this.root, filename); + const partial = `${target}.${process.pid}.partial`; + fs.rmSync(partial, { force: true }); + try { + const response = await this.fetch(entry.url, { redirect: "follow" }); + if (!response.ok || !response.body) throw new Error(`Download failed with HTTP ${response.status}.`); + const handle = fs.createWriteStream(partial, { flags: "wx", mode: 0o600 }); + for await (const chunk of response.body) { + if (!handle.write(chunk)) await new Promise((resolve) => handle.once("drain", resolve)); + } + await new Promise((resolve, reject) => handle.end((error) => error ? reject(error) : resolve())); + const actual = sha256File(partial); + if (actual !== entry.sha256) throw new Error(`Checksum mismatch for ${entry.id}.`); + fs.renameSync(partial, target); + return this.status({ ...entry, filename }); + } finally { fs.rmSync(partial, { force: true }); } + } + installZip(entry, archivePath) { + validateManifestEntry(entry); + if (sha256File(archivePath) !== entry.sha256) throw new Error(`Checksum mismatch for ${entry.id}.`); + const target = path.join(this.root, entry.id); + const staged = `${target}.${process.pid}.staged`; + fs.rmSync(staged, { recursive: true, force: true }); + fs.mkdirSync(staged, { recursive: true }); + const zip = new AdmZip(archivePath); + for (const item of zip.getEntries()) { + const portable = String(item.entryName).replace(/\\/g, "/"); + const relative = path.posix.normalize(portable).replace(/^\/+/, ""); + if (!relative || relative === ".." || relative.startsWith("../") || /^[A-Za-z]:/.test(relative)) throw new Error("Runtime archive contains an unsafe path."); + } + zip.extractAllTo(staged, true); + for (const expected of entry.expected_paths || []) if (!findBasename(staged, expected)) throw new Error(`Runtime is missing ${expected}.`); + fs.rmSync(target, { recursive: true, force: true }); + fs.renameSync(staged, target); + return { installed: true, path: target, backend: entry.backend, version: entry.id }; + } +} + +function validateManifestEntry(entry) { if (!entry?.id || !/^https:\/\//.test(entry.url || "") || !/^[a-f0-9]{64}$/.test(entry.sha256 || "")) throw new Error("Artifact manifest entry is invalid."); } +function sha256File(target) { const hash = crypto.createHash("sha256"); const fd = fs.openSync(target, "r"); const buffer = Buffer.alloc(1024 * 1024); try { let read; while ((read = fs.readSync(fd, buffer, 0, buffer.length, null)) > 0) hash.update(buffer.subarray(0, read)); } finally { fs.closeSync(fd); } return hash.digest("hex"); } +function findBasename(root, basename) { const pending = [root]; while (pending.length) { const current = pending.pop(); for (const entry of fs.readdirSync(current, { withFileTypes: true })) { const target = path.join(current, entry.name); if (entry.isDirectory()) pending.push(target); else if (entry.name === basename) return target; } } return null; } + +module.exports = { ArtifactManager, sha256File, validateManifestEntry }; diff --git a/plugins/lumi_transcription/backend/paths.js b/plugins/lumi_transcription/backend/paths.js new file mode 100644 index 0000000..8fa23df --- /dev/null +++ b/plugins/lumi_transcription/backend/paths.js @@ -0,0 +1,7 @@ +const fs = require("fs"); +const path = require("path"); +const ROOT = path.resolve(__dirname, ".."); +const DATA = path.join(ROOT, "data"); +function ensureDataDirs() { for (const name of ["logs", "models", "runtime", "tmp"]) fs.mkdirSync(path.join(DATA, name), { recursive: true }); } +function dataPath(...parts) { const target = path.resolve(DATA, ...parts); if (target !== DATA && !target.startsWith(`${DATA}${path.sep}`)) throw new Error("Path escapes transcription plugin data."); return target; } +module.exports = { ROOT, DATA, ensureDataDirs, dataPath }; diff --git a/plugins/lumi_transcription/backend/sessions/bounded_queue.js b/plugins/lumi_transcription/backend/sessions/bounded_queue.js new file mode 100644 index 0000000..4df7e72 --- /dev/null +++ b/plugins/lumi_transcription/backend/sessions/bounded_queue.js @@ -0,0 +1,77 @@ +class BoundedQueue { + constructor(options = {}) { + this.maxItems = positive(options.maxItems, 250); + this.maxBytes = positive(options.maxBytes, 5 * 32000); + this.maxAgeMs = positive(options.maxAgeMs, 5000); + this.now = options.now || Date.now; + this.items = []; + this.bytes = 0; + this.dropped = { capacity: 0, stale: 0 }; + } + + push(value, options = {}) { + const bytes = Math.max(0, Number(options.bytes ?? value?.length ?? value?.pcm?.length ?? 0)); + const capturedAt = Number(options.capturedAt ?? this.now()); + if (bytes > this.maxBytes) { + this.dropped.capacity += 1; + return false; + } + this.prune(); + while (this.items.length >= this.maxItems || this.bytes + bytes > this.maxBytes) this.dropOldest("capacity"); + this.items.push({ value, bytes, capturedAt }); + this.bytes += bytes; + return true; + } + + shift() { + this.prune(); + const entry = this.items.shift(); + if (!entry) return null; + this.bytes -= entry.bytes; + return entry.value; + } + + prune() { + const cutoff = this.now() - this.maxAgeMs; + while (this.items[0] && this.items[0].capturedAt < cutoff) this.dropOldest("stale"); + } + + clear() { this.items.length = 0; this.bytes = 0; } + size() { this.prune(); return this.items.length; } + metrics() { this.prune(); return { items: this.items.length, bytes: this.bytes, dropped: { ...this.dropped } }; } + + dropOldest(reason) { + const entry = this.items.shift(); + if (!entry) return; + this.bytes -= entry.bytes; + this.dropped[reason] += 1; + } +} + +class SequenceTracker { + constructor() { this.last = null; this.gaps = 0; this.outOfOrder = 0; } + accept(sequence) { + const current = Number(sequence) >>> 0; + if (this.last == null) { this.last = current; return { accepted: true, gap: 0 }; } + if (current <= this.last) { this.outOfOrder += 1; return { accepted: false, gap: 0 }; } + const gap = current - this.last - 1; + if (gap) this.gaps += gap; + this.last = current; + return { accepted: true, gap }; + } + metrics() { return { last_sequence: this.last, sequence_gaps: this.gaps, out_of_order: this.outOfOrder }; } +} + +class RollingPcmBuffer { + constructor(options = {}) { + this.queue = new BoundedQueue({ maxItems: options.maxItems || 300, maxBytes: (options.seconds || 5) * 32000, maxAgeMs: (options.seconds || 5) * 1000, now: options.now }); + } + push(frame, capturedAt) { return this.queue.push(Buffer.from(frame), { bytes: frame.length, capturedAt }); } + snapshot() { this.queue.prune(); return Buffer.concat(this.queue.items.map((entry) => entry.value), this.queue.bytes); } + metrics() { return this.queue.metrics(); } + clear() { this.queue.clear(); } +} + +function positive(value, fallback) { const number = Number(value); return Number.isFinite(number) && number > 0 ? number : fallback; } + +module.exports = { BoundedQueue, SequenceTracker, RollingPcmBuffer }; diff --git a/plugins/lumi_transcription/backend/sessions/session_coordinator.js b/plugins/lumi_transcription/backend/sessions/session_coordinator.js new file mode 100644 index 0000000..a53cce9 --- /dev/null +++ b/plugins/lumi_transcription/backend/sessions/session_coordinator.js @@ -0,0 +1,152 @@ +const crypto = require("crypto"); +const { RollingPcmBuffer, SequenceTracker } = require("./bounded_queue"); +const { CaptionStabilizer } = require("../transcription/stabilizer"); + +class SessionCoordinator { + constructor(options) { + this.provider = options.provider; + this.deliveryFactory = options.deliveryFactory; + this.log = options.log || { append() {} }; + this.now = options.now || Date.now; + this.graceMs = options.graceMs || 30000; + this.sessions = new Map(); + this.stabilizer = new CaptionStabilizer({ now: this.now }); + this.provider.on?.("hypothesis", (event) => this.onHypothesis(event)); + } + create(device, send, resumeSessionId = null) { + const resumable = resumeSessionId && this.sessions.get(resumeSessionId); + if (resumable && resumable.deviceId === device.id && resumable.graceUntil > this.now()) { + clearTimeout(resumable.graceTimer); + resumable.graceTimer = null; + resumable.graceUntil = 0; + resumable.connected = true; + resumable.send = send; + resumable.delivery = this.deliveryFactory(send); + this.log.append({ kind: "session", state: "resumed", session_id: resumable.id, device_id: device.id }); + return { session: resumable, resumed: true }; + } + const session = { + id: crypto.randomUUID(), deviceId: device.id, connected: true, send, + state: "idle", mode: null, obs: { streaming: false, recording: false }, + tracks: new Map(), delivery: this.deliveryFactory(send), createdAt: this.now(), graceUntil: 0, graceTimer: null + }; + this.sessions.set(session.id, session); + this.log.append({ kind: "session", state: "created", session_id: session.id, device_id: device.id }); + return { session, resumed: false }; + } + updateSource(sessionId, input) { + const session = this.require(sessionId); + if (!isUuid(input.source_uuid)) throw new Error("Source UUID is invalid."); + const existing = session.tracks.get(input.source_uuid); + const track = existing || { sequence: new SequenceTracker(), buffer: new RollingPcmBuffer({ seconds: 5, now: this.now }), lastSpeechAt: 0 }; + Object.assign(track, { + source_uuid: input.source_uuid, display_name: text(input.display_name, 160) || "OBS source", + speaker_label: text(input.speaker_label, 80) || null, enabled: input.enabled !== false, + primary: Boolean(input.primary), single_speaker: input.single_speaker !== false, + delivery_enabled: input.delivery_enabled !== false, program_active: Boolean(input.program_active), + source_missing: Boolean(input.source_missing), last_activity_at: this.now() + }); + if (track.primary) for (const other of session.tracks.values()) other.primary = false; + session.tracks.set(track.source_uuid, track); + return serializeTrack(track); + } + async updateObsState(sessionId, obs) { + const session = this.require(sessionId); + const wasStreaming = session.obs.streaming; + session.obs = { streaming: Boolean(obs.streaming), recording: Boolean(obs.recording), version: text(obs.version, 32) || null }; + if (!wasStreaming && session.obs.streaming && session.state === "idle" && obs.auto_start !== false) await this.start(sessionId, { mode: "live" }); + if (wasStreaming && !session.obs.streaming && session.state === "running" && session.mode === "live") this.beginGrace(session); + if (!wasStreaming && session.obs.streaming && session.state === "grace") await this.resume(session); + return { ...session.obs, state: session.state }; + } + async start(sessionId, options = {}) { + const session = this.require(sessionId); + const mode = options.mode === "test" ? "test" : "live"; + if (mode === "live" && !session.obs.streaming) throw Object.assign(new Error("Live transcription requires an active OBS stream."), { code: "OBS_NOT_STREAMING" }); + const tracks = Array.from(session.tracks.values()).filter((track) => track.enabled && !track.source_missing); + if (!tracks.length) throw Object.assign(new Error("Select an available OBS audio source first."), { code: "NO_TRACKS" }); + const providerHealth = await this.provider.health(); + if (!providerHealth.healthy) throw Object.assign(new Error("The whisper.cpp worker is not ready. Install and load a model first."), { code: "PROVIDER_UNAVAILABLE", details: providerHealth }); + await this.provider.startSession({ id: session.id, mode }); + for (const track of tracks) await this.provider.addTrack(session.id, serializeTrack(track)); + await session.delivery.start({ testMode: mode === "test" }); + session.mode = mode; + session.state = "running"; + this.log.append({ kind: "session", state: "running", mode, session_id: session.id, tracks: tracks.map((track) => track.source_uuid) }); + return this.status(session.id); + } + async stop(sessionId, reason = "requested") { + const session = this.require(sessionId); + clearTimeout(session.graceTimer); + session.graceTimer = null; + await session.delivery.stop(); + if (session.state !== "idle") await this.provider.stopSession(session.id); + session.state = "idle"; + session.mode = null; + for (const track of session.tracks.values()) track.buffer.clear(); + this.log.append({ kind: "session", state: "stopped", reason, session_id: session.id }); + return this.status(session.id); + } + async audio(sessionId, frame) { + const session = this.require(sessionId); + if (frame.session_id !== session.id) throw new Error("Audio session does not match the negotiated session."); + const track = session.tracks.get(frame.source_uuid); + if (!track) return { accepted: false, reason: "unselected_source" }; + const sequence = track.sequence.accept(frame.sequence); + if (!sequence.accepted) return { accepted: false, reason: "out_of_order" }; + if (sequence.gap) this.log.append({ kind: "audio_gap", session_id: session.id, source_uuid: track.source_uuid, missing_frames: sequence.gap }); + if (session.state !== "running" || !track.enabled || !track.program_active || track.source_missing || frame.muted || !frame.active) return { accepted: false, reason: "inactive" }; + track.buffer.push(frame.pcm, this.now()); + const accepted = await this.provider.pushAudio(session.id, track.source_uuid, frame); + return { accepted: accepted !== false, gap: sequence.gap, buffer: track.buffer.metrics() }; + } + disconnect(sessionId) { + const session = this.sessions.get(sessionId); + if (!session) return; + session.connected = false; + if (session.state === "running") this.beginGrace(session); + else this.expireLater(session); + } + status(sessionId) { + const session = this.require(sessionId); + return { session_id: session.id, state: session.state, mode: session.mode, obs: { ...session.obs }, grace_until: session.graceUntil || null, tracks: Array.from(session.tracks.values()).map(serializeTrack) }; + } + async close() { for (const session of Array.from(this.sessions.values())) await this.stop(session.id, "plugin_shutdown"); this.sessions.clear(); } + require(id) { const session = this.sessions.get(id); if (!session) throw new Error("Session was not found."); return session; } + beginGrace(session) { + clearTimeout(session.graceTimer); + session.state = "grace"; + session.graceUntil = this.now() + this.graceMs; + session.delivery.pause(); + session.graceTimer = setTimeout(() => this.stop(session.id, "stream_grace_expired").catch(() => {}), this.graceMs); + session.graceTimer.unref?.(); + this.log.append({ kind: "session", state: "grace", session_id: session.id, grace_until: session.graceUntil }); + } + async resume(session) { clearTimeout(session.graceTimer); session.graceTimer = null; session.graceUntil = 0; session.state = "running"; await session.delivery.resume(); this.log.append({ kind: "session", state: "resumed", session_id: session.id }); } + expireLater(session) { clearTimeout(session.graceTimer); session.graceUntil = this.now() + this.graceMs; session.graceTimer = setTimeout(() => this.sessions.delete(session.id), this.graceMs); session.graceTimer.unref?.(); } + async onHypothesis(raw) { + const session = this.sessions.get(raw.session_id); + const track = session?.tracks.get(raw.track_id || raw.source_uuid); + if (!session || !track || session.state !== "running") return; + track.lastSpeechAt = this.now(); + const primary = Array.from(session.tracks.values()).find((candidate) => candidate.primary); + if (primary && track !== primary && this.now() - primary.lastSpeechAt < 800) return; + const stable = this.stabilizer.update(track.source_uuid, raw.text, { final: raw.final, newUtterance: raw.new_utterance, trailingIncomplete: raw.incomplete_word }); + const event = { + session_id: session.id, source_uuid: track.source_uuid, + speaker_label: speakerLabel(session, track), ...stable, + audio: { start_us: raw.audio_start_us || 0, end_us: raw.audio_end_us || 0 }, + latency: { capture_ms: raw.capture_ms || 0, network_ms: raw.network_ms || 0, queue_ms: raw.queue_ms || 0, inference_ms: raw.inference_ms || 0, stabilization_ms: stable.stabilization_ms, total_ms: raw.total_ms || 0 }, + model: { id: raw.model_id || "unknown", provider: "whisper.cpp", backend: raw.backend || "unknown" } + }; + const result = await session.delivery.deliver(event); + this.log.append({ kind: "caption", session_id: session.id, source_uuid: track.source_uuid, caption_text: event.stable_text, uncertain_text: event.uncertain_text, revision: event.revision, final: event.final, delivery: result.disposition, latency: event.latency, model: event.model }); + } +} + +function speakerLabel(session, track) { const enabled = Array.from(session.tracks.values()).filter((candidate) => candidate.enabled); return enabled.length === 1 && track.single_speaker ? null : track.speaker_label || track.display_name; } +function serializeTrack(track) { return { source_uuid: track.source_uuid, display_name: track.display_name, speaker_label: track.speaker_label, enabled: track.enabled, primary: track.primary, single_speaker: track.single_speaker, delivery_enabled: track.delivery_enabled, program_active: track.program_active, source_missing: track.source_missing, last_activity_at: track.last_activity_at, sequence: track.sequence?.metrics?.() }; } +function text(value, max) { return String(value || "").trim().slice(0, max); } +function isUuid(value) { return /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(String(value || "")); } + +module.exports = { SessionCoordinator, serializeTrack }; diff --git a/plugins/lumi_transcription/backend/transcription/provider.js b/plugins/lumi_transcription/backend/transcription/provider.js new file mode 100644 index 0000000..8c50a0c --- /dev/null +++ b/plugins/lumi_transcription/backend/transcription/provider.js @@ -0,0 +1,122 @@ +const { EventEmitter } = require("events"); +const { spawn } = require("child_process"); +const { BoundedQueue } = require("../sessions/bounded_queue"); + +class TranscriptionProvider extends EventEmitter { + async loadModel() { throw new Error("Model loading is not implemented."); } + async benchmark() { throw new Error("Benchmarking is not implemented."); } + async startSession() { throw new Error("Session start is not implemented."); } + async addTrack() { throw new Error("Track add is not implemented."); } + async pushAudio() { throw new Error("Audio input is not implemented."); } + async removeTrack() { throw new Error("Track removal is not implemented."); } + async stopSession() { throw new Error("Session stop is not implemented."); } + async health() { return { healthy: false, state: "unavailable" }; } +} + +class WhisperWorkerSupervisor extends EventEmitter { + constructor(options = {}) { + super(); + this.executable = options.executable; + this.args = options.args || []; + this.spawn = options.spawn || spawn; + this.maxRestarts = options.maxRestarts ?? 3; + this.restartWindowMs = options.restartWindowMs || 60000; + this.queue = new BoundedQueue({ maxItems: options.maxQueuedPackets || 250, maxBytes: options.maxQueuedBytes || 5 * 32000, maxAgeMs: 5000 }); + this.child = null; + this.state = "stopped"; + this.stopping = false; + this.restartTimes = []; + this.stdoutBuffer = ""; + } + start() { + if (this.child) return; + if (!this.executable) throw new Error("Whisper worker executable is not configured."); + this.stopping = false; + this.state = "starting"; + const child = this.spawn(this.executable, this.args, { stdio: ["pipe", "pipe", "pipe"], windowsHide: true }); + this.child = child; + child.stdout.on("data", (chunk) => this.onStdout(chunk)); + child.stderr.on("data", (chunk) => this.emit("diagnostic", { level: "warning", message: String(chunk).trim().slice(0, 1000) })); + child.on("error", (error) => this.emit("error", error)); + child.on("exit", (code, signal) => this.onExit(code, signal)); + child.stdin.on("drain", () => this.flush()); + this.state = "running"; + this.emit("state", { state: this.state }); + this.flush(); + } + send(metadata, pcm = null) { + const body = Buffer.from(JSON.stringify(metadata), "utf8"); + const audio = pcm ? Buffer.from(pcm) : Buffer.alloc(0); + if (body.length > 64 * 1024 || audio.length > 5 * 32000) throw new Error("Worker packet exceeds its limit."); + const header = Buffer.alloc(8); + header.writeUInt32LE(body.length, 0); + header.writeUInt32LE(audio.length, 4); + const packet = Buffer.concat([header, body, audio]); + const accepted = this.queue.push(packet, { bytes: packet.length, capturedAt: metadata.captured_at || Date.now() }); + if (accepted) this.flush(); + return accepted; + } + flush() { + if (!this.child || this.state !== "running") return; + let packet; + while ((packet = this.queue.shift())) if (!this.child.stdin.write(packet)) break; + } + async stop() { + this.stopping = true; + this.queue.clear(); + const child = this.child; + if (!child) { this.state = "stopped"; return; } + this.sendDirect({ type: "shutdown" }); + await new Promise((resolve) => { + const timer = setTimeout(() => { child.kill(); resolve(); }, 2000); + child.once("exit", () => { clearTimeout(timer); resolve(); }); + }); + this.child = null; + this.state = "stopped"; + } + sendDirect(metadata) { if (!this.child) return false; const body = Buffer.from(JSON.stringify(metadata)); const header = Buffer.alloc(8); header.writeUInt32LE(body.length, 0); return this.child.stdin.write(Buffer.concat([header, body])); } + onStdout(chunk) { + this.stdoutBuffer += String(chunk); + if (this.stdoutBuffer.length > 256 * 1024) this.stdoutBuffer = this.stdoutBuffer.slice(-64 * 1024); + let index; + while ((index = this.stdoutBuffer.indexOf("\n")) >= 0) { + const line = this.stdoutBuffer.slice(0, index).trim(); + this.stdoutBuffer = this.stdoutBuffer.slice(index + 1); + if (!line) continue; + try { this.emit("message", JSON.parse(line)); } + catch { this.emit("diagnostic", { level: "warning", message: "Whisper worker emitted malformed output." }); } + } + } + onExit(code, signal) { + this.child = null; + if (this.stopping) { this.state = "stopped"; return; } + this.state = "failed"; + this.emit("crash", { code, signal }); + const now = Date.now(); + this.restartTimes = this.restartTimes.filter((time) => now - time < this.restartWindowMs); + if (this.restartTimes.length >= this.maxRestarts) return; + this.restartTimes.push(now); + setTimeout(() => { try { this.start(); } catch (error) { this.emit("error", error); } }, 250).unref?.(); + } + health() { return { healthy: this.state === "running", state: this.state, queue: this.queue.metrics(), restarts_in_window: this.restartTimes.length }; } +} + +class WhisperCppServerProvider extends TranscriptionProvider { + constructor(supervisor) { + super(); this.worker = supervisor; this.model = null; this.sessions = new Set(); + supervisor.on("message", (message) => this.emit(message.type || "message", message)); + supervisor.on("crash", (event) => this.emit("provider_error", Object.assign(new Error("Whisper worker crashed."), { details: event }))); + supervisor.on("error", (error) => this.emit("provider_error", error)); + } + async loadModel(model) { this.worker.start(); this.model = model; this.worker.send({ type: "load_model", model }); return this.health(); } + async benchmark(options = {}) { this.worker.send({ type: "benchmark", options }); return { accepted: true, model: this.model?.id || null }; } + async startSession(session) { this.sessions.add(session.id); this.worker.send({ type: "start_session", session }); } + async addTrack(sessionId, track) { this.worker.send({ type: "add_track", session_id: sessionId, track }); } + async pushAudio(sessionId, trackId, frame) { return this.worker.send({ type: "audio", session_id: sessionId, track_id: trackId, sequence: frame.sequence, capture_timestamp_us: frame.capture_timestamp_us, captured_at: Date.now() }, frame.pcm); } + async removeTrack(sessionId, trackId) { this.worker.send({ type: "remove_track", session_id: sessionId, track_id: trackId }); } + async stopSession(sessionId) { this.sessions.delete(sessionId); this.worker.send({ type: "stop_session", session_id: sessionId }); } + async health() { return { provider: "whisper_cpp_server", model: this.model?.id || null, sessions: this.sessions.size, ...this.worker.health() }; } + async stop() { this.sessions.clear(); await this.worker.stop(); } +} + +module.exports = { TranscriptionProvider, WhisperWorkerSupervisor, WhisperCppServerProvider }; diff --git a/plugins/lumi_transcription/backend/transcription/stabilizer.js b/plugins/lumi_transcription/backend/transcription/stabilizer.js new file mode 100644 index 0000000..c6b1452 --- /dev/null +++ b/plugins/lumi_transcription/backend/transcription/stabilizer.js @@ -0,0 +1,67 @@ +const crypto = require("crypto"); + +class CaptionStabilizer { + constructor(options = {}) { + this.maxChars = options.maxChars || 80; + this.fragmentAfterMs = options.fragmentAfterMs || 1000; + this.now = options.now || Date.now; + this.tracks = new Map(); + } + + update(sourceUuid, hypothesis, details = {}) { + const now = this.now(); + const words = tokenize(hypothesis); + let state = this.tracks.get(sourceUuid); + if (!state || details.newUtterance) { + state = { captionId: crypto.randomUUID(), revision: 0, stable: [], previous: [], startedAt: now, tailSince: now }; + this.tracks.set(sourceUuid, state); + } + const common = commonPrefix(state.previous, words); + const agreement = commonPrefix(state.stable, words); + if (agreement.length < state.stable.length) { + words.splice(0, agreement.length, ...state.stable.slice(0, agreement.length)); + } + const stableLimit = details.trailingIncomplete && !details.final + ? Math.max(state.stable.length, common.length - 1) + : Math.max(state.stable.length, common.length); + const agreedCandidate = words.slice(0, stableLimit); + if (agreedCandidate.length > state.stable.length) state.stable = agreedCandidate; + const tail = words.slice(state.stable.length); + if (tail.join(" ") !== state.previous.slice(state.stable.length).join(" ")) state.tailSince = now; + const incompleteWord = Boolean(details.trailingIncomplete && tail.length && now - state.tailSince >= this.fragmentAfterMs && !details.final); + if (details.final) state.stable = words; + state.previous = words; + state.revision += 1; + const stableText = clip(state.stable.join(" "), this.maxChars); + const uncertainText = details.final ? "" : clip(tail.join(" "), Math.max(0, this.maxChars - stableText.length - 1)); + const event = { + caption_id: state.captionId, + revision: state.revision, + stable_text: stableText, + uncertain_text: uncertainText, + final: Boolean(details.final), + incomplete_word: incompleteWord, + stabilization_ms: now - state.startedAt + }; + if (details.final) this.tracks.delete(sourceUuid); + return event; + } +} + +class LatestCaptionGate { + constructor() { this.revisions = new Map(); } + accept(event) { + const key = `${event.session_id}:${event.caption_id}`; + const current = this.revisions.get(key) || 0; + if (event.revision <= current) return false; + this.revisions.set(key, event.revision); + return true; + } + clearSession(sessionId) { for (const key of this.revisions.keys()) if (key.startsWith(`${sessionId}:`)) this.revisions.delete(key); } +} + +function tokenize(value) { return String(value || "").trim().split(/\s+/).filter(Boolean); } +function commonPrefix(left, right) { let index = 0; while (index < left.length && index < right.length && left[index] === right[index]) index += 1; return right.slice(0, index); } +function clip(value, max) { if (value.length <= max) return value; return value.slice(0, max).trimEnd(); } + +module.exports = { CaptionStabilizer, LatestCaptionGate, tokenize, commonPrefix }; diff --git a/plugins/lumi_transcription/data/.gitkeep b/plugins/lumi_transcription/data/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/plugins/lumi_transcription/data/.gitkeep @@ -0,0 +1 @@ + diff --git a/plugins/lumi_transcription/index.js b/plugins/lumi_transcription/index.js new file mode 100644 index 0000000..4810cac --- /dev/null +++ b/plugins/lumi_transcription/index.js @@ -0,0 +1,134 @@ +const express = require("express"); +const fs = require("fs"); +const path = require("path"); +const { DeviceStore } = require("./backend/companion/device_store"); +const { CompanionGateway } = require("./backend/companion/gateway"); +const { RevisionStore } = require("./backend/config/revision_store"); +const { CompanionCaptionDeliveryAdapter } = require("./backend/delivery/caption_delivery"); +const { JsonlDiagnosticLog } = require("./backend/logs/jsonl_log"); +const { ArtifactManager } = require("./backend/models/artifact_manager"); +const { SessionCoordinator } = require("./backend/sessions/session_coordinator"); +const { WhisperWorkerSupervisor, WhisperCppServerProvider } = require("./backend/transcription/provider"); +const { ensureDataDirs, dataPath } = require("./backend/paths"); +const modelManifest = require("./models_manifest.json"); +const runtimeManifest = require("./runtime_manifest.json"); +const manifest = require("./plugin.json"); + +const PLUGIN_ID = "lumi_transcription"; + +module.exports = { + id: PLUGIN_ID, + init({ web, db, logger }) { + ensureDataDirs(); + const allowInsecure = process.env.LUMI_COMPANION_DEV_ALLOW_INSECURE === "1"; + const devices = new DeviceStore(db, { allowInsecure }); + const revisions = new RevisionStore(db); + const diagnosticLog = new JsonlDiagnosticLog(dataPath("logs")); + diagnosticLog.cleanup(); + const cleanupTimer = setInterval(() => diagnosticLog.cleanup(), 60 * 60 * 1000); + cleanupTimer.unref?.(); + const supervisor = new WhisperWorkerSupervisor({ + executable: process.env.LUMI_TRANSCRIPTION_WORKER || "", + args: process.env.LUMI_TRANSCRIPTION_WORKER_ARGS ? JSON.parse(process.env.LUMI_TRANSCRIPTION_WORKER_ARGS) : [] + }); + supervisor.on("diagnostic", (entry) => diagnosticLog.append({ kind: "worker", ...entry })); + supervisor.on("error", (error) => diagnosticLog.append({ kind: "worker", state: "error", message: error.message })); + const provider = new WhisperCppServerProvider(supervisor); + const sessions = new SessionCoordinator({ + provider, + deliveryFactory: (send) => new CompanionCaptionDeliveryAdapter(send), + log: diagnosticLog + }); + const gateway = new CompanionGateway({ devices, sessions, log: diagnosticLog, allowInsecure }); + const unregisterUpgrade = web.addUpgradeHandler("/plugins/lumi_transcription/live", (request, socket, head) => gateway.upgrade(request, socket, head)); + const models = new ArtifactManager(dataPath("models")); + const runtimeArchives = new ArtifactManager(dataPath("tmp")); + const runtimes = new ArtifactManager(dataPath("runtime")); + + const router = web.createRouter(); + router.use("/assets", express.static(path.join(__dirname, "public"))); + router.get("/", requireAdmin, async (_req, res) => { + res.render(path.join(__dirname, "views", "settings.ejs"), { + title: "Lumi Transcription", + pluginVersion: manifest.version, + providerHealth: await provider.health(), + devices: devices.list(), + settings: revisions.list(), + models: modelManifest.models.map((entry) => ({ ...entry, status: models.status(entry) })), + runtimeManifest, + logs: diagnosticLog.files() + }); + }); + router.get("/api/status", requireAdmin, async (_req, res) => res.json({ + ok: true, plugin: { id: PLUGIN_ID, version: manifest.version }, protocol_version: 1, + provider: await provider.health(), devices: devices.list(), settings: revisions.list(), + models: modelManifest.models.map((entry) => ({ id: entry.id, ...models.status(entry) })), + runtime: runtimeManifest + })); + router.post("/api/pairing-package", requireAdmin, (req, res) => { + try { + const host = requestHost(req); + const pairing = devices.issuePairing({ userId: req.session.user.id, host }); + const bootstrap = { format: "lumi-companion-bootstrap-v1", ...pairing, exchange_url: `${host}/plugins/${PLUGIN_ID}/api/pair` }; + res.set("Cache-Control", "no-store"); + res.attachment(`lumi-companion-${pairing.pairing_id}.lumi-pairing.json`); + res.send(`${JSON.stringify(bootstrap, null, 2)}\n`); + } catch (error) { res.status(400).json({ ok: false, error: error.message }); } + }); + router.post("/api/pair", requireSecureRequest(allowInsecure), (req, res) => { + try { res.set("Cache-Control", "no-store"); res.status(201).json({ ok: true, ...devices.exchange(req.body || {}) }); } + catch (error) { res.status(error.code === "PAIRING_ALREADY_USED" ? 409 : 400).json({ ok: false, code: error.code, error: error.message }); } + }); + router.get("/api/devices", requireAdmin, (_req, res) => res.json({ devices: devices.list() })); + router.post("/api/devices/:id/revoke", requireAdmin, (req, res) => res.json({ ok: devices.revoke(req.params.id) })); + router.post("/api/devices/:id/capabilities", requireAdmin, (req, res) => { + const capabilities = devices.setCapabilities(req.params.id, req.body.capabilities); + if (!capabilities) return res.status(404).json({ ok: false, error: "Device was not found or is revoked." }); + res.json({ ok: true, capabilities }); + }); + router.get("/api/settings", requireSettingsAccess(devices, allowInsecure), (_req, res) => res.json({ fields: revisions.list() })); + router.patch("/api/settings", requireSettingsAccess(devices, allowInsecure), (req, res) => { + try { + const actor = req.session?.user?.id || req.lumiDevice?.id; + const result = revisions.apply(req.body.changes, actor); + if (result.applied.length) web.emitEvent?.("transcription:settings_changed", { fields: result.applied.map((entry) => entry.key) }, { role: "admin" }); + res.status(result.conflicts.length ? 409 : 200).json({ ok: !result.conflicts.length, ...result }); + } catch (error) { res.status(400).json({ ok: false, error: error.message }); } + }); + router.post("/api/models/:id/download", requireAdmin, async (req, res) => { + const entry = modelManifest.models.find((candidate) => candidate.id === req.params.id); + if (!entry) return res.status(404).json({ ok: false, error: "Model was not found." }); + try { res.status(201).json({ ok: true, status: await models.download(entry, { confirmed: req.body.confirmed === true }) }); } + catch (error) { res.status(400).json({ ok: false, error: error.message }); } + }); + router.post("/api/runtime/:id/install", requireAdmin, async (req, res) => { + const entry = runtimeManifest.artifacts.find((candidate) => candidate.id === req.params.id); + if (!entry) return res.status(404).json({ ok: false, error: "Runtime was not found." }); + try { + const filename = `${entry.id}.zip`; + const archive = await runtimeArchives.download({ ...entry, filename }, { confirmed: req.body.confirmed === true }); + res.status(201).json({ ok: true, status: runtimes.installZip(entry, archive.path) }); + } catch (error) { res.status(400).json({ ok: false, error: error.message }); } + }); + router.get("/api/logs", requireAdmin, (_req, res) => res.json({ files: diagnosticLog.files().map(({ path: _path, ...entry }) => entry) })); + + web.mount(`/plugins/${PLUGIN_ID}`, router, { label: "Transcription", role: "admin", section: "plugins" }); + global.lumiFrameworks = global.lumiFrameworks || {}; + global.lumiFrameworks.transcription = { version: manifest.version, protocol_version: 1, health: () => provider.health() }; + + return async () => { + clearInterval(cleanupTimer); + unregisterUpgrade(); + await gateway.close(); + await sessions.close(); + await provider.stop(); + if (global.lumiFrameworks?.transcription?.version === manifest.version) delete global.lumiFrameworks.transcription; + logger?.info?.("Lumi transcription stopped", {}, { event: "transcription_stopped" }); + }; + } +}; + +function requireAdmin(req, res, next) { if (req.session?.user?.isAdmin) return next(); return res.status(403).json({ error: "Administrator access is required." }); } +function requireSettingsAccess(devices, allowInsecure) { return (req, res, next) => { if (req.session?.user?.isAdmin) return next(); const forwarded = String(req.headers["x-forwarded-proto"] || "").split(",")[0].trim(); const local = ["127.0.0.1", "::1", "::ffff:127.0.0.1"].includes(req.socket.remoteAddress); if (!req.secure && forwarded !== "https" && !(allowInsecure && local)) return res.status(426).json({ error: "Device settings synchronization requires HTTPS." }); const auth = devices.authenticate(req.headers.authorization, "transcription.settings.v1"); if (!auth.allowed) return res.status(401).json({ error: "A paired device credential is required." }); req.lumiDevice = auth.device; next(); }; } +function requestHost(req) { const forwarded = String(req.headers["x-forwarded-proto"] || "").split(",")[0].trim(); const protocol = forwarded === "https" ? "https" : req.protocol; return `${protocol}://${req.get("host")}`; } +function requireSecureRequest(allowInsecure) { return (req, res, next) => { const forwarded = String(req.headers["x-forwarded-proto"] || "").split(",")[0].trim(); const local = ["127.0.0.1", "::1", "::ffff:127.0.0.1"].includes(req.socket.remoteAddress); if (req.secure || forwarded === "https" || (allowInsecure && local)) return next(); return res.status(426).json({ error: "Lumi Companion pairing requires HTTPS." }); }; } diff --git a/plugins/lumi_transcription/models_manifest.json b/plugins/lumi_transcription/models_manifest.json new file mode 100644 index 0000000..232ceb6 --- /dev/null +++ b/plugins/lumi_transcription/models_manifest.json @@ -0,0 +1,33 @@ +{ + "schema_version": 1, + "repository_commit": "5359861c739e955e79d9a303bcbc70fb988958b1", + "models": [ + { + "id": "small.en", + "label": "Small English", + "recommended": true, + "url": "https://huggingface.co/ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/ggml-small.en.bin", + "sha256": "c6138d6d58ecc8322097e0f987c32f1be8bb0a18532a3f88f734d1bbf9c41e5d", + "bytes": 487614201, + "filename": "ggml-small.en.bin" + }, + { + "id": "small.en-q5_1", + "label": "Small English quantized (Q5_1)", + "recommended": false, + "url": "https://huggingface.co/ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/ggml-small.en-q5_1.bin", + "sha256": "bfdff4894dcb76bbf647d56263ea2a96645423f1669176f4844a1bf8e478ad30", + "bytes": 190098681, + "filename": "ggml-small.en-q5_1.bin" + }, + { + "id": "base.en", + "label": "Base English", + "recommended": false, + "url": "https://huggingface.co/ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/ggml-base.en.bin", + "sha256": "a03779c86df3323075f5e796cb2ce5029f00ec8869eee3fdfb897afe36c6d002", + "bytes": 147964211, + "filename": "ggml-base.en.bin" + } + ] +} diff --git a/plugins/lumi_transcription/plugin.json b/plugins/lumi_transcription/plugin.json new file mode 100644 index 0000000..4f3e895 --- /dev/null +++ b/plugins/lumi_transcription/plugin.json @@ -0,0 +1,10 @@ +{ + "id": "lumi_transcription", + "name": "Lumi Transcription", + "version": "0.1.0-experimental.1", + "description": "Server-hosted whisper.cpp transcription for Lumi Companion and OBS closed captions.", + "main": "index.js", + "channel": "experimental", + "compatible_from": "0.1.0-experimental.1", + "rollback_safe": true +} diff --git a/plugins/lumi_transcription/public/transcription.css b/plugins/lumi_transcription/public/transcription.css new file mode 100644 index 0000000..2d6cf70 --- /dev/null +++ b/plugins/lumi_transcription/public/transcription.css @@ -0,0 +1 @@ +.transcription-shell{--accent:#85e3c4;--warn:#f5c56b;max-width:1120px;margin:0 auto;padding:clamp(1rem,3vw,2.5rem);display:grid;gap:1.35rem}.transcription-shell section{padding:1.35rem 0;border-top:1px solid color-mix(in srgb,currentColor 16%,transparent)}.transcription-hero{display:flex;justify-content:space-between;align-items:flex-start;gap:2rem;padding:1rem 0 2rem}.transcription-hero h1{font-size:clamp(2rem,5vw,4.25rem);line-height:.95;margin:.25rem 0 1rem}.transcription-hero p:not(.eyebrow){max-width:67ch;opacity:.78}.eyebrow{text-transform:uppercase;letter-spacing:.13em;font-size:.72rem;font-weight:750;opacity:.65;margin:0}.health-pill{display:inline-flex;align-items:center;gap:.55rem;padding:.55rem .8rem;border-radius:999px;white-space:nowrap;background:color-mix(in srgb,var(--warn) 16%,transparent)}.health-pill span{width:.65rem;height:.65rem;border-radius:50%;background:var(--warn)}.health-pill.is-ready{background:color-mix(in srgb,var(--accent) 16%,transparent)}.health-pill.is-ready span{background:var(--accent)}.section-heading{display:flex;justify-content:space-between;align-items:end;gap:1rem;margin-bottom:1rem}.section-heading h2{margin:.2rem 0 0;font-size:1.35rem}.setup-path ol{display:grid;grid-template-columns:repeat(4,1fr);list-style:none;padding:0;margin:0 0 1.25rem;counter-reset:steps}.setup-path li{counter-increment:steps;padding:0 1rem 1rem 2.25rem;position:relative;opacity:.62}.setup-path li:before{content:counter(steps);position:absolute;left:0;top:-.2rem;width:1.6rem;height:1.6rem;border:1px solid currentColor;border-radius:50%;display:grid;place-items:center;font-size:.75rem}.setup-path li.is-current{opacity:1}.setup-path li.is-current:before{background:var(--accent);color:#10251f;border-color:var(--accent)}.setup-path li span,.plain-list span{display:block;font-size:.88rem;opacity:.7;margin-top:.25rem}.primary-action{border:0;border-radius:.6rem;background:var(--accent);color:#10251f;font-weight:750;padding:.7rem 1rem;cursor:pointer}.primary-action:disabled{opacity:.55;cursor:wait}.inline-status{display:inline;margin-left:.8rem}.transcription-grid{display:grid;grid-template-columns:1fr 1fr;gap:2.5rem}.health-list{margin:0}.health-list div{display:flex;justify-content:space-between;gap:1rem;padding:.55rem 0}.health-list dt{opacity:.65}.notice,.privacy-note,.empty-state{padding:1rem;border-radius:.65rem;background:color-mix(in srgb,currentColor 6%,transparent);font-size:.9rem}.plain-list{list-style:none;padding:0;margin:0}.plain-list li{display:flex;justify-content:space-between;gap:1rem;padding:.7rem 0}.plain-list code{font-size:.72rem;opacity:.6}.model-row{display:grid;gap:.5rem}.model-row article{display:flex;justify-content:space-between;align-items:center;gap:1rem;padding:.9rem 0}.model-row h3,.model-row p{margin:0}.model-row p{font-size:.88rem;opacity:.7;margin-top:.2rem}.state-label{font-size:.78rem;font-weight:700;white-space:nowrap}@media(max-width:760px){.transcription-hero,.transcription-grid{display:grid;grid-template-columns:1fr}.setup-path ol{grid-template-columns:1fr}.health-pill{justify-self:start}.plain-list li{display:block}.plain-list code{display:block;margin-top:.5rem;overflow-wrap:anywhere}} diff --git a/plugins/lumi_transcription/public/transcription.js b/plugins/lumi_transcription/public/transcription.js new file mode 100644 index 0000000..1d973f2 --- /dev/null +++ b/plugins/lumi_transcription/public/transcription.js @@ -0,0 +1,22 @@ +(() => { + const root = document.querySelector("[data-transcription-admin]"); + if (!root) return; + const button = root.querySelector("[data-create-pairing]"); + const status = root.querySelector("[data-status]"); + button?.addEventListener("click", async () => { + button.disabled = true; + status.textContent = "Creating a one-time pairing package…"; + try { + const response = await fetch("/plugins/lumi_transcription/api/pairing-package", { method: "POST", headers: { Accept: "application/json" } }); + if (!response.ok) throw new Error((await response.json()).error || "Pairing package could not be created."); + const blob = await response.blob(); + const disposition = response.headers.get("content-disposition") || ""; + const filename = /filename="?([^";]+)"?/i.exec(disposition)?.[1] || "lumi-companion.lumi-pairing.json"; + const link = document.createElement("a"); + link.href = URL.createObjectURL(blob); link.download = filename; link.click(); + setTimeout(() => URL.revokeObjectURL(link.href), 1000); + status.textContent = "Pairing package created. It expires in 15 minutes and works once."; + } catch (error) { status.textContent = error.message; } + finally { button.disabled = false; } + }); +})(); diff --git a/plugins/lumi_transcription/runtime_manifest.json b/plugins/lumi_transcription/runtime_manifest.json new file mode 100644 index 0000000..39bd69c --- /dev/null +++ b/plugins/lumi_transcription/runtime_manifest.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "provider": "whisper.cpp", + "tested_version": "v1.9.1", + "tested_commit": "f049fff95a089aa9969deb009cdd4892b3e74916", + "artifacts": [ + { + "id": "windows-x64-cuda-12.4", + "platform": "win32", + "architecture": "x64", + "backend": "cuda", + "cuda": "12.4", + "url": "https://github.com/ggml-org/whisper.cpp/releases/download/v1.9.1/whisper-cublas-12.4.0-bin-x64.zip", + "sha256": "106a2030eff8998e4ef320fe72e263a78449e9040386ee27c41ea80b001b601b", + "expected_paths": ["whisper-cli.exe"], + "tested": true + }, + { + "id": "windows-x64-cpu", + "platform": "win32", + "architecture": "x64", + "backend": "cpu", + "url": "https://github.com/ggml-org/whisper.cpp/releases/download/v1.9.1/whisper-bin-x64.zip", + "sha256": "7d8be46ecd31828e1eb7a2ecdd0d6b314feafd82163038ab6092594b0a063539", + "expected_paths": ["whisper-cli.exe"], + "tested": false, + "note": "CPU fallback remains disabled until the host benchmark meets the latency target." + } + ] +} diff --git a/plugins/lumi_transcription/tests/verify.js b/plugins/lumi_transcription/tests/verify.js new file mode 100644 index 0000000..8035a90 --- /dev/null +++ b/plugins/lumi_transcription/tests/verify.js @@ -0,0 +1,241 @@ +const assert = require("assert"); +const crypto = require("crypto"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +const http = require("http"); +const { EventEmitter } = require("events"); +const { PassThrough } = require("stream"); +const Database = require("better-sqlite3"); +const express = require("express"); +const { WebSocket } = require("ws"); +const { DeviceStore } = require("../backend/companion/device_store"); +const { CompanionGateway } = require("../backend/companion/gateway"); +const protocol = require("../backend/companion/protocol"); +const { RevisionStore } = require("../backend/config/revision_store"); +const { JsonlDiagnosticLog, sanitize } = require("../backend/logs/jsonl_log"); +const { ArtifactManager, sha256File } = require("../backend/models/artifact_manager"); +const { BoundedQueue, SequenceTracker, RollingPcmBuffer } = require("../backend/sessions/bounded_queue"); +const { SessionCoordinator } = require("../backend/sessions/session_coordinator"); +const { CaptionStabilizer, LatestCaptionGate } = require("../backend/transcription/stabilizer"); +const { WhisperWorkerSupervisor } = require("../backend/transcription/provider"); +const plugin = require("../index"); +const { createWebUpgradeRegistry } = require("../../../src/services/web-upgrades"); + +async function run() { + const temp = fs.mkdtempSync(path.join(os.tmpdir(), "lumi-transcription-")); + try { + verifyProtocol(); + verifyPairingAndRevocation(); + verifyRevisions(); + verifyQueues(); + verifyStabilization(); + await verifySessionLifecycle(); + await verifyWorkerRestart(); + await verifyAuthenticatedGateway(); + verifyArtifactsAndLogs(temp); + await verifyPluginIsolation(); + console.log("Lumi transcription verification passed: protocol, pairing, revocation, revisions, queues, stabilization, lifecycle, worker recovery, artifacts, logs, and plugin isolation."); + } finally { fs.rmSync(temp, { recursive: true, force: true }); } +} + +function verifyProtocol() { + const sessionId = crypto.randomUUID(); + const sourceUuid = crypto.randomUUID(); + const encoded = protocol.encodeAudioFrame({ session_id: sessionId, source_uuid: sourceUuid, sequence: 42, capture_timestamp_us: 987654321, active: true, muted: false, pcm: Buffer.alloc(640, 3) }); + const decoded = protocol.parseAudioFrame(encoded); + assert.equal(decoded.session_id, sessionId); + assert.equal(decoded.source_uuid, sourceUuid); + assert.equal(decoded.sequence, 42); + assert.equal(decoded.pcm.length, 640); + assert.throws(() => protocol.parseAudioFrame(Buffer.alloc(64)), /magic/i); + const hello = protocol.envelope("hello", { companion_version: "0.1.0", plugin_version: "0.1.0", capabilities: ["transcription.capture.v1"], audio: { codec: "pcm_s16le", sample_rate: 16000, channels: 1, bits: 16 } }); + assert.equal(protocol.validateHello(protocol.parseEnvelope(JSON.stringify(hello))).audio.codec, "pcm_s16le"); + assert.throws(() => protocol.parseEnvelope(JSON.stringify({ ...hello, version: 2 })), /unsupported/i); + assert.throws(() => protocol.parseEnvelope(Buffer.alloc(protocol.MAX_JSON_BYTES + 1)), /size/i); +} + +function verifyPairingAndRevocation() { + const db = new Database(":memory:"); + let now = 1000; + let randomSeed = 6; + const store = new DeviceStore(db, { now: () => now, randomBytes: (size) => Buffer.alloc(size, ++randomSeed) }); + const pairing = store.issuePairing({ userId: "admin", host: "https://lumi.example" }); + const issued = store.exchange({ token: pairing.token, device: { install_id: "install", name: "Stream PC", companion_version: "0.1.0" } }); + assert.equal(store.authenticate(`LumiDevice ${issued.device_id}.${issued.device_secret}`, "transcription.capture.v1").allowed, true); + assert.throws(() => store.exchange({ token: pairing.token, device: {} }), (error) => error.code === "PAIRING_ALREADY_USED"); + assert.deepEqual(store.setCapabilities(issued.device_id, ["transcription.settings.v1", "made.up"]), ["transcription.settings.v1"]); + assert.equal(store.authenticate(`LumiDevice ${issued.device_id}.${issued.device_secret}`, "transcription.capture.v1").reason, "capability_revoked"); + assert.equal(store.revoke(issued.device_id), true); + assert.equal(store.authenticate(`LumiDevice ${issued.device_id}.${issued.device_secret}`).reason, "device_revoked"); + const expired = store.issuePairing({ userId: "admin", host: "https://lumi.example", ttlMs: 50 }); + now += 51; + assert.throws(() => store.exchange({ token: expired.token, device: {} }), (error) => error.code === "PAIRING_INVALID"); + db.close(); +} + +function verifyRevisions() { + const db = new Database(":memory:"); + const store = new RevisionStore(db, { now: () => 1234 }); + const first = store.apply([{ key: "selected_model_id", value: "small.en", base_revision: 0 }], "admin"); + assert.equal(first.applied[0].revision, 1); + const merged = store.apply([{ key: "caption_max_chars", value: 80, base_revision: 0 }], "companion"); + assert.equal(merged.conflicts.length, 0); + const conflict = store.apply([{ key: "selected_model_id", value: "base.en", base_revision: 0 }], "companion"); + assert.equal(conflict.conflicts[0].server.value, "small.en"); + assert.equal(store.list().selected_model_id.value, "small.en"); + db.close(); +} + +function verifyQueues() { + let now = 0; + const queue = new BoundedQueue({ maxItems: 2, maxBytes: 8, maxAgeMs: 50, now: () => now }); + queue.push(Buffer.alloc(4), { capturedAt: now }); queue.push(Buffer.alloc(4), { capturedAt: now }); queue.push(Buffer.alloc(4), { capturedAt: now }); + assert.equal(queue.metrics().dropped.capacity, 1); + now = 51; assert.equal(queue.size(), 0); assert.equal(queue.metrics().dropped.stale, 2); + const sequence = new SequenceTracker(); + assert.equal(sequence.accept(5).accepted, true); assert.equal(sequence.accept(8).gap, 2); assert.equal(sequence.accept(7).accepted, false); + const rolling = new RollingPcmBuffer({ seconds: 1, now: () => now }); + rolling.push(Buffer.alloc(20000), now); rolling.push(Buffer.alloc(20000), now); + assert.equal(rolling.snapshot().length, 20000); +} + +function verifyStabilization() { + let now = 0; + const stabilizer = new CaptionStabilizer({ now: () => now, fragmentAfterMs: 1000 }); + const source = crypto.randomUUID(); + stabilizer.update(source, "hello wor"); + now += 600; + const second = stabilizer.update(source, "hello world"); + assert.equal(second.stable_text, "hello"); + now += 600; + const third = stabilizer.update(source, "hello world again"); + assert.equal(third.stable_text, "hello world"); + const fragmentSource = crypto.randomUUID(); + stabilizer.update(fragmentSource, "extraord", { trailingIncomplete: true }); + now += 1001; + const fragment = stabilizer.update(fragmentSource, "extraord", { trailingIncomplete: true }); + assert.equal(fragment.incomplete_word, true); + assert.equal(fragment.stable_text, ""); + const gate = new LatestCaptionGate(); + assert.equal(gate.accept({ session_id: source, caption_id: fragment.caption_id, revision: 2 }), true); + assert.equal(gate.accept({ session_id: source, caption_id: fragment.caption_id, revision: 1 }), false); +} + +async function verifySessionLifecycle() { + class Provider extends EventEmitter { + constructor() { super(); this.audio = []; this.stops = 0; } + async health() { return { healthy: true }; } + async startSession() {} async addTrack() {} async pushAudio(_session, _track, frame) { this.audio.push(frame); return true; } + async stopSession() { this.stops += 1; } + } + const provider = new Provider(); + const delivered = []; + const deliveryFactory = () => ({ start: async () => {}, stop: async () => {}, pause: async () => {}, resume: async () => {}, deliver: async (event) => { delivered.push(event); return { disposition: "simulated" }; } }); + const coordinator = new SessionCoordinator({ provider, deliveryFactory, graceMs: 25 }); + const { session } = coordinator.create({ id: "device" }, () => {}); + const primary = crypto.randomUUID(); + coordinator.updateSource(session.id, { source_uuid: primary, display_name: "Mic", primary: true, program_active: true }); + await assert.rejects(coordinator.start(session.id, { mode: "live" }), (error) => error.code === "OBS_NOT_STREAMING"); + await coordinator.updateObsState(session.id, { streaming: true, auto_start: false }); + await coordinator.start(session.id, { mode: "live" }); + const frame = protocol.parseAudioFrame(protocol.encodeAudioFrame({ session_id: session.id, source_uuid: primary, sequence: 1, capture_timestamp_us: 1, pcm: Buffer.alloc(640) })); + assert.equal((await coordinator.audio(session.id, frame)).accepted, true); + provider.emit("hypothesis", { session_id: session.id, track_id: primary, text: "hello", final: true, model_id: "small.en", backend: "cuda" }); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(delivered[0].stable_text, "hello"); + await coordinator.updateObsState(session.id, { streaming: false }); + assert.equal(coordinator.status(session.id).state, "grace"); + await new Promise((resolve) => setTimeout(resolve, 40)); + assert.equal(coordinator.status(session.id).state, "idle"); + assert.equal(provider.stops, 1); + await coordinator.close(); +} + +async function verifyWorkerRestart() { + const children = []; + const fakeSpawn = () => { + const child = new EventEmitter(); + child.stdin = new PassThrough(); child.stdout = new PassThrough(); child.stderr = new PassThrough(); child.kill = () => child.emit("exit", null, "SIGTERM"); + children.push(child); return child; + }; + const supervisor = new WhisperWorkerSupervisor({ executable: "fake-worker", spawn: fakeSpawn, maxRestarts: 1 }); + supervisor.on("error", () => {}); + supervisor.start(); + assert.equal(supervisor.send({ type: "audio", captured_at: Date.now() }, Buffer.alloc(640)), true); + children[0].emit("exit", 1, null); + await new Promise((resolve) => setTimeout(resolve, 320)); + assert.equal(children.length, 2); + supervisor.stopping = true; children[1].emit("exit", 0, null); +} + +async function verifyAuthenticatedGateway() { + const db = new Database(":memory:"); + const devices = new DeviceStore(db); + const pairing = devices.issuePairing({ userId: "admin", host: "https://lumi.example" }); + const credential = devices.exchange({ token: pairing.token, device: { name: "Stream PC" } }); + const sessionId = crypto.randomUUID(); + let disconnected = false; + const sessions = { + create: (_device, send) => ({ session: { id: sessionId, state: "idle", send }, resumed: false }), + disconnect: () => { disconnected = true; }, + audio: async () => ({ accepted: true }), updateSource: () => ({}), updateObsState: async () => ({}), start: async () => ({}), stop: async () => ({}) + }; + const gateway = new CompanionGateway({ devices, sessions, allowInsecure: true }); + const registry = createWebUpgradeRegistry(); + registry.add("/plugins/lumi_transcription/live", (request, socket, head) => gateway.upgrade(request, socket, head)); + const server = http.createServer((_req, res) => res.end("ok")); + registry.attach(server); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const client = new WebSocket(`ws://127.0.0.1:${server.address().port}/plugins/lumi_transcription/live`, { + headers: { Authorization: `LumiDevice ${credential.device_id}.${credential.device_secret}` } + }); + await new Promise((resolve, reject) => { client.once("open", resolve); client.once("error", reject); }); + client.send(JSON.stringify(protocol.envelope("hello", { companion_version: "0.1.0", plugin_version: "0.1.0", capabilities: ["transcription.capture.v1"], audio: { codec: "pcm_s16le", sample_rate: 16000, channels: 1, bits: 16 } }))); + const response = await new Promise((resolve, reject) => { client.once("message", (data) => resolve(JSON.parse(String(data)))); client.once("error", reject); }); + assert.equal(response.type, "hello_ack"); + assert.equal(response.session_id, sessionId); + await new Promise((resolve) => { client.once("close", resolve); client.close(); }); + for (let attempt = 0; attempt < 20 && !disconnected; attempt += 1) await new Promise((resolve) => setTimeout(resolve, 5)); + assert.equal(disconnected, true); + await gateway.close(); registry.close(); + await new Promise((resolve) => server.close(resolve)); + db.close(); +} + +function verifyArtifactsAndLogs(temp) { + const artifactRoot = path.join(temp, "artifacts"); fs.mkdirSync(artifactRoot); + const file = path.join(artifactRoot, "model.bin"); fs.writeFileSync(file, "verified"); + const entry = { id: "model", filename: "model.bin", url: "https://example.invalid/model.bin", sha256: sha256File(file) }; + assert.equal(new ArtifactManager(artifactRoot).status(entry).valid, true); + assert.equal(sanitize({ pcm: Buffer.alloc(10), device_secret: "secret", stable_text: "hello" }, false).pcm, "[redacted]"); + const logsRoot = path.join(temp, "logs"); + const logs = new JsonlDiagnosticLog(logsRoot, { retentionDays: 1, maxBytes: 100 }); + logs.append({ kind: "caption", stable_text: "hello", pcm: Buffer.alloc(10) }); + const old = path.join(logsRoot, "transcription-2000-01-01.jsonl"); fs.writeFileSync(old, `${"x".repeat(150)}\n`); fs.utimesSync(old, new Date(0), new Date(0)); + assert.ok(logs.cleanup().removed >= 1); + assert.equal(fs.readdirSync(logsRoot).some((name) => name.endsWith(".wav") || name.endsWith(".pcm")), false); +} + +async function verifyPluginIsolation() { + const db = new Database(":memory:"); + const mounts = []; + let upgradeRemoved = false; + const cleanup = plugin.init({ + db, logger: { info() {} }, + web: { + createRouter: () => express.Router(), + addUpgradeHandler: () => () => { upgradeRemoved = true; }, + emitEvent() {}, + mount: (mountPath) => mounts.push(mountPath) + } + }); + assert.deepEqual(mounts, ["/plugins/lumi_transcription"]); + assert.equal(typeof global.lumiFrameworks.transcription.health, "function"); + await cleanup(); + assert.equal(upgradeRemoved, true); + assert.equal(global.lumiFrameworks.transcription, undefined); + db.close(); +} + +run().catch((error) => { console.error(error); process.exit(1); }); diff --git a/plugins/lumi_transcription/views/settings.ejs b/plugins/lumi_transcription/views/settings.ejs new file mode 100644 index 0000000..9182321 --- /dev/null +++ b/plugins/lumi_transcription/views/settings.ejs @@ -0,0 +1,59 @@ + +
+
+
+

Experimental companion

+

Lumi Transcription

+

Server-hosted speech recognition for selected OBS sources. Live Twitch delivery remains blocked until the native OBS caption compatibility test passes on the target setup.

+
+ + <%= providerHealth.healthy ? 'Inference ready' : 'Inference setup required' %> + +
+ +
+

First usable path

Setup progress

+
    +
  1. Pair a companionCreate a single-use package for the streaming computer.
  2. +
  3. Install the OBS bridgeManaged by Lumi Companion; no separate bridge settings.
  4. +
  5. Install and benchmark a modelSmall English is recommended. Nothing downloads without confirmation.
  6. +
  7. Select and test a microphoneTest mode must pass before live delivery is enabled.
  8. +
+ +

+
+ +
+
+

Lumi host

Inference

+
+
Provider
whisper.cpp <%= runtimeManifest.tested_version %>
+
Worker
<%= providerHealth.state %>
+
Selected model
<%= providerHealth.model || 'Not loaded' %>
+
Active sessions
<%= providerHealth.sessions || 0 %>
+
+

Lumi AI may already occupy most GPU memory. The MVP warns and benchmarks; it does not unload Lumi AI models automatically.

+
+ +
+

Access

Paired devices

<%= devices.length %>
+ <% if (!devices.length) { %>

No companion is paired yet.

<% } %> +
    + <% devices.forEach((device) => { %> +
  • <%= device.name %><%= device.revoked_at ? 'Revoked' : 'Last connected ' + new Date(device.last_connected_at).toLocaleString() %>
    <%= device.id %>
  • + <% }) %> +
+
+
+ +
+

Curated choices

Speech models

+
+ <% models.forEach((model) => { %> +

<%= model.label %>

<%= model.recommended ? 'Recommended starting point' : 'Fallback option' %> · <%= Math.round(model.bytes / 1048576) %> MiB

<%= model.status.valid ? 'Verified' : model.status.installed ? 'Checksum failed' : 'Not installed' %>
+ <% }) %> +
+

Privacy: raw audio is held only in bounded memory and is never written to disk by default. Diagnostic caption text is retained for seven days unless disabled.

+
+
+ diff --git a/protocol/companion-protocol-v1.md b/protocol/companion-protocol-v1.md new file mode 100644 index 0000000..a84aea0 --- /dev/null +++ b/protocol/companion-protocol-v1.md @@ -0,0 +1,50 @@ +# Lumi Companion Protocol v1 + +Status: experimental. The protocol is versioned independently from Lumi core, the server plugin, companion core, and companion transcription plugin. + +## Trust boundaries + +The OBS bridge accepts only same-user named-pipe clients and speaks only the local IPC subset. The companion owns pairing and connects to Lumi over TLS WebSockets. Lumi authenticates a device before accepting a WebSocket upgrade. The bridge never receives a Lumi device credential and never connects to the network. + +## Server connection + +- Endpoint: `wss:///plugins/lumi_transcription/live` +- Header: `Authorization: LumiDevice .` +- Maximum structured message: 64 KiB. +- Maximum binary audio message: 6,464 bytes (64-byte header plus up to 200 ms of PCM). +- The client sends `hello` first. Lumi replies with `hello_ack` or closes with an application error. +- Protocol v1 supports `pcm_s16le`; codec negotiation is still explicit so Opus can be added later. +- Heartbeats use `ping`/`pong`. A new connection is a new session unless `resume_session_id` names a resumable session owned by that device. + +Structured messages use the envelope in `schemas/envelope.schema.json`. Supported client events are `hello`, `ping`, `source_update`, `obs_state`, `start`, `stop`, and `ack`. Server events are `hello_ack`, `pong`, `status`, `caption`, `metric`, and `error`. + +## Binary PCM frame + +All integers are little-endian. The fixed header is 64 bytes: + +| Offset | Size | Meaning | +| --- | ---: | --- | +| 0 | 4 | ASCII `LACP` | +| 4 | 1 | protocol version (`1`) | +| 5 | 1 | flags: active `0x01`, muted `0x02` | +| 6 | 2 | header bytes (`64`) | +| 8 | 4 | sequence number | +| 12 | 8 | monotonic capture timestamp, microseconds | +| 20 | 16 | session UUID bytes | +| 36 | 16 | OBS source UUID bytes | +| 52 | 4 | sample rate (`16000`) | +| 56 | 2 | channels (`1`) | +| 58 | 2 | PCM bits (`16`) | +| 60 | 4 | payload bytes | + +The payload is mono signed 16-bit little-endian PCM. Receivers reject malformed sizes and unsupported audio formats. Sequence gaps are reported; out-of-order frames are discarded. Every queue drops obsolete data rather than blocking a capture thread. + +## Local OBS IPC + +Windows transport is a per-user named pipe named `Lumi.Companion.ObsBridge.v1.`. Messages are length-prefixed (32-bit little-endian), capped at 64 KiB for JSON and 6,464 bytes for audio, and begin with a v1 handshake. The bridge accepts only this allowlist: `hello`, `select_sources`, `caption`, `delivery_state`, `shutdown_notice`, and PCM frames. It cannot execute arbitrary commands. + +Caption events are revision-aware. Consumers retain the greatest revision for each `(session_id, caption_id)` and discard older revisions, including after reconnect. + +## Compatibility + +Protocol v1 requires companion capability `transcription.capture.v1` and server capability `transcription.server.v1`. OBS delivery is separately negotiated as `obs.caption.native.v1`; its absence keeps test-mode simulation available but blocks live delivery. diff --git a/protocol/schemas/caption.schema.json b/protocol/schemas/caption.schema.json new file mode 100644 index 0000000..dfa3c01 --- /dev/null +++ b/protocol/schemas/caption.schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://lumi.local/protocol/v1/caption.schema.json", + "type": "object", + "required": ["session_id", "source_uuid", "caption_id", "revision", "stable_text", "uncertain_text", "final", "incomplete_word", "audio", "latency", "delivery", "model"], + "properties": { + "session_id": { "type": "string", "format": "uuid" }, + "source_uuid": { "type": "string", "format": "uuid" }, + "speaker_label": { "type": ["string", "null"], "maxLength": 80 }, + "caption_id": { "type": "string", "format": "uuid" }, + "revision": { "type": "integer", "minimum": 1 }, + "stable_text": { "type": "string", "maxLength": 80 }, + "uncertain_text": { "type": "string", "maxLength": 80 }, + "final": { "type": "boolean" }, + "incomplete_word": { "type": "boolean" }, + "audio": { "type": "object", "required": ["start_us", "end_us"] }, + "latency": { "type": "object", "required": ["capture_ms", "network_ms", "queue_ms", "inference_ms", "stabilization_ms", "total_ms"] }, + "delivery": { "type": "object", "required": ["disposition"] }, + "model": { "type": "object", "required": ["id", "provider", "backend"] } + }, + "additionalProperties": false +} diff --git a/protocol/schemas/envelope.schema.json b/protocol/schemas/envelope.schema.json new file mode 100644 index 0000000..1cefe3f --- /dev/null +++ b/protocol/schemas/envelope.schema.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://lumi.local/protocol/v1/envelope.schema.json", + "type": "object", + "required": ["version", "type", "id", "sent_at"], + "properties": { + "version": { "const": 1 }, + "type": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[a-z][a-z0-9_]*$" }, + "id": { "type": "string", "format": "uuid" }, + "sent_at": { "type": "string", "format": "date-time" }, + "session_id": { "type": ["string", "null"], "format": "uuid" }, + "payload": { "type": "object" } + }, + "additionalProperties": false +} diff --git a/protocol/schemas/hello.schema.json b/protocol/schemas/hello.schema.json new file mode 100644 index 0000000..2e085af --- /dev/null +++ b/protocol/schemas/hello.schema.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://lumi.local/protocol/v1/hello.schema.json", + "allOf": [ + { "$ref": "envelope.schema.json" }, + { + "properties": { + "type": { "const": "hello" }, + "payload": { + "type": "object", + "required": ["companion_version", "plugin_version", "capabilities", "audio"], + "properties": { + "companion_version": { "type": "string", "maxLength": 32 }, + "plugin_version": { "type": "string", "maxLength": 32 }, + "obs_version": { "type": ["string", "null"], "maxLength": 32 }, + "resume_session_id": { "type": ["string", "null"], "format": "uuid" }, + "capabilities": { "type": "array", "maxItems": 32, "items": { "type": "string", "maxLength": 64 } }, + "audio": { + "type": "object", + "required": ["codec", "sample_rate", "channels", "bits"], + "properties": { + "codec": { "enum": ["pcm_s16le"] }, + "sample_rate": { "const": 16000 }, + "channels": { "const": 1 }, + "bits": { "const": 16 } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + } + } + ] +} diff --git a/scripts/verify-all.js b/scripts/verify-all.js index e789a7f..dc0df07 100644 --- a/scripts/verify-all.js +++ b/scripts/verify-all.js @@ -21,6 +21,7 @@ const checks = [ "plugins/lumi_ai/tests/verify.js", "plugins/lumi_ai/tests/verify-tools.js", "plugins/lumi_ai_web_search/tests/verify.js", + "plugins/lumi_transcription/tests/verify.js", "scripts/verify-assistant-panels.js", "scripts/verify-command-preview-confirmations.js", "scripts/verify-command-policies.js", diff --git a/src/main.js b/src/main.js index 4ec5d7b..26770e2 100644 --- a/src/main.js +++ b/src/main.js @@ -125,6 +125,7 @@ async function main() { console.log(`WebUI listening on http://localhost:${port}`, { port }); }); }); + app.locals.lumiUpgradeRegistry?.attach(webServer); const autoUpdateEnabled = getSetting("auto_update_enabled", false); const intervalMinutes = getSetting("auto_update_interval_minutes", 60); @@ -155,6 +156,7 @@ async function main() { return; } shuttingDown = true; + app.locals.lumiUpgradeRegistry?.close(); runtimeLog.info("Lumi shutdown started", { exit_code: exitCode }, { event: exitCode === 10 ? "restart" : "shutdown" }); const closeWebServer = new Promise((resolve) => webServer.close(resolve)); for (const stop of [ diff --git a/src/services/web-upgrades.js b/src/services/web-upgrades.js new file mode 100644 index 0000000..b1b6fac --- /dev/null +++ b/src/services/web-upgrades.js @@ -0,0 +1,62 @@ +function createWebUpgradeRegistry() { + const handlers = []; + let attachedServer = null; + + function add(pathname, handler) { + const path = normalizePath(pathname); + if (typeof handler !== "function") throw new Error("An upgrade handler is required."); + const entry = { path, handler }; + handlers.push(entry); + return () => { + const index = handlers.indexOf(entry); + if (index >= 0) handlers.splice(index, 1); + }; + } + + function dispatch(request, socket, head) { + let pathname = "/"; + try { + pathname = new URL(request.url || "/", "http://lumi.local").pathname; + } catch {} + const entry = handlers.find((candidate) => candidate.path === pathname); + if (!entry) { + socket.write("HTTP/1.1 404 Not Found\r\nConnection: close\r\n\r\n"); + socket.destroy(); + return; + } + try { + entry.handler(request, socket, head); + } catch { + if (!socket.destroyed) { + socket.write("HTTP/1.1 500 Internal Server Error\r\nConnection: close\r\n\r\n"); + socket.destroy(); + } + } + } + + function attach(server) { + if (!server || typeof server.on !== "function") throw new Error("An HTTP server is required."); + if (attachedServer === server) return; + if (attachedServer) attachedServer.off("upgrade", dispatch); + attachedServer = server; + server.on("upgrade", dispatch); + } + + function close() { + if (attachedServer) attachedServer.off("upgrade", dispatch); + attachedServer = null; + handlers.length = 0; + } + + return { add, attach, close, count: () => handlers.length }; +} + +function normalizePath(value) { + const path = String(value || "").trim(); + if (!path.startsWith("/") || path.includes("?") || path.includes("#")) { + throw new Error("Upgrade paths must be absolute URL paths."); + } + return path; +} + +module.exports = { createWebUpgradeRegistry }; diff --git a/src/web/server.js b/src/web/server.js index 47e70d1..960e284 100644 --- a/src/web/server.js +++ b/src/web/server.js @@ -28,6 +28,7 @@ const { } = require("../services/themes"); const { getRoleFlags, hasAccess } = require("../services/rbac"); const { createWebAuth } = require("../services/web-auth"); +const { createWebUpgradeRegistry } = require("../services/web-upgrades"); const { cleanupUploadedFiles, safeDownloadFilename, @@ -3100,6 +3101,8 @@ async function verifyYouTubeSettings(settings) { function createWebServer({ loadPlugins, discordClient, commandRouter }) { const app = express(); + const upgradeRegistry = createWebUpgradeRegistry(); + app.locals.lumiUpgradeRegistry = upgradeRegistry; // Only trust forwarding headers from a reverse proxy on this machine. This // lets the diagnostics endpoint recognize HTTPS without trusting arbitrary // client-supplied X-Forwarded-Proto headers. @@ -3448,6 +3451,7 @@ function createWebServer({ loadPlugins, discordClient, commandRouter }) { registerPlaceholder: placeholders.registerPlaceholder, registerPlaceholders: placeholders.registerPlaceholders, registerPlaceholderField: placeholders.registerFieldPolicy, + addUpgradeHandler: upgradeRegistry.add, emitEvent: publishWebEvent };