Add Lumi transcription server and protocol foundation
This commit is contained in:
Franz Rolfsvaag
parent
162c3b1df6
commit
134d3efa74
5
.gitignore
vendored
5
.gitignore
vendored
@ -16,6 +16,11 @@ plugins/*/data/**
|
|||||||
*.sqlite-*
|
*.sqlite-*
|
||||||
npm-debug.log
|
npm-debug.log
|
||||||
/dist/
|
/dist/
|
||||||
|
companion/**/bin/
|
||||||
|
companion/**/obj/
|
||||||
|
companion/installer/output/
|
||||||
|
companion/**/logs/
|
||||||
|
*.lumi-pairing.json
|
||||||
security-audit-*.json
|
security-audit-*.json
|
||||||
security-audit-*.md
|
security-audit-*.md
|
||||||
taskfile.txt
|
taskfile.txt
|
||||||
|
|||||||
@ -21,7 +21,8 @@
|
|||||||
"test:ui:update": "playwright test --update-snapshots",
|
"test:ui:update": "playwright test --update-snapshots",
|
||||||
"verify:webui": "node scripts/verify-webui.js && node scripts/verify-destructive-actions.js",
|
"verify:webui": "node scripts/verify-webui.js && node scripts/verify-destructive-actions.js",
|
||||||
"benchmark:okf": "node scripts/benchmark-okf-search.js",
|
"benchmark:okf": "node scripts/benchmark-okf-search.js",
|
||||||
"verify:content": "node scripts/verify-content-library.js"
|
"verify:content": "node scripts/verify-content-library.js",
|
||||||
|
"verify:transcription": "node plugins/lumi_transcription/tests/verify.js"
|
||||||
},
|
},
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">=18"
|
"node": ">=18"
|
||||||
|
|||||||
90
plugins/lumi_transcription/backend/companion/device_store.js
Normal file
90
plugins/lumi_transcription/backend/companion/device_store.js
Normal file
@ -0,0 +1,90 @@
|
|||||||
|
const crypto = require("crypto");
|
||||||
|
|
||||||
|
const DEFAULT_CAPABILITIES = Object.freeze(["transcription.capture.v1", "transcription.settings.v1", "obs.caption.native.v1"]);
|
||||||
|
|
||||||
|
class DeviceStore {
|
||||||
|
constructor(db, options = {}) {
|
||||||
|
this.db = db;
|
||||||
|
this.now = options.now || Date.now;
|
||||||
|
this.randomBytes = options.randomBytes || crypto.randomBytes;
|
||||||
|
this.allowInsecure = options.allowInsecure === true;
|
||||||
|
this.migrate();
|
||||||
|
}
|
||||||
|
|
||||||
|
migrate() {
|
||||||
|
this.db.exec(`
|
||||||
|
CREATE TABLE IF NOT EXISTS transcription_pairing_tokens (
|
||||||
|
id TEXT PRIMARY KEY, token_hash TEXT NOT NULL UNIQUE, lumi_user_id TEXT NOT NULL,
|
||||||
|
host TEXT NOT NULL, expires_at INTEGER NOT NULL, activated_at INTEGER, created_at INTEGER NOT NULL
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS transcription_devices (
|
||||||
|
id TEXT PRIMARY KEY, install_id TEXT, name TEXT NOT NULL, lumi_user_id TEXT NOT NULL,
|
||||||
|
credential_hash TEXT NOT NULL, capabilities_json TEXT NOT NULL, metadata_json TEXT NOT NULL,
|
||||||
|
first_connected_at INTEGER NOT NULL, last_connected_at INTEGER NOT NULL, revoked_at INTEGER
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS transcription_devices_user_idx ON transcription_devices(lumi_user_id);
|
||||||
|
`);
|
||||||
|
}
|
||||||
|
|
||||||
|
issuePairing({ userId, host, ttlMs = 15 * 60 * 1000 }) {
|
||||||
|
if (!userId || !host) throw new Error("A Lumi user and host are required.");
|
||||||
|
const id = crypto.randomUUID();
|
||||||
|
const token = tokenValue(this.randomBytes(32));
|
||||||
|
const now = this.now();
|
||||||
|
this.db.prepare("INSERT INTO transcription_pairing_tokens (id, token_hash, lumi_user_id, host, expires_at, activated_at, created_at) VALUES (?, ?, ?, ?, ?, NULL, ?)")
|
||||||
|
.run(id, digest(token), String(userId), normalizeHost(host, this.allowInsecure), now + ttlMs, now);
|
||||||
|
return { pairing_id: id, token, host: normalizeHost(host, this.allowInsecure), expires_at: now + ttlMs, protocol_version: 1 };
|
||||||
|
}
|
||||||
|
|
||||||
|
exchange({ token, device = {} }) {
|
||||||
|
const hash = digest(token);
|
||||||
|
const row = this.db.prepare("SELECT * FROM transcription_pairing_tokens WHERE token_hash = ?").get(hash);
|
||||||
|
if (!row || row.activated_at || row.expires_at < this.now()) {
|
||||||
|
const error = new Error(row?.activated_at ? "This pairing package was already activated. Download a new companion package." : "This pairing package is invalid or expired. Download a new companion package.");
|
||||||
|
error.code = row?.activated_at ? "PAIRING_ALREADY_USED" : "PAIRING_INVALID";
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
const deviceId = crypto.randomUUID();
|
||||||
|
const secret = tokenValue(this.randomBytes(32));
|
||||||
|
const now = this.now();
|
||||||
|
const capabilities = DEFAULT_CAPABILITIES.slice();
|
||||||
|
const transaction = this.db.transaction(() => {
|
||||||
|
const consumed = this.db.prepare("UPDATE transcription_pairing_tokens SET activated_at = ? WHERE id = ? AND activated_at IS NULL").run(now, row.id);
|
||||||
|
if (consumed.changes !== 1) { const error = new Error("This pairing package was already activated. Download a new companion package."); error.code = "PAIRING_ALREADY_USED"; throw error; }
|
||||||
|
this.db.prepare("INSERT INTO transcription_devices (id, install_id, name, lumi_user_id, credential_hash, capabilities_json, metadata_json, first_connected_at, last_connected_at, revoked_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL)")
|
||||||
|
.run(deviceId, clean(device.install_id, 128) || null, clean(device.name, 160) || "Lumi Companion", row.lumi_user_id, digest(secret), JSON.stringify(capabilities), JSON.stringify(safeMetadata(device)), now, now);
|
||||||
|
});
|
||||||
|
transaction();
|
||||||
|
return { device_id: deviceId, device_secret: secret, host: row.host, capabilities, protocol_version: 1 };
|
||||||
|
}
|
||||||
|
|
||||||
|
authenticate(header, requiredCapability = null) {
|
||||||
|
const match = /^LumiDevice\s+([0-9a-f-]{36})\.([A-Za-z0-9_-]{40,})$/i.exec(String(header || ""));
|
||||||
|
if (!match) return { allowed: false, reason: "missing_credentials" };
|
||||||
|
const row = this.db.prepare("SELECT * FROM transcription_devices WHERE id = ?").get(match[1]);
|
||||||
|
if (!row || row.revoked_at || !safeEqual(row.credential_hash, digest(match[2]))) return { allowed: false, reason: row?.revoked_at ? "device_revoked" : "invalid_credentials" };
|
||||||
|
const capabilities = parseArray(row.capabilities_json);
|
||||||
|
if (requiredCapability && !capabilities.includes(requiredCapability)) return { allowed: false, reason: "capability_revoked" };
|
||||||
|
this.db.prepare("UPDATE transcription_devices SET last_connected_at = ? WHERE id = ?").run(this.now(), row.id);
|
||||||
|
return { allowed: true, device: serialize(row, capabilities) };
|
||||||
|
}
|
||||||
|
|
||||||
|
list() { return this.db.prepare("SELECT * FROM transcription_devices ORDER BY last_connected_at DESC").all().map((row) => serialize(row, parseArray(row.capabilities_json))); }
|
||||||
|
revoke(deviceId) { return this.db.prepare("UPDATE transcription_devices SET revoked_at = ? WHERE id = ? AND revoked_at IS NULL").run(this.now(), deviceId).changes === 1; }
|
||||||
|
setCapabilities(deviceId, capabilities) {
|
||||||
|
const allowed = DEFAULT_CAPABILITIES.filter((capability) => new Set(capabilities || []).has(capability));
|
||||||
|
const changed = this.db.prepare("UPDATE transcription_devices SET capabilities_json = ? WHERE id = ? AND revoked_at IS NULL").run(JSON.stringify(allowed), deviceId).changes;
|
||||||
|
return changed ? allowed : null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function digest(value) { return crypto.createHash("sha256").update(String(value || ""), "utf8").digest("hex"); }
|
||||||
|
function safeEqual(left, right) { const a = Buffer.from(String(left)); const b = Buffer.from(String(right)); return a.length === b.length && crypto.timingSafeEqual(a, b); }
|
||||||
|
function tokenValue(buffer) { return Buffer.from(buffer).toString("base64url"); }
|
||||||
|
function normalizeHost(value, allowInsecure = false) { const url = new URL(String(value)); if (url.protocol !== "https:" && !(allowInsecure && url.protocol === "http:" && ["localhost", "127.0.0.1", "::1"].includes(url.hostname))) throw new Error("Lumi Companion requires HTTPS. Insecure HTTP is allowed only for explicit localhost development."); return url.origin; }
|
||||||
|
function clean(value, max) { return String(value || "").trim().slice(0, max); }
|
||||||
|
function parseArray(value) { try { const parsed = JSON.parse(value); return Array.isArray(parsed) ? parsed : []; } catch { return []; } }
|
||||||
|
function safeMetadata(device) { return { companion_version: clean(device.companion_version, 32), obs_version: clean(device.obs_version, 32), os: clean(device.os, 120), architecture: clean(device.architecture, 32), hardware: clean(device.hardware, 500) }; }
|
||||||
|
function serialize(row, capabilities) { return { id: row.id, install_id: row.install_id, name: row.name, lumi_user_id: row.lumi_user_id, capabilities, metadata: JSON.parse(row.metadata_json || "{}"), first_connected_at: row.first_connected_at, last_connected_at: row.last_connected_at, revoked_at: row.revoked_at }; }
|
||||||
|
|
||||||
|
module.exports = { DeviceStore, DEFAULT_CAPABILITIES, digest };
|
||||||
102
plugins/lumi_transcription/backend/companion/gateway.js
Normal file
102
plugins/lumi_transcription/backend/companion/gateway.js
Normal file
@ -0,0 +1,102 @@
|
|||||||
|
const { WebSocketServer, WebSocket } = require("ws");
|
||||||
|
const { MAX_JSON_BYTES, MAX_AUDIO_PAYLOAD_BYTES, AUDIO_HEADER_BYTES, parseEnvelope, validateHello, parseAudioFrame, envelope } = require("./protocol");
|
||||||
|
|
||||||
|
class CompanionGateway {
|
||||||
|
constructor(options) {
|
||||||
|
this.devices = options.devices;
|
||||||
|
this.sessions = options.sessions;
|
||||||
|
this.log = options.log || { append() {} };
|
||||||
|
this.allowInsecure = options.allowInsecure === true;
|
||||||
|
this.wss = new WebSocketServer({ noServer: true, maxPayload: Math.max(MAX_JSON_BYTES, AUDIO_HEADER_BYTES + MAX_AUDIO_PAYLOAD_BYTES), perMessageDeflate: false, clientTracking: true });
|
||||||
|
this.wss.on("connection", (socket, request, device) => this.connection(socket, request, device));
|
||||||
|
}
|
||||||
|
upgrade(request, socket, head) {
|
||||||
|
const forwardedProto = String(request.headers["x-forwarded-proto"] || "").split(",")[0].trim();
|
||||||
|
const secure = Boolean(request.socket.encrypted) || forwardedProto === "https";
|
||||||
|
const local = ["127.0.0.1", "::1", "::ffff:127.0.0.1"].includes(request.socket.remoteAddress);
|
||||||
|
if (!secure && !(this.allowInsecure && local)) return reject(socket, 426, "tls_required");
|
||||||
|
const auth = this.devices.authenticate(request.headers.authorization, "transcription.capture.v1");
|
||||||
|
if (!auth.allowed) return reject(socket, auth.reason === "capability_revoked" ? 403 : 401, auth.reason);
|
||||||
|
const origin = request.headers.origin;
|
||||||
|
if (origin && !sameHostOrigin(origin, request.headers.host)) return reject(socket, 403, "origin_rejected");
|
||||||
|
this.wss.handleUpgrade(request, socket, head, (ws) => this.wss.emit("connection", ws, request, auth.device));
|
||||||
|
}
|
||||||
|
connection(socket, _request, device) {
|
||||||
|
let session = null;
|
||||||
|
let helloComplete = false;
|
||||||
|
let lastPong = Date.now();
|
||||||
|
let windowStarted = Date.now();
|
||||||
|
let messagesInWindow = 0;
|
||||||
|
const send = (type, payload, sessionId = session?.id || null) => {
|
||||||
|
if (socket.readyState === WebSocket.OPEN) socket.send(JSON.stringify(envelope(type, payload, sessionId)));
|
||||||
|
};
|
||||||
|
const helloTimer = setTimeout(() => closeWith(socket, 4408, "hello_timeout"), 5000);
|
||||||
|
const heartbeat = setInterval(() => {
|
||||||
|
if (Date.now() - lastPong > 30000) return closeWith(socket, 4408, "heartbeat_timeout");
|
||||||
|
send("status", { kind: "heartbeat", state: session?.state || "connecting" });
|
||||||
|
}, 10000);
|
||||||
|
helloTimer.unref?.(); heartbeat.unref?.();
|
||||||
|
socket.on("message", async (data, isBinary) => {
|
||||||
|
try {
|
||||||
|
if (Date.now() - windowStarted >= 1000) { windowStarted = Date.now(); messagesInWindow = 0; }
|
||||||
|
messagesInWindow += 1;
|
||||||
|
if (messagesInWindow > 300) throw coded("RATE_LIMIT", "Companion message rate exceeded its limit.");
|
||||||
|
if (isBinary) {
|
||||||
|
if (!helloComplete || !session) throw coded("HELLO_REQUIRED", "Complete the handshake before sending audio.");
|
||||||
|
const result = await this.sessions.audio(session.id, parseAudioFrame(data));
|
||||||
|
if (result.gap) send("metric", { kind: "sequence_gap", missing_frames: result.gap });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const message = parseEnvelope(data);
|
||||||
|
if (!helloComplete) {
|
||||||
|
const hello = validateHello(message);
|
||||||
|
const created = this.sessions.create(device, send, hello.resume_session_id);
|
||||||
|
session = created.session;
|
||||||
|
helloComplete = true;
|
||||||
|
clearTimeout(helloTimer);
|
||||||
|
send("hello_ack", {
|
||||||
|
protocol_version: 1, resumed: created.resumed,
|
||||||
|
server_plugin_version: require("../../plugin.json").version,
|
||||||
|
capabilities: ["transcription.server.v1", "settings.revision.v1", "pcm_s16le"],
|
||||||
|
heartbeat_ms: 10000, recovery_window_ms: 5000
|
||||||
|
});
|
||||||
|
this.log.append({ kind: "connection", state: "authenticated", device_id: device.id, session_id: session.id });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await this.structured(session, message, send, () => { lastPong = Date.now(); });
|
||||||
|
} catch (error) {
|
||||||
|
send("error", { code: error.code || "INVALID_MESSAGE", message: error.message, recoverable: !["INCOMPATIBLE_VERSION", "HELLO_REQUIRED"].includes(error.code) });
|
||||||
|
this.log.append({ kind: "protocol_error", device_id: device.id, session_id: session?.id, code: error.code || "INVALID_MESSAGE", message: error.message });
|
||||||
|
if (["INCOMPATIBLE_VERSION", "HELLO_REQUIRED", "RATE_LIMIT"].includes(error.code)) closeWith(socket, 4400, error.code);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
socket.on("close", () => {
|
||||||
|
clearTimeout(helloTimer); clearInterval(heartbeat);
|
||||||
|
if (session) this.sessions.disconnect(session.id);
|
||||||
|
this.log.append({ kind: "connection", state: "closed", device_id: device.id, session_id: session?.id });
|
||||||
|
});
|
||||||
|
socket.on("error", (error) => this.log.append({ kind: "connection", state: "error", device_id: device.id, session_id: session?.id, message: error.message }));
|
||||||
|
}
|
||||||
|
async structured(session, message, send, pong) {
|
||||||
|
switch (message.type) {
|
||||||
|
case "ping": pong(); send("pong", { received_id: message.id }); break;
|
||||||
|
case "source_update": send("status", { kind: "source", source: this.sessions.updateSource(session.id, message.payload || {}) }); break;
|
||||||
|
case "obs_state": send("status", { kind: "obs", ...(await this.sessions.updateObsState(session.id, message.payload || {})) }); break;
|
||||||
|
case "start": send("status", { kind: "session", ...(await this.sessions.start(session.id, message.payload || {})) }); break;
|
||||||
|
case "stop": send("status", { kind: "session", ...(await this.sessions.stop(session.id)) }); break;
|
||||||
|
case "ack": break;
|
||||||
|
default: throw coded("UNEXPECTED_MESSAGE", `Message ${message.type} is not valid after the handshake.`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
async close() {
|
||||||
|
for (const client of this.wss.clients) closeWith(client, 1001, "plugin_shutdown");
|
||||||
|
await new Promise((resolve) => this.wss.close(resolve));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function reject(socket, status, reason) { const labels = { 401: "Unauthorized", 403: "Forbidden", 426: "Upgrade Required" }; socket.write(`HTTP/1.1 ${status} ${labels[status] || "Rejected"}\r\nConnection: close\r\nContent-Type: application/json\r\n\r\n${JSON.stringify({ error: reason })}`); socket.destroy(); }
|
||||||
|
function closeWith(socket, code, reason) { if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) socket.close(code, String(reason).slice(0, 120)); }
|
||||||
|
function sameHostOrigin(origin, host) { try { return new URL(origin).host === host; } catch { return false; } }
|
||||||
|
function coded(code, message) { return Object.assign(new Error(message), { code }); }
|
||||||
|
|
||||||
|
module.exports = { CompanionGateway, sameHostOrigin };
|
||||||
110
plugins/lumi_transcription/backend/companion/protocol.js
Normal file
110
plugins/lumi_transcription/backend/companion/protocol.js
Normal file
@ -0,0 +1,110 @@
|
|||||||
|
const crypto = require("crypto");
|
||||||
|
|
||||||
|
const PROTOCOL_VERSION = 1;
|
||||||
|
const AUDIO_MAGIC = Buffer.from("LACP");
|
||||||
|
const AUDIO_HEADER_BYTES = 64;
|
||||||
|
const MAX_JSON_BYTES = 64 * 1024;
|
||||||
|
const MAX_AUDIO_PAYLOAD_BYTES = 6400;
|
||||||
|
const CLIENT_TYPES = new Set(["hello", "ping", "source_update", "obs_state", "start", "stop", "ack"]);
|
||||||
|
|
||||||
|
function parseEnvelope(input) {
|
||||||
|
const bytes = Buffer.isBuffer(input) ? input : Buffer.from(String(input || ""), "utf8");
|
||||||
|
if (!bytes.length || bytes.length > MAX_JSON_BYTES) throw protocolError("MESSAGE_SIZE", "Structured message size is invalid.");
|
||||||
|
let value;
|
||||||
|
try { value = JSON.parse(bytes.toString("utf8")); }
|
||||||
|
catch { throw protocolError("INVALID_JSON", "Structured message is not valid JSON."); }
|
||||||
|
if (!value || Array.isArray(value) || typeof value !== "object") throw protocolError("INVALID_ENVELOPE", "Message envelope must be an object.");
|
||||||
|
if (value.version !== PROTOCOL_VERSION) throw protocolError("INCOMPATIBLE_VERSION", `Protocol version ${value.version} is unsupported.`);
|
||||||
|
if (!CLIENT_TYPES.has(value.type)) throw protocolError("UNKNOWN_TYPE", "Message type is not allowed.");
|
||||||
|
if (!isUuid(value.id)) throw protocolError("INVALID_ID", "Message id must be a UUID.");
|
||||||
|
if (!value.sent_at || !Number.isFinite(Date.parse(value.sent_at))) throw protocolError("INVALID_TIME", "Message sent_at must be an ISO timestamp.");
|
||||||
|
if (value.session_id != null && !isUuid(value.session_id)) throw protocolError("INVALID_SESSION", "Session id must be a UUID.");
|
||||||
|
if (value.payload != null && (Array.isArray(value.payload) || typeof value.payload !== "object")) throw protocolError("INVALID_PAYLOAD", "Message payload must be an object.");
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
function validateHello(envelope) {
|
||||||
|
if (envelope.type !== "hello") throw protocolError("HELLO_REQUIRED", "The first message must be hello.");
|
||||||
|
const payload = envelope.payload || {};
|
||||||
|
if (!shortVersion(payload.companion_version) || !shortVersion(payload.plugin_version)) throw protocolError("INVALID_HELLO", "Companion and plugin versions are required.");
|
||||||
|
if (!Array.isArray(payload.capabilities) || payload.capabilities.length > 32) throw protocolError("INVALID_CAPABILITIES", "Capabilities are invalid.");
|
||||||
|
if (!payload.capabilities.includes("transcription.capture.v1")) throw protocolError("MISSING_CAPABILITY", "The transcription capture capability is required.");
|
||||||
|
const audio = payload.audio || {};
|
||||||
|
if (audio.codec !== "pcm_s16le" || audio.sample_rate !== 16000 || audio.channels !== 1 || audio.bits !== 16) {
|
||||||
|
throw protocolError("UNSUPPORTED_AUDIO", "Protocol v1 requires 16 kHz mono signed 16-bit PCM.");
|
||||||
|
}
|
||||||
|
if (payload.resume_session_id != null && !isUuid(payload.resume_session_id)) throw protocolError("INVALID_RESUME", "Resume session id is invalid.");
|
||||||
|
return payload;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseAudioFrame(input) {
|
||||||
|
const frame = Buffer.from(input || []);
|
||||||
|
if (frame.length < AUDIO_HEADER_BYTES || frame.length > AUDIO_HEADER_BYTES + MAX_AUDIO_PAYLOAD_BYTES) {
|
||||||
|
throw protocolError("AUDIO_SIZE", "Audio frame size is invalid.");
|
||||||
|
}
|
||||||
|
if (!frame.subarray(0, 4).equals(AUDIO_MAGIC)) throw protocolError("AUDIO_MAGIC", "Audio frame magic is invalid.");
|
||||||
|
const version = frame.readUInt8(4);
|
||||||
|
const flags = frame.readUInt8(5);
|
||||||
|
const headerBytes = frame.readUInt16LE(6);
|
||||||
|
const payloadBytes = frame.readUInt32LE(60);
|
||||||
|
if (version !== PROTOCOL_VERSION || headerBytes !== AUDIO_HEADER_BYTES || frame.length !== headerBytes + payloadBytes) {
|
||||||
|
throw protocolError("AUDIO_HEADER", "Audio frame header is invalid.");
|
||||||
|
}
|
||||||
|
if (payloadBytes > MAX_AUDIO_PAYLOAD_BYTES || payloadBytes % 2 !== 0) throw protocolError("AUDIO_PAYLOAD", "PCM payload length is invalid.");
|
||||||
|
if (frame.readUInt32LE(52) !== 16000 || frame.readUInt16LE(56) !== 1 || frame.readUInt16LE(58) !== 16) {
|
||||||
|
throw protocolError("UNSUPPORTED_AUDIO", "Audio format is unsupported.");
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
version,
|
||||||
|
active: Boolean(flags & 1),
|
||||||
|
muted: Boolean(flags & 2),
|
||||||
|
sequence: frame.readUInt32LE(8),
|
||||||
|
capture_timestamp_us: Number(frame.readBigUInt64LE(12)),
|
||||||
|
session_id: bytesToUuid(frame.subarray(20, 36)),
|
||||||
|
source_uuid: bytesToUuid(frame.subarray(36, 52)),
|
||||||
|
sample_rate: 16000,
|
||||||
|
channels: 1,
|
||||||
|
bits: 16,
|
||||||
|
pcm: frame.subarray(AUDIO_HEADER_BYTES)
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function encodeAudioFrame(value) {
|
||||||
|
const pcm = Buffer.from(value.pcm || []);
|
||||||
|
if (pcm.length > MAX_AUDIO_PAYLOAD_BYTES || pcm.length % 2) throw protocolError("AUDIO_PAYLOAD", "PCM payload length is invalid.");
|
||||||
|
const frame = Buffer.alloc(AUDIO_HEADER_BYTES + pcm.length);
|
||||||
|
AUDIO_MAGIC.copy(frame, 0);
|
||||||
|
frame.writeUInt8(PROTOCOL_VERSION, 4);
|
||||||
|
frame.writeUInt8((value.active === false ? 0 : 1) | (value.muted ? 2 : 0), 5);
|
||||||
|
frame.writeUInt16LE(AUDIO_HEADER_BYTES, 6);
|
||||||
|
frame.writeUInt32LE(Number(value.sequence) >>> 0, 8);
|
||||||
|
frame.writeBigUInt64LE(BigInt(value.capture_timestamp_us || 0), 12);
|
||||||
|
uuidToBytes(value.session_id).copy(frame, 20);
|
||||||
|
uuidToBytes(value.source_uuid).copy(frame, 36);
|
||||||
|
frame.writeUInt32LE(16000, 52);
|
||||||
|
frame.writeUInt16LE(1, 56);
|
||||||
|
frame.writeUInt16LE(16, 58);
|
||||||
|
frame.writeUInt32LE(pcm.length, 60);
|
||||||
|
pcm.copy(frame, AUDIO_HEADER_BYTES);
|
||||||
|
return frame;
|
||||||
|
}
|
||||||
|
|
||||||
|
function envelope(type, payload = {}, sessionId = null) {
|
||||||
|
return { version: 1, type, id: crypto.randomUUID(), sent_at: new Date().toISOString(), session_id: sessionId, payload };
|
||||||
|
}
|
||||||
|
|
||||||
|
function protocolError(code, message) {
|
||||||
|
const error = new Error(message);
|
||||||
|
error.code = code;
|
||||||
|
return error;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isUuid(value) {
|
||||||
|
return /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(String(value || ""));
|
||||||
|
}
|
||||||
|
|
||||||
|
function shortVersion(value) { return typeof value === "string" && value.length > 0 && value.length <= 32; }
|
||||||
|
function uuidToBytes(value) { if (!isUuid(value)) throw protocolError("INVALID_UUID", "UUID is invalid."); return Buffer.from(value.replace(/-/g, ""), "hex"); }
|
||||||
|
function bytesToUuid(value) { const hex = Buffer.from(value).toString("hex"); return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; }
|
||||||
|
|
||||||
|
module.exports = { PROTOCOL_VERSION, AUDIO_HEADER_BYTES, MAX_JSON_BYTES, MAX_AUDIO_PAYLOAD_BYTES, parseEnvelope, validateHello, parseAudioFrame, encodeAudioFrame, envelope, protocolError, isUuid };
|
||||||
52
plugins/lumi_transcription/backend/config/revision_store.js
Normal file
52
plugins/lumi_transcription/backend/config/revision_store.js
Normal file
@ -0,0 +1,52 @@
|
|||||||
|
const ALLOWED_KEYS = new Set([
|
||||||
|
"selected_model_id", "fallback_order", "decode_interval_ms", "silence_finalize_ms",
|
||||||
|
"rolling_context_ms", "caption_max_chars", "minimum_display_ms", "auto_start_stream",
|
||||||
|
"offline_authorization_severity", "diagnostic_caption_text", "tracks"
|
||||||
|
]);
|
||||||
|
|
||||||
|
class RevisionStore {
|
||||||
|
constructor(db, options = {}) { this.db = db; this.now = options.now || Date.now; this.migrate(); }
|
||||||
|
migrate() {
|
||||||
|
this.db.exec(`CREATE TABLE IF NOT EXISTS transcription_settings (
|
||||||
|
key TEXT PRIMARY KEY, value_json TEXT NOT NULL, revision INTEGER NOT NULL,
|
||||||
|
actor_id TEXT NOT NULL, updated_at INTEGER NOT NULL
|
||||||
|
);`);
|
||||||
|
}
|
||||||
|
list() {
|
||||||
|
return Object.fromEntries(this.db.prepare("SELECT * FROM transcription_settings ORDER BY key").all().map((row) => [row.key, decode(row)]));
|
||||||
|
}
|
||||||
|
apply(changes, actorId) {
|
||||||
|
if (!Array.isArray(changes) || !changes.length || changes.length > 50) throw new Error("One to fifty field changes are required.");
|
||||||
|
const normalized = changes.map(validateChange);
|
||||||
|
const applied = [];
|
||||||
|
const conflicts = [];
|
||||||
|
this.db.transaction(() => {
|
||||||
|
for (const change of normalized) {
|
||||||
|
const current = this.db.prepare("SELECT * FROM transcription_settings WHERE key = ?").get(change.key);
|
||||||
|
const currentRevision = current?.revision || 0;
|
||||||
|
if (change.base_revision !== currentRevision) {
|
||||||
|
conflicts.push({ key: change.key, local_value: change.value, local_base_revision: change.base_revision, server: current ? decode(current) : { value: null, revision: 0, actor_id: null, updated_at: null } });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const revision = currentRevision + 1;
|
||||||
|
const updatedAt = this.now();
|
||||||
|
this.db.prepare("INSERT INTO transcription_settings (key, value_json, revision, actor_id, updated_at) VALUES (?, ?, ?, ?, ?) ON CONFLICT(key) DO UPDATE SET value_json = excluded.value_json, revision = excluded.revision, actor_id = excluded.actor_id, updated_at = excluded.updated_at")
|
||||||
|
.run(change.key, JSON.stringify(change.value), revision, String(actorId || "unknown"), updatedAt);
|
||||||
|
applied.push({ key: change.key, value: change.value, revision, actor_id: String(actorId || "unknown"), updated_at: updatedAt });
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
return { applied, conflicts, current: this.list() };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function validateChange(change) {
|
||||||
|
if (!change || !ALLOWED_KEYS.has(change.key)) throw new Error(`Setting key ${change?.key || "(missing)"} is not allowed.`);
|
||||||
|
const baseRevision = Number(change.base_revision);
|
||||||
|
if (!Number.isInteger(baseRevision) || baseRevision < 0) throw new Error("A non-negative base revision is required for every changed field.");
|
||||||
|
const encoded = JSON.stringify(change.value);
|
||||||
|
if (encoded == null || Buffer.byteLength(encoded) > 64 * 1024) throw new Error("Setting value is too large.");
|
||||||
|
return { key: change.key, value: change.value, base_revision: baseRevision };
|
||||||
|
}
|
||||||
|
function decode(row) { return { value: JSON.parse(row.value_json), revision: row.revision, actor_id: row.actor_id, updated_at: row.updated_at }; }
|
||||||
|
|
||||||
|
module.exports = { RevisionStore, ALLOWED_KEYS };
|
||||||
@ -0,0 +1,28 @@
|
|||||||
|
const { LatestCaptionGate } = require("../transcription/stabilizer");
|
||||||
|
|
||||||
|
class CaptionDeliveryAdapter {
|
||||||
|
async test() { throw new Error("Caption delivery test is not implemented."); }
|
||||||
|
async start() { throw new Error("Caption delivery start is not implemented."); }
|
||||||
|
async deliver() { throw new Error("Caption delivery is not implemented."); }
|
||||||
|
async pause() {}
|
||||||
|
async resume() {}
|
||||||
|
async stop() {}
|
||||||
|
async health() { return { healthy: false, state: "unavailable" }; }
|
||||||
|
}
|
||||||
|
|
||||||
|
class CompanionCaptionDeliveryAdapter extends CaptionDeliveryAdapter {
|
||||||
|
constructor(send) { super(); this.send = send; this.gate = new LatestCaptionGate(); this.state = "idle"; this.testMode = false; }
|
||||||
|
async test() { return { supported: true, mode: "simulated", note: "The companion simulates the exact outgoing native-caption stream without sending it to Twitch." }; }
|
||||||
|
async start(options = {}) { this.testMode = Boolean(options.testMode); this.state = "running"; return this.health(); }
|
||||||
|
async deliver(event) {
|
||||||
|
if (this.state !== "running" || !this.gate.accept(event)) return { disposition: "obsolete_or_paused" };
|
||||||
|
this.send("caption", { ...event, delivery: { disposition: this.testMode ? "simulated" : "forwarded_to_obs" } }, event.session_id);
|
||||||
|
return { disposition: this.testMode ? "simulated" : "forwarded_to_obs" };
|
||||||
|
}
|
||||||
|
async pause() { this.state = "paused"; }
|
||||||
|
async resume() { this.state = "running"; }
|
||||||
|
async stop() { this.state = "idle"; }
|
||||||
|
async health() { return { healthy: this.state !== "idle", state: this.state, adapter: "companion_obs_native", test_mode: this.testMode }; }
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { CaptionDeliveryAdapter, CompanionCaptionDeliveryAdapter };
|
||||||
55
plugins/lumi_transcription/backend/logs/jsonl_log.js
Normal file
55
plugins/lumi_transcription/backend/logs/jsonl_log.js
Normal file
@ -0,0 +1,55 @@
|
|||||||
|
const fs = require("fs");
|
||||||
|
const path = require("path");
|
||||||
|
|
||||||
|
class JsonlDiagnosticLog {
|
||||||
|
constructor(directory, options = {}) {
|
||||||
|
this.directory = directory;
|
||||||
|
this.retentionMs = (options.retentionDays || 7) * 86400000;
|
||||||
|
this.maxBytes = options.maxBytes || 256 * 1024 * 1024;
|
||||||
|
this.includeCaptionText = options.includeCaptionText !== false;
|
||||||
|
fs.mkdirSync(directory, { recursive: true });
|
||||||
|
}
|
||||||
|
append(entry) {
|
||||||
|
const safe = sanitize({ timestamp: new Date().toISOString(), ...entry }, this.includeCaptionText);
|
||||||
|
fs.appendFileSync(this.fileForToday(), `${JSON.stringify(safe)}\n`, { encoding: "utf8", mode: 0o600 });
|
||||||
|
}
|
||||||
|
cleanup(now = Date.now()) {
|
||||||
|
const files = this.files();
|
||||||
|
let removed = 0;
|
||||||
|
for (const file of files) {
|
||||||
|
if (now - file.mtimeMs > this.retentionMs) { fs.rmSync(file.path, { force: true }); removed += 1; }
|
||||||
|
}
|
||||||
|
const retained = this.files();
|
||||||
|
let total = retained.reduce((sum, file) => sum + file.size, 0);
|
||||||
|
for (const file of retained) {
|
||||||
|
if (total <= this.maxBytes) break;
|
||||||
|
fs.rmSync(file.path, { force: true });
|
||||||
|
total -= file.size;
|
||||||
|
removed += 1;
|
||||||
|
}
|
||||||
|
return { removed, bytes: total };
|
||||||
|
}
|
||||||
|
files() {
|
||||||
|
return fs.readdirSync(this.directory).filter((name) => /^transcription-\d{4}-\d{2}-\d{2}\.jsonl$/.test(name)).map((name) => {
|
||||||
|
const target = path.join(this.directory, name);
|
||||||
|
const stat = fs.statSync(target);
|
||||||
|
return { path: target, name, size: stat.size, mtimeMs: stat.mtimeMs };
|
||||||
|
}).sort((a, b) => a.mtimeMs - b.mtimeMs);
|
||||||
|
}
|
||||||
|
fileForToday() { return path.join(this.directory, `transcription-${new Date().toISOString().slice(0, 10)}.jsonl`); }
|
||||||
|
}
|
||||||
|
|
||||||
|
function sanitize(value, includeCaptionText) {
|
||||||
|
if (Buffer.isBuffer(value)) return "[binary omitted]";
|
||||||
|
if (Array.isArray(value)) return value.map((entry) => sanitize(entry, includeCaptionText));
|
||||||
|
if (!value || typeof value !== "object") return value;
|
||||||
|
const result = {};
|
||||||
|
for (const [key, child] of Object.entries(value)) {
|
||||||
|
if (/audio|pcm|credential|secret|token/i.test(key)) result[key] = "[redacted]";
|
||||||
|
else if (!includeCaptionText && /(?:stable|uncertain|caption)_text/i.test(key)) result[key] = "[caption text disabled]";
|
||||||
|
else result[key] = sanitize(child, includeCaptionText);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { JsonlDiagnosticLog, sanitize };
|
||||||
@ -0,0 +1,59 @@
|
|||||||
|
const crypto = require("crypto");
|
||||||
|
const fs = require("fs");
|
||||||
|
const path = require("path");
|
||||||
|
const AdmZip = require("adm-zip");
|
||||||
|
|
||||||
|
class ArtifactManager {
|
||||||
|
constructor(root, options = {}) { this.root = root; this.fetch = options.fetch || global.fetch; fs.mkdirSync(root, { recursive: true }); }
|
||||||
|
status(entry) {
|
||||||
|
const target = path.join(this.root, entry.filename || entry.id);
|
||||||
|
if (!fs.existsSync(target)) return { installed: false, valid: false, path: target };
|
||||||
|
return { installed: true, valid: sha256File(target) === entry.sha256, path: target, bytes: fs.statSync(target).size };
|
||||||
|
}
|
||||||
|
async download(entry, options = {}) {
|
||||||
|
validateManifestEntry(entry);
|
||||||
|
if (options.confirmed !== true) throw new Error("Download requires explicit setup confirmation.");
|
||||||
|
const filename = entry.filename || path.basename(new URL(entry.url).pathname);
|
||||||
|
const target = path.join(this.root, filename);
|
||||||
|
const partial = `${target}.${process.pid}.partial`;
|
||||||
|
fs.rmSync(partial, { force: true });
|
||||||
|
try {
|
||||||
|
const response = await this.fetch(entry.url, { redirect: "follow" });
|
||||||
|
if (!response.ok || !response.body) throw new Error(`Download failed with HTTP ${response.status}.`);
|
||||||
|
const handle = fs.createWriteStream(partial, { flags: "wx", mode: 0o600 });
|
||||||
|
for await (const chunk of response.body) {
|
||||||
|
if (!handle.write(chunk)) await new Promise((resolve) => handle.once("drain", resolve));
|
||||||
|
}
|
||||||
|
await new Promise((resolve, reject) => handle.end((error) => error ? reject(error) : resolve()));
|
||||||
|
const actual = sha256File(partial);
|
||||||
|
if (actual !== entry.sha256) throw new Error(`Checksum mismatch for ${entry.id}.`);
|
||||||
|
fs.renameSync(partial, target);
|
||||||
|
return this.status({ ...entry, filename });
|
||||||
|
} finally { fs.rmSync(partial, { force: true }); }
|
||||||
|
}
|
||||||
|
installZip(entry, archivePath) {
|
||||||
|
validateManifestEntry(entry);
|
||||||
|
if (sha256File(archivePath) !== entry.sha256) throw new Error(`Checksum mismatch for ${entry.id}.`);
|
||||||
|
const target = path.join(this.root, entry.id);
|
||||||
|
const staged = `${target}.${process.pid}.staged`;
|
||||||
|
fs.rmSync(staged, { recursive: true, force: true });
|
||||||
|
fs.mkdirSync(staged, { recursive: true });
|
||||||
|
const zip = new AdmZip(archivePath);
|
||||||
|
for (const item of zip.getEntries()) {
|
||||||
|
const portable = String(item.entryName).replace(/\\/g, "/");
|
||||||
|
const relative = path.posix.normalize(portable).replace(/^\/+/, "");
|
||||||
|
if (!relative || relative === ".." || relative.startsWith("../") || /^[A-Za-z]:/.test(relative)) throw new Error("Runtime archive contains an unsafe path.");
|
||||||
|
}
|
||||||
|
zip.extractAllTo(staged, true);
|
||||||
|
for (const expected of entry.expected_paths || []) if (!findBasename(staged, expected)) throw new Error(`Runtime is missing ${expected}.`);
|
||||||
|
fs.rmSync(target, { recursive: true, force: true });
|
||||||
|
fs.renameSync(staged, target);
|
||||||
|
return { installed: true, path: target, backend: entry.backend, version: entry.id };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function validateManifestEntry(entry) { if (!entry?.id || !/^https:\/\//.test(entry.url || "") || !/^[a-f0-9]{64}$/.test(entry.sha256 || "")) throw new Error("Artifact manifest entry is invalid."); }
|
||||||
|
function sha256File(target) { const hash = crypto.createHash("sha256"); const fd = fs.openSync(target, "r"); const buffer = Buffer.alloc(1024 * 1024); try { let read; while ((read = fs.readSync(fd, buffer, 0, buffer.length, null)) > 0) hash.update(buffer.subarray(0, read)); } finally { fs.closeSync(fd); } return hash.digest("hex"); }
|
||||||
|
function findBasename(root, basename) { const pending = [root]; while (pending.length) { const current = pending.pop(); for (const entry of fs.readdirSync(current, { withFileTypes: true })) { const target = path.join(current, entry.name); if (entry.isDirectory()) pending.push(target); else if (entry.name === basename) return target; } } return null; }
|
||||||
|
|
||||||
|
module.exports = { ArtifactManager, sha256File, validateManifestEntry };
|
||||||
7
plugins/lumi_transcription/backend/paths.js
Normal file
7
plugins/lumi_transcription/backend/paths.js
Normal file
@ -0,0 +1,7 @@
|
|||||||
|
const fs = require("fs");
|
||||||
|
const path = require("path");
|
||||||
|
const ROOT = path.resolve(__dirname, "..");
|
||||||
|
const DATA = path.join(ROOT, "data");
|
||||||
|
function ensureDataDirs() { for (const name of ["logs", "models", "runtime", "tmp"]) fs.mkdirSync(path.join(DATA, name), { recursive: true }); }
|
||||||
|
function dataPath(...parts) { const target = path.resolve(DATA, ...parts); if (target !== DATA && !target.startsWith(`${DATA}${path.sep}`)) throw new Error("Path escapes transcription plugin data."); return target; }
|
||||||
|
module.exports = { ROOT, DATA, ensureDataDirs, dataPath };
|
||||||
77
plugins/lumi_transcription/backend/sessions/bounded_queue.js
Normal file
77
plugins/lumi_transcription/backend/sessions/bounded_queue.js
Normal file
@ -0,0 +1,77 @@
|
|||||||
|
class BoundedQueue {
|
||||||
|
constructor(options = {}) {
|
||||||
|
this.maxItems = positive(options.maxItems, 250);
|
||||||
|
this.maxBytes = positive(options.maxBytes, 5 * 32000);
|
||||||
|
this.maxAgeMs = positive(options.maxAgeMs, 5000);
|
||||||
|
this.now = options.now || Date.now;
|
||||||
|
this.items = [];
|
||||||
|
this.bytes = 0;
|
||||||
|
this.dropped = { capacity: 0, stale: 0 };
|
||||||
|
}
|
||||||
|
|
||||||
|
push(value, options = {}) {
|
||||||
|
const bytes = Math.max(0, Number(options.bytes ?? value?.length ?? value?.pcm?.length ?? 0));
|
||||||
|
const capturedAt = Number(options.capturedAt ?? this.now());
|
||||||
|
if (bytes > this.maxBytes) {
|
||||||
|
this.dropped.capacity += 1;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
this.prune();
|
||||||
|
while (this.items.length >= this.maxItems || this.bytes + bytes > this.maxBytes) this.dropOldest("capacity");
|
||||||
|
this.items.push({ value, bytes, capturedAt });
|
||||||
|
this.bytes += bytes;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
shift() {
|
||||||
|
this.prune();
|
||||||
|
const entry = this.items.shift();
|
||||||
|
if (!entry) return null;
|
||||||
|
this.bytes -= entry.bytes;
|
||||||
|
return entry.value;
|
||||||
|
}
|
||||||
|
|
||||||
|
prune() {
|
||||||
|
const cutoff = this.now() - this.maxAgeMs;
|
||||||
|
while (this.items[0] && this.items[0].capturedAt < cutoff) this.dropOldest("stale");
|
||||||
|
}
|
||||||
|
|
||||||
|
clear() { this.items.length = 0; this.bytes = 0; }
|
||||||
|
size() { this.prune(); return this.items.length; }
|
||||||
|
metrics() { this.prune(); return { items: this.items.length, bytes: this.bytes, dropped: { ...this.dropped } }; }
|
||||||
|
|
||||||
|
dropOldest(reason) {
|
||||||
|
const entry = this.items.shift();
|
||||||
|
if (!entry) return;
|
||||||
|
this.bytes -= entry.bytes;
|
||||||
|
this.dropped[reason] += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class SequenceTracker {
|
||||||
|
constructor() { this.last = null; this.gaps = 0; this.outOfOrder = 0; }
|
||||||
|
accept(sequence) {
|
||||||
|
const current = Number(sequence) >>> 0;
|
||||||
|
if (this.last == null) { this.last = current; return { accepted: true, gap: 0 }; }
|
||||||
|
if (current <= this.last) { this.outOfOrder += 1; return { accepted: false, gap: 0 }; }
|
||||||
|
const gap = current - this.last - 1;
|
||||||
|
if (gap) this.gaps += gap;
|
||||||
|
this.last = current;
|
||||||
|
return { accepted: true, gap };
|
||||||
|
}
|
||||||
|
metrics() { return { last_sequence: this.last, sequence_gaps: this.gaps, out_of_order: this.outOfOrder }; }
|
||||||
|
}
|
||||||
|
|
||||||
|
class RollingPcmBuffer {
|
||||||
|
constructor(options = {}) {
|
||||||
|
this.queue = new BoundedQueue({ maxItems: options.maxItems || 300, maxBytes: (options.seconds || 5) * 32000, maxAgeMs: (options.seconds || 5) * 1000, now: options.now });
|
||||||
|
}
|
||||||
|
push(frame, capturedAt) { return this.queue.push(Buffer.from(frame), { bytes: frame.length, capturedAt }); }
|
||||||
|
snapshot() { this.queue.prune(); return Buffer.concat(this.queue.items.map((entry) => entry.value), this.queue.bytes); }
|
||||||
|
metrics() { return this.queue.metrics(); }
|
||||||
|
clear() { this.queue.clear(); }
|
||||||
|
}
|
||||||
|
|
||||||
|
function positive(value, fallback) { const number = Number(value); return Number.isFinite(number) && number > 0 ? number : fallback; }
|
||||||
|
|
||||||
|
module.exports = { BoundedQueue, SequenceTracker, RollingPcmBuffer };
|
||||||
@ -0,0 +1,152 @@
|
|||||||
|
const crypto = require("crypto");
|
||||||
|
const { RollingPcmBuffer, SequenceTracker } = require("./bounded_queue");
|
||||||
|
const { CaptionStabilizer } = require("../transcription/stabilizer");
|
||||||
|
|
||||||
|
class SessionCoordinator {
|
||||||
|
constructor(options) {
|
||||||
|
this.provider = options.provider;
|
||||||
|
this.deliveryFactory = options.deliveryFactory;
|
||||||
|
this.log = options.log || { append() {} };
|
||||||
|
this.now = options.now || Date.now;
|
||||||
|
this.graceMs = options.graceMs || 30000;
|
||||||
|
this.sessions = new Map();
|
||||||
|
this.stabilizer = new CaptionStabilizer({ now: this.now });
|
||||||
|
this.provider.on?.("hypothesis", (event) => this.onHypothesis(event));
|
||||||
|
}
|
||||||
|
create(device, send, resumeSessionId = null) {
|
||||||
|
const resumable = resumeSessionId && this.sessions.get(resumeSessionId);
|
||||||
|
if (resumable && resumable.deviceId === device.id && resumable.graceUntil > this.now()) {
|
||||||
|
clearTimeout(resumable.graceTimer);
|
||||||
|
resumable.graceTimer = null;
|
||||||
|
resumable.graceUntil = 0;
|
||||||
|
resumable.connected = true;
|
||||||
|
resumable.send = send;
|
||||||
|
resumable.delivery = this.deliveryFactory(send);
|
||||||
|
this.log.append({ kind: "session", state: "resumed", session_id: resumable.id, device_id: device.id });
|
||||||
|
return { session: resumable, resumed: true };
|
||||||
|
}
|
||||||
|
const session = {
|
||||||
|
id: crypto.randomUUID(), deviceId: device.id, connected: true, send,
|
||||||
|
state: "idle", mode: null, obs: { streaming: false, recording: false },
|
||||||
|
tracks: new Map(), delivery: this.deliveryFactory(send), createdAt: this.now(), graceUntil: 0, graceTimer: null
|
||||||
|
};
|
||||||
|
this.sessions.set(session.id, session);
|
||||||
|
this.log.append({ kind: "session", state: "created", session_id: session.id, device_id: device.id });
|
||||||
|
return { session, resumed: false };
|
||||||
|
}
|
||||||
|
updateSource(sessionId, input) {
|
||||||
|
const session = this.require(sessionId);
|
||||||
|
if (!isUuid(input.source_uuid)) throw new Error("Source UUID is invalid.");
|
||||||
|
const existing = session.tracks.get(input.source_uuid);
|
||||||
|
const track = existing || { sequence: new SequenceTracker(), buffer: new RollingPcmBuffer({ seconds: 5, now: this.now }), lastSpeechAt: 0 };
|
||||||
|
Object.assign(track, {
|
||||||
|
source_uuid: input.source_uuid, display_name: text(input.display_name, 160) || "OBS source",
|
||||||
|
speaker_label: text(input.speaker_label, 80) || null, enabled: input.enabled !== false,
|
||||||
|
primary: Boolean(input.primary), single_speaker: input.single_speaker !== false,
|
||||||
|
delivery_enabled: input.delivery_enabled !== false, program_active: Boolean(input.program_active),
|
||||||
|
source_missing: Boolean(input.source_missing), last_activity_at: this.now()
|
||||||
|
});
|
||||||
|
if (track.primary) for (const other of session.tracks.values()) other.primary = false;
|
||||||
|
session.tracks.set(track.source_uuid, track);
|
||||||
|
return serializeTrack(track);
|
||||||
|
}
|
||||||
|
async updateObsState(sessionId, obs) {
|
||||||
|
const session = this.require(sessionId);
|
||||||
|
const wasStreaming = session.obs.streaming;
|
||||||
|
session.obs = { streaming: Boolean(obs.streaming), recording: Boolean(obs.recording), version: text(obs.version, 32) || null };
|
||||||
|
if (!wasStreaming && session.obs.streaming && session.state === "idle" && obs.auto_start !== false) await this.start(sessionId, { mode: "live" });
|
||||||
|
if (wasStreaming && !session.obs.streaming && session.state === "running" && session.mode === "live") this.beginGrace(session);
|
||||||
|
if (!wasStreaming && session.obs.streaming && session.state === "grace") await this.resume(session);
|
||||||
|
return { ...session.obs, state: session.state };
|
||||||
|
}
|
||||||
|
async start(sessionId, options = {}) {
|
||||||
|
const session = this.require(sessionId);
|
||||||
|
const mode = options.mode === "test" ? "test" : "live";
|
||||||
|
if (mode === "live" && !session.obs.streaming) throw Object.assign(new Error("Live transcription requires an active OBS stream."), { code: "OBS_NOT_STREAMING" });
|
||||||
|
const tracks = Array.from(session.tracks.values()).filter((track) => track.enabled && !track.source_missing);
|
||||||
|
if (!tracks.length) throw Object.assign(new Error("Select an available OBS audio source first."), { code: "NO_TRACKS" });
|
||||||
|
const providerHealth = await this.provider.health();
|
||||||
|
if (!providerHealth.healthy) throw Object.assign(new Error("The whisper.cpp worker is not ready. Install and load a model first."), { code: "PROVIDER_UNAVAILABLE", details: providerHealth });
|
||||||
|
await this.provider.startSession({ id: session.id, mode });
|
||||||
|
for (const track of tracks) await this.provider.addTrack(session.id, serializeTrack(track));
|
||||||
|
await session.delivery.start({ testMode: mode === "test" });
|
||||||
|
session.mode = mode;
|
||||||
|
session.state = "running";
|
||||||
|
this.log.append({ kind: "session", state: "running", mode, session_id: session.id, tracks: tracks.map((track) => track.source_uuid) });
|
||||||
|
return this.status(session.id);
|
||||||
|
}
|
||||||
|
async stop(sessionId, reason = "requested") {
|
||||||
|
const session = this.require(sessionId);
|
||||||
|
clearTimeout(session.graceTimer);
|
||||||
|
session.graceTimer = null;
|
||||||
|
await session.delivery.stop();
|
||||||
|
if (session.state !== "idle") await this.provider.stopSession(session.id);
|
||||||
|
session.state = "idle";
|
||||||
|
session.mode = null;
|
||||||
|
for (const track of session.tracks.values()) track.buffer.clear();
|
||||||
|
this.log.append({ kind: "session", state: "stopped", reason, session_id: session.id });
|
||||||
|
return this.status(session.id);
|
||||||
|
}
|
||||||
|
async audio(sessionId, frame) {
|
||||||
|
const session = this.require(sessionId);
|
||||||
|
if (frame.session_id !== session.id) throw new Error("Audio session does not match the negotiated session.");
|
||||||
|
const track = session.tracks.get(frame.source_uuid);
|
||||||
|
if (!track) return { accepted: false, reason: "unselected_source" };
|
||||||
|
const sequence = track.sequence.accept(frame.sequence);
|
||||||
|
if (!sequence.accepted) return { accepted: false, reason: "out_of_order" };
|
||||||
|
if (sequence.gap) this.log.append({ kind: "audio_gap", session_id: session.id, source_uuid: track.source_uuid, missing_frames: sequence.gap });
|
||||||
|
if (session.state !== "running" || !track.enabled || !track.program_active || track.source_missing || frame.muted || !frame.active) return { accepted: false, reason: "inactive" };
|
||||||
|
track.buffer.push(frame.pcm, this.now());
|
||||||
|
const accepted = await this.provider.pushAudio(session.id, track.source_uuid, frame);
|
||||||
|
return { accepted: accepted !== false, gap: sequence.gap, buffer: track.buffer.metrics() };
|
||||||
|
}
|
||||||
|
disconnect(sessionId) {
|
||||||
|
const session = this.sessions.get(sessionId);
|
||||||
|
if (!session) return;
|
||||||
|
session.connected = false;
|
||||||
|
if (session.state === "running") this.beginGrace(session);
|
||||||
|
else this.expireLater(session);
|
||||||
|
}
|
||||||
|
status(sessionId) {
|
||||||
|
const session = this.require(sessionId);
|
||||||
|
return { session_id: session.id, state: session.state, mode: session.mode, obs: { ...session.obs }, grace_until: session.graceUntil || null, tracks: Array.from(session.tracks.values()).map(serializeTrack) };
|
||||||
|
}
|
||||||
|
async close() { for (const session of Array.from(this.sessions.values())) await this.stop(session.id, "plugin_shutdown"); this.sessions.clear(); }
|
||||||
|
require(id) { const session = this.sessions.get(id); if (!session) throw new Error("Session was not found."); return session; }
|
||||||
|
beginGrace(session) {
|
||||||
|
clearTimeout(session.graceTimer);
|
||||||
|
session.state = "grace";
|
||||||
|
session.graceUntil = this.now() + this.graceMs;
|
||||||
|
session.delivery.pause();
|
||||||
|
session.graceTimer = setTimeout(() => this.stop(session.id, "stream_grace_expired").catch(() => {}), this.graceMs);
|
||||||
|
session.graceTimer.unref?.();
|
||||||
|
this.log.append({ kind: "session", state: "grace", session_id: session.id, grace_until: session.graceUntil });
|
||||||
|
}
|
||||||
|
async resume(session) { clearTimeout(session.graceTimer); session.graceTimer = null; session.graceUntil = 0; session.state = "running"; await session.delivery.resume(); this.log.append({ kind: "session", state: "resumed", session_id: session.id }); }
|
||||||
|
expireLater(session) { clearTimeout(session.graceTimer); session.graceUntil = this.now() + this.graceMs; session.graceTimer = setTimeout(() => this.sessions.delete(session.id), this.graceMs); session.graceTimer.unref?.(); }
|
||||||
|
async onHypothesis(raw) {
|
||||||
|
const session = this.sessions.get(raw.session_id);
|
||||||
|
const track = session?.tracks.get(raw.track_id || raw.source_uuid);
|
||||||
|
if (!session || !track || session.state !== "running") return;
|
||||||
|
track.lastSpeechAt = this.now();
|
||||||
|
const primary = Array.from(session.tracks.values()).find((candidate) => candidate.primary);
|
||||||
|
if (primary && track !== primary && this.now() - primary.lastSpeechAt < 800) return;
|
||||||
|
const stable = this.stabilizer.update(track.source_uuid, raw.text, { final: raw.final, newUtterance: raw.new_utterance, trailingIncomplete: raw.incomplete_word });
|
||||||
|
const event = {
|
||||||
|
session_id: session.id, source_uuid: track.source_uuid,
|
||||||
|
speaker_label: speakerLabel(session, track), ...stable,
|
||||||
|
audio: { start_us: raw.audio_start_us || 0, end_us: raw.audio_end_us || 0 },
|
||||||
|
latency: { capture_ms: raw.capture_ms || 0, network_ms: raw.network_ms || 0, queue_ms: raw.queue_ms || 0, inference_ms: raw.inference_ms || 0, stabilization_ms: stable.stabilization_ms, total_ms: raw.total_ms || 0 },
|
||||||
|
model: { id: raw.model_id || "unknown", provider: "whisper.cpp", backend: raw.backend || "unknown" }
|
||||||
|
};
|
||||||
|
const result = await session.delivery.deliver(event);
|
||||||
|
this.log.append({ kind: "caption", session_id: session.id, source_uuid: track.source_uuid, caption_text: event.stable_text, uncertain_text: event.uncertain_text, revision: event.revision, final: event.final, delivery: result.disposition, latency: event.latency, model: event.model });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function speakerLabel(session, track) { const enabled = Array.from(session.tracks.values()).filter((candidate) => candidate.enabled); return enabled.length === 1 && track.single_speaker ? null : track.speaker_label || track.display_name; }
|
||||||
|
function serializeTrack(track) { return { source_uuid: track.source_uuid, display_name: track.display_name, speaker_label: track.speaker_label, enabled: track.enabled, primary: track.primary, single_speaker: track.single_speaker, delivery_enabled: track.delivery_enabled, program_active: track.program_active, source_missing: track.source_missing, last_activity_at: track.last_activity_at, sequence: track.sequence?.metrics?.() }; }
|
||||||
|
function text(value, max) { return String(value || "").trim().slice(0, max); }
|
||||||
|
function isUuid(value) { return /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(String(value || "")); }
|
||||||
|
|
||||||
|
module.exports = { SessionCoordinator, serializeTrack };
|
||||||
122
plugins/lumi_transcription/backend/transcription/provider.js
Normal file
122
plugins/lumi_transcription/backend/transcription/provider.js
Normal file
@ -0,0 +1,122 @@
|
|||||||
|
const { EventEmitter } = require("events");
|
||||||
|
const { spawn } = require("child_process");
|
||||||
|
const { BoundedQueue } = require("../sessions/bounded_queue");
|
||||||
|
|
||||||
|
class TranscriptionProvider extends EventEmitter {
|
||||||
|
async loadModel() { throw new Error("Model loading is not implemented."); }
|
||||||
|
async benchmark() { throw new Error("Benchmarking is not implemented."); }
|
||||||
|
async startSession() { throw new Error("Session start is not implemented."); }
|
||||||
|
async addTrack() { throw new Error("Track add is not implemented."); }
|
||||||
|
async pushAudio() { throw new Error("Audio input is not implemented."); }
|
||||||
|
async removeTrack() { throw new Error("Track removal is not implemented."); }
|
||||||
|
async stopSession() { throw new Error("Session stop is not implemented."); }
|
||||||
|
async health() { return { healthy: false, state: "unavailable" }; }
|
||||||
|
}
|
||||||
|
|
||||||
|
class WhisperWorkerSupervisor extends EventEmitter {
|
||||||
|
constructor(options = {}) {
|
||||||
|
super();
|
||||||
|
this.executable = options.executable;
|
||||||
|
this.args = options.args || [];
|
||||||
|
this.spawn = options.spawn || spawn;
|
||||||
|
this.maxRestarts = options.maxRestarts ?? 3;
|
||||||
|
this.restartWindowMs = options.restartWindowMs || 60000;
|
||||||
|
this.queue = new BoundedQueue({ maxItems: options.maxQueuedPackets || 250, maxBytes: options.maxQueuedBytes || 5 * 32000, maxAgeMs: 5000 });
|
||||||
|
this.child = null;
|
||||||
|
this.state = "stopped";
|
||||||
|
this.stopping = false;
|
||||||
|
this.restartTimes = [];
|
||||||
|
this.stdoutBuffer = "";
|
||||||
|
}
|
||||||
|
start() {
|
||||||
|
if (this.child) return;
|
||||||
|
if (!this.executable) throw new Error("Whisper worker executable is not configured.");
|
||||||
|
this.stopping = false;
|
||||||
|
this.state = "starting";
|
||||||
|
const child = this.spawn(this.executable, this.args, { stdio: ["pipe", "pipe", "pipe"], windowsHide: true });
|
||||||
|
this.child = child;
|
||||||
|
child.stdout.on("data", (chunk) => this.onStdout(chunk));
|
||||||
|
child.stderr.on("data", (chunk) => this.emit("diagnostic", { level: "warning", message: String(chunk).trim().slice(0, 1000) }));
|
||||||
|
child.on("error", (error) => this.emit("error", error));
|
||||||
|
child.on("exit", (code, signal) => this.onExit(code, signal));
|
||||||
|
child.stdin.on("drain", () => this.flush());
|
||||||
|
this.state = "running";
|
||||||
|
this.emit("state", { state: this.state });
|
||||||
|
this.flush();
|
||||||
|
}
|
||||||
|
send(metadata, pcm = null) {
|
||||||
|
const body = Buffer.from(JSON.stringify(metadata), "utf8");
|
||||||
|
const audio = pcm ? Buffer.from(pcm) : Buffer.alloc(0);
|
||||||
|
if (body.length > 64 * 1024 || audio.length > 5 * 32000) throw new Error("Worker packet exceeds its limit.");
|
||||||
|
const header = Buffer.alloc(8);
|
||||||
|
header.writeUInt32LE(body.length, 0);
|
||||||
|
header.writeUInt32LE(audio.length, 4);
|
||||||
|
const packet = Buffer.concat([header, body, audio]);
|
||||||
|
const accepted = this.queue.push(packet, { bytes: packet.length, capturedAt: metadata.captured_at || Date.now() });
|
||||||
|
if (accepted) this.flush();
|
||||||
|
return accepted;
|
||||||
|
}
|
||||||
|
flush() {
|
||||||
|
if (!this.child || this.state !== "running") return;
|
||||||
|
let packet;
|
||||||
|
while ((packet = this.queue.shift())) if (!this.child.stdin.write(packet)) break;
|
||||||
|
}
|
||||||
|
async stop() {
|
||||||
|
this.stopping = true;
|
||||||
|
this.queue.clear();
|
||||||
|
const child = this.child;
|
||||||
|
if (!child) { this.state = "stopped"; return; }
|
||||||
|
this.sendDirect({ type: "shutdown" });
|
||||||
|
await new Promise((resolve) => {
|
||||||
|
const timer = setTimeout(() => { child.kill(); resolve(); }, 2000);
|
||||||
|
child.once("exit", () => { clearTimeout(timer); resolve(); });
|
||||||
|
});
|
||||||
|
this.child = null;
|
||||||
|
this.state = "stopped";
|
||||||
|
}
|
||||||
|
sendDirect(metadata) { if (!this.child) return false; const body = Buffer.from(JSON.stringify(metadata)); const header = Buffer.alloc(8); header.writeUInt32LE(body.length, 0); return this.child.stdin.write(Buffer.concat([header, body])); }
|
||||||
|
onStdout(chunk) {
|
||||||
|
this.stdoutBuffer += String(chunk);
|
||||||
|
if (this.stdoutBuffer.length > 256 * 1024) this.stdoutBuffer = this.stdoutBuffer.slice(-64 * 1024);
|
||||||
|
let index;
|
||||||
|
while ((index = this.stdoutBuffer.indexOf("\n")) >= 0) {
|
||||||
|
const line = this.stdoutBuffer.slice(0, index).trim();
|
||||||
|
this.stdoutBuffer = this.stdoutBuffer.slice(index + 1);
|
||||||
|
if (!line) continue;
|
||||||
|
try { this.emit("message", JSON.parse(line)); }
|
||||||
|
catch { this.emit("diagnostic", { level: "warning", message: "Whisper worker emitted malformed output." }); }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
onExit(code, signal) {
|
||||||
|
this.child = null;
|
||||||
|
if (this.stopping) { this.state = "stopped"; return; }
|
||||||
|
this.state = "failed";
|
||||||
|
this.emit("crash", { code, signal });
|
||||||
|
const now = Date.now();
|
||||||
|
this.restartTimes = this.restartTimes.filter((time) => now - time < this.restartWindowMs);
|
||||||
|
if (this.restartTimes.length >= this.maxRestarts) return;
|
||||||
|
this.restartTimes.push(now);
|
||||||
|
setTimeout(() => { try { this.start(); } catch (error) { this.emit("error", error); } }, 250).unref?.();
|
||||||
|
}
|
||||||
|
health() { return { healthy: this.state === "running", state: this.state, queue: this.queue.metrics(), restarts_in_window: this.restartTimes.length }; }
|
||||||
|
}
|
||||||
|
|
||||||
|
class WhisperCppServerProvider extends TranscriptionProvider {
|
||||||
|
constructor(supervisor) {
|
||||||
|
super(); this.worker = supervisor; this.model = null; this.sessions = new Set();
|
||||||
|
supervisor.on("message", (message) => this.emit(message.type || "message", message));
|
||||||
|
supervisor.on("crash", (event) => this.emit("provider_error", Object.assign(new Error("Whisper worker crashed."), { details: event })));
|
||||||
|
supervisor.on("error", (error) => this.emit("provider_error", error));
|
||||||
|
}
|
||||||
|
async loadModel(model) { this.worker.start(); this.model = model; this.worker.send({ type: "load_model", model }); return this.health(); }
|
||||||
|
async benchmark(options = {}) { this.worker.send({ type: "benchmark", options }); return { accepted: true, model: this.model?.id || null }; }
|
||||||
|
async startSession(session) { this.sessions.add(session.id); this.worker.send({ type: "start_session", session }); }
|
||||||
|
async addTrack(sessionId, track) { this.worker.send({ type: "add_track", session_id: sessionId, track }); }
|
||||||
|
async pushAudio(sessionId, trackId, frame) { return this.worker.send({ type: "audio", session_id: sessionId, track_id: trackId, sequence: frame.sequence, capture_timestamp_us: frame.capture_timestamp_us, captured_at: Date.now() }, frame.pcm); }
|
||||||
|
async removeTrack(sessionId, trackId) { this.worker.send({ type: "remove_track", session_id: sessionId, track_id: trackId }); }
|
||||||
|
async stopSession(sessionId) { this.sessions.delete(sessionId); this.worker.send({ type: "stop_session", session_id: sessionId }); }
|
||||||
|
async health() { return { provider: "whisper_cpp_server", model: this.model?.id || null, sessions: this.sessions.size, ...this.worker.health() }; }
|
||||||
|
async stop() { this.sessions.clear(); await this.worker.stop(); }
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { TranscriptionProvider, WhisperWorkerSupervisor, WhisperCppServerProvider };
|
||||||
@ -0,0 +1,67 @@
|
|||||||
|
const crypto = require("crypto");
|
||||||
|
|
||||||
|
class CaptionStabilizer {
|
||||||
|
constructor(options = {}) {
|
||||||
|
this.maxChars = options.maxChars || 80;
|
||||||
|
this.fragmentAfterMs = options.fragmentAfterMs || 1000;
|
||||||
|
this.now = options.now || Date.now;
|
||||||
|
this.tracks = new Map();
|
||||||
|
}
|
||||||
|
|
||||||
|
update(sourceUuid, hypothesis, details = {}) {
|
||||||
|
const now = this.now();
|
||||||
|
const words = tokenize(hypothesis);
|
||||||
|
let state = this.tracks.get(sourceUuid);
|
||||||
|
if (!state || details.newUtterance) {
|
||||||
|
state = { captionId: crypto.randomUUID(), revision: 0, stable: [], previous: [], startedAt: now, tailSince: now };
|
||||||
|
this.tracks.set(sourceUuid, state);
|
||||||
|
}
|
||||||
|
const common = commonPrefix(state.previous, words);
|
||||||
|
const agreement = commonPrefix(state.stable, words);
|
||||||
|
if (agreement.length < state.stable.length) {
|
||||||
|
words.splice(0, agreement.length, ...state.stable.slice(0, agreement.length));
|
||||||
|
}
|
||||||
|
const stableLimit = details.trailingIncomplete && !details.final
|
||||||
|
? Math.max(state.stable.length, common.length - 1)
|
||||||
|
: Math.max(state.stable.length, common.length);
|
||||||
|
const agreedCandidate = words.slice(0, stableLimit);
|
||||||
|
if (agreedCandidate.length > state.stable.length) state.stable = agreedCandidate;
|
||||||
|
const tail = words.slice(state.stable.length);
|
||||||
|
if (tail.join(" ") !== state.previous.slice(state.stable.length).join(" ")) state.tailSince = now;
|
||||||
|
const incompleteWord = Boolean(details.trailingIncomplete && tail.length && now - state.tailSince >= this.fragmentAfterMs && !details.final);
|
||||||
|
if (details.final) state.stable = words;
|
||||||
|
state.previous = words;
|
||||||
|
state.revision += 1;
|
||||||
|
const stableText = clip(state.stable.join(" "), this.maxChars);
|
||||||
|
const uncertainText = details.final ? "" : clip(tail.join(" "), Math.max(0, this.maxChars - stableText.length - 1));
|
||||||
|
const event = {
|
||||||
|
caption_id: state.captionId,
|
||||||
|
revision: state.revision,
|
||||||
|
stable_text: stableText,
|
||||||
|
uncertain_text: uncertainText,
|
||||||
|
final: Boolean(details.final),
|
||||||
|
incomplete_word: incompleteWord,
|
||||||
|
stabilization_ms: now - state.startedAt
|
||||||
|
};
|
||||||
|
if (details.final) this.tracks.delete(sourceUuid);
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class LatestCaptionGate {
|
||||||
|
constructor() { this.revisions = new Map(); }
|
||||||
|
accept(event) {
|
||||||
|
const key = `${event.session_id}:${event.caption_id}`;
|
||||||
|
const current = this.revisions.get(key) || 0;
|
||||||
|
if (event.revision <= current) return false;
|
||||||
|
this.revisions.set(key, event.revision);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
clearSession(sessionId) { for (const key of this.revisions.keys()) if (key.startsWith(`${sessionId}:`)) this.revisions.delete(key); }
|
||||||
|
}
|
||||||
|
|
||||||
|
function tokenize(value) { return String(value || "").trim().split(/\s+/).filter(Boolean); }
|
||||||
|
function commonPrefix(left, right) { let index = 0; while (index < left.length && index < right.length && left[index] === right[index]) index += 1; return right.slice(0, index); }
|
||||||
|
function clip(value, max) { if (value.length <= max) return value; return value.slice(0, max).trimEnd(); }
|
||||||
|
|
||||||
|
module.exports = { CaptionStabilizer, LatestCaptionGate, tokenize, commonPrefix };
|
||||||
1
plugins/lumi_transcription/data/.gitkeep
Normal file
1
plugins/lumi_transcription/data/.gitkeep
Normal file
@ -0,0 +1 @@
|
|||||||
|
|
||||||
134
plugins/lumi_transcription/index.js
Normal file
134
plugins/lumi_transcription/index.js
Normal file
@ -0,0 +1,134 @@
|
|||||||
|
const express = require("express");
|
||||||
|
const fs = require("fs");
|
||||||
|
const path = require("path");
|
||||||
|
const { DeviceStore } = require("./backend/companion/device_store");
|
||||||
|
const { CompanionGateway } = require("./backend/companion/gateway");
|
||||||
|
const { RevisionStore } = require("./backend/config/revision_store");
|
||||||
|
const { CompanionCaptionDeliveryAdapter } = require("./backend/delivery/caption_delivery");
|
||||||
|
const { JsonlDiagnosticLog } = require("./backend/logs/jsonl_log");
|
||||||
|
const { ArtifactManager } = require("./backend/models/artifact_manager");
|
||||||
|
const { SessionCoordinator } = require("./backend/sessions/session_coordinator");
|
||||||
|
const { WhisperWorkerSupervisor, WhisperCppServerProvider } = require("./backend/transcription/provider");
|
||||||
|
const { ensureDataDirs, dataPath } = require("./backend/paths");
|
||||||
|
const modelManifest = require("./models_manifest.json");
|
||||||
|
const runtimeManifest = require("./runtime_manifest.json");
|
||||||
|
const manifest = require("./plugin.json");
|
||||||
|
|
||||||
|
const PLUGIN_ID = "lumi_transcription";
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
id: PLUGIN_ID,
|
||||||
|
init({ web, db, logger }) {
|
||||||
|
ensureDataDirs();
|
||||||
|
const allowInsecure = process.env.LUMI_COMPANION_DEV_ALLOW_INSECURE === "1";
|
||||||
|
const devices = new DeviceStore(db, { allowInsecure });
|
||||||
|
const revisions = new RevisionStore(db);
|
||||||
|
const diagnosticLog = new JsonlDiagnosticLog(dataPath("logs"));
|
||||||
|
diagnosticLog.cleanup();
|
||||||
|
const cleanupTimer = setInterval(() => diagnosticLog.cleanup(), 60 * 60 * 1000);
|
||||||
|
cleanupTimer.unref?.();
|
||||||
|
const supervisor = new WhisperWorkerSupervisor({
|
||||||
|
executable: process.env.LUMI_TRANSCRIPTION_WORKER || "",
|
||||||
|
args: process.env.LUMI_TRANSCRIPTION_WORKER_ARGS ? JSON.parse(process.env.LUMI_TRANSCRIPTION_WORKER_ARGS) : []
|
||||||
|
});
|
||||||
|
supervisor.on("diagnostic", (entry) => diagnosticLog.append({ kind: "worker", ...entry }));
|
||||||
|
supervisor.on("error", (error) => diagnosticLog.append({ kind: "worker", state: "error", message: error.message }));
|
||||||
|
const provider = new WhisperCppServerProvider(supervisor);
|
||||||
|
const sessions = new SessionCoordinator({
|
||||||
|
provider,
|
||||||
|
deliveryFactory: (send) => new CompanionCaptionDeliveryAdapter(send),
|
||||||
|
log: diagnosticLog
|
||||||
|
});
|
||||||
|
const gateway = new CompanionGateway({ devices, sessions, log: diagnosticLog, allowInsecure });
|
||||||
|
const unregisterUpgrade = web.addUpgradeHandler("/plugins/lumi_transcription/live", (request, socket, head) => gateway.upgrade(request, socket, head));
|
||||||
|
const models = new ArtifactManager(dataPath("models"));
|
||||||
|
const runtimeArchives = new ArtifactManager(dataPath("tmp"));
|
||||||
|
const runtimes = new ArtifactManager(dataPath("runtime"));
|
||||||
|
|
||||||
|
const router = web.createRouter();
|
||||||
|
router.use("/assets", express.static(path.join(__dirname, "public")));
|
||||||
|
router.get("/", requireAdmin, async (_req, res) => {
|
||||||
|
res.render(path.join(__dirname, "views", "settings.ejs"), {
|
||||||
|
title: "Lumi Transcription",
|
||||||
|
pluginVersion: manifest.version,
|
||||||
|
providerHealth: await provider.health(),
|
||||||
|
devices: devices.list(),
|
||||||
|
settings: revisions.list(),
|
||||||
|
models: modelManifest.models.map((entry) => ({ ...entry, status: models.status(entry) })),
|
||||||
|
runtimeManifest,
|
||||||
|
logs: diagnosticLog.files()
|
||||||
|
});
|
||||||
|
});
|
||||||
|
router.get("/api/status", requireAdmin, async (_req, res) => res.json({
|
||||||
|
ok: true, plugin: { id: PLUGIN_ID, version: manifest.version }, protocol_version: 1,
|
||||||
|
provider: await provider.health(), devices: devices.list(), settings: revisions.list(),
|
||||||
|
models: modelManifest.models.map((entry) => ({ id: entry.id, ...models.status(entry) })),
|
||||||
|
runtime: runtimeManifest
|
||||||
|
}));
|
||||||
|
router.post("/api/pairing-package", requireAdmin, (req, res) => {
|
||||||
|
try {
|
||||||
|
const host = requestHost(req);
|
||||||
|
const pairing = devices.issuePairing({ userId: req.session.user.id, host });
|
||||||
|
const bootstrap = { format: "lumi-companion-bootstrap-v1", ...pairing, exchange_url: `${host}/plugins/${PLUGIN_ID}/api/pair` };
|
||||||
|
res.set("Cache-Control", "no-store");
|
||||||
|
res.attachment(`lumi-companion-${pairing.pairing_id}.lumi-pairing.json`);
|
||||||
|
res.send(`${JSON.stringify(bootstrap, null, 2)}\n`);
|
||||||
|
} catch (error) { res.status(400).json({ ok: false, error: error.message }); }
|
||||||
|
});
|
||||||
|
router.post("/api/pair", requireSecureRequest(allowInsecure), (req, res) => {
|
||||||
|
try { res.set("Cache-Control", "no-store"); res.status(201).json({ ok: true, ...devices.exchange(req.body || {}) }); }
|
||||||
|
catch (error) { res.status(error.code === "PAIRING_ALREADY_USED" ? 409 : 400).json({ ok: false, code: error.code, error: error.message }); }
|
||||||
|
});
|
||||||
|
router.get("/api/devices", requireAdmin, (_req, res) => res.json({ devices: devices.list() }));
|
||||||
|
router.post("/api/devices/:id/revoke", requireAdmin, (req, res) => res.json({ ok: devices.revoke(req.params.id) }));
|
||||||
|
router.post("/api/devices/:id/capabilities", requireAdmin, (req, res) => {
|
||||||
|
const capabilities = devices.setCapabilities(req.params.id, req.body.capabilities);
|
||||||
|
if (!capabilities) return res.status(404).json({ ok: false, error: "Device was not found or is revoked." });
|
||||||
|
res.json({ ok: true, capabilities });
|
||||||
|
});
|
||||||
|
router.get("/api/settings", requireSettingsAccess(devices, allowInsecure), (_req, res) => res.json({ fields: revisions.list() }));
|
||||||
|
router.patch("/api/settings", requireSettingsAccess(devices, allowInsecure), (req, res) => {
|
||||||
|
try {
|
||||||
|
const actor = req.session?.user?.id || req.lumiDevice?.id;
|
||||||
|
const result = revisions.apply(req.body.changes, actor);
|
||||||
|
if (result.applied.length) web.emitEvent?.("transcription:settings_changed", { fields: result.applied.map((entry) => entry.key) }, { role: "admin" });
|
||||||
|
res.status(result.conflicts.length ? 409 : 200).json({ ok: !result.conflicts.length, ...result });
|
||||||
|
} catch (error) { res.status(400).json({ ok: false, error: error.message }); }
|
||||||
|
});
|
||||||
|
router.post("/api/models/:id/download", requireAdmin, async (req, res) => {
|
||||||
|
const entry = modelManifest.models.find((candidate) => candidate.id === req.params.id);
|
||||||
|
if (!entry) return res.status(404).json({ ok: false, error: "Model was not found." });
|
||||||
|
try { res.status(201).json({ ok: true, status: await models.download(entry, { confirmed: req.body.confirmed === true }) }); }
|
||||||
|
catch (error) { res.status(400).json({ ok: false, error: error.message }); }
|
||||||
|
});
|
||||||
|
router.post("/api/runtime/:id/install", requireAdmin, async (req, res) => {
|
||||||
|
const entry = runtimeManifest.artifacts.find((candidate) => candidate.id === req.params.id);
|
||||||
|
if (!entry) return res.status(404).json({ ok: false, error: "Runtime was not found." });
|
||||||
|
try {
|
||||||
|
const filename = `${entry.id}.zip`;
|
||||||
|
const archive = await runtimeArchives.download({ ...entry, filename }, { confirmed: req.body.confirmed === true });
|
||||||
|
res.status(201).json({ ok: true, status: runtimes.installZip(entry, archive.path) });
|
||||||
|
} catch (error) { res.status(400).json({ ok: false, error: error.message }); }
|
||||||
|
});
|
||||||
|
router.get("/api/logs", requireAdmin, (_req, res) => res.json({ files: diagnosticLog.files().map(({ path: _path, ...entry }) => entry) }));
|
||||||
|
|
||||||
|
web.mount(`/plugins/${PLUGIN_ID}`, router, { label: "Transcription", role: "admin", section: "plugins" });
|
||||||
|
global.lumiFrameworks = global.lumiFrameworks || {};
|
||||||
|
global.lumiFrameworks.transcription = { version: manifest.version, protocol_version: 1, health: () => provider.health() };
|
||||||
|
|
||||||
|
return async () => {
|
||||||
|
clearInterval(cleanupTimer);
|
||||||
|
unregisterUpgrade();
|
||||||
|
await gateway.close();
|
||||||
|
await sessions.close();
|
||||||
|
await provider.stop();
|
||||||
|
if (global.lumiFrameworks?.transcription?.version === manifest.version) delete global.lumiFrameworks.transcription;
|
||||||
|
logger?.info?.("Lumi transcription stopped", {}, { event: "transcription_stopped" });
|
||||||
|
};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
function requireAdmin(req, res, next) { if (req.session?.user?.isAdmin) return next(); return res.status(403).json({ error: "Administrator access is required." }); }
|
||||||
|
function requireSettingsAccess(devices, allowInsecure) { return (req, res, next) => { if (req.session?.user?.isAdmin) return next(); const forwarded = String(req.headers["x-forwarded-proto"] || "").split(",")[0].trim(); const local = ["127.0.0.1", "::1", "::ffff:127.0.0.1"].includes(req.socket.remoteAddress); if (!req.secure && forwarded !== "https" && !(allowInsecure && local)) return res.status(426).json({ error: "Device settings synchronization requires HTTPS." }); const auth = devices.authenticate(req.headers.authorization, "transcription.settings.v1"); if (!auth.allowed) return res.status(401).json({ error: "A paired device credential is required." }); req.lumiDevice = auth.device; next(); }; }
|
||||||
|
function requestHost(req) { const forwarded = String(req.headers["x-forwarded-proto"] || "").split(",")[0].trim(); const protocol = forwarded === "https" ? "https" : req.protocol; return `${protocol}://${req.get("host")}`; }
|
||||||
|
function requireSecureRequest(allowInsecure) { return (req, res, next) => { const forwarded = String(req.headers["x-forwarded-proto"] || "").split(",")[0].trim(); const local = ["127.0.0.1", "::1", "::ffff:127.0.0.1"].includes(req.socket.remoteAddress); if (req.secure || forwarded === "https" || (allowInsecure && local)) return next(); return res.status(426).json({ error: "Lumi Companion pairing requires HTTPS." }); }; }
|
||||||
33
plugins/lumi_transcription/models_manifest.json
Normal file
33
plugins/lumi_transcription/models_manifest.json
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"repository_commit": "5359861c739e955e79d9a303bcbc70fb988958b1",
|
||||||
|
"models": [
|
||||||
|
{
|
||||||
|
"id": "small.en",
|
||||||
|
"label": "Small English",
|
||||||
|
"recommended": true,
|
||||||
|
"url": "https://huggingface.co/ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/ggml-small.en.bin",
|
||||||
|
"sha256": "c6138d6d58ecc8322097e0f987c32f1be8bb0a18532a3f88f734d1bbf9c41e5d",
|
||||||
|
"bytes": 487614201,
|
||||||
|
"filename": "ggml-small.en.bin"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "small.en-q5_1",
|
||||||
|
"label": "Small English quantized (Q5_1)",
|
||||||
|
"recommended": false,
|
||||||
|
"url": "https://huggingface.co/ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/ggml-small.en-q5_1.bin",
|
||||||
|
"sha256": "bfdff4894dcb76bbf647d56263ea2a96645423f1669176f4844a1bf8e478ad30",
|
||||||
|
"bytes": 190098681,
|
||||||
|
"filename": "ggml-small.en-q5_1.bin"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "base.en",
|
||||||
|
"label": "Base English",
|
||||||
|
"recommended": false,
|
||||||
|
"url": "https://huggingface.co/ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/ggml-base.en.bin",
|
||||||
|
"sha256": "a03779c86df3323075f5e796cb2ce5029f00ec8869eee3fdfb897afe36c6d002",
|
||||||
|
"bytes": 147964211,
|
||||||
|
"filename": "ggml-base.en.bin"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
10
plugins/lumi_transcription/plugin.json
Normal file
10
plugins/lumi_transcription/plugin.json
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
{
|
||||||
|
"id": "lumi_transcription",
|
||||||
|
"name": "Lumi Transcription",
|
||||||
|
"version": "0.1.0-experimental.1",
|
||||||
|
"description": "Server-hosted whisper.cpp transcription for Lumi Companion and OBS closed captions.",
|
||||||
|
"main": "index.js",
|
||||||
|
"channel": "experimental",
|
||||||
|
"compatible_from": "0.1.0-experimental.1",
|
||||||
|
"rollback_safe": true
|
||||||
|
}
|
||||||
1
plugins/lumi_transcription/public/transcription.css
Normal file
1
plugins/lumi_transcription/public/transcription.css
Normal file
@ -0,0 +1 @@
|
|||||||
|
.transcription-shell{--accent:#85e3c4;--warn:#f5c56b;max-width:1120px;margin:0 auto;padding:clamp(1rem,3vw,2.5rem);display:grid;gap:1.35rem}.transcription-shell section{padding:1.35rem 0;border-top:1px solid color-mix(in srgb,currentColor 16%,transparent)}.transcription-hero{display:flex;justify-content:space-between;align-items:flex-start;gap:2rem;padding:1rem 0 2rem}.transcription-hero h1{font-size:clamp(2rem,5vw,4.25rem);line-height:.95;margin:.25rem 0 1rem}.transcription-hero p:not(.eyebrow){max-width:67ch;opacity:.78}.eyebrow{text-transform:uppercase;letter-spacing:.13em;font-size:.72rem;font-weight:750;opacity:.65;margin:0}.health-pill{display:inline-flex;align-items:center;gap:.55rem;padding:.55rem .8rem;border-radius:999px;white-space:nowrap;background:color-mix(in srgb,var(--warn) 16%,transparent)}.health-pill span{width:.65rem;height:.65rem;border-radius:50%;background:var(--warn)}.health-pill.is-ready{background:color-mix(in srgb,var(--accent) 16%,transparent)}.health-pill.is-ready span{background:var(--accent)}.section-heading{display:flex;justify-content:space-between;align-items:end;gap:1rem;margin-bottom:1rem}.section-heading h2{margin:.2rem 0 0;font-size:1.35rem}.setup-path ol{display:grid;grid-template-columns:repeat(4,1fr);list-style:none;padding:0;margin:0 0 1.25rem;counter-reset:steps}.setup-path li{counter-increment:steps;padding:0 1rem 1rem 2.25rem;position:relative;opacity:.62}.setup-path li:before{content:counter(steps);position:absolute;left:0;top:-.2rem;width:1.6rem;height:1.6rem;border:1px solid currentColor;border-radius:50%;display:grid;place-items:center;font-size:.75rem}.setup-path li.is-current{opacity:1}.setup-path li.is-current:before{background:var(--accent);color:#10251f;border-color:var(--accent)}.setup-path li span,.plain-list span{display:block;font-size:.88rem;opacity:.7;margin-top:.25rem}.primary-action{border:0;border-radius:.6rem;background:var(--accent);color:#10251f;font-weight:750;padding:.7rem 1rem;cursor:pointer}.primary-action:disabled{opacity:.55;cursor:wait}.inline-status{display:inline;margin-left:.8rem}.transcription-grid{display:grid;grid-template-columns:1fr 1fr;gap:2.5rem}.health-list{margin:0}.health-list div{display:flex;justify-content:space-between;gap:1rem;padding:.55rem 0}.health-list dt{opacity:.65}.notice,.privacy-note,.empty-state{padding:1rem;border-radius:.65rem;background:color-mix(in srgb,currentColor 6%,transparent);font-size:.9rem}.plain-list{list-style:none;padding:0;margin:0}.plain-list li{display:flex;justify-content:space-between;gap:1rem;padding:.7rem 0}.plain-list code{font-size:.72rem;opacity:.6}.model-row{display:grid;gap:.5rem}.model-row article{display:flex;justify-content:space-between;align-items:center;gap:1rem;padding:.9rem 0}.model-row h3,.model-row p{margin:0}.model-row p{font-size:.88rem;opacity:.7;margin-top:.2rem}.state-label{font-size:.78rem;font-weight:700;white-space:nowrap}@media(max-width:760px){.transcription-hero,.transcription-grid{display:grid;grid-template-columns:1fr}.setup-path ol{grid-template-columns:1fr}.health-pill{justify-self:start}.plain-list li{display:block}.plain-list code{display:block;margin-top:.5rem;overflow-wrap:anywhere}}
|
||||||
22
plugins/lumi_transcription/public/transcription.js
Normal file
22
plugins/lumi_transcription/public/transcription.js
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
(() => {
|
||||||
|
const root = document.querySelector("[data-transcription-admin]");
|
||||||
|
if (!root) return;
|
||||||
|
const button = root.querySelector("[data-create-pairing]");
|
||||||
|
const status = root.querySelector("[data-status]");
|
||||||
|
button?.addEventListener("click", async () => {
|
||||||
|
button.disabled = true;
|
||||||
|
status.textContent = "Creating a one-time pairing package…";
|
||||||
|
try {
|
||||||
|
const response = await fetch("/plugins/lumi_transcription/api/pairing-package", { method: "POST", headers: { Accept: "application/json" } });
|
||||||
|
if (!response.ok) throw new Error((await response.json()).error || "Pairing package could not be created.");
|
||||||
|
const blob = await response.blob();
|
||||||
|
const disposition = response.headers.get("content-disposition") || "";
|
||||||
|
const filename = /filename="?([^";]+)"?/i.exec(disposition)?.[1] || "lumi-companion.lumi-pairing.json";
|
||||||
|
const link = document.createElement("a");
|
||||||
|
link.href = URL.createObjectURL(blob); link.download = filename; link.click();
|
||||||
|
setTimeout(() => URL.revokeObjectURL(link.href), 1000);
|
||||||
|
status.textContent = "Pairing package created. It expires in 15 minutes and works once.";
|
||||||
|
} catch (error) { status.textContent = error.message; }
|
||||||
|
finally { button.disabled = false; }
|
||||||
|
});
|
||||||
|
})();
|
||||||
30
plugins/lumi_transcription/runtime_manifest.json
Normal file
30
plugins/lumi_transcription/runtime_manifest.json
Normal file
@ -0,0 +1,30 @@
|
|||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"provider": "whisper.cpp",
|
||||||
|
"tested_version": "v1.9.1",
|
||||||
|
"tested_commit": "f049fff95a089aa9969deb009cdd4892b3e74916",
|
||||||
|
"artifacts": [
|
||||||
|
{
|
||||||
|
"id": "windows-x64-cuda-12.4",
|
||||||
|
"platform": "win32",
|
||||||
|
"architecture": "x64",
|
||||||
|
"backend": "cuda",
|
||||||
|
"cuda": "12.4",
|
||||||
|
"url": "https://github.com/ggml-org/whisper.cpp/releases/download/v1.9.1/whisper-cublas-12.4.0-bin-x64.zip",
|
||||||
|
"sha256": "106a2030eff8998e4ef320fe72e263a78449e9040386ee27c41ea80b001b601b",
|
||||||
|
"expected_paths": ["whisper-cli.exe"],
|
||||||
|
"tested": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "windows-x64-cpu",
|
||||||
|
"platform": "win32",
|
||||||
|
"architecture": "x64",
|
||||||
|
"backend": "cpu",
|
||||||
|
"url": "https://github.com/ggml-org/whisper.cpp/releases/download/v1.9.1/whisper-bin-x64.zip",
|
||||||
|
"sha256": "7d8be46ecd31828e1eb7a2ecdd0d6b314feafd82163038ab6092594b0a063539",
|
||||||
|
"expected_paths": ["whisper-cli.exe"],
|
||||||
|
"tested": false,
|
||||||
|
"note": "CPU fallback remains disabled until the host benchmark meets the latency target."
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
241
plugins/lumi_transcription/tests/verify.js
Normal file
241
plugins/lumi_transcription/tests/verify.js
Normal file
@ -0,0 +1,241 @@
|
|||||||
|
const assert = require("assert");
|
||||||
|
const crypto = require("crypto");
|
||||||
|
const fs = require("fs");
|
||||||
|
const os = require("os");
|
||||||
|
const path = require("path");
|
||||||
|
const http = require("http");
|
||||||
|
const { EventEmitter } = require("events");
|
||||||
|
const { PassThrough } = require("stream");
|
||||||
|
const Database = require("better-sqlite3");
|
||||||
|
const express = require("express");
|
||||||
|
const { WebSocket } = require("ws");
|
||||||
|
const { DeviceStore } = require("../backend/companion/device_store");
|
||||||
|
const { CompanionGateway } = require("../backend/companion/gateway");
|
||||||
|
const protocol = require("../backend/companion/protocol");
|
||||||
|
const { RevisionStore } = require("../backend/config/revision_store");
|
||||||
|
const { JsonlDiagnosticLog, sanitize } = require("../backend/logs/jsonl_log");
|
||||||
|
const { ArtifactManager, sha256File } = require("../backend/models/artifact_manager");
|
||||||
|
const { BoundedQueue, SequenceTracker, RollingPcmBuffer } = require("../backend/sessions/bounded_queue");
|
||||||
|
const { SessionCoordinator } = require("../backend/sessions/session_coordinator");
|
||||||
|
const { CaptionStabilizer, LatestCaptionGate } = require("../backend/transcription/stabilizer");
|
||||||
|
const { WhisperWorkerSupervisor } = require("../backend/transcription/provider");
|
||||||
|
const plugin = require("../index");
|
||||||
|
const { createWebUpgradeRegistry } = require("../../../src/services/web-upgrades");
|
||||||
|
|
||||||
|
async function run() {
|
||||||
|
const temp = fs.mkdtempSync(path.join(os.tmpdir(), "lumi-transcription-"));
|
||||||
|
try {
|
||||||
|
verifyProtocol();
|
||||||
|
verifyPairingAndRevocation();
|
||||||
|
verifyRevisions();
|
||||||
|
verifyQueues();
|
||||||
|
verifyStabilization();
|
||||||
|
await verifySessionLifecycle();
|
||||||
|
await verifyWorkerRestart();
|
||||||
|
await verifyAuthenticatedGateway();
|
||||||
|
verifyArtifactsAndLogs(temp);
|
||||||
|
await verifyPluginIsolation();
|
||||||
|
console.log("Lumi transcription verification passed: protocol, pairing, revocation, revisions, queues, stabilization, lifecycle, worker recovery, artifacts, logs, and plugin isolation.");
|
||||||
|
} finally { fs.rmSync(temp, { recursive: true, force: true }); }
|
||||||
|
}
|
||||||
|
|
||||||
|
function verifyProtocol() {
|
||||||
|
const sessionId = crypto.randomUUID();
|
||||||
|
const sourceUuid = crypto.randomUUID();
|
||||||
|
const encoded = protocol.encodeAudioFrame({ session_id: sessionId, source_uuid: sourceUuid, sequence: 42, capture_timestamp_us: 987654321, active: true, muted: false, pcm: Buffer.alloc(640, 3) });
|
||||||
|
const decoded = protocol.parseAudioFrame(encoded);
|
||||||
|
assert.equal(decoded.session_id, sessionId);
|
||||||
|
assert.equal(decoded.source_uuid, sourceUuid);
|
||||||
|
assert.equal(decoded.sequence, 42);
|
||||||
|
assert.equal(decoded.pcm.length, 640);
|
||||||
|
assert.throws(() => protocol.parseAudioFrame(Buffer.alloc(64)), /magic/i);
|
||||||
|
const hello = protocol.envelope("hello", { companion_version: "0.1.0", plugin_version: "0.1.0", capabilities: ["transcription.capture.v1"], audio: { codec: "pcm_s16le", sample_rate: 16000, channels: 1, bits: 16 } });
|
||||||
|
assert.equal(protocol.validateHello(protocol.parseEnvelope(JSON.stringify(hello))).audio.codec, "pcm_s16le");
|
||||||
|
assert.throws(() => protocol.parseEnvelope(JSON.stringify({ ...hello, version: 2 })), /unsupported/i);
|
||||||
|
assert.throws(() => protocol.parseEnvelope(Buffer.alloc(protocol.MAX_JSON_BYTES + 1)), /size/i);
|
||||||
|
}
|
||||||
|
|
||||||
|
function verifyPairingAndRevocation() {
|
||||||
|
const db = new Database(":memory:");
|
||||||
|
let now = 1000;
|
||||||
|
let randomSeed = 6;
|
||||||
|
const store = new DeviceStore(db, { now: () => now, randomBytes: (size) => Buffer.alloc(size, ++randomSeed) });
|
||||||
|
const pairing = store.issuePairing({ userId: "admin", host: "https://lumi.example" });
|
||||||
|
const issued = store.exchange({ token: pairing.token, device: { install_id: "install", name: "Stream PC", companion_version: "0.1.0" } });
|
||||||
|
assert.equal(store.authenticate(`LumiDevice ${issued.device_id}.${issued.device_secret}`, "transcription.capture.v1").allowed, true);
|
||||||
|
assert.throws(() => store.exchange({ token: pairing.token, device: {} }), (error) => error.code === "PAIRING_ALREADY_USED");
|
||||||
|
assert.deepEqual(store.setCapabilities(issued.device_id, ["transcription.settings.v1", "made.up"]), ["transcription.settings.v1"]);
|
||||||
|
assert.equal(store.authenticate(`LumiDevice ${issued.device_id}.${issued.device_secret}`, "transcription.capture.v1").reason, "capability_revoked");
|
||||||
|
assert.equal(store.revoke(issued.device_id), true);
|
||||||
|
assert.equal(store.authenticate(`LumiDevice ${issued.device_id}.${issued.device_secret}`).reason, "device_revoked");
|
||||||
|
const expired = store.issuePairing({ userId: "admin", host: "https://lumi.example", ttlMs: 50 });
|
||||||
|
now += 51;
|
||||||
|
assert.throws(() => store.exchange({ token: expired.token, device: {} }), (error) => error.code === "PAIRING_INVALID");
|
||||||
|
db.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
function verifyRevisions() {
|
||||||
|
const db = new Database(":memory:");
|
||||||
|
const store = new RevisionStore(db, { now: () => 1234 });
|
||||||
|
const first = store.apply([{ key: "selected_model_id", value: "small.en", base_revision: 0 }], "admin");
|
||||||
|
assert.equal(first.applied[0].revision, 1);
|
||||||
|
const merged = store.apply([{ key: "caption_max_chars", value: 80, base_revision: 0 }], "companion");
|
||||||
|
assert.equal(merged.conflicts.length, 0);
|
||||||
|
const conflict = store.apply([{ key: "selected_model_id", value: "base.en", base_revision: 0 }], "companion");
|
||||||
|
assert.equal(conflict.conflicts[0].server.value, "small.en");
|
||||||
|
assert.equal(store.list().selected_model_id.value, "small.en");
|
||||||
|
db.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
function verifyQueues() {
|
||||||
|
let now = 0;
|
||||||
|
const queue = new BoundedQueue({ maxItems: 2, maxBytes: 8, maxAgeMs: 50, now: () => now });
|
||||||
|
queue.push(Buffer.alloc(4), { capturedAt: now }); queue.push(Buffer.alloc(4), { capturedAt: now }); queue.push(Buffer.alloc(4), { capturedAt: now });
|
||||||
|
assert.equal(queue.metrics().dropped.capacity, 1);
|
||||||
|
now = 51; assert.equal(queue.size(), 0); assert.equal(queue.metrics().dropped.stale, 2);
|
||||||
|
const sequence = new SequenceTracker();
|
||||||
|
assert.equal(sequence.accept(5).accepted, true); assert.equal(sequence.accept(8).gap, 2); assert.equal(sequence.accept(7).accepted, false);
|
||||||
|
const rolling = new RollingPcmBuffer({ seconds: 1, now: () => now });
|
||||||
|
rolling.push(Buffer.alloc(20000), now); rolling.push(Buffer.alloc(20000), now);
|
||||||
|
assert.equal(rolling.snapshot().length, 20000);
|
||||||
|
}
|
||||||
|
|
||||||
|
function verifyStabilization() {
|
||||||
|
let now = 0;
|
||||||
|
const stabilizer = new CaptionStabilizer({ now: () => now, fragmentAfterMs: 1000 });
|
||||||
|
const source = crypto.randomUUID();
|
||||||
|
stabilizer.update(source, "hello wor");
|
||||||
|
now += 600;
|
||||||
|
const second = stabilizer.update(source, "hello world");
|
||||||
|
assert.equal(second.stable_text, "hello");
|
||||||
|
now += 600;
|
||||||
|
const third = stabilizer.update(source, "hello world again");
|
||||||
|
assert.equal(third.stable_text, "hello world");
|
||||||
|
const fragmentSource = crypto.randomUUID();
|
||||||
|
stabilizer.update(fragmentSource, "extraord", { trailingIncomplete: true });
|
||||||
|
now += 1001;
|
||||||
|
const fragment = stabilizer.update(fragmentSource, "extraord", { trailingIncomplete: true });
|
||||||
|
assert.equal(fragment.incomplete_word, true);
|
||||||
|
assert.equal(fragment.stable_text, "");
|
||||||
|
const gate = new LatestCaptionGate();
|
||||||
|
assert.equal(gate.accept({ session_id: source, caption_id: fragment.caption_id, revision: 2 }), true);
|
||||||
|
assert.equal(gate.accept({ session_id: source, caption_id: fragment.caption_id, revision: 1 }), false);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function verifySessionLifecycle() {
|
||||||
|
class Provider extends EventEmitter {
|
||||||
|
constructor() { super(); this.audio = []; this.stops = 0; }
|
||||||
|
async health() { return { healthy: true }; }
|
||||||
|
async startSession() {} async addTrack() {} async pushAudio(_session, _track, frame) { this.audio.push(frame); return true; }
|
||||||
|
async stopSession() { this.stops += 1; }
|
||||||
|
}
|
||||||
|
const provider = new Provider();
|
||||||
|
const delivered = [];
|
||||||
|
const deliveryFactory = () => ({ start: async () => {}, stop: async () => {}, pause: async () => {}, resume: async () => {}, deliver: async (event) => { delivered.push(event); return { disposition: "simulated" }; } });
|
||||||
|
const coordinator = new SessionCoordinator({ provider, deliveryFactory, graceMs: 25 });
|
||||||
|
const { session } = coordinator.create({ id: "device" }, () => {});
|
||||||
|
const primary = crypto.randomUUID();
|
||||||
|
coordinator.updateSource(session.id, { source_uuid: primary, display_name: "Mic", primary: true, program_active: true });
|
||||||
|
await assert.rejects(coordinator.start(session.id, { mode: "live" }), (error) => error.code === "OBS_NOT_STREAMING");
|
||||||
|
await coordinator.updateObsState(session.id, { streaming: true, auto_start: false });
|
||||||
|
await coordinator.start(session.id, { mode: "live" });
|
||||||
|
const frame = protocol.parseAudioFrame(protocol.encodeAudioFrame({ session_id: session.id, source_uuid: primary, sequence: 1, capture_timestamp_us: 1, pcm: Buffer.alloc(640) }));
|
||||||
|
assert.equal((await coordinator.audio(session.id, frame)).accepted, true);
|
||||||
|
provider.emit("hypothesis", { session_id: session.id, track_id: primary, text: "hello", final: true, model_id: "small.en", backend: "cuda" });
|
||||||
|
await new Promise((resolve) => setImmediate(resolve));
|
||||||
|
assert.equal(delivered[0].stable_text, "hello");
|
||||||
|
await coordinator.updateObsState(session.id, { streaming: false });
|
||||||
|
assert.equal(coordinator.status(session.id).state, "grace");
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 40));
|
||||||
|
assert.equal(coordinator.status(session.id).state, "idle");
|
||||||
|
assert.equal(provider.stops, 1);
|
||||||
|
await coordinator.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function verifyWorkerRestart() {
|
||||||
|
const children = [];
|
||||||
|
const fakeSpawn = () => {
|
||||||
|
const child = new EventEmitter();
|
||||||
|
child.stdin = new PassThrough(); child.stdout = new PassThrough(); child.stderr = new PassThrough(); child.kill = () => child.emit("exit", null, "SIGTERM");
|
||||||
|
children.push(child); return child;
|
||||||
|
};
|
||||||
|
const supervisor = new WhisperWorkerSupervisor({ executable: "fake-worker", spawn: fakeSpawn, maxRestarts: 1 });
|
||||||
|
supervisor.on("error", () => {});
|
||||||
|
supervisor.start();
|
||||||
|
assert.equal(supervisor.send({ type: "audio", captured_at: Date.now() }, Buffer.alloc(640)), true);
|
||||||
|
children[0].emit("exit", 1, null);
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 320));
|
||||||
|
assert.equal(children.length, 2);
|
||||||
|
supervisor.stopping = true; children[1].emit("exit", 0, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function verifyAuthenticatedGateway() {
|
||||||
|
const db = new Database(":memory:");
|
||||||
|
const devices = new DeviceStore(db);
|
||||||
|
const pairing = devices.issuePairing({ userId: "admin", host: "https://lumi.example" });
|
||||||
|
const credential = devices.exchange({ token: pairing.token, device: { name: "Stream PC" } });
|
||||||
|
const sessionId = crypto.randomUUID();
|
||||||
|
let disconnected = false;
|
||||||
|
const sessions = {
|
||||||
|
create: (_device, send) => ({ session: { id: sessionId, state: "idle", send }, resumed: false }),
|
||||||
|
disconnect: () => { disconnected = true; },
|
||||||
|
audio: async () => ({ accepted: true }), updateSource: () => ({}), updateObsState: async () => ({}), start: async () => ({}), stop: async () => ({})
|
||||||
|
};
|
||||||
|
const gateway = new CompanionGateway({ devices, sessions, allowInsecure: true });
|
||||||
|
const registry = createWebUpgradeRegistry();
|
||||||
|
registry.add("/plugins/lumi_transcription/live", (request, socket, head) => gateway.upgrade(request, socket, head));
|
||||||
|
const server = http.createServer((_req, res) => res.end("ok"));
|
||||||
|
registry.attach(server);
|
||||||
|
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||||
|
const client = new WebSocket(`ws://127.0.0.1:${server.address().port}/plugins/lumi_transcription/live`, {
|
||||||
|
headers: { Authorization: `LumiDevice ${credential.device_id}.${credential.device_secret}` }
|
||||||
|
});
|
||||||
|
await new Promise((resolve, reject) => { client.once("open", resolve); client.once("error", reject); });
|
||||||
|
client.send(JSON.stringify(protocol.envelope("hello", { companion_version: "0.1.0", plugin_version: "0.1.0", capabilities: ["transcription.capture.v1"], audio: { codec: "pcm_s16le", sample_rate: 16000, channels: 1, bits: 16 } })));
|
||||||
|
const response = await new Promise((resolve, reject) => { client.once("message", (data) => resolve(JSON.parse(String(data)))); client.once("error", reject); });
|
||||||
|
assert.equal(response.type, "hello_ack");
|
||||||
|
assert.equal(response.session_id, sessionId);
|
||||||
|
await new Promise((resolve) => { client.once("close", resolve); client.close(); });
|
||||||
|
for (let attempt = 0; attempt < 20 && !disconnected; attempt += 1) await new Promise((resolve) => setTimeout(resolve, 5));
|
||||||
|
assert.equal(disconnected, true);
|
||||||
|
await gateway.close(); registry.close();
|
||||||
|
await new Promise((resolve) => server.close(resolve));
|
||||||
|
db.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
function verifyArtifactsAndLogs(temp) {
|
||||||
|
const artifactRoot = path.join(temp, "artifacts"); fs.mkdirSync(artifactRoot);
|
||||||
|
const file = path.join(artifactRoot, "model.bin"); fs.writeFileSync(file, "verified");
|
||||||
|
const entry = { id: "model", filename: "model.bin", url: "https://example.invalid/model.bin", sha256: sha256File(file) };
|
||||||
|
assert.equal(new ArtifactManager(artifactRoot).status(entry).valid, true);
|
||||||
|
assert.equal(sanitize({ pcm: Buffer.alloc(10), device_secret: "secret", stable_text: "hello" }, false).pcm, "[redacted]");
|
||||||
|
const logsRoot = path.join(temp, "logs");
|
||||||
|
const logs = new JsonlDiagnosticLog(logsRoot, { retentionDays: 1, maxBytes: 100 });
|
||||||
|
logs.append({ kind: "caption", stable_text: "hello", pcm: Buffer.alloc(10) });
|
||||||
|
const old = path.join(logsRoot, "transcription-2000-01-01.jsonl"); fs.writeFileSync(old, `${"x".repeat(150)}\n`); fs.utimesSync(old, new Date(0), new Date(0));
|
||||||
|
assert.ok(logs.cleanup().removed >= 1);
|
||||||
|
assert.equal(fs.readdirSync(logsRoot).some((name) => name.endsWith(".wav") || name.endsWith(".pcm")), false);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function verifyPluginIsolation() {
|
||||||
|
const db = new Database(":memory:");
|
||||||
|
const mounts = [];
|
||||||
|
let upgradeRemoved = false;
|
||||||
|
const cleanup = plugin.init({
|
||||||
|
db, logger: { info() {} },
|
||||||
|
web: {
|
||||||
|
createRouter: () => express.Router(),
|
||||||
|
addUpgradeHandler: () => () => { upgradeRemoved = true; },
|
||||||
|
emitEvent() {},
|
||||||
|
mount: (mountPath) => mounts.push(mountPath)
|
||||||
|
}
|
||||||
|
});
|
||||||
|
assert.deepEqual(mounts, ["/plugins/lumi_transcription"]);
|
||||||
|
assert.equal(typeof global.lumiFrameworks.transcription.health, "function");
|
||||||
|
await cleanup();
|
||||||
|
assert.equal(upgradeRemoved, true);
|
||||||
|
assert.equal(global.lumiFrameworks.transcription, undefined);
|
||||||
|
db.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
run().catch((error) => { console.error(error); process.exit(1); });
|
||||||
59
plugins/lumi_transcription/views/settings.ejs
Normal file
59
plugins/lumi_transcription/views/settings.ejs
Normal file
@ -0,0 +1,59 @@
|
|||||||
|
<link rel="stylesheet" href="/plugins/lumi_transcription/assets/transcription.css">
|
||||||
|
<main class="transcription-shell" data-transcription-admin>
|
||||||
|
<header class="transcription-hero">
|
||||||
|
<div>
|
||||||
|
<p class="eyebrow">Experimental companion</p>
|
||||||
|
<h1>Lumi Transcription</h1>
|
||||||
|
<p>Server-hosted speech recognition for selected OBS sources. Live Twitch delivery remains blocked until the native OBS caption compatibility test passes on the target setup.</p>
|
||||||
|
</div>
|
||||||
|
<span class="health-pill <%= providerHealth.healthy ? 'is-ready' : 'is-blocked' %>">
|
||||||
|
<span aria-hidden="true"></span><%= providerHealth.healthy ? 'Inference ready' : 'Inference setup required' %>
|
||||||
|
</span>
|
||||||
|
</header>
|
||||||
|
|
||||||
|
<section class="setup-path" aria-labelledby="setup-title">
|
||||||
|
<div class="section-heading"><div><p class="eyebrow">First usable path</p><h2 id="setup-title">Setup progress</h2></div></div>
|
||||||
|
<ol>
|
||||||
|
<li class="is-current"><strong>Pair a companion</strong><span>Create a single-use package for the streaming computer.</span></li>
|
||||||
|
<li><strong>Install the OBS bridge</strong><span>Managed by Lumi Companion; no separate bridge settings.</span></li>
|
||||||
|
<li><strong>Install and benchmark a model</strong><span>Small English is recommended. Nothing downloads without confirmation.</span></li>
|
||||||
|
<li><strong>Select and test a microphone</strong><span>Test mode must pass before live delivery is enabled.</span></li>
|
||||||
|
</ol>
|
||||||
|
<button class="primary-action" type="button" data-create-pairing>Create pairing package</button>
|
||||||
|
<p class="inline-status" role="status" data-status></p>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<div class="transcription-grid">
|
||||||
|
<section aria-labelledby="runtime-title">
|
||||||
|
<div class="section-heading"><div><p class="eyebrow">Lumi host</p><h2 id="runtime-title">Inference</h2></div></div>
|
||||||
|
<dl class="health-list">
|
||||||
|
<div><dt>Provider</dt><dd>whisper.cpp <%= runtimeManifest.tested_version %></dd></div>
|
||||||
|
<div><dt>Worker</dt><dd><%= providerHealth.state %></dd></div>
|
||||||
|
<div><dt>Selected model</dt><dd><%= providerHealth.model || 'Not loaded' %></dd></div>
|
||||||
|
<div><dt>Active sessions</dt><dd><%= providerHealth.sessions || 0 %></dd></div>
|
||||||
|
</dl>
|
||||||
|
<p class="notice">Lumi AI may already occupy most GPU memory. The MVP warns and benchmarks; it does not unload Lumi AI models automatically.</p>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section aria-labelledby="devices-title">
|
||||||
|
<div class="section-heading"><div><p class="eyebrow">Access</p><h2 id="devices-title">Paired devices</h2></div><span><%= devices.length %></span></div>
|
||||||
|
<% if (!devices.length) { %><p class="empty-state">No companion is paired yet.</p><% } %>
|
||||||
|
<ul class="plain-list">
|
||||||
|
<% devices.forEach((device) => { %>
|
||||||
|
<li><div><strong><%= device.name %></strong><span><%= device.revoked_at ? 'Revoked' : 'Last connected ' + new Date(device.last_connected_at).toLocaleString() %></span></div><code><%= device.id %></code></li>
|
||||||
|
<% }) %>
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<section aria-labelledby="models-title">
|
||||||
|
<div class="section-heading"><div><p class="eyebrow">Curated choices</p><h2 id="models-title">Speech models</h2></div></div>
|
||||||
|
<div class="model-row">
|
||||||
|
<% models.forEach((model) => { %>
|
||||||
|
<article><div><h3><%= model.label %></h3><p><%= model.recommended ? 'Recommended starting point' : 'Fallback option' %> · <%= Math.round(model.bytes / 1048576) %> MiB</p></div><span class="state-label"><%= model.status.valid ? 'Verified' : model.status.installed ? 'Checksum failed' : 'Not installed' %></span></article>
|
||||||
|
<% }) %>
|
||||||
|
</div>
|
||||||
|
<p class="privacy-note"><strong>Privacy:</strong> raw audio is held only in bounded memory and is never written to disk by default. Diagnostic caption text is retained for seven days unless disabled.</p>
|
||||||
|
</section>
|
||||||
|
</main>
|
||||||
|
<script src="/plugins/lumi_transcription/assets/transcription.js" defer></script>
|
||||||
50
protocol/companion-protocol-v1.md
Normal file
50
protocol/companion-protocol-v1.md
Normal file
@ -0,0 +1,50 @@
|
|||||||
|
# Lumi Companion Protocol v1
|
||||||
|
|
||||||
|
Status: experimental. The protocol is versioned independently from Lumi core, the server plugin, companion core, and companion transcription plugin.
|
||||||
|
|
||||||
|
## Trust boundaries
|
||||||
|
|
||||||
|
The OBS bridge accepts only same-user named-pipe clients and speaks only the local IPC subset. The companion owns pairing and connects to Lumi over TLS WebSockets. Lumi authenticates a device before accepting a WebSocket upgrade. The bridge never receives a Lumi device credential and never connects to the network.
|
||||||
|
|
||||||
|
## Server connection
|
||||||
|
|
||||||
|
- Endpoint: `wss://<lumi-host>/plugins/lumi_transcription/live`
|
||||||
|
- Header: `Authorization: LumiDevice <device-id>.<device-secret>`
|
||||||
|
- Maximum structured message: 64 KiB.
|
||||||
|
- Maximum binary audio message: 6,464 bytes (64-byte header plus up to 200 ms of PCM).
|
||||||
|
- The client sends `hello` first. Lumi replies with `hello_ack` or closes with an application error.
|
||||||
|
- Protocol v1 supports `pcm_s16le`; codec negotiation is still explicit so Opus can be added later.
|
||||||
|
- Heartbeats use `ping`/`pong`. A new connection is a new session unless `resume_session_id` names a resumable session owned by that device.
|
||||||
|
|
||||||
|
Structured messages use the envelope in `schemas/envelope.schema.json`. Supported client events are `hello`, `ping`, `source_update`, `obs_state`, `start`, `stop`, and `ack`. Server events are `hello_ack`, `pong`, `status`, `caption`, `metric`, and `error`.
|
||||||
|
|
||||||
|
## Binary PCM frame
|
||||||
|
|
||||||
|
All integers are little-endian. The fixed header is 64 bytes:
|
||||||
|
|
||||||
|
| Offset | Size | Meaning |
|
||||||
|
| --- | ---: | --- |
|
||||||
|
| 0 | 4 | ASCII `LACP` |
|
||||||
|
| 4 | 1 | protocol version (`1`) |
|
||||||
|
| 5 | 1 | flags: active `0x01`, muted `0x02` |
|
||||||
|
| 6 | 2 | header bytes (`64`) |
|
||||||
|
| 8 | 4 | sequence number |
|
||||||
|
| 12 | 8 | monotonic capture timestamp, microseconds |
|
||||||
|
| 20 | 16 | session UUID bytes |
|
||||||
|
| 36 | 16 | OBS source UUID bytes |
|
||||||
|
| 52 | 4 | sample rate (`16000`) |
|
||||||
|
| 56 | 2 | channels (`1`) |
|
||||||
|
| 58 | 2 | PCM bits (`16`) |
|
||||||
|
| 60 | 4 | payload bytes |
|
||||||
|
|
||||||
|
The payload is mono signed 16-bit little-endian PCM. Receivers reject malformed sizes and unsupported audio formats. Sequence gaps are reported; out-of-order frames are discarded. Every queue drops obsolete data rather than blocking a capture thread.
|
||||||
|
|
||||||
|
## Local OBS IPC
|
||||||
|
|
||||||
|
Windows transport is a per-user named pipe named `Lumi.Companion.ObsBridge.v1.<user-sid-hash>`. Messages are length-prefixed (32-bit little-endian), capped at 64 KiB for JSON and 6,464 bytes for audio, and begin with a v1 handshake. The bridge accepts only this allowlist: `hello`, `select_sources`, `caption`, `delivery_state`, `shutdown_notice`, and PCM frames. It cannot execute arbitrary commands.
|
||||||
|
|
||||||
|
Caption events are revision-aware. Consumers retain the greatest revision for each `(session_id, caption_id)` and discard older revisions, including after reconnect.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
Protocol v1 requires companion capability `transcription.capture.v1` and server capability `transcription.server.v1`. OBS delivery is separately negotiated as `obs.caption.native.v1`; its absence keeps test-mode simulation available but blocks live delivery.
|
||||||
22
protocol/schemas/caption.schema.json
Normal file
22
protocol/schemas/caption.schema.json
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"$id": "https://lumi.local/protocol/v1/caption.schema.json",
|
||||||
|
"type": "object",
|
||||||
|
"required": ["session_id", "source_uuid", "caption_id", "revision", "stable_text", "uncertain_text", "final", "incomplete_word", "audio", "latency", "delivery", "model"],
|
||||||
|
"properties": {
|
||||||
|
"session_id": { "type": "string", "format": "uuid" },
|
||||||
|
"source_uuid": { "type": "string", "format": "uuid" },
|
||||||
|
"speaker_label": { "type": ["string", "null"], "maxLength": 80 },
|
||||||
|
"caption_id": { "type": "string", "format": "uuid" },
|
||||||
|
"revision": { "type": "integer", "minimum": 1 },
|
||||||
|
"stable_text": { "type": "string", "maxLength": 80 },
|
||||||
|
"uncertain_text": { "type": "string", "maxLength": 80 },
|
||||||
|
"final": { "type": "boolean" },
|
||||||
|
"incomplete_word": { "type": "boolean" },
|
||||||
|
"audio": { "type": "object", "required": ["start_us", "end_us"] },
|
||||||
|
"latency": { "type": "object", "required": ["capture_ms", "network_ms", "queue_ms", "inference_ms", "stabilization_ms", "total_ms"] },
|
||||||
|
"delivery": { "type": "object", "required": ["disposition"] },
|
||||||
|
"model": { "type": "object", "required": ["id", "provider", "backend"] }
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
15
protocol/schemas/envelope.schema.json
Normal file
15
protocol/schemas/envelope.schema.json
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"$id": "https://lumi.local/protocol/v1/envelope.schema.json",
|
||||||
|
"type": "object",
|
||||||
|
"required": ["version", "type", "id", "sent_at"],
|
||||||
|
"properties": {
|
||||||
|
"version": { "const": 1 },
|
||||||
|
"type": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[a-z][a-z0-9_]*$" },
|
||||||
|
"id": { "type": "string", "format": "uuid" },
|
||||||
|
"sent_at": { "type": "string", "format": "date-time" },
|
||||||
|
"session_id": { "type": ["string", "null"], "format": "uuid" },
|
||||||
|
"payload": { "type": "object" }
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
35
protocol/schemas/hello.schema.json
Normal file
35
protocol/schemas/hello.schema.json
Normal file
@ -0,0 +1,35 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"$id": "https://lumi.local/protocol/v1/hello.schema.json",
|
||||||
|
"allOf": [
|
||||||
|
{ "$ref": "envelope.schema.json" },
|
||||||
|
{
|
||||||
|
"properties": {
|
||||||
|
"type": { "const": "hello" },
|
||||||
|
"payload": {
|
||||||
|
"type": "object",
|
||||||
|
"required": ["companion_version", "plugin_version", "capabilities", "audio"],
|
||||||
|
"properties": {
|
||||||
|
"companion_version": { "type": "string", "maxLength": 32 },
|
||||||
|
"plugin_version": { "type": "string", "maxLength": 32 },
|
||||||
|
"obs_version": { "type": ["string", "null"], "maxLength": 32 },
|
||||||
|
"resume_session_id": { "type": ["string", "null"], "format": "uuid" },
|
||||||
|
"capabilities": { "type": "array", "maxItems": 32, "items": { "type": "string", "maxLength": 64 } },
|
||||||
|
"audio": {
|
||||||
|
"type": "object",
|
||||||
|
"required": ["codec", "sample_rate", "channels", "bits"],
|
||||||
|
"properties": {
|
||||||
|
"codec": { "enum": ["pcm_s16le"] },
|
||||||
|
"sample_rate": { "const": 16000 },
|
||||||
|
"channels": { "const": 1 },
|
||||||
|
"bits": { "const": 16 }
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@ -21,6 +21,7 @@ const checks = [
|
|||||||
"plugins/lumi_ai/tests/verify.js",
|
"plugins/lumi_ai/tests/verify.js",
|
||||||
"plugins/lumi_ai/tests/verify-tools.js",
|
"plugins/lumi_ai/tests/verify-tools.js",
|
||||||
"plugins/lumi_ai_web_search/tests/verify.js",
|
"plugins/lumi_ai_web_search/tests/verify.js",
|
||||||
|
"plugins/lumi_transcription/tests/verify.js",
|
||||||
"scripts/verify-assistant-panels.js",
|
"scripts/verify-assistant-panels.js",
|
||||||
"scripts/verify-command-preview-confirmations.js",
|
"scripts/verify-command-preview-confirmations.js",
|
||||||
"scripts/verify-command-policies.js",
|
"scripts/verify-command-policies.js",
|
||||||
|
|||||||
@ -125,6 +125,7 @@ async function main() {
|
|||||||
console.log(`WebUI listening on http://localhost:${port}`, { port });
|
console.log(`WebUI listening on http://localhost:${port}`, { port });
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
app.locals.lumiUpgradeRegistry?.attach(webServer);
|
||||||
|
|
||||||
const autoUpdateEnabled = getSetting("auto_update_enabled", false);
|
const autoUpdateEnabled = getSetting("auto_update_enabled", false);
|
||||||
const intervalMinutes = getSetting("auto_update_interval_minutes", 60);
|
const intervalMinutes = getSetting("auto_update_interval_minutes", 60);
|
||||||
@ -155,6 +156,7 @@ async function main() {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
shuttingDown = true;
|
shuttingDown = true;
|
||||||
|
app.locals.lumiUpgradeRegistry?.close();
|
||||||
runtimeLog.info("Lumi shutdown started", { exit_code: exitCode }, { event: exitCode === 10 ? "restart" : "shutdown" });
|
runtimeLog.info("Lumi shutdown started", { exit_code: exitCode }, { event: exitCode === 10 ? "restart" : "shutdown" });
|
||||||
const closeWebServer = new Promise((resolve) => webServer.close(resolve));
|
const closeWebServer = new Promise((resolve) => webServer.close(resolve));
|
||||||
for (const stop of [
|
for (const stop of [
|
||||||
|
|||||||
62
src/services/web-upgrades.js
Normal file
62
src/services/web-upgrades.js
Normal file
@ -0,0 +1,62 @@
|
|||||||
|
function createWebUpgradeRegistry() {
|
||||||
|
const handlers = [];
|
||||||
|
let attachedServer = null;
|
||||||
|
|
||||||
|
function add(pathname, handler) {
|
||||||
|
const path = normalizePath(pathname);
|
||||||
|
if (typeof handler !== "function") throw new Error("An upgrade handler is required.");
|
||||||
|
const entry = { path, handler };
|
||||||
|
handlers.push(entry);
|
||||||
|
return () => {
|
||||||
|
const index = handlers.indexOf(entry);
|
||||||
|
if (index >= 0) handlers.splice(index, 1);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function dispatch(request, socket, head) {
|
||||||
|
let pathname = "/";
|
||||||
|
try {
|
||||||
|
pathname = new URL(request.url || "/", "http://lumi.local").pathname;
|
||||||
|
} catch {}
|
||||||
|
const entry = handlers.find((candidate) => candidate.path === pathname);
|
||||||
|
if (!entry) {
|
||||||
|
socket.write("HTTP/1.1 404 Not Found\r\nConnection: close\r\n\r\n");
|
||||||
|
socket.destroy();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
entry.handler(request, socket, head);
|
||||||
|
} catch {
|
||||||
|
if (!socket.destroyed) {
|
||||||
|
socket.write("HTTP/1.1 500 Internal Server Error\r\nConnection: close\r\n\r\n");
|
||||||
|
socket.destroy();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function attach(server) {
|
||||||
|
if (!server || typeof server.on !== "function") throw new Error("An HTTP server is required.");
|
||||||
|
if (attachedServer === server) return;
|
||||||
|
if (attachedServer) attachedServer.off("upgrade", dispatch);
|
||||||
|
attachedServer = server;
|
||||||
|
server.on("upgrade", dispatch);
|
||||||
|
}
|
||||||
|
|
||||||
|
function close() {
|
||||||
|
if (attachedServer) attachedServer.off("upgrade", dispatch);
|
||||||
|
attachedServer = null;
|
||||||
|
handlers.length = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
return { add, attach, close, count: () => handlers.length };
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizePath(value) {
|
||||||
|
const path = String(value || "").trim();
|
||||||
|
if (!path.startsWith("/") || path.includes("?") || path.includes("#")) {
|
||||||
|
throw new Error("Upgrade paths must be absolute URL paths.");
|
||||||
|
}
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { createWebUpgradeRegistry };
|
||||||
@ -28,6 +28,7 @@ const {
|
|||||||
} = require("../services/themes");
|
} = require("../services/themes");
|
||||||
const { getRoleFlags, hasAccess } = require("../services/rbac");
|
const { getRoleFlags, hasAccess } = require("../services/rbac");
|
||||||
const { createWebAuth } = require("../services/web-auth");
|
const { createWebAuth } = require("../services/web-auth");
|
||||||
|
const { createWebUpgradeRegistry } = require("../services/web-upgrades");
|
||||||
const {
|
const {
|
||||||
cleanupUploadedFiles,
|
cleanupUploadedFiles,
|
||||||
safeDownloadFilename,
|
safeDownloadFilename,
|
||||||
@ -3100,6 +3101,8 @@ async function verifyYouTubeSettings(settings) {
|
|||||||
|
|
||||||
function createWebServer({ loadPlugins, discordClient, commandRouter }) {
|
function createWebServer({ loadPlugins, discordClient, commandRouter }) {
|
||||||
const app = express();
|
const app = express();
|
||||||
|
const upgradeRegistry = createWebUpgradeRegistry();
|
||||||
|
app.locals.lumiUpgradeRegistry = upgradeRegistry;
|
||||||
// Only trust forwarding headers from a reverse proxy on this machine. This
|
// Only trust forwarding headers from a reverse proxy on this machine. This
|
||||||
// lets the diagnostics endpoint recognize HTTPS without trusting arbitrary
|
// lets the diagnostics endpoint recognize HTTPS without trusting arbitrary
|
||||||
// client-supplied X-Forwarded-Proto headers.
|
// client-supplied X-Forwarded-Proto headers.
|
||||||
@ -3448,6 +3451,7 @@ function createWebServer({ loadPlugins, discordClient, commandRouter }) {
|
|||||||
registerPlaceholder: placeholders.registerPlaceholder,
|
registerPlaceholder: placeholders.registerPlaceholder,
|
||||||
registerPlaceholders: placeholders.registerPlaceholders,
|
registerPlaceholders: placeholders.registerPlaceholders,
|
||||||
registerPlaceholderField: placeholders.registerFieldPolicy,
|
registerPlaceholderField: placeholders.registerFieldPolicy,
|
||||||
|
addUpgradeHandler: upgradeRegistry.add,
|
||||||
emitEvent: publishWebEvent
|
emitEvent: publishWebEvent
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user