Lumi/plugins/lumi_transcription/index.js
2026-07-24 14:44:27 +02:00

411 lines
26 KiB
JavaScript

const express = require("express");
const ejs = require("ejs");
const fs = require("fs");
const path = require("path");
const { DeviceStore } = require("./backend/companion/device_store");
const { insecureDeviceAllowed, sameLocalhostOrigin } = require("./backend/companion/device_store");
const { CompanionGateway } = require("./backend/companion/gateway");
const { CompanionPackageService } = require("./backend/companion/package_service");
const { RevisionStore } = require("./backend/config/revision_store");
const { CompanionCaptionDeliveryAdapter } = require("./backend/delivery/caption_delivery");
const { JsonlDiagnosticLog } = require("./backend/logs/jsonl_log");
const { ArtifactManager } = require("./backend/models/artifact_manager");
const { SessionCoordinator } = require("./backend/sessions/session_coordinator");
const { WhisperWorkerSupervisor, WhisperCppServerProvider } = require("./backend/transcription/provider");
const { BenchmarkStore } = require("./backend/tests/benchmark_store");
const { ensureDataDirs, dataPath, resolveWorkerExecutable, setActiveWorkerExecutable } = require("./backend/paths");
const modelManifest = require("./models_manifest.json");
const runtimeManifest = require("./runtime_manifest.json");
const manifest = require("./plugin.json");
const PLUGIN_ID = "lumi_transcription";
module.exports = {
id: PLUGIN_ID,
compareVersions,
init({ web, db, logger }) {
ensureDataDirs();
const devices = new DeviceStore(db);
const revisions = new RevisionStore(db);
const benchmarks = new BenchmarkStore(db);
const diagnosticLog = new JsonlDiagnosticLog(dataPath("logs"));
diagnosticLog.cleanup();
const cleanupTimer = setInterval(() => { diagnosticLog.cleanup(); devices.cleanup(); benchmarks.cleanup(); }, 60 * 60 * 1000);
cleanupTimer.unref?.();
const supervisor = new WhisperWorkerSupervisor({
executable: resolveWorkerExecutable(process.env.LUMI_TRANSCRIPTION_WORKER),
args: process.env.LUMI_TRANSCRIPTION_WORKER_ARGS ? JSON.parse(process.env.LUMI_TRANSCRIPTION_WORKER_ARGS) : []
});
supervisor.on("diagnostic", (entry) => diagnosticLog.append({ kind: "worker", ...entry }));
supervisor.on("error", (error) => diagnosticLog.append({ kind: "worker", state: "error", message: error.message }));
supervisor.on("crash", (entry) => diagnosticLog.append({ kind: "worker", state: "crashed", exit_code: entry.code, signal: entry.signal, occurred_at: entry.at }));
supervisor.on("state", (entry) => diagnosticLog.append({ kind: "worker", state: entry.state }));
const provider = new WhisperCppServerProvider(supervisor);
provider.on("diagnostic", (entry) => diagnosticLog.append({ kind: "worker", state: entry.state || "diagnostic", level: entry.level || null, message: entry.message || null }));
provider.on("provider_error", (error) => diagnosticLog.append({ kind: "worker", state: "provider_error", code: error.code || null, message: error.message, details: error.details || null }));
provider.on("recovered", (entry) => diagnosticLog.append({ kind: "worker", state: "recovered", ...entry }));
const sessions = new SessionCoordinator({
provider,
deliveryFactory: (send) => new CompanionCaptionDeliveryAdapter(send),
log: diagnosticLog,
benchmarks,
readinessContext: () => {
const current = revisions.list();
return Object.fromEntries(["selected_model_id", "fallback_order", "decode_interval_ms", "silence_finalize_ms", "rolling_context_ms", "caption_max_chars", "minimum_display_ms", "tracks"]
.filter((key) => current[key]).map((key) => [key, current[key].revision]));
}
});
const gateway = new CompanionGateway({ devices, sessions, log: diagnosticLog });
const unregisterUpgrade = web.addUpgradeHandler("/plugins/lumi_transcription/live", (request, socket, head) => gateway.upgrade(request, socket, head));
const models = new ArtifactManager(dataPath("models"));
const runtimeArchives = new ArtifactManager(dataPath("tmp"));
const runtimes = new ArtifactManager(dataPath("runtime"));
const companionPackages = new CompanionPackageService(dataPath("companion"), path.join(__dirname, "companion_manifest.json"));
const selectedModelId = revisions.list().selected_model_id?.value;
const selectedModel = modelManifest.models.find((entry) => entry.id === selectedModelId);
if (supervisor.executable && selectedModel) {
const selectedStatus = models.status(selectedModel);
if (selectedStatus.valid) provider.loadModel({ ...selectedModel, path: selectedStatus.path })
.catch((error) => diagnosticLog.append({ kind: "worker", state: "model_restore_failed", model_id: selectedModel.id, message: error.message }));
}
const router = web.createRouter();
router.use("/assets", express.static(path.join(__dirname, "public")));
router.get("/", requireAdmin, async (_req, res) => {
const locals = {
...res.locals,
title: "Lumi Transcription",
pageWidth: "wide",
pageId: "lumi-transcription",
extraStyles: [`/plugins/${PLUGIN_ID}/assets/transcription.css?v=${manifest.version}`],
extraScripts: [`/plugins/${PLUGIN_ID}/assets/transcription.js?v=${manifest.version}`],
pluginVersion: manifest.version,
providerHealth: await provider.health(),
devices: devices.list(),
revokedDevices: devices.list({ status: "revoked" }),
benchmarks: benchmarks.list(),
settings: revisions.list(),
models: modelManifest.models.map((entry) => ({ ...entry, status: models.status(entry) })),
runtimeManifest,
logs: diagnosticLog.files()
};
res.send(await renderLumiPage(locals));
});
router.get("/api/status", requireAdmin, async (_req, res) => res.json({
ok: true, plugin: { id: PLUGIN_ID, version: manifest.version }, protocol_version: 1,
provider: await provider.health(), devices: devices.list(), revoked_devices: devices.list({ status: "revoked" }), settings: revisions.list(),
models: modelManifest.models.map((entry) => ({ id: entry.id, ...models.status(entry) })),
runtime: runtimeManifest
}));
router.post("/api/pairing-package", requireAdmin, (req, res) => {
try {
const host = requestHost(req);
const pairing = devices.issuePairing({ userId: req.session.user.id, host });
const bootstrap = { format: "lumi-companion-bootstrap-v1", ...pairing, exchange_url: `${host}/plugins/${PLUGIN_ID}/api/pair`, ...companionOperatorMetadata(host) };
res.set("Cache-Control", "no-store");
res.attachment(`lumi-companion-${pairing.pairing_id}.lumi-pairing.json`);
res.send(`${JSON.stringify(bootstrap, null, 2)}\n`);
} catch (error) { res.status(400).json({ ok: false, error: error.message }); }
});
router.post("/api/companion/download", requireAdmin, async (req, res) => {
try {
const host = requestHost(req);
const pairing = devices.issuePairing({ userId: req.session.user.id, host });
const bootstrap = { format: "lumi-companion-bootstrap-v1", ...pairing, exchange_url: `${host}/plugins/${PLUGIN_ID}/api/pair`, ...companionOperatorMetadata(host) };
const bundle = await companionPackages.build({ ...pairing, bootstrap });
res.set("Cache-Control", "no-store");
res.attachment(bundle.filename);
res.send(bundle.buffer);
} catch (error) { res.status(503).json({ ok: false, error: error.message }); }
});
const companionUpdateHandler = async (req, res) => {
try {
const updateManifest = companionPackages.currentManifest();
const currentVersion = String(req.body?.current_version || req.query.current_version || "").slice(0, 40);
const developmentUpdates = global.lumiFrameworks?.developmentUpdates;
const localDevelopment = Boolean(
req.method === "POST" &&
developmentUpdates?.enabled?.() &&
global.lumiRuntime?.allowsLocalDevelopmentUpdates?.(req) &&
sameLocalhostOrigin(req.lumiDevice?.pairing_host, requestHost(req))
);
res.set("Cache-Control", "no-store");
if (localDevelopment) {
const comparison = developmentUpdates.compare({
build_checksum: req.body?.build_checksum || req.query.build_checksum,
component_checksums: req.body?.component_checksums || {}
});
let artifact = null;
let buildPending = false;
if (comparison.update_available) {
let built = developmentUpdates.readArtifact(comparison.aggregate_checksum);
if (!built && req.body?.supports_pending_build === true) {
buildPending = true;
developmentUpdates.buildCompanionArtifact(comparison).catch((error) => {
diagnosticLog.append({ kind: "companion_development_build", state: "failed", message: error.message });
});
} else if (!built) {
// Compatibility for the first development-update adoption. Older
// Companion builds do not understand build_pending and require a
// complete artifact response.
built = await developmentUpdates.buildCompanionArtifact(comparison);
}
if (built) {
artifact = {
url: `${requestHost(req)}/plugins/${PLUGIN_ID}/api/companion/dev-artifact/${built.build_id}`,
sha256: built.sha256,
bytes: built.bytes,
entrypoint: built.entrypoint
};
}
}
return res.json({
ok: true,
version: updateManifest.version,
current_version: currentVersion,
update_available: comparison.update_available,
development: true,
build_checksum: comparison.aggregate_checksum,
component_checksums: Object.fromEntries(comparison.updatable_components.map((item) => [item.id, item.checksum])),
changed_components: comparison.changed_components,
server_components: comparison.server_components,
artifact,
build_pending: buildPending,
retry_after_ms: buildPending ? 2000 : null,
signed: false,
release_notes: comparison.update_available
? `Local development changes detected: ${comparison.changed_components.map((item) => item.id).join(", ") || "Companion source"}.`
: "The localhost Companion build matches the current Lumi source checksums."
});
}
const entry = companionPackages.entry(updateManifest);
if (!entry) return res.status(503).json({ ok: false, error: "No Windows Companion update is configured." });
return res.json({
ok: true,
version: updateManifest.version,
current_version: currentVersion,
update_available: compareVersions(updateManifest.version, currentVersion) > 0,
development: false,
build_checksum: null,
component_checksums: {},
changed_components: [],
server_components: [],
artifact: { url: entry.url, sha256: entry.sha256, bytes: entry.bytes, entrypoint: entry.entrypoint },
signed: updateManifest.signed === true,
release_notes: String(updateManifest.release_notes || "Companion reliability and integration improvements.").slice(0, 500)
});
} catch (error) {
diagnosticLog.append({ kind: "companion_update", state: "failed", message: error.message });
return res.status(503).json({ ok: false, error: error.message });
}
};
router.get("/api/companion/update", requireDeviceAccess(devices), companionUpdateHandler);
router.post("/api/companion/update", requireDeviceAccess(devices), companionUpdateHandler);
router.get("/api/companion/dev-artifact/:buildId", requireDeviceAccess(devices), (req, res) => {
const developmentUpdates = global.lumiFrameworks?.developmentUpdates;
const allowed = Boolean(
developmentUpdates?.enabled?.() &&
global.lumiRuntime?.allowsLocalDevelopmentUpdates?.(req) &&
sameLocalhostOrigin(req.lumiDevice?.pairing_host, requestHost(req))
);
if (!allowed) return res.status(404).json({ ok: false, error: "Local development artifacts are available only through the paired localhost origin." });
const artifact = developmentUpdates.readArtifact(req.params.buildId);
if (!artifact) return res.status(404).json({ ok: false, error: "The requested local development artifact is unavailable or stale." });
res.set("Cache-Control", "no-store");
res.set("X-Lumi-Development-Build", artifact.build_id);
res.download(artifact.path, "Lumi.Companion-win-x64.zip");
});
router.post("/api/pair", requirePairingTransport(devices), (req, res) => {
try { res.set("Cache-Control", "no-store"); res.status(201).json({ ok: true, ...devices.exchange(req.body || {}) }); }
catch (error) { res.status(error.code === "PAIRING_ALREADY_USED" ? 409 : 400).json({ ok: false, code: error.code, error: error.message }); }
});
router.get("/api/devices", requireAdmin, (req, res) => res.json({ devices: devices.list({ status: req.query.status }) }));
router.post("/api/devices/:id/revoke", requireAdmin, (req, res) => {
if (!devices.revoke(req.params.id)) return res.status(404).json({ ok: false, error: "Active device was not found." });
res.json({ ok: true });
});
router.post("/api/devices/:id/capabilities", requireAdmin, (req, res) => {
const capabilities = devices.setCapabilities(req.params.id, req.body.capabilities);
if (!capabilities) return res.status(404).json({ ok: false, error: "Device was not found or is revoked." });
res.json({ ok: true, capabilities });
});
router.get("/api/tests", requireAdmin, (_req, res) => res.json({ tests: benchmarks.list(), retention_ms: benchmarks.retentionMs }));
router.get("/api/tests/:id", requireAdmin, (req, res) => {
const test = benchmarks.list().find((entry) => entry.id === req.params.id);
if (!test) return res.status(404).json({ error: "Transcription test was not found or has expired." });
res.json({ test });
});
router.get("/api/settings", requireSettingsAccess(devices), (_req, res) => res.json({ fields: revisions.list() }));
router.patch("/api/settings", requireSettingsAccess(devices), (req, res) => {
try {
const actor = req.session?.user?.id || req.lumiDevice?.id;
const result = revisions.apply(req.body.changes, actor);
if (result.applied.length) web.emitEvent?.("transcription:settings_changed", { fields: result.applied.map((entry) => entry.key) }, { role: "admin" });
res.status(result.conflicts.length ? 409 : 200).json({ ok: !result.conflicts.length, ...result });
} catch (error) { res.status(400).json({ ok: false, error: error.message }); }
});
router.post("/api/models/:id/download", requireAdmin, async (req, res) => {
const entry = modelManifest.models.find((candidate) => candidate.id === req.params.id);
if (!entry) return res.status(404).json({ ok: false, error: "Model was not found." });
try { res.status(201).json({ ok: true, status: await models.download(entry, { confirmed: req.body.confirmed === true }) }); }
catch (error) { res.status(400).json({ ok: false, error: error.message }); }
});
router.post("/api/models/:id/load", requireAdmin, async (req, res) => {
const entry = modelManifest.models.find((candidate) => candidate.id === req.params.id);
if (!entry) return res.status(404).json({ ok: false, error: "Model was not found." });
const status = models.status(entry);
if (!status.valid) return res.status(409).json({ ok: false, error: "Download and verify this model before loading it." });
try {
const providerStatus = await provider.loadModel({ ...entry, path: status.path });
const current = revisions.list().selected_model_id;
revisions.apply([{ key: "selected_model_id", value: entry.id, base_revision: current?.revision || 0 }], req.session.user.id);
res.json({ ok: true, status: providerStatus });
}
catch (error) { res.status(error.code === "WORKER_NOT_CONFIGURED" ? 409 : 500).json({ ok: false, code: error.code || "MODEL_LOAD_FAILED", error: error.message }); }
});
router.post("/api/runtime/:id/install", requireAdmin, async (req, res) => {
const entry = runtimeManifest.artifacts.find((candidate) => candidate.id === req.params.id);
if (!entry) return res.status(404).json({ ok: false, error: "Runtime was not found." });
try {
if ((await provider.health()).sessions > 0) return res.status(409).json({ ok: false, error: "End active transcription sessions before changing the inference runtime." });
const filename = `${entry.id}.zip`;
const archive = await runtimeArchives.download({ ...entry, filename }, { confirmed: req.body.confirmed === true });
const selected = modelManifest.models.find((candidate) => candidate.id === revisions.list().selected_model_id?.value);
const selectedStatus = selected ? models.status(selected) : null;
const previousExecutable = supervisor.executable;
let activated;
try {
await provider.stop();
const installed = runtimes.installZip(entry, archive.path);
const executable = path.join(installed.path, "bin", process.platform === "win32" ? "lumi-whisper-worker.exe" : "lumi-whisper-worker");
if (!fs.existsSync(executable)) throw new Error("Installed runtime did not contain the Lumi transcription worker.");
supervisor.executable = executable;
activated = selected && selectedStatus?.valid
? await provider.loadModel({ ...selected, path: selectedStatus.path })
: await provider.health();
setActiveWorkerExecutable(executable);
res.status(201).json({ ok: true, status: installed, activated });
} catch (activationError) {
await provider.stop().catch(() => {});
supervisor.executable = previousExecutable;
if (previousExecutable && selected && selectedStatus?.valid)
await provider.loadModel({ ...selected, path: selectedStatus.path }).catch(() => {});
throw activationError;
}
} catch (error) { res.status(400).json({ ok: false, error: error.message }); }
});
router.get("/api/logs", requireAdmin, (_req, res) => res.json({ files: diagnosticLog.files().map(({ path: _path, ...entry }) => entry) }));
web.mount(`/plugins/${PLUGIN_ID}`, router, { label: "Transcription", role: "admin", section: "plugins" });
global.lumiFrameworks = global.lumiFrameworks || {};
const companionFramework = {
version: 1,
requireDevice: requireDeviceAccess(devices),
authenticate: (header, requiredCapability = null) => devices.authenticate(header, requiredCapability)
};
global.lumiFrameworks.companion = companionFramework;
global.lumiFrameworks.transcription = {
version: manifest.version,
protocol_version: 1,
health: () => provider.health(),
dashboardSummary: () => buildDashboardSummary({ provider, devices, sessions, companionPackages })
};
return async () => {
clearInterval(cleanupTimer);
unregisterUpgrade();
await gateway.close();
await sessions.close();
await provider.stop();
if (global.lumiFrameworks?.companion === companionFramework) delete global.lumiFrameworks.companion;
if (global.lumiFrameworks?.transcription?.version === manifest.version) delete global.lumiFrameworks.transcription;
logger?.info?.("Lumi transcription stopped", {}, { event: "transcription_stopped" });
};
}
};
function requireAdmin(req, res, next) { if (req.session?.user?.isAdmin) return next(); return res.status(403).json({ error: "Administrator access is required." }); }
function companionOperatorMetadata(host) {
const value = (name, max) => String(process.env[name] || "").replace(/[\r\n\0"\\]/g, " ").trim().slice(0, max) || undefined;
const privacyCandidate = value("LUMI_OPERATOR_PRIVACY_URL", 500);
let privacyUrl;
if (privacyCandidate) {
try {
const parsed = new URL(privacyCandidate, host);
if (["http:", "https:"].includes(parsed.protocol)) privacyUrl = parsed.toString();
} catch { }
}
return {
operator_name: value("LUMI_OPERATOR_NAME", 200),
operator_contact: value("LUMI_OPERATOR_CONTACT", 300),
privacy_url: privacyUrl
};
}
function requireDeviceAccess(devices) { return (req, res, next) => { const auth = devices.authenticate(req.headers.authorization); if (!auth.allowed) return res.status(401).json({ error: "A paired device credential is required." }); if (!req.secure && !insecureDeviceAllowed(auth.device, requestHost(req), req.socket.remoteAddress)) return res.status(426).json({ error: "Companion requests require HTTPS unless this device was paired from the same localhost origin." }); req.lumiDevice = auth.device; next(); }; }
function compareVersions(left, right) {
const parse = (value) => {
const [version] = String(value || "0").split("+", 1);
const separator = version.indexOf("-");
const core = (separator < 0 ? version : version.slice(0, separator)).split(".").map((part) => Number(part) || 0);
const prerelease = separator < 0 ? null : version.slice(separator + 1).split(/[.\-]/).filter(Boolean);
return { core, prerelease };
};
const a = parse(left); const b = parse(right);
for (let index = 0; index < Math.max(a.core.length, b.core.length); index += 1) {
if ((a.core[index] || 0) !== (b.core[index] || 0)) return (a.core[index] || 0) > (b.core[index] || 0) ? 1 : -1;
}
if (a.prerelease === null || b.prerelease === null) return a.prerelease === b.prerelease ? 0 : a.prerelease === null ? 1 : -1;
for (let index = 0; index < Math.max(a.prerelease.length, b.prerelease.length); index += 1) {
if (a.prerelease[index] === undefined || b.prerelease[index] === undefined) return a.prerelease[index] === b.prerelease[index] ? 0 : a.prerelease[index] === undefined ? -1 : 1;
const numericA = /^\d+$/.test(a.prerelease[index]); const numericB = /^\d+$/.test(b.prerelease[index]);
if (numericA && numericB && Number(a.prerelease[index]) !== Number(b.prerelease[index])) return Number(a.prerelease[index]) > Number(b.prerelease[index]) ? 1 : -1;
if (numericA !== numericB) return numericA ? -1 : 1;
const compared = a.prerelease[index].localeCompare(b.prerelease[index]);
if (compared) return compared > 0 ? 1 : -1;
}
return 0;
}
async function buildDashboardSummary({ provider, devices, sessions, companionPackages }) {
const inference = await provider.health();
const activeDevices = devices.list();
const connections = sessions.summary();
const packageStatus = companionPackages.status();
const issues = [];
if (!inference.healthy) issues.push("Server-hosted speech recognition is not ready.");
if (!inference.model) issues.push("No speech model is loaded.");
if (!activeDevices.length) issues.push("No Companion device is paired.");
else if (!connections.connected) issues.push("No paired Companion is currently connected.");
if (!packageStatus.available) issues.push("The Windows Companion download is not configured.");
const tone = inference.state === "failed" ? "danger" : issues.length ? "warning" : "success";
return {
id: "lumi-companion", eyebrow: "Streaming computer", title: "Lumi Companion",
description: "Download the paired Windows Companion and review the live transcription path at a glance.",
status: { tone, label: issues.length ? `${issues.length} issue${issues.length === 1 ? "" : "s"} discovered` : "Everything healthy" },
metrics: [
{ label: "Connection", value: connections.connected ? "Active" : "Offline" },
{ label: "Paired devices", value: activeDevices.length },
{ label: "Inference", value: inference.healthy ? "Healthy" : "Needs setup" },
{ label: "Active sessions", value: connections.running }
],
issues,
actions: [
{ label: "Download Companion", href: `/plugins/${PLUGIN_ID}/api/companion/download`, method: "post", primary: true, disabled: !packageStatus.available, disabledReason: packageStatus.reason },
{ label: "Open transcription settings", href: `/plugins/${PLUGIN_ID}`, method: "get" }
]
};
}
async function renderLumiPage(locals) {
const coreViews = path.join(__dirname, "..", "..", "src", "web", "views");
const page = path.join(__dirname, "views", "settings.ejs");
const [top, body, bottom] = await Promise.all([
ejs.renderFile(path.join(coreViews, "partials", "layout-top.ejs"), locals),
ejs.renderFile(page, locals),
ejs.renderFile(path.join(coreViews, "partials", "layout-bottom.ejs"), locals)
]);
return `${top}${body}${bottom}`;
}
function requireSettingsAccess(devices) { return (req, res, next) => { if (req.session?.user?.isAdmin) return next(); const auth = devices.authenticate(req.headers.authorization, "transcription.settings.v1"); if (!auth.allowed) return res.status(401).json({ error: "A paired device credential is required." }); if (!req.secure && !insecureDeviceAllowed(auth.device, requestHost(req), req.socket.remoteAddress)) return res.status(426).json({ error: "Device settings synchronization requires HTTPS unless this device was paired from the same localhost origin." }); req.lumiDevice = auth.device; next(); }; }
function requestHost(req) { return `${req.protocol === "https" ? "https" : "http"}://${req.get("host")}`; }
function requirePairingTransport(devices) { return (req, res, next) => { if (req.secure) return next(); const local = ["127.0.0.1", "::1", "::ffff:127.0.0.1"].includes(req.socket.remoteAddress); if (local && devices.pairingAllowsHttp(req.body?.token, requestHost(req))) return next(); return res.status(426).json({ error: "Lumi Companion pairing requires HTTPS unless the package was generated from this exact localhost URL." }); }; }