const crypto = require("crypto"); class BenchmarkStore { constructor(db, options = {}) { this.db = db; this.now = options.now || Date.now; this.retentionMs = options.retentionMs || 60 * 60 * 1000; this.migrate(); this.db.prepare("UPDATE transcription_benchmark_tests SET status = 'aborted', ended_at = ? WHERE status = 'running'").run(this.now()); this.cleanup(); } migrate() { this.db.exec(` CREATE TABLE IF NOT EXISTS transcription_benchmark_tests ( id TEXT PRIMARY KEY, session_id TEXT NOT NULL UNIQUE, device_id TEXT NOT NULL, source_uuid TEXT NOT NULL, source_name TEXT NOT NULL, started_at INTEGER NOT NULL, ended_at INTEGER, status TEXT NOT NULL, model_id TEXT, backend TEXT ); CREATE TABLE IF NOT EXISTS transcription_benchmark_revisions ( id INTEGER PRIMARY KEY AUTOINCREMENT, test_id TEXT NOT NULL, caption_id TEXT NOT NULL, revision INTEGER NOT NULL, received_at INTEGER NOT NULL, final INTEGER NOT NULL, text TEXT NOT NULL, words_json TEXT NOT NULL, inference_ms REAL NOT NULL, model_id TEXT, backend TEXT, UNIQUE(test_id, caption_id, revision) ); CREATE INDEX IF NOT EXISTS transcription_benchmark_started_idx ON transcription_benchmark_tests(started_at DESC); CREATE INDEX IF NOT EXISTS transcription_benchmark_revision_test_idx ON transcription_benchmark_revisions(test_id, received_at); `); } start({ sessionId, deviceId, source }) { this.cleanup(); const id = crypto.randomUUID(); this.db.prepare(`INSERT INTO transcription_benchmark_tests (id, session_id, device_id, source_uuid, source_name, started_at, ended_at, status, model_id, backend) VALUES (?, ?, ?, ?, ?, ?, NULL, 'running', NULL, NULL)`) .run(id, sessionId, deviceId, source.source_uuid, source.display_name || "OBS source", this.now()); return id; } record(sessionId, event) { const test = this.db.prepare("SELECT id FROM transcription_benchmark_tests WHERE session_id = ? AND status = 'running'").get(sessionId); if (!test) return false; const count = this.db.prepare("SELECT COUNT(*) AS count FROM transcription_benchmark_revisions WHERE test_id = ?").get(test.id).count; if (count >= 10000) return false; const words = Array.isArray(event.analysis?.words) ? event.analysis.words.slice(0, 500).map(safeWord) : []; this.db.prepare(`INSERT OR IGNORE INTO transcription_benchmark_revisions (test_id, caption_id, revision, received_at, final, text, words_json, inference_ms, model_id, backend) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`) .run(test.id, event.caption_id, event.revision, this.now(), event.final ? 1 : 0, String(event.analysis?.transcript || event.stable_text || event.uncertain_text || "").slice(0, 8000), JSON.stringify(words), finite(event.latency?.inference_ms), event.model?.id || null, event.model?.backend || null); this.db.prepare("UPDATE transcription_benchmark_tests SET model_id = COALESCE(?, model_id), backend = COALESCE(?, backend) WHERE id = ?") .run(event.model?.id || null, event.model?.backend || null, test.id); return true; } finish(sessionId, status = "completed") { const allowed = ["completed", "silence_timeout", "aborted", "failed"].includes(status) ? status : "completed"; this.db.prepare("UPDATE transcription_benchmark_tests SET ended_at = ?, status = ? WHERE session_id = ? AND status = 'running'") .run(this.now(), allowed, sessionId); return this.bySession(sessionId); } bySession(sessionId) { const row = this.db.prepare("SELECT * FROM transcription_benchmark_tests WHERE session_id = ?").get(sessionId); return row ? this.snapshot(row) : null; } list() { this.cleanup(); return this.db.prepare("SELECT * FROM transcription_benchmark_tests ORDER BY started_at DESC").all().map((row) => this.snapshot(row)); } snapshot(row) { const revisions = this.db.prepare("SELECT * FROM transcription_benchmark_revisions WHERE test_id = ? ORDER BY received_at, revision").all(row.id); const captions = new Map(); for (const revision of revisions) { const existing = captions.get(revision.caption_id); if (!existing || (revision.final && !existing.final) || revision.final === existing.final && revision.revision > existing.revision) captions.set(revision.caption_id, revision); } const selected = Array.from(captions.values()).sort((a, b) => a.received_at - b.received_at); const words = []; for (const revision of selected) { for (const word of parseWords(revision.words_json)) words.push({ ...word, confidence: revision.final ? word.confidence : null, final: Boolean(revision.final) }); } const latency = metricStats(words.map((word) => word.latency_ms)); const confidence = metricStats(words.map((word) => word.confidence).filter(Number.isFinite)); return { id: row.id, session_id: row.session_id, device_id: row.device_id, source_uuid: row.source_uuid, source_name: row.source_name, started_at: row.started_at, ended_at: row.ended_at, duration_ms: Math.max(0, (row.ended_at || this.now()) - row.started_at), status: row.status, model_id: row.model_id, backend: row.backend, transcript: selected.map((revision) => revision.text).filter(Boolean).join(" "), words, stats: { latency, confidence }, revisions: revisions.length }; } cleanup(now = this.now()) { const cutoff = now - this.retentionMs; const ids = this.db.prepare("SELECT id FROM transcription_benchmark_tests WHERE ended_at IS NOT NULL AND ended_at < ?").all(cutoff).map((row) => row.id); const removeRevisions = this.db.prepare("DELETE FROM transcription_benchmark_revisions WHERE test_id = ?"); const removeTest = this.db.prepare("DELETE FROM transcription_benchmark_tests WHERE id = ?"); this.db.transaction(() => { for (const id of ids) { removeRevisions.run(id); removeTest.run(id); } })(); return ids.length; } } function safeWord(word) { return { text: String(word?.text || "").slice(0, 120), latency_ms: finite(word?.latency_ms), confidence: clamp(word?.confidence, 0, 1), audio_start_ms: finite(word?.audio_start_ms), audio_end_ms: finite(word?.audio_end_ms), captured_at_ms: finite(word?.captured_at_ms) }; } function parseWords(value) { try { const parsed = JSON.parse(value); return Array.isArray(parsed) ? parsed.map(safeWord).filter((word) => word.text) : []; } catch { return []; } } function finite(value) { const number = Number(value); return Number.isFinite(number) ? Math.max(0, number) : 0; } function clamp(value, min, max) { return Math.min(max, Math.max(min, finite(value))); } function metricStats(input) { const values = input.map(Number).filter(Number.isFinite).sort((a, b) => a - b); if (!values.length) return { count: 0, min: null, low_1_average: null, median: null, average: null, p99: null, high_1_average: null, max: null }; const tail = Math.max(1, Math.ceil(values.length * 0.01)); return { count: values.length, min: values[0], low_1_average: average(values.slice(0, tail)), median: percentile(values, 0.5), average: average(values), p99: percentile(values, 0.99), high_1_average: average(values.slice(-tail)), max: values.at(-1) }; } function average(values) { return values.reduce((sum, value) => sum + value, 0) / values.length; } function percentile(values, ratio) { const position = (values.length - 1) * ratio; const lower = Math.floor(position); const upper = Math.ceil(position); return values[lower] + (values[upper] - values[lower]) * (position - lower); } module.exports = { BenchmarkStore, metricStats };