192 lines
11 KiB
JavaScript
192 lines
11 KiB
JavaScript
const crypto = require("crypto");
|
|
|
|
class BenchmarkStore {
|
|
constructor(db, options = {}) {
|
|
this.db = db;
|
|
this.now = options.now || Date.now;
|
|
this.retentionMs = options.retentionMs || 60 * 60 * 1000;
|
|
this.migrate();
|
|
this.db.prepare("UPDATE transcription_benchmark_tests SET status = 'aborted', ended_at = ? WHERE status = 'running'").run(this.now());
|
|
this.cleanup();
|
|
}
|
|
|
|
migrate() {
|
|
this.db.exec(`
|
|
CREATE TABLE IF NOT EXISTS transcription_benchmark_tests (
|
|
id TEXT PRIMARY KEY, session_id TEXT NOT NULL, device_id TEXT NOT NULL,
|
|
source_uuid TEXT NOT NULL, source_name TEXT NOT NULL, started_at INTEGER NOT NULL,
|
|
ended_at INTEGER, status TEXT NOT NULL, model_id TEXT, backend TEXT
|
|
);
|
|
CREATE TABLE IF NOT EXISTS transcription_benchmark_revisions (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT, test_id TEXT NOT NULL, caption_id TEXT NOT NULL,
|
|
revision INTEGER NOT NULL, received_at INTEGER NOT NULL, final INTEGER NOT NULL,
|
|
text TEXT NOT NULL, words_json TEXT NOT NULL, inference_ms REAL NOT NULL,
|
|
model_id TEXT, backend TEXT,
|
|
UNIQUE(test_id, caption_id, revision)
|
|
);
|
|
`);
|
|
const schema = this.db.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'transcription_benchmark_tests'").get()?.sql || "";
|
|
if (/session_id\s+TEXT\s+NOT\s+NULL\s+UNIQUE/i.test(schema)) {
|
|
this.db.transaction(() => {
|
|
this.db.exec(`
|
|
ALTER TABLE transcription_benchmark_tests RENAME TO transcription_benchmark_tests_legacy;
|
|
CREATE TABLE transcription_benchmark_tests (
|
|
id TEXT PRIMARY KEY, session_id TEXT NOT NULL, device_id TEXT NOT NULL,
|
|
source_uuid TEXT NOT NULL, source_name TEXT NOT NULL, started_at INTEGER NOT NULL,
|
|
ended_at INTEGER, status TEXT NOT NULL, model_id TEXT, backend TEXT
|
|
);
|
|
INSERT INTO transcription_benchmark_tests
|
|
(id, session_id, device_id, source_uuid, source_name, started_at, ended_at, status, model_id, backend)
|
|
SELECT id, session_id, device_id, source_uuid, source_name, started_at, ended_at, status, model_id, backend
|
|
FROM transcription_benchmark_tests_legacy;
|
|
DROP TABLE transcription_benchmark_tests_legacy;
|
|
`);
|
|
})();
|
|
}
|
|
this.db.exec(`
|
|
CREATE INDEX IF NOT EXISTS transcription_benchmark_started_idx ON transcription_benchmark_tests(started_at DESC);
|
|
CREATE INDEX IF NOT EXISTS transcription_benchmark_session_idx ON transcription_benchmark_tests(session_id, started_at DESC);
|
|
CREATE INDEX IF NOT EXISTS transcription_benchmark_revision_test_idx ON transcription_benchmark_revisions(test_id, received_at);
|
|
`);
|
|
}
|
|
|
|
start({ sessionId, deviceId, source }) {
|
|
this.cleanup();
|
|
const id = crypto.randomUUID();
|
|
this.db.prepare(`INSERT INTO transcription_benchmark_tests
|
|
(id, session_id, device_id, source_uuid, source_name, started_at, ended_at, status, model_id, backend)
|
|
VALUES (?, ?, ?, ?, ?, ?, NULL, 'running', NULL, NULL)`)
|
|
.run(id, sessionId, deviceId, source.source_uuid, source.display_name || "OBS source", this.now());
|
|
return id;
|
|
}
|
|
|
|
record(sessionId, event) {
|
|
const test = this.db.prepare("SELECT id FROM transcription_benchmark_tests WHERE session_id = ? AND status = 'running' ORDER BY started_at DESC LIMIT 1").get(sessionId);
|
|
if (!test) return false;
|
|
const count = this.db.prepare("SELECT COUNT(*) AS count FROM transcription_benchmark_revisions WHERE test_id = ?").get(test.id).count;
|
|
if (count >= 10000) return false;
|
|
const words = Array.isArray(event.analysis?.words) ? event.analysis.words.slice(0, 500).map(safeWord) : [];
|
|
this.db.prepare(`INSERT OR IGNORE INTO transcription_benchmark_revisions
|
|
(test_id, caption_id, revision, received_at, final, text, words_json, inference_ms, model_id, backend)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
.run(test.id, event.caption_id, event.revision, this.now(), event.final ? 1 : 0,
|
|
String(event.analysis?.transcript || event.stable_text || event.uncertain_text || "").slice(0, 8000), JSON.stringify(words),
|
|
finite(event.latency?.inference_ms), event.model?.id || null, event.model?.backend || null);
|
|
this.db.prepare("UPDATE transcription_benchmark_tests SET model_id = COALESCE(?, model_id), backend = COALESCE(?, backend) WHERE id = ?")
|
|
.run(event.model?.id || null, event.model?.backend || null, test.id);
|
|
return true;
|
|
}
|
|
|
|
finish(sessionId, status = "completed") {
|
|
const allowed = ["completed", "silence_timeout", "aborted", "failed"].includes(status) ? status : "completed";
|
|
const test = this.db.prepare("SELECT id FROM transcription_benchmark_tests WHERE session_id = ? AND status = 'running' ORDER BY started_at DESC LIMIT 1").get(sessionId);
|
|
if (!test) return this.bySession(sessionId);
|
|
this.db.prepare("UPDATE transcription_benchmark_tests SET ended_at = ?, status = ? WHERE id = ?").run(this.now(), allowed, test.id);
|
|
return this.byId(test.id);
|
|
}
|
|
|
|
bySession(sessionId) {
|
|
const row = this.db.prepare("SELECT * FROM transcription_benchmark_tests WHERE session_id = ? ORDER BY started_at DESC LIMIT 1").get(sessionId);
|
|
return row ? this.snapshot(row) : null;
|
|
}
|
|
|
|
byId(id) {
|
|
const row = this.db.prepare("SELECT * FROM transcription_benchmark_tests WHERE id = ?").get(id);
|
|
return row ? this.snapshot(row) : null;
|
|
}
|
|
|
|
list() {
|
|
this.cleanup();
|
|
return this.db.prepare("SELECT * FROM transcription_benchmark_tests ORDER BY started_at DESC").all().map((row) => this.snapshot(row));
|
|
}
|
|
|
|
snapshot(row) {
|
|
const revisions = this.db.prepare("SELECT * FROM transcription_benchmark_revisions WHERE test_id = ? ORDER BY received_at, revision").all(row.id);
|
|
const captions = new Map();
|
|
for (const revision of revisions) {
|
|
const existing = captions.get(revision.caption_id);
|
|
if (!existing || (revision.final && !existing.final) || revision.final === existing.final && revision.revision > existing.revision) captions.set(revision.caption_id, revision);
|
|
}
|
|
const selected = Array.from(captions.values()).sort((a, b) => a.received_at - b.received_at);
|
|
const words = [];
|
|
for (const revision of selected) {
|
|
const history = revisions.filter((candidate) => candidate.caption_id === revision.caption_id);
|
|
for (const word of mergeRevisionWords(revision, history)) words.push({ ...word, confidence: revision.final ? word.confidence : null, final: Boolean(revision.final) });
|
|
}
|
|
const latency = metricStats(words.map((word) => word.latency_ms));
|
|
const confidence = metricStats(words.map((word) => word.confidence).filter(Number.isFinite));
|
|
return {
|
|
id: row.id, session_id: row.session_id, device_id: row.device_id,
|
|
source_uuid: row.source_uuid, source_name: row.source_name, started_at: row.started_at,
|
|
ended_at: row.ended_at, duration_ms: Math.max(0, (row.ended_at || this.now()) - row.started_at),
|
|
status: row.status, model_id: row.model_id, backend: row.backend,
|
|
transcript: selected.map((revision) => cleanTranscript(revision.text)).filter(Boolean).join(" "), words,
|
|
stats: { latency, confidence }, revisions: revisions.length
|
|
};
|
|
}
|
|
|
|
cleanup(now = this.now()) {
|
|
const cutoff = now - this.retentionMs;
|
|
const ids = this.db.prepare("SELECT id FROM transcription_benchmark_tests WHERE ended_at IS NOT NULL AND ended_at < ?").all(cutoff).map((row) => row.id);
|
|
const removeRevisions = this.db.prepare("DELETE FROM transcription_benchmark_revisions WHERE test_id = ?");
|
|
const removeTest = this.db.prepare("DELETE FROM transcription_benchmark_tests WHERE id = ?");
|
|
this.db.transaction(() => { for (const id of ids) { removeRevisions.run(id); removeTest.run(id); } })();
|
|
return ids.length;
|
|
}
|
|
}
|
|
|
|
function safeWord(word) {
|
|
return {
|
|
text: cleanWordText(word?.text).slice(0, 120),
|
|
latency_ms: finite(word?.latency_ms), confidence: clamp(word?.confidence, 0, 1),
|
|
audio_start_ms: finite(word?.audio_start_ms), audio_end_ms: finite(word?.audio_end_ms),
|
|
captured_at_ms: finite(word?.captured_at_ms)
|
|
};
|
|
}
|
|
function parseWords(value) { try { const parsed = JSON.parse(value); return Array.isArray(parsed) ? parsed.map(safeWord).filter((word) => word.text) : []; } catch { return []; } }
|
|
function mergeRevisionWords(selected, revisions) {
|
|
const words = withOccurrences(parseWords(selected.words_json));
|
|
const candidates = revisions.flatMap((revision) => withOccurrences(parseWords(revision.words_json)));
|
|
return words.map((word) => {
|
|
const matches = candidates.filter((candidate) => candidate.key === word.key && candidate.occurrence === word.occurrence &&
|
|
Math.abs(candidate.captured_at_ms - word.captured_at_ms) <= 1500);
|
|
const { key: _key, occurrence: _occurrence, ...plain } = word;
|
|
return matches.length ? { ...plain, latency_ms: Math.min(...matches.map((candidate) => candidate.latency_ms)) } : plain;
|
|
});
|
|
}
|
|
function withOccurrences(words) {
|
|
const seen = new Map();
|
|
return words.map((word) => {
|
|
const key = comparableWord(word.text);
|
|
const occurrence = seen.get(key) || 0;
|
|
seen.set(key, occurrence + 1);
|
|
return { ...word, key, occurrence };
|
|
});
|
|
}
|
|
function cleanWordText(value) {
|
|
return String(value || "")
|
|
.replace(/\[_(?:BEG_|TT_\d+)\]/gi, "")
|
|
.replace(/\[BLANK_AUDIO\]/gi, "")
|
|
.trim();
|
|
}
|
|
function cleanTranscript(value) {
|
|
return cleanWordText(value).replace(/\s+/g, " ").trim();
|
|
}
|
|
function comparableWord(value) { return cleanWordText(value).toLocaleLowerCase().replace(/[^\p{L}\p{N}]+/gu, ""); }
|
|
function finite(value) { const number = Number(value); return Number.isFinite(number) ? Math.max(0, number) : 0; }
|
|
function clamp(value, min, max) { return Math.min(max, Math.max(min, finite(value))); }
|
|
function metricStats(input) {
|
|
const values = input.map(Number).filter(Number.isFinite).sort((a, b) => a - b);
|
|
if (!values.length) return { count: 0, min: null, low_1_average: null, median: null, average: null, p99: null, high_1_average: null, max: null };
|
|
const tail = Math.max(1, Math.ceil(values.length * 0.01));
|
|
return {
|
|
count: values.length, min: values[0], low_1_average: average(values.slice(0, tail)),
|
|
median: percentile(values, 0.5), average: average(values), p99: percentile(values, 0.99),
|
|
high_1_average: average(values.slice(-tail)), max: values.at(-1)
|
|
};
|
|
}
|
|
function average(values) { return values.reduce((sum, value) => sum + value, 0) / values.length; }
|
|
function percentile(values, ratio) { const position = (values.length - 1) * ratio; const lower = Math.floor(position); const upper = Math.ceil(position); return values[lower] + (values[upper] - values[lower]) * (position - lower); }
|
|
|
|
module.exports = { BenchmarkStore, metricStats };
|