135 lines
7.5 KiB
JavaScript
135 lines
7.5 KiB
JavaScript
const crypto = require("crypto");
|
|
|
|
class BenchmarkStore {
|
|
constructor(db, options = {}) {
|
|
this.db = db;
|
|
this.now = options.now || Date.now;
|
|
this.retentionMs = options.retentionMs || 60 * 60 * 1000;
|
|
this.migrate();
|
|
this.db.prepare("UPDATE transcription_benchmark_tests SET status = 'aborted', ended_at = ? WHERE status = 'running'").run(this.now());
|
|
this.cleanup();
|
|
}
|
|
|
|
migrate() {
|
|
this.db.exec(`
|
|
CREATE TABLE IF NOT EXISTS transcription_benchmark_tests (
|
|
id TEXT PRIMARY KEY, session_id TEXT NOT NULL UNIQUE, device_id TEXT NOT NULL,
|
|
source_uuid TEXT NOT NULL, source_name TEXT NOT NULL, started_at INTEGER NOT NULL,
|
|
ended_at INTEGER, status TEXT NOT NULL, model_id TEXT, backend TEXT
|
|
);
|
|
CREATE TABLE IF NOT EXISTS transcription_benchmark_revisions (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT, test_id TEXT NOT NULL, caption_id TEXT NOT NULL,
|
|
revision INTEGER NOT NULL, received_at INTEGER NOT NULL, final INTEGER NOT NULL,
|
|
text TEXT NOT NULL, words_json TEXT NOT NULL, inference_ms REAL NOT NULL,
|
|
model_id TEXT, backend TEXT,
|
|
UNIQUE(test_id, caption_id, revision)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS transcription_benchmark_started_idx ON transcription_benchmark_tests(started_at DESC);
|
|
CREATE INDEX IF NOT EXISTS transcription_benchmark_revision_test_idx ON transcription_benchmark_revisions(test_id, received_at);
|
|
`);
|
|
}
|
|
|
|
start({ sessionId, deviceId, source }) {
|
|
this.cleanup();
|
|
const id = crypto.randomUUID();
|
|
this.db.prepare(`INSERT INTO transcription_benchmark_tests
|
|
(id, session_id, device_id, source_uuid, source_name, started_at, ended_at, status, model_id, backend)
|
|
VALUES (?, ?, ?, ?, ?, ?, NULL, 'running', NULL, NULL)`)
|
|
.run(id, sessionId, deviceId, source.source_uuid, source.display_name || "OBS source", this.now());
|
|
return id;
|
|
}
|
|
|
|
record(sessionId, event) {
|
|
const test = this.db.prepare("SELECT id FROM transcription_benchmark_tests WHERE session_id = ? AND status = 'running'").get(sessionId);
|
|
if (!test) return false;
|
|
const count = this.db.prepare("SELECT COUNT(*) AS count FROM transcription_benchmark_revisions WHERE test_id = ?").get(test.id).count;
|
|
if (count >= 10000) return false;
|
|
const words = Array.isArray(event.analysis?.words) ? event.analysis.words.slice(0, 500).map(safeWord) : [];
|
|
this.db.prepare(`INSERT OR IGNORE INTO transcription_benchmark_revisions
|
|
(test_id, caption_id, revision, received_at, final, text, words_json, inference_ms, model_id, backend)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
.run(test.id, event.caption_id, event.revision, this.now(), event.final ? 1 : 0,
|
|
String(event.analysis?.transcript || event.stable_text || event.uncertain_text || "").slice(0, 8000), JSON.stringify(words),
|
|
finite(event.latency?.inference_ms), event.model?.id || null, event.model?.backend || null);
|
|
this.db.prepare("UPDATE transcription_benchmark_tests SET model_id = COALESCE(?, model_id), backend = COALESCE(?, backend) WHERE id = ?")
|
|
.run(event.model?.id || null, event.model?.backend || null, test.id);
|
|
return true;
|
|
}
|
|
|
|
finish(sessionId, status = "completed") {
|
|
const allowed = ["completed", "silence_timeout", "aborted", "failed"].includes(status) ? status : "completed";
|
|
this.db.prepare("UPDATE transcription_benchmark_tests SET ended_at = ?, status = ? WHERE session_id = ? AND status = 'running'")
|
|
.run(this.now(), allowed, sessionId);
|
|
return this.bySession(sessionId);
|
|
}
|
|
|
|
bySession(sessionId) {
|
|
const row = this.db.prepare("SELECT * FROM transcription_benchmark_tests WHERE session_id = ?").get(sessionId);
|
|
return row ? this.snapshot(row) : null;
|
|
}
|
|
|
|
list() {
|
|
this.cleanup();
|
|
return this.db.prepare("SELECT * FROM transcription_benchmark_tests ORDER BY started_at DESC").all().map((row) => this.snapshot(row));
|
|
}
|
|
|
|
snapshot(row) {
|
|
const revisions = this.db.prepare("SELECT * FROM transcription_benchmark_revisions WHERE test_id = ? ORDER BY received_at, revision").all(row.id);
|
|
const captions = new Map();
|
|
for (const revision of revisions) {
|
|
const existing = captions.get(revision.caption_id);
|
|
if (!existing || (revision.final && !existing.final) || revision.final === existing.final && revision.revision > existing.revision) captions.set(revision.caption_id, revision);
|
|
}
|
|
const selected = Array.from(captions.values()).sort((a, b) => a.received_at - b.received_at);
|
|
const words = [];
|
|
for (const revision of selected) {
|
|
for (const word of parseWords(revision.words_json)) words.push({ ...word, confidence: revision.final ? word.confidence : null, final: Boolean(revision.final) });
|
|
}
|
|
const latency = metricStats(words.map((word) => word.latency_ms));
|
|
const confidence = metricStats(words.map((word) => word.confidence).filter(Number.isFinite));
|
|
return {
|
|
id: row.id, session_id: row.session_id, device_id: row.device_id,
|
|
source_uuid: row.source_uuid, source_name: row.source_name, started_at: row.started_at,
|
|
ended_at: row.ended_at, duration_ms: Math.max(0, (row.ended_at || this.now()) - row.started_at),
|
|
status: row.status, model_id: row.model_id, backend: row.backend,
|
|
transcript: selected.map((revision) => revision.text).filter(Boolean).join(" "), words,
|
|
stats: { latency, confidence }, revisions: revisions.length
|
|
};
|
|
}
|
|
|
|
cleanup(now = this.now()) {
|
|
const cutoff = now - this.retentionMs;
|
|
const ids = this.db.prepare("SELECT id FROM transcription_benchmark_tests WHERE ended_at IS NOT NULL AND ended_at < ?").all(cutoff).map((row) => row.id);
|
|
const removeRevisions = this.db.prepare("DELETE FROM transcription_benchmark_revisions WHERE test_id = ?");
|
|
const removeTest = this.db.prepare("DELETE FROM transcription_benchmark_tests WHERE id = ?");
|
|
this.db.transaction(() => { for (const id of ids) { removeRevisions.run(id); removeTest.run(id); } })();
|
|
return ids.length;
|
|
}
|
|
}
|
|
|
|
function safeWord(word) {
|
|
return {
|
|
text: String(word?.text || "").slice(0, 120),
|
|
latency_ms: finite(word?.latency_ms), confidence: clamp(word?.confidence, 0, 1),
|
|
audio_start_ms: finite(word?.audio_start_ms), audio_end_ms: finite(word?.audio_end_ms),
|
|
captured_at_ms: finite(word?.captured_at_ms)
|
|
};
|
|
}
|
|
function parseWords(value) { try { const parsed = JSON.parse(value); return Array.isArray(parsed) ? parsed.map(safeWord).filter((word) => word.text) : []; } catch { return []; } }
|
|
function finite(value) { const number = Number(value); return Number.isFinite(number) ? Math.max(0, number) : 0; }
|
|
function clamp(value, min, max) { return Math.min(max, Math.max(min, finite(value))); }
|
|
function metricStats(input) {
|
|
const values = input.map(Number).filter(Number.isFinite).sort((a, b) => a - b);
|
|
if (!values.length) return { count: 0, min: null, low_1_average: null, median: null, average: null, p99: null, high_1_average: null, max: null };
|
|
const tail = Math.max(1, Math.ceil(values.length * 0.01));
|
|
return {
|
|
count: values.length, min: values[0], low_1_average: average(values.slice(0, tail)),
|
|
median: percentile(values, 0.5), average: average(values), p99: percentile(values, 0.99),
|
|
high_1_average: average(values.slice(-tail)), max: values.at(-1)
|
|
};
|
|
}
|
|
function average(values) { return values.reduce((sum, value) => sum + value, 0) / values.length; }
|
|
function percentile(values, ratio) { const position = (values.length - 1) * ratio; const lower = Math.floor(position); const upper = Math.ceil(position); return values[lower] + (values[upper] - values[lower]) * (position - lower); }
|
|
|
|
module.exports = { BenchmarkStore, metricStats };
|