const fs = require("fs"); const { resolveData } = require("./paths"); const historyFile = () => resolveData("metrics", "history.jsonl"); const stateFile = () => resolveData("metrics", "summary.json"); function getSummary() { try { return JSON.parse(fs.readFileSync(stateFile(), "utf8")); } catch { return { total_requests:0, successful:0, failed:0, refusals:0, gate_decisions:0, tool_decisions:0, tool_suggestions:0, tool_executions:0, tool_denials:0, confirmation_cancellations:0, timeout_count:0, runtime_crash_count:0, runtime_self_test_total:0, runtime_self_test_failed_total:0, runtime_start_attempt_total:0, runtime_start_failed_total:0, verified_downloads:0, failed_downloads:0, requests_by_role:{}, requests_by_scope:{}, requests_by_route:{}, gate_reason_codes:{}, tool_rejections_by_reason:{}, runtime_exit_code_counts:{}, stage_totals:{}, stage_samples:0, slow_requests:[], durations:[], queue_wait_total_ms:0 }; } } function record(entry) { const summary = getSummary(); summary.requests_by_role ||= {}; summary.requests_by_scope ||= {}; summary.requests_by_route ||= {}; if (entry.kind === "request") { summary.total_requests += 1; if (entry.status === "success") summary.successful += 1; if (entry.status === "failed") summary.failed += 1; if (entry.status === "refused") summary.refusals += 1; if (entry.role) summary.requests_by_role[entry.role] = (summary.requests_by_role[entry.role] || 0) + 1; if (entry.scope) summary.requests_by_scope[entry.scope] = (summary.requests_by_scope[entry.scope] || 0) + 1; } if (entry.kind === "gate_decision") { summary.gate_decisions = (summary.gate_decisions || 0) + 1; summary.gate_reason_codes ||= {}; if (entry.reason_code) { summary.gate_reason_codes[entry.reason_code] = (summary.gate_reason_codes[entry.reason_code] || 0) + 1; } } if (entry.kind === "tool_decision") { summary.tool_decisions = (summary.tool_decisions || 0) + 1; summary.tool_rejections_by_reason ||= {}; if (entry.status === "rejected" && entry.rejected_reason) { summary.tool_rejections_by_reason[entry.rejected_reason] = (summary.tool_rejections_by_reason[entry.rejected_reason] || 0) + 1; } } if (entry.route_used) { summary.requests_by_route[entry.route_used] = (summary.requests_by_route[entry.route_used] || 0) + 1; } if (entry.tool_requested) summary.tool_suggestions += 1; if (entry.tool_executed) summary.tool_executions += 1; if (entry.kind === "tool" && entry.status === "failed") summary.tool_denials += 1; if (entry.kind === "tool" && entry.status === "cancelled") summary.confirmation_cancellations += 1; if (entry.timeout) summary.timeout_count += 1; if (entry.runtime_crash) summary.runtime_crash_count += 1; if (entry.kind === "runtime_self_test") { summary.runtime_self_test_total += 1; if (entry.status === "failed") summary.runtime_self_test_failed_total += 1; } if (entry.kind === "runtime_start") { if (entry.status === "attempt") summary.runtime_start_attempt_total += 1; if (entry.status === "failed") summary.runtime_start_failed_total += 1; } if (entry.code) { summary.runtime_exit_code_counts ||= {}; summary.runtime_exit_code_counts[entry.code] = (summary.runtime_exit_code_counts[entry.code] || 0) + 1; } if (entry.kind === "download" && entry.status === "success") summary.verified_downloads += 1; if (entry.kind === "download" && entry.status === "failed") summary.failed_downloads += 1; summary.durations = (Array.isArray(summary.durations) ? summary.durations : []) .filter(isValidTiming); if (isValidTiming(entry.duration_ms)) summary.durations.push(Number(entry.duration_ms)); summary.durations = summary.durations.slice(-500); if (isValidTiming(entry.queue_wait_ms)) { summary.queue_wait_total_ms = Math.max(0, Number(summary.queue_wait_total_ms) || 0) + Number(entry.queue_wait_ms); } const stageKeys = [ "deterministic_ms", "gate_ms", "queue_ms", "prompt_eval_ms", "generation_ms", "main_queue_ms", "main_generate_ms", "total_ms" ]; if (entry.kind === "request" && stageKeys.some((key) => isValidTiming(entry[key]))) { summary.stage_totals ||= {}; if (!summary.stage_counts) { const legacySamples = Math.max(0, Number(summary.stage_samples) || 0); summary.stage_counts = Object.fromEntries( Object.keys(summary.stage_totals) .filter((key) => isValidTiming(summary.stage_totals[key])) .map((key) => [key, legacySamples]) ); } for (const key of stageKeys) { if (!isValidTiming(entry[key])) continue; summary.stage_totals[key] = Math.max(0, Number(summary.stage_totals[key]) || 0) + Number(entry[key]); summary.stage_counts[key] = Math.max(0, Number(summary.stage_counts[key]) || 0) + 1; } } const totalCandidate = entry.total_ms ?? entry.duration_ms; const totalMs = isValidTiming(totalCandidate) ? Number(totalCandidate) : null; if (entry.kind === "request" && totalMs != null && totalMs >= 30000) { summary.slow_requests ||= []; summary.slow_requests.unshift({ timestamp: new Date().toISOString(), request_id: entry.request_id || null, route_used: entry.route_used || null, route_class: entry.route_class || null, reason_code: entry.gate_reason_code || entry.reason_code || null, deterministic_ms: entry.deterministic_ms || 0, gate_ms: entry.gate_ms || 0, queue_ms: entry.queue_ms ?? entry.main_queue_ms ?? 0, prompt_eval_ms: entry.prompt_eval_ms || 0, generation_ms: entry.generation_ms ?? entry.main_generate_ms ?? 0, main_queue_ms: entry.main_queue_ms ?? entry.queue_ms ?? 0, main_generate_ms: entry.main_generate_ms ?? entry.generation_ms ?? 0, prompt_tokens: entry.prompt_tokens || 0, generated_tokens: entry.generated_tokens || 0, prompt_tps: entry.prompt_tps || 0, generation_tps: entry.generation_tps || 0, backend: entry.backend || null, gpu_layers: entry.gpu_layers || 0, context_size: entry.context_size || 0, max_output_tokens_used: entry.max_output_tokens_used ?? entry.max_output_tokens ?? 0, frontend_soft_timeout: Boolean(entry.frontend_soft_timeout), total_ms: totalMs, risk_504: totalMs >= 45000 }); summary.slow_requests = summary.slow_requests.slice(0, 25); } fs.writeFileSync(stateFile(), JSON.stringify(summary, null, 2)); fs.appendFileSync(historyFile(), `${JSON.stringify({ timestamp:new Date().toISOString(), ...entry })}\n`); } function report() { return summarizeMetrics(getSummary()); } function summarizeMetrics(s = {}) { const sorted = (Array.isArray(s.durations) ? s.durations : []) .filter(isValidTiming) .map(Number) .sort((a,b)=>a-b); const average_stage_ms = Object.fromEntries( Object.entries(s.stage_totals || {}) .filter(([, value]) => isValidTiming(value)) .map(([key, value]) => { const legacyCount = Math.max(0, Number(s.stage_samples) || 0); const count = Math.max(0, Number(s.stage_counts?.[key]) || legacyCount); return [key, count ? Math.max(0, Math.round(Number(value) / count)) : 0]; }) ); return { ...s, durations: sorted, average_stage_ms, average_response_ms: sorted.length ? Math.max(0, Math.round(sorted.reduce((a,b)=>a+b,0)/sorted.length)) : 0, median_response_ms: sorted.length ? Math.max(0, sorted[Math.floor(sorted.length/2)]) : 0 }; } function history(limit=100) { try { return fs.readFileSync(historyFile(),"utf8").trim().split(/\r?\n/) .filter(Boolean) .map(parseHistoryRow) .filter(Boolean) .slice(-limit) .reverse(); } catch { return []; } } function historyPage(page = 1, pageSize = 25) { const safePage = Math.max(1, Number.parseInt(page, 10) || 1); const safeSize = Math.max(1, Math.min(100, Number.parseInt(pageSize, 10) || 25)); try { const rows = fs.readFileSync(historyFile(), "utf8").trim().split(/\r?\n/) .filter(Boolean) .map(parseHistoryRow) .filter(Boolean); return paginateRows(rows, safePage, safeSize); } catch { return { entries: [], page: 1, pages: 1, page_size: safeSize, total: 0 }; } } function slowRequestsPage(page = 1, pageSize = 15) { const safePage = Math.max(1, Number.parseInt(page, 10) || 1); const safeSize = Math.max(1, Math.min(100, Number.parseInt(pageSize, 10) || 15)); try { const rows = fs.readFileSync(historyFile(), "utf8").trim().split(/\r?\n/) .filter(Boolean) .map(parseHistoryRow) .filter((entry) => { const total = entry?.total_ms ?? entry?.duration_ms; return entry?.kind === "request" && isValidTiming(total) && Number(total) >= 30000; }) .map(normalizeSlowEntry); return paginateRows(rows, safePage, safeSize); } catch { return { entries: [], page: 1, pages: 1, page_size: safeSize, total: 0 }; } } function paginateRows(rows, page = 1, pageSize = 25, map = (value) => value) { const total = rows.length; const pages = Math.max(1, Math.ceil(total / pageSize)); const current = Math.min(Math.max(1, page), pages); const end = total - (current - 1) * pageSize; const start = Math.max(0, end - pageSize); return { entries: rows.slice(start, end).reverse().map(map), page: current, pages, page_size: pageSize, total }; } function parseHistoryRow(value) { try { return JSON.parse(value); } catch { return null; } } function normalizeSlowEntry(entry) { const totalMs = Number(entry.total_ms ?? entry.duration_ms); return { ...entry, total_ms: totalMs, queue_ms: validOrZero(entry.queue_ms ?? entry.main_queue_ms), prompt_eval_ms: validOrZero(entry.prompt_eval_ms), generation_ms: validOrZero(entry.generation_ms ?? entry.main_generate_ms), gate_ms: validOrZero(entry.gate_ms), prompt_tokens: validOrZero(entry.prompt_tokens), generated_tokens: validOrZero(entry.generated_tokens), prompt_tps: validOrZero(entry.prompt_tps), generation_tps: validOrZero(entry.generation_tps), max_output_tokens_used: validOrZero(entry.max_output_tokens_used ?? entry.max_output_tokens), risk_504: totalMs >= 45000 }; } function isValidTiming(value) { const number = Number(value); return Number.isFinite(number) && number >= 0; } function validOrZero(value) { return isValidTiming(value) ? Number(value) : 0; } module.exports = { record, report, history, historyPage, slowRequestsPage, paginateRows, isValidTiming, summarizeMetrics };