236 lines
9.9 KiB
JavaScript
236 lines
9.9 KiB
JavaScript
const fs = require("fs");
|
|
const { resolveData } = require("./paths");
|
|
|
|
const historyFile = () => resolveData("metrics", "history.jsonl");
|
|
const stateFile = () => resolveData("metrics", "summary.json");
|
|
function getSummary() {
|
|
try { return JSON.parse(fs.readFileSync(stateFile(), "utf8")); }
|
|
catch { return { total_requests:0, successful:0, failed:0, refusals:0, gate_decisions:0, tool_suggestions:0, tool_executions:0, tool_denials:0, confirmation_cancellations:0, timeout_count:0, runtime_crash_count:0, runtime_self_test_total:0, runtime_self_test_failed_total:0, runtime_start_attempt_total:0, runtime_start_failed_total:0, verified_downloads:0, failed_downloads:0, requests_by_role:{}, requests_by_scope:{}, requests_by_route:{}, gate_reason_codes:{}, runtime_exit_code_counts:{}, stage_totals:{}, stage_samples:0, slow_requests:[], durations:[], queue_wait_total_ms:0 }; }
|
|
}
|
|
function record(entry) {
|
|
const summary = getSummary();
|
|
summary.requests_by_role ||= {};
|
|
summary.requests_by_scope ||= {};
|
|
summary.requests_by_route ||= {};
|
|
if (entry.kind === "request") {
|
|
summary.total_requests += 1;
|
|
if (entry.status === "success") summary.successful += 1;
|
|
if (entry.status === "failed") summary.failed += 1;
|
|
if (entry.status === "refused") summary.refusals += 1;
|
|
if (entry.role) summary.requests_by_role[entry.role] = (summary.requests_by_role[entry.role] || 0) + 1;
|
|
if (entry.scope) summary.requests_by_scope[entry.scope] = (summary.requests_by_scope[entry.scope] || 0) + 1;
|
|
}
|
|
if (entry.kind === "gate_decision") {
|
|
summary.gate_decisions = (summary.gate_decisions || 0) + 1;
|
|
summary.gate_reason_codes ||= {};
|
|
if (entry.reason_code) {
|
|
summary.gate_reason_codes[entry.reason_code] = (summary.gate_reason_codes[entry.reason_code] || 0) + 1;
|
|
}
|
|
}
|
|
if (entry.route_used) {
|
|
summary.requests_by_route[entry.route_used] = (summary.requests_by_route[entry.route_used] || 0) + 1;
|
|
}
|
|
if (entry.tool_requested) summary.tool_suggestions += 1;
|
|
if (entry.tool_executed) summary.tool_executions += 1;
|
|
if (entry.kind === "tool" && entry.status === "failed") summary.tool_denials += 1;
|
|
if (entry.kind === "tool" && entry.status === "cancelled") summary.confirmation_cancellations += 1;
|
|
if (entry.timeout) summary.timeout_count += 1;
|
|
if (entry.runtime_crash) summary.runtime_crash_count += 1;
|
|
if (entry.kind === "runtime_self_test") {
|
|
summary.runtime_self_test_total += 1;
|
|
if (entry.status === "failed") summary.runtime_self_test_failed_total += 1;
|
|
}
|
|
if (entry.kind === "runtime_start") {
|
|
if (entry.status === "attempt") summary.runtime_start_attempt_total += 1;
|
|
if (entry.status === "failed") summary.runtime_start_failed_total += 1;
|
|
}
|
|
if (entry.code) {
|
|
summary.runtime_exit_code_counts ||= {};
|
|
summary.runtime_exit_code_counts[entry.code] = (summary.runtime_exit_code_counts[entry.code] || 0) + 1;
|
|
}
|
|
if (entry.kind === "download" && entry.status === "success") summary.verified_downloads += 1;
|
|
if (entry.kind === "download" && entry.status === "failed") summary.failed_downloads += 1;
|
|
summary.durations = (Array.isArray(summary.durations) ? summary.durations : [])
|
|
.filter(isValidTiming);
|
|
if (isValidTiming(entry.duration_ms)) summary.durations.push(Number(entry.duration_ms));
|
|
summary.durations = summary.durations.slice(-500);
|
|
if (isValidTiming(entry.queue_wait_ms)) {
|
|
summary.queue_wait_total_ms = Math.max(0, Number(summary.queue_wait_total_ms) || 0) + Number(entry.queue_wait_ms);
|
|
}
|
|
const stageKeys = [
|
|
"deterministic_ms", "gate_ms", "queue_ms", "prompt_eval_ms", "generation_ms",
|
|
"main_queue_ms", "main_generate_ms", "total_ms"
|
|
];
|
|
if (entry.kind === "request" && stageKeys.some((key) => isValidTiming(entry[key]))) {
|
|
summary.stage_totals ||= {};
|
|
if (!summary.stage_counts) {
|
|
const legacySamples = Math.max(0, Number(summary.stage_samples) || 0);
|
|
summary.stage_counts = Object.fromEntries(
|
|
Object.keys(summary.stage_totals)
|
|
.filter((key) => isValidTiming(summary.stage_totals[key]))
|
|
.map((key) => [key, legacySamples])
|
|
);
|
|
}
|
|
for (const key of stageKeys) {
|
|
if (!isValidTiming(entry[key])) continue;
|
|
summary.stage_totals[key] = Math.max(0, Number(summary.stage_totals[key]) || 0) + Number(entry[key]);
|
|
summary.stage_counts[key] = Math.max(0, Number(summary.stage_counts[key]) || 0) + 1;
|
|
}
|
|
}
|
|
const totalCandidate = entry.total_ms ?? entry.duration_ms;
|
|
const totalMs = isValidTiming(totalCandidate) ? Number(totalCandidate) : null;
|
|
if (entry.kind === "request" && totalMs != null && totalMs >= 30000) {
|
|
summary.slow_requests ||= [];
|
|
summary.slow_requests.unshift({
|
|
timestamp: new Date().toISOString(),
|
|
request_id: entry.request_id || null,
|
|
route_used: entry.route_used || null,
|
|
route_class: entry.route_class || null,
|
|
reason_code: entry.gate_reason_code || entry.reason_code || null,
|
|
deterministic_ms: entry.deterministic_ms || 0,
|
|
gate_ms: entry.gate_ms || 0,
|
|
queue_ms: entry.queue_ms ?? entry.main_queue_ms ?? 0,
|
|
prompt_eval_ms: entry.prompt_eval_ms || 0,
|
|
generation_ms: entry.generation_ms ?? entry.main_generate_ms ?? 0,
|
|
main_queue_ms: entry.main_queue_ms ?? entry.queue_ms ?? 0,
|
|
main_generate_ms: entry.main_generate_ms ?? entry.generation_ms ?? 0,
|
|
prompt_tokens: entry.prompt_tokens || 0,
|
|
generated_tokens: entry.generated_tokens || 0,
|
|
prompt_tps: entry.prompt_tps || 0,
|
|
generation_tps: entry.generation_tps || 0,
|
|
backend: entry.backend || null,
|
|
gpu_layers: entry.gpu_layers || 0,
|
|
context_size: entry.context_size || 0,
|
|
max_output_tokens_used: entry.max_output_tokens_used ?? entry.max_output_tokens ?? 0,
|
|
frontend_soft_timeout: Boolean(entry.frontend_soft_timeout),
|
|
total_ms: totalMs,
|
|
risk_504: totalMs >= 45000
|
|
});
|
|
summary.slow_requests = summary.slow_requests.slice(0, 25);
|
|
}
|
|
fs.writeFileSync(stateFile(), JSON.stringify(summary, null, 2));
|
|
fs.appendFileSync(historyFile(), `${JSON.stringify({ timestamp:new Date().toISOString(), ...entry })}\n`);
|
|
}
|
|
function report() {
|
|
return summarizeMetrics(getSummary());
|
|
}
|
|
function summarizeMetrics(s = {}) {
|
|
const sorted = (Array.isArray(s.durations) ? s.durations : [])
|
|
.filter(isValidTiming)
|
|
.map(Number)
|
|
.sort((a,b)=>a-b);
|
|
const average_stage_ms = Object.fromEntries(
|
|
Object.entries(s.stage_totals || {})
|
|
.filter(([, value]) => isValidTiming(value))
|
|
.map(([key, value]) => {
|
|
const legacyCount = Math.max(0, Number(s.stage_samples) || 0);
|
|
const count = Math.max(0, Number(s.stage_counts?.[key]) || legacyCount);
|
|
return [key, count ? Math.max(0, Math.round(Number(value) / count)) : 0];
|
|
})
|
|
);
|
|
return {
|
|
...s,
|
|
durations: sorted,
|
|
average_stage_ms,
|
|
average_response_ms: sorted.length ? Math.max(0, Math.round(sorted.reduce((a,b)=>a+b,0)/sorted.length)) : 0,
|
|
median_response_ms: sorted.length ? Math.max(0, sorted[Math.floor(sorted.length/2)]) : 0
|
|
};
|
|
}
|
|
function history(limit=100) {
|
|
try {
|
|
return fs.readFileSync(historyFile(),"utf8").trim().split(/\r?\n/)
|
|
.filter(Boolean)
|
|
.map(parseHistoryRow)
|
|
.filter(Boolean)
|
|
.slice(-limit)
|
|
.reverse();
|
|
} catch { return []; }
|
|
}
|
|
function historyPage(page = 1, pageSize = 25) {
|
|
const safePage = Math.max(1, Number.parseInt(page, 10) || 1);
|
|
const safeSize = Math.max(1, Math.min(100, Number.parseInt(pageSize, 10) || 25));
|
|
try {
|
|
const rows = fs.readFileSync(historyFile(), "utf8").trim().split(/\r?\n/)
|
|
.filter(Boolean)
|
|
.map(parseHistoryRow)
|
|
.filter(Boolean);
|
|
return paginateRows(rows, safePage, safeSize);
|
|
} catch {
|
|
return { entries: [], page: 1, pages: 1, page_size: safeSize, total: 0 };
|
|
}
|
|
}
|
|
function slowRequestsPage(page = 1, pageSize = 15) {
|
|
const safePage = Math.max(1, Number.parseInt(page, 10) || 1);
|
|
const safeSize = Math.max(1, Math.min(100, Number.parseInt(pageSize, 10) || 15));
|
|
try {
|
|
const rows = fs.readFileSync(historyFile(), "utf8").trim().split(/\r?\n/)
|
|
.filter(Boolean)
|
|
.map(parseHistoryRow)
|
|
.filter((entry) => {
|
|
const total = entry?.total_ms ?? entry?.duration_ms;
|
|
return entry?.kind === "request" && isValidTiming(total) && Number(total) >= 30000;
|
|
})
|
|
.map(normalizeSlowEntry);
|
|
return paginateRows(rows, safePage, safeSize);
|
|
} catch {
|
|
return { entries: [], page: 1, pages: 1, page_size: safeSize, total: 0 };
|
|
}
|
|
}
|
|
function paginateRows(rows, page = 1, pageSize = 25, map = (value) => value) {
|
|
const total = rows.length;
|
|
const pages = Math.max(1, Math.ceil(total / pageSize));
|
|
const current = Math.min(Math.max(1, page), pages);
|
|
const end = total - (current - 1) * pageSize;
|
|
const start = Math.max(0, end - pageSize);
|
|
return {
|
|
entries: rows.slice(start, end).reverse().map(map),
|
|
page: current,
|
|
pages,
|
|
page_size: pageSize,
|
|
total
|
|
};
|
|
}
|
|
|
|
function parseHistoryRow(value) {
|
|
try { return JSON.parse(value); }
|
|
catch { return null; }
|
|
}
|
|
|
|
function normalizeSlowEntry(entry) {
|
|
const totalMs = Number(entry.total_ms ?? entry.duration_ms);
|
|
return {
|
|
...entry,
|
|
total_ms: totalMs,
|
|
queue_ms: validOrZero(entry.queue_ms ?? entry.main_queue_ms),
|
|
prompt_eval_ms: validOrZero(entry.prompt_eval_ms),
|
|
generation_ms: validOrZero(entry.generation_ms ?? entry.main_generate_ms),
|
|
gate_ms: validOrZero(entry.gate_ms),
|
|
prompt_tokens: validOrZero(entry.prompt_tokens),
|
|
generated_tokens: validOrZero(entry.generated_tokens),
|
|
prompt_tps: validOrZero(entry.prompt_tps),
|
|
generation_tps: validOrZero(entry.generation_tps),
|
|
max_output_tokens_used: validOrZero(entry.max_output_tokens_used ?? entry.max_output_tokens),
|
|
risk_504: totalMs >= 45000
|
|
};
|
|
}
|
|
|
|
function isValidTiming(value) {
|
|
const number = Number(value);
|
|
return Number.isFinite(number) && number >= 0;
|
|
}
|
|
|
|
function validOrZero(value) {
|
|
return isValidTiming(value) ? Number(value) : 0;
|
|
}
|
|
|
|
module.exports = {
|
|
record,
|
|
report,
|
|
history,
|
|
historyPage,
|
|
slowRequestsPage,
|
|
paginateRows,
|
|
isValidTiming,
|
|
summarizeMetrics
|
|
};
|