Lumi/plugins/lumi_ai/backend/metrics.js
2026-06-25 14:10:04 +02:00

482 lines
21 KiB
JavaScript

const fs = require("fs");
const { resolveData } = require("./paths");
const historyFile = () => resolveData("metrics", "history.jsonl");
const stateFile = () => resolveData("metrics", "summary.json");
let retention = { mode: "count", count: 500, age_value: 30, age_unit: "days" };
function getSummary() {
try { return JSON.parse(fs.readFileSync(stateFile(), "utf8")); }
catch { return { total_requests:0, successful:0, failed:0, refusals:0, gate_decisions:0, tool_decisions:0, tool_suggestions:0, tool_executions:0, tool_denials:0, confirmation_cancellations:0, timeout_count:0, runtime_crash_count:0, runtime_self_test_total:0, runtime_self_test_failed_total:0, runtime_start_attempt_total:0, runtime_start_failed_total:0, verified_downloads:0, failed_downloads:0, requests_by_role:{}, requests_by_scope:{}, requests_by_route:{}, gate_reason_codes:{}, tool_rejections_by_reason:{}, runtime_exit_code_counts:{}, stage_totals:{}, stage_samples:0, slow_requests:[], durations:[], queue_wait_total_ms:0 }; }
}
function record(entry) {
maybeApplyRetention();
const summary = getSummary();
summary.requests_by_role ||= {};
summary.requests_by_scope ||= {};
summary.requests_by_route ||= {};
if (entry.kind === "request") {
summary.total_requests += 1;
if (entry.status === "success") summary.successful += 1;
if (entry.status === "failed") summary.failed += 1;
if (entry.status === "refused") summary.refusals += 1;
if (entry.role) summary.requests_by_role[entry.role] = (summary.requests_by_role[entry.role] || 0) + 1;
if (entry.scope) summary.requests_by_scope[entry.scope] = (summary.requests_by_scope[entry.scope] || 0) + 1;
}
if (entry.kind === "gate_decision") {
summary.gate_decisions = (summary.gate_decisions || 0) + 1;
summary.gate_reason_codes ||= {};
if (entry.reason_code) {
summary.gate_reason_codes[entry.reason_code] = (summary.gate_reason_codes[entry.reason_code] || 0) + 1;
}
}
if (entry.kind === "tool_decision") {
summary.tool_decisions = (summary.tool_decisions || 0) + 1;
summary.tool_rejections_by_reason ||= {};
if (entry.status === "rejected" && entry.rejected_reason) {
summary.tool_rejections_by_reason[entry.rejected_reason] =
(summary.tool_rejections_by_reason[entry.rejected_reason] || 0) + 1;
}
}
if (entry.route_used) {
summary.requests_by_route[entry.route_used] = (summary.requests_by_route[entry.route_used] || 0) + 1;
}
if (entry.tool_requested) summary.tool_suggestions += 1;
if (entry.tool_executed) summary.tool_executions += 1;
if (entry.kind === "tool" && entry.status === "failed") summary.tool_denials += 1;
if (entry.kind === "tool" && entry.status === "cancelled") summary.confirmation_cancellations += 1;
if (entry.timeout) summary.timeout_count += 1;
if (entry.runtime_crash) summary.runtime_crash_count += 1;
if (entry.kind === "runtime_self_test") {
summary.runtime_self_test_total += 1;
if (entry.status === "failed") summary.runtime_self_test_failed_total += 1;
}
if (entry.kind === "runtime_start") {
if (entry.status === "attempt") summary.runtime_start_attempt_total += 1;
if (entry.status === "failed") summary.runtime_start_failed_total += 1;
}
if (entry.code) {
summary.runtime_exit_code_counts ||= {};
summary.runtime_exit_code_counts[entry.code] = (summary.runtime_exit_code_counts[entry.code] || 0) + 1;
}
if (entry.kind === "download" && entry.status === "success") summary.verified_downloads += 1;
if (entry.kind === "download" && entry.status === "failed") summary.failed_downloads += 1;
summary.durations = (Array.isArray(summary.durations) ? summary.durations : [])
.filter(isValidTiming);
if (isValidTiming(entry.duration_ms)) summary.durations.push(Number(entry.duration_ms));
summary.durations = summary.durations.slice(-500);
if (isValidTiming(entry.queue_wait_ms)) {
summary.queue_wait_total_ms = Math.max(0, Number(summary.queue_wait_total_ms) || 0) + Number(entry.queue_wait_ms);
}
const stageKeys = [
"deterministic_ms", "gate_ms", "queue_ms", "prompt_eval_ms", "generation_ms",
"main_queue_ms", "main_generate_ms", "total_ms"
];
if (entry.kind === "request" && stageKeys.some((key) => isValidTiming(entry[key]))) {
summary.stage_totals ||= {};
if (!summary.stage_counts) {
const legacySamples = Math.max(0, Number(summary.stage_samples) || 0);
summary.stage_counts = Object.fromEntries(
Object.keys(summary.stage_totals)
.filter((key) => isValidTiming(summary.stage_totals[key]))
.map((key) => [key, legacySamples])
);
}
for (const key of stageKeys) {
if (!isValidTiming(entry[key])) continue;
summary.stage_totals[key] = Math.max(0, Number(summary.stage_totals[key]) || 0) + Number(entry[key]);
summary.stage_counts[key] = Math.max(0, Number(summary.stage_counts[key]) || 0) + 1;
}
}
const totalCandidate = entry.total_ms ?? entry.duration_ms;
const totalMs = isValidTiming(totalCandidate) ? Number(totalCandidate) : null;
if (entry.kind === "request" && totalMs != null && totalMs >= 30000) {
summary.slow_requests ||= [];
summary.slow_requests.unshift({
timestamp: new Date().toISOString(),
request_id: entry.request_id || null,
route_used: entry.route_used || null,
route_class: entry.route_class || null,
reason_code: entry.gate_reason_code || entry.reason_code || null,
deterministic_ms: entry.deterministic_ms || 0,
gate_ms: entry.gate_ms || 0,
queue_ms: entry.queue_ms ?? entry.main_queue_ms ?? 0,
prompt_eval_ms: entry.prompt_eval_ms || 0,
generation_ms: entry.generation_ms ?? entry.main_generate_ms ?? 0,
main_queue_ms: entry.main_queue_ms ?? entry.queue_ms ?? 0,
main_generate_ms: entry.main_generate_ms ?? entry.generation_ms ?? 0,
prompt_tokens: entry.prompt_tokens || 0,
generated_tokens: entry.generated_tokens || 0,
prompt_tps: entry.prompt_tps || 0,
generation_tps: entry.generation_tps || 0,
backend: entry.backend || null,
gpu_layers: entry.gpu_layers || 0,
context_size: entry.context_size || 0,
max_output_tokens_used: entry.max_output_tokens_used ?? entry.max_output_tokens ?? 0,
frontend_soft_timeout: Boolean(entry.frontend_soft_timeout),
total_ms: totalMs,
risk_504: totalMs >= 45000
});
summary.slow_requests = summary.slow_requests.slice(0, 25);
}
fs.writeFileSync(stateFile(), JSON.stringify(summary, null, 2));
fs.appendFileSync(historyFile(), `${JSON.stringify({ timestamp:new Date().toISOString(), ...entry })}\n`);
}
function report() {
return summarizeMetrics(getSummary());
}
function summarizeMetrics(s = {}) {
const sorted = (Array.isArray(s.durations) ? s.durations : [])
.filter(isValidTiming)
.map(Number)
.sort((a,b)=>a-b);
const average_stage_ms = Object.fromEntries(
Object.entries(s.stage_totals || {})
.filter(([, value]) => isValidTiming(value))
.map(([key, value]) => {
const legacyCount = Math.max(0, Number(s.stage_samples) || 0);
const count = Math.max(0, Number(s.stage_counts?.[key]) || legacyCount);
return [key, count ? Math.max(0, Math.round(Number(value) / count)) : 0];
})
);
return {
...s,
durations: sorted,
average_stage_ms,
average_response_ms: sorted.length ? Math.max(0, Math.round(sorted.reduce((a,b)=>a+b,0)/sorted.length)) : 0,
median_response_ms: sorted.length ? Math.max(0, sorted[Math.floor(sorted.length/2)]) : 0
};
}
function history(limit=100) {
try {
return fs.readFileSync(historyFile(),"utf8").trim().split(/\r?\n/)
.filter(Boolean)
.map(parseHistoryRow)
.filter(Boolean)
.slice(-limit)
.reverse();
} catch { return []; }
}
function historyPage(page = 1, pageSize = 25) {
const safePage = Math.max(1, Number.parseInt(page, 10) || 1);
const safeSize = Math.max(1, Math.min(100, Number.parseInt(pageSize, 10) || 25));
try {
const rows = fs.readFileSync(historyFile(), "utf8").trim().split(/\r?\n/)
.filter(Boolean)
.map(parseHistoryRow)
.filter(Boolean);
return paginateRows(rows, safePage, safeSize);
} catch {
return { entries: [], page: 1, pages: 1, page_size: safeSize, total: 0 };
}
}
function workHistoryPage(query = {}, page = 1, pageSize = 20) {
const rows = readHistoryRows();
const filters = normalizeWorkFilters(query);
const grouped = groupWorkRows(rows)
.filter((work) => matchesWorkFilters(work, filters));
return paginateRows(grouped, page, pageSize);
}
function slowRequestsPage(page = 1, pageSize = 15) {
const safePage = Math.max(1, Number.parseInt(page, 10) || 1);
const safeSize = Math.max(1, Math.min(100, Number.parseInt(pageSize, 10) || 15));
try {
const rows = fs.readFileSync(historyFile(), "utf8").trim().split(/\r?\n/)
.filter(Boolean)
.map(parseHistoryRow)
.filter((entry) => {
const total = entry?.total_ms ?? entry?.duration_ms;
return entry?.kind === "request" && isValidTiming(total) && Number(total) >= 30000;
})
.map(normalizeSlowEntry);
return paginateRows(rows, safePage, safeSize);
} catch {
return { entries: [], page: 1, pages: 1, page_size: safeSize, total: 0 };
}
}
function configureRetention(value = {}) {
retention = normalizeRetention(value);
applyRetention();
}
function applyRetention() {
const rows = readHistoryRows();
if (!rows.length) return;
const keepIds = retainedWorkIds(groupWorkRows(rows), retention);
if (!keepIds) return;
const kept = rows.filter((row) => {
const id = row.request_id || row.work_id;
return !id || keepIds.has(id);
});
if (kept.length === rows.length) return;
fs.writeFileSync(historyFile(), kept.map((row) => JSON.stringify(row)).join("\n") + (kept.length ? "\n" : ""));
}
function maybeApplyRetention() {
if (Math.random() > 0.02) return;
try { applyRetention(); } catch {}
}
function readHistoryRows() {
try {
return fs.readFileSync(historyFile(), "utf8").trim().split(/\r?\n/)
.filter(Boolean)
.map(parseHistoryRow)
.filter(Boolean);
} catch {
return [];
}
}
function groupWorkRows(rows) {
const groups = new Map();
for (const row of rows) {
const id = row.request_id || row.work_id;
if (!id) continue;
if (!groups.has(id)) {
groups.set(id, {
work_id: id,
started_at: row.timestamp,
finished_at: row.timestamp,
status: "success",
source: row.origin || row.platform || "webui",
user_id: row.user_id || "",
role: row.role || "",
internal_mode: row.controller_complexity || "",
okf_retrieval: row.okf_retrieval_depth || "",
reason_code: row.controller_reason_code || row.gate_reason_code || row.reason_code || "",
processed_tokens: 0,
final_tokens: 0,
delivered_tokens: 0,
duration_ms: 0,
prompt: "",
prompt_tokens: 0,
has_error: false,
has_refusal: false,
has_fallback: false,
has_truncation: false,
has_okf_context: false,
events: []
});
}
const group = groups.get(id);
group.started_at = earlierIso(group.started_at, row.timestamp);
group.finished_at = laterIso(group.finished_at, row.timestamp);
group.source = row.origin || row.platform || group.source;
group.user_id = row.user_id || group.user_id;
group.role = row.role || group.role;
group.internal_mode = row.controller_complexity || group.internal_mode;
group.okf_retrieval = row.okf_retrieval_depth || group.okf_retrieval;
group.reason_code = row.controller_reason_code || row.gate_reason_code || row.reason_code || group.reason_code;
group.has_error ||= row.status === "failed" || row.kind === "error" || Boolean(row.error_code);
group.has_refusal ||= row.status === "refused" || row.route_used === "refusal";
group.has_fallback ||= Boolean(row.controller_fallback_used || row.fallback_reason || row.rejected_reason);
group.has_truncation ||= Boolean(row.truncated);
group.has_okf_context ||= row.kind === "okf_retrieval" || row.okf_retrieval_depth === "light" || row.okf_retrieval_depth === "deep" || Number(row.okf_match_count) > 0;
if (row.kind === "prompt") {
group.prompt = row.prompt || group.prompt;
group.prompt_tokens = validOrZero(row.prompt_tokens);
}
const promptTokens = validOrZero(row.prompt_tokens);
const generatedTokens = validOrZero(row.generated_tokens);
group.processed_tokens += promptTokens + generatedTokens;
if (row.final_tokens != null) group.final_tokens = validOrZero(row.final_tokens);
else if (row.final_reply_length != null || row.original_final_length != null) {
group.final_tokens = estimateTokensFromChars(row.final_reply_length ?? row.original_final_length);
}
if (row.delivered_tokens != null) group.delivered_tokens = validOrZero(row.delivered_tokens);
else if (row.delivered_length != null) group.delivered_tokens = estimateTokensFromChars(row.delivered_length);
group.duration_ms = Math.max(group.duration_ms, validOrZero(row.total_ms ?? row.duration_ms));
group.events.push(workEvent(row));
}
return [...groups.values()]
.map((group) => {
if (!group.processed_tokens) group.processed_tokens = group.prompt_tokens + group.final_tokens;
if (group.has_error && !group.delivered_tokens) group.status = "failed";
else if (group.has_error || group.has_refusal || group.has_fallback || group.has_truncation) group.status = "partial";
else group.status = "success";
group.events.sort((left, right) => String(left.timestamp).localeCompare(String(right.timestamp)));
return group;
})
.sort((left, right) => String(left.started_at).localeCompare(String(right.started_at)));
}
function workEvent(row) {
return {
timestamp: row.timestamp,
type: normalizeWorkEventType(row),
status: row.status || "",
summary: workEventSummary(row),
prompt: row.kind === "prompt" ? row.prompt || "" : "",
tokens: validOrZero(row.prompt_tokens) + validOrZero(row.generated_tokens),
data: compactWorkData(row)
};
}
function normalizeWorkEventType(row) {
if (row.kind === "prompt") return "prompt";
if (row.kind === "controller_decision") return "controller";
if (row.kind === "gate_summary" || row.kind === "gate_decision") return "controller";
if (row.kind === "okf_retrieval") return "okf_retrieval";
if (row.kind === "prompt_build") return "prompt_build";
if (row.kind === "model_request") return "model_request";
if (row.kind === "request") return row.status === "failed" ? "error" : "model_response";
if (row.kind === "delivery") return row.truncated ? "source_limit" : "delivery";
if (row.kind === "tool_exposure") return "context";
return row.kind || "metrics";
}
function workEventSummary(row) {
if (row.kind === "prompt") return `Prompt from ${row.origin || row.platform || "webui"} as ${row.role || "unknown role"}`;
if (row.kind === "controller_decision") {
return `Controller selected ${row.controller_complexity || "-"} / ${row.okf_retrieval_depth || "-"} (${row.controller_reason_code || row.reason_code || "no reason"})`;
}
if (row.kind === "gate_summary" || row.kind === "gate_decision") {
return `Gate route ${row.route_used || "-"} (${row.reason_code || "no reason"})`;
}
if (row.kind === "okf_retrieval") return `OKF retrieval ${row.status || "success"} with ${row.okf_match_count || 0} match(es)`;
if (row.kind === "prompt_build") return `Prompt built with ${row.context_block_count || 0} context block(s)`;
if (row.kind === "model_request") return `Main model request in ${row.controller_complexity || "dynamic"} mode`;
if (row.kind === "delivery") return row.truncated ? "Delivered after source limit truncation" : "Delivered reply";
if (row.kind === "request" && row.status === "failed") return row.message || row.error_code || "Request failed";
return row.reason_code || row.controller_reason_code || row.status || row.kind || "Event";
}
function compactWorkData(row) {
const keys = [
"route_used", "route_class", "reason_code", "gate_reason_code", "controller_intent",
"controller_complexity", "okf_retrieval_depth", "answer_style", "source_profile",
"controller_confidence", "okf_match_count", "context_block_count", "prompt_tokens",
"generated_tokens", "final_tokens", "delivered_tokens", "final_reply_length",
"delivered_length", "truncated", "delivery_action", "fallback_reason", "rejected_reason", "error_code"
];
return Object.fromEntries(keys.filter((key) => row[key] != null).map((key) => [key, row[key]]));
}
function normalizeWorkFilters(query = {}) {
return {
q: String(query.work_q || query.q || "").trim().toLowerCase(),
status: String(query.work_status || "").trim(),
source: String(query.work_source || "").trim(),
role: String(query.work_role || "").trim(),
mode: String(query.work_mode || "").trim(),
okf: String(query.work_okf || "").trim(),
flag: String(query.work_flag || "").trim(),
from: parseTime(query.work_from),
to: parseTime(query.work_to, true)
};
}
function matchesWorkFilters(work, filters) {
if (filters.status && work.status !== filters.status) return false;
if (filters.source && work.source !== filters.source) return false;
if (filters.role && work.role !== filters.role) return false;
if (filters.mode && work.internal_mode !== filters.mode) return false;
if (filters.okf && work.okf_retrieval !== filters.okf) return false;
if (filters.flag === "error" && !work.has_error) return false;
if (filters.flag === "refusal" && !work.has_refusal) return false;
if (filters.flag === "fallback" && !work.has_fallback) return false;
if (filters.flag === "truncation" && !work.has_truncation) return false;
if (filters.flag === "okf" && !work.has_okf_context) return false;
const started = Date.parse(work.started_at);
if (filters.from && started < filters.from) return false;
if (filters.to && started > filters.to) return false;
if (!filters.q) return true;
const haystack = [
work.work_id, work.prompt, work.source, work.user_id, work.role, work.status, work.internal_mode,
work.okf_retrieval, work.reason_code,
...work.events.map((event) => `${event.type} ${event.status} ${event.summary} ${JSON.stringify(event.data)}`)
].join("\n").toLowerCase();
return haystack.includes(filters.q);
}
function normalizeRetention(value = {}) {
const mode = value.mode === "age" ? "age" : "count";
return {
mode,
count: Math.max(50, Math.min(10000, Number.parseInt(value.count, 10) || 500)),
age_value: Math.max(1, Math.min(1000, Number.parseInt(value.age_value, 10) || 30)),
age_unit: ["hours", "days", "weeks", "months", "years"].includes(value.age_unit) ? value.age_unit : "days"
};
}
function retainedWorkIds(groups, config) {
if (config.mode === "count") {
return new Set(groups.slice(-config.count).map((group) => group.work_id));
}
const cutoff = Date.now() - ageMs(config.age_value, config.age_unit);
return new Set(groups.filter((group) => Date.parse(group.started_at) >= cutoff).map((group) => group.work_id));
}
function ageMs(value, unit) {
const hours = unit === "hours" ? value
: unit === "weeks" ? value * 24 * 7
: unit === "months" ? value * 24 * 30
: unit === "years" ? value * 24 * 365
: value * 24;
return hours * 60 * 60 * 1000;
}
function parseTime(value, endOfDay = false) {
if (!value) return null;
const date = new Date(value);
if (Number.isNaN(date.getTime())) return null;
if (endOfDay && /^\d{4}-\d{2}-\d{2}$/.test(String(value))) date.setHours(23, 59, 59, 999);
return date.getTime();
}
function earlierIso(left, right) {
return Date.parse(left) <= Date.parse(right) ? left : right;
}
function laterIso(left, right) {
return Date.parse(left) >= Date.parse(right) ? left : right;
}
function estimateTokensFromChars(value) {
const number = Number(value);
return Number.isFinite(number) && number > 0 ? Math.max(1, Math.ceil(number / 4)) : 0;
}
function paginateRows(rows, page = 1, pageSize = 25, map = (value) => value) {
const total = rows.length;
const pages = Math.max(1, Math.ceil(total / pageSize));
const current = Math.min(Math.max(1, page), pages);
const end = total - (current - 1) * pageSize;
const start = Math.max(0, end - pageSize);
return {
entries: rows.slice(start, end).reverse().map(map),
page: current,
pages,
page_size: pageSize,
total
};
}
function parseHistoryRow(value) {
try { return JSON.parse(value); }
catch { return null; }
}
function normalizeSlowEntry(entry) {
const totalMs = Number(entry.total_ms ?? entry.duration_ms);
return {
...entry,
total_ms: totalMs,
queue_ms: validOrZero(entry.queue_ms ?? entry.main_queue_ms),
prompt_eval_ms: validOrZero(entry.prompt_eval_ms),
generation_ms: validOrZero(entry.generation_ms ?? entry.main_generate_ms),
gate_ms: validOrZero(entry.gate_ms),
prompt_tokens: validOrZero(entry.prompt_tokens),
generated_tokens: validOrZero(entry.generated_tokens),
prompt_tps: validOrZero(entry.prompt_tps),
generation_tps: validOrZero(entry.generation_tps),
max_output_tokens_used: validOrZero(entry.max_output_tokens_used ?? entry.max_output_tokens),
risk_504: totalMs >= 45000
};
}
function isValidTiming(value) {
const number = Number(value);
return Number.isFinite(number) && number >= 0;
}
function validOrZero(value) {
return isValidTiming(value) ? Number(value) : 0;
}
module.exports = {
configureRetention,
applyRetention,
record,
report,
history,
historyPage,
workHistoryPage,
slowRequestsPage,
paginateRows,
isValidTiming,
summarizeMetrics
};