Lumi/plugins/okf/backend/generate_knowledge.js
2026-06-25 14:10:04 +02:00

520 lines
22 KiB
JavaScript

const fs = require("fs");
const path = require("path");
const { ensureKnowledgeDirs, knowledgeRoot } = require("./file_knowledge");
function generateKnowledgeFiles(rootDir = process.cwd()) {
const root = ensureKnowledgeDirs(rootDir);
const generatedAt = new Date().toISOString();
const written = [];
const coreFile = path.join(root, "core", "lumi-core.md");
const coreMarkdown = buildCoreKnowledge(rootDir, generatedAt);
if (writeGeneratedFile(coreFile, coreMarkdown)) {
written.push(path.relative(rootDir, coreFile));
}
for (const plugin of discoverPlugins(rootDir)) {
const pluginFile = path.join(root, "plugins", `${slugify(plugin.id)}.md`);
const pluginMarkdown = buildPluginKnowledge(plugin, generatedAt);
if (writeGeneratedFile(pluginFile, pluginMarkdown)) {
written.push(path.relative(rootDir, pluginFile));
}
}
return written;
}
function buildCoreKnowledge(rootDir, generatedAt) {
const packageJson = readJson(path.join(rootDir, "package.json"));
const readme = readText(path.join(rootDir, "README.md"));
const routes = discoverCoreRoutes(rootDir);
const routeReference = buildRouteReference(routes);
return frontmatter({
id: "core.lumi",
title: packageJson.name || "Lumi Core",
scope: "core",
status: "active",
priority: 20,
visibility: "user",
category: "Core",
tags: "core, routes, commands, settings",
generated: true,
editable: false,
updated_at: generatedAt
}) + [
`# ${packageJson.name || "Lumi Core"}`,
"",
packageJson.description || "Lumi is the core web UI and bot runtime.",
"",
"## Runtime",
"",
`Package: ${packageJson.name || "unknown"}`,
packageJson.version ? `Version: ${packageJson.version}` : "",
packageJson.main ? `Entry point: ${packageJson.main}` : "",
"",
"## Routes",
"",
routes.length ? routes.map((route) => `- ${route.method.toUpperCase()} ${route.path}`).join("\n") : "- No core routes detected.",
"",
"## Route Reference",
"",
routeReference || "No route details detected.",
"",
"## README Summary",
"",
excerpt(readme, 1800) || "No README summary available."
].filter((line) => line !== "").join("\n");
}
function buildPluginKnowledge(plugin, generatedAt) {
const routeLines = plugin.routes.length
? plugin.routes.map((route) => `- ${route.method ? `${route.method.toUpperCase()} ` : ""}${route.path}`).join("\n")
: "- No plugin routes detected.";
const routeReference = buildRouteReference(plugin.routes);
const commandLines = plugin.commands.length
? plugin.commands.map((command) => `- ${command}`).join("\n")
: "- No plugin command triggers detected.";
return frontmatter({
id: `plugin.${plugin.id}`,
title: plugin.name || plugin.id,
scope: "plugins",
status: "active",
priority: 10,
visibility: "user",
category: "Plugin",
tags: ["plugin", plugin.id, ...(plugin.keywords || [])].join(", "),
generated: true,
editable: false,
updated_at: generatedAt
}) + [
`# ${plugin.name || plugin.id}`,
"",
plugin.description || "No plugin description is available.",
"",
"## Metadata",
"",
`Plugin ID: ${plugin.id}`,
plugin.version ? `Version: ${plugin.version}` : "",
plugin.author ? `Author: ${plugin.author}` : "",
plugin.enabled === false ? "Default state: disabled" : "Default state: enabled",
"",
"## Web Routes",
"",
routeLines,
"",
"## Route Reference",
"",
routeReference || "No route details detected.",
"",
"## Commands",
"",
commandLines,
"",
"## Source",
"",
`Plugin folder: plugins/${plugin.folder}`
].filter((line) => line !== "").join("\n");
}
function discoverPlugins(rootDir) {
const pluginsDir = path.join(rootDir, "plugins");
if (!fs.existsSync(pluginsDir)) return [];
return fs.readdirSync(pluginsDir, { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.map((entry) => {
const dir = path.join(pluginsDir, entry.name);
const manifest = readJson(path.join(dir, "plugin.json"));
if (!manifest.id && !manifest.name) return null;
const source = readText(path.join(dir, "index.js"));
return {
id: cleanText(manifest.id || entry.name, 120),
folder: entry.name,
name: cleanText(manifest.name || manifest.label || manifest.id || entry.name, 180),
version: cleanText(manifest.version, 80),
description: cleanText(manifest.description, 800),
author: cleanText(manifest.author, 180),
enabled: manifest.enabled,
keywords: Array.isArray(manifest.keywords) ? manifest.keywords.map((item) => cleanText(item, 80)).filter(Boolean) : [],
routes: discoverPluginRoutes(source, manifest.id || entry.name),
commands: discoverCommandTriggers(source)
};
})
.filter(Boolean)
.sort((a, b) => a.id.localeCompare(b.id));
}
function discoverCoreRoutes(rootDir) {
const server = readText(path.join(rootDir, "src", "web", "server.js"));
const routes = [];
for (const match of server.matchAll(/\bapp\.(get|post|put|patch|delete)\(\s*["'`]([^"'`]+)["'`]/g)) {
const source = extractCallExpression(server, match.index);
routes.push({
method: match[1],
path: match[2],
details: analyzeRouteSource(source, match[1], match[2])
});
}
return uniqueRoutes(routes).slice(0, 200);
}
function discoverPluginRoutes(source, pluginId) {
const routes = [];
const mountPaths = discoverMountPaths(source, pluginId);
for (const match of source.matchAll(/\bweb\.mount\(\s*["'`]([^"'`]+)["'`]/g)) {
routes.push({
method: "",
path: normalizeTemplatePath(match[1], pluginId),
details: {
purpose: "Mounts the plugin router at this base WebUI path.",
inputs: [],
response: "Plugin router mount point.",
access: "Access is controlled by the mount options and individual plugin routes.",
sideEffects: "No direct route action; child routes handle requests.",
limits: "Mount metadata is inferred from static source scanning."
}
});
}
for (const match of source.matchAll(/\brouter\.(get|post|put|patch|delete)\(\s*["'`]([^"'`]+)["'`]/g)) {
const sourceSnippet = extractCallExpression(source, match.index);
const routePath = normalizeTemplatePath(match[2], pluginId);
const bases = mountPaths.length ? mountPaths : [`/plugins/${pluginId}`];
for (const base of bases) {
const fullPath = joinRoutePath(base, routePath);
routes.push({
method: match[1],
path: fullPath,
details: analyzeRouteSource(sourceSnippet, match[1], fullPath)
});
}
}
return uniqueRoutes(routes).slice(0, 100);
}
function discoverMountPaths(source, pluginId) {
const paths = [];
for (const match of source.matchAll(/\bweb\.mount\(\s*["'`]([^"'`]+)["'`]/g)) {
paths.push(normalizeTemplatePath(match[1], pluginId));
}
return Array.from(new Set(paths)).filter(Boolean);
}
function normalizeTemplatePath(value, pluginId) {
return String(value || "").replace(/\$\{PLUGIN_ID\}/g, pluginId);
}
function joinRoutePath(base, child) {
const normalizedBase = String(base || "").replace(/\/+$/, "") || "/";
const normalizedChild = String(child || "").replace(/^\/+/, "");
if (!normalizedChild) return normalizedBase;
if (normalizedChild === "/") return normalizedBase;
return `${normalizedBase}/${normalizedChild}`.replace(/\/+/g, "/");
}
function discoverCommandTriggers(source) {
const triggers = new Set();
for (const match of source.matchAll(/\btrigger\s*:\s*["'`]([^"'`]+)["'`]/g)) {
triggers.add(match[1]);
}
for (const match of source.matchAll(/\bcommand\s*:\s*["'`]([^"'`]+)["'`]/g)) {
triggers.add(match[1]);
}
return Array.from(triggers).sort().slice(0, 100);
}
function writeGeneratedFile(filePath, content) {
if (!canOverwriteGeneratedFile(filePath)) return false;
fs.mkdirSync(path.dirname(filePath), { recursive: true });
fs.writeFileSync(filePath, `${content.trim()}\n`);
return true;
}
function canOverwriteGeneratedFile(filePath) {
if (!fs.existsSync(filePath)) return true;
const text = readText(filePath);
if (/generated\s*:\s*false/i.test(text)) return false;
if (/editable\s*:\s*true/i.test(text)) return false;
return true;
}
function uniqueRoutes(routes) {
const seen = new Set();
return routes.filter((route) => {
const key = `${route.method}:${route.path}`;
if (seen.has(key)) return false;
seen.add(key);
return true;
});
}
function buildRouteReference(routes) {
if (!routes.length) return "";
return routes.map((route) => {
const details = route.details || analyzeRouteSource("", route.method, route.path);
return [
`### ${route.method ? route.method.toUpperCase() : "MOUNT"} ${route.path}`,
"",
`- Purpose: ${details.purpose}`,
`- Inputs: ${details.inputs.length ? details.inputs.join("; ") : "No request parameters detected by static analysis."}`,
`- Response format: ${details.response}`,
`- Access: ${details.access}`,
`- Side effects: ${details.sideEffects}`,
`- Limits/notes: ${details.limits}`
].join("\n");
}).join("\n\n");
}
function analyzeRouteSource(source, method, routePath) {
const bodyFields = findRequestFields(source, "body");
const queryFields = findRequestFields(source, "query");
const paramFields = Array.from(new Set([
...extractPathParams(routePath),
...findRequestFields(source, "params")
])).sort();
const inputs = [];
if (paramFields.length) inputs.push(`path params: ${paramFields.map((field) => `\`${field}\``).join(", ")}`);
if (queryFields.length) inputs.push(`query: ${queryFields.map((field) => `\`${field}\``).join(", ")}`);
if (bodyFields.length) inputs.push(`body: ${bodyFields.map((field) => `\`${field}\``).join(", ")}`);
if (/\breq\.file\b|\breq\.files\b|\bupload\.(single|array|fields)\(/.test(source)) {
inputs.push("file upload: multipart form file data");
}
if (!bodyFields.length && /\breq\.body\b/.test(source)) {
inputs.push("body: full submitted body is passed to a helper; exact fields are defined by the matching form/service");
}
if (!queryFields.length && /\breq\.query\b/.test(source)) {
inputs.push("query: full query object is passed to a helper; exact fields are defined by the matching view/service");
}
if (!paramFields.length && /\breq\.params\b/.test(source)) {
inputs.push("path params: full params object is used; exact keys come from the route path");
}
const response = describeResponseFormat(source, method, routePath);
return {
purpose: describeRoutePurpose(source, method, routePath, response),
inputs,
response,
access: describeAccess(source, routePath),
sideEffects: describeSideEffects(source, method, bodyFields),
limits: describeRouteLimits(source, method, routePath)
};
}
function findRequestFields(source, bucket) {
const fields = new Set();
const dotPattern = new RegExp(`\\breq\\.${bucket}\\.([A-Za-z_$][A-Za-z0-9_$]*)`, "g");
for (const match of source.matchAll(dotPattern)) {
fields.add(match[1]);
}
const bracketPattern = new RegExp(`\\breq\\.${bucket}\\s*\\[\\s*["'\`]([^"'\`]+)["'\`]\\s*\\]`, "g");
for (const match of source.matchAll(bracketPattern)) {
fields.add(match[1]);
}
return Array.from(fields).sort().slice(0, 40);
}
function extractPathParams(routePath) {
const fields = new Set();
for (const match of String(routePath || "").matchAll(/:([A-Za-z0-9_]+)/g)) {
fields.add(match[1]);
}
return Array.from(fields);
}
function describeResponseFormat(source, method, routePath) {
const formats = [];
if (/\bres\.render\(/.test(source)) formats.push("HTML page rendered from an EJS view");
if (/\brenderAdmin\(/.test(source)) formats.push("HTML page rendered by a helper");
if (/\bres\.json\(/.test(source)) formats.push("JSON response");
if (/\bres\.redirect\(/.test(source)) formats.push("HTTP redirect after handling the request");
if (/\bres\.download\(/.test(source)) formats.push("file download");
if (/\bres\.sendFile\(/.test(source)) formats.push("static file response");
if (/\bres\.write\(/.test(source) || /text\/event-stream|subscribeWebEvents/.test(source)) formats.push("streaming event response");
if (/\bres\.send\(/.test(source) && !formats.length) formats.push("plain or HTML response");
if (!formats.length && !method) return "Router mount; child routes provide concrete responses.";
if (!formats.length && String(routePath).startsWith("/api/")) return "API response; exact schema was not detected statically.";
if (!formats.length && method === "get") return "HTML or data response; exact format was not detected statically.";
if (!formats.length) return "Form/action response; exact format was not detected statically.";
return Array.from(new Set(formats)).join("; ");
}
function describeRoutePurpose(source, method, routePath, response) {
const pathText = String(routePath || "");
const normalized = pathText.replace(/[:/._-]+/g, " ").trim();
if (!method) return "Registers a plugin route namespace.";
if (/\/api\/events/.test(pathText)) return "Streams live WebUI event notifications to the browser.";
const pluginAdminMatch = pathText.match(/^\/plugins\/([^/]+)\/admin(?:\/(.+))?$/);
if (pluginAdminMatch) {
const pluginId = pluginAdminMatch[1];
const action = pluginAdminMatch[2] ? pluginAdminMatch[2].replace(/[:/._-]+/g, " ").trim() : "";
if (method === "get") return `Renders the ${pluginId} plugin administration page${action ? ` for ${action}` : ""}.`;
return `Processes the ${pluginId} plugin administration action${action ? ` for ${action}` : ""}.`;
}
const pluginPageMatch = pathText.match(/^\/plugins\/([^/]+)(?:\/(.+))?$/);
if (pluginPageMatch) {
const pluginId = pluginPageMatch[1];
const action = pluginPageMatch[2] ? pluginPageMatch[2].replace(/[:/._-]+/g, " ").trim() : "";
if (method === "get") return `Renders or serves the ${pluginId} plugin page${action ? ` for ${action}` : ""}.`;
return `Processes the ${pluginId} plugin action${action ? ` for ${action}` : ""}.`;
}
if (/\/api\/users\/search/.test(pathText)) return "Searches known linked Lumi users for shared user-lookup fields.";
if (/\/api\/placeholders\/catalog/.test(pathText)) return "Returns placeholder suggestions available to the current user and field policy.";
if (/\/api\/placeholders\/preview/.test(pathText)) return "Previews placeholder rendering for a submitted template without saving it.";
if (/\/feedback/.test(pathText) && method === "post") return "Creates, updates, comments on, exports, or manages feedback records depending on the action path.";
if (/\/admin\/updates/.test(pathText)) return "Checks, applies, reverts, or reports update state for core or plugin updates.";
if (/\/admin\/settings/.test(pathText)) return method === "get" ? "Renders the core settings page." : "Saves core settings from the admin settings form.";
if (/\/auth\//.test(pathText)) return "Starts, completes, or cancels a platform authentication/linking flow.";
if (/\/setup/.test(pathText)) return "Guides first-time platform setup and verification.";
if (/\/commands/.test(pathText)) return method === "get" ? "Displays configured bot commands." : "Creates, updates, previews, toggles, or deletes custom commands.";
if (/^\/admin\/plugins/.test(pathText)) return "Manages installed plugins, plugin uploads, installs, updates, or plugin enablement.";
if (/\/plugins/.test(pathText)) return "Displays or handles plugin-specific WebUI functionality.";
if (/\/health/.test(pathText)) return "Returns runtime health information.";
if (/\/logs/.test(pathText)) return "Displays, downloads, or manages application logs.";
if (/\/stats/.test(pathText)) return "Displays user or community statistics.";
if (/\/leaderboards/.test(pathText)) return "Displays leaderboard data.";
if (/\bres\.render\(/.test(source)) return `Renders the ${normalized || "requested"} WebUI page.`;
if (/\bres\.json\(/.test(source)) return `Provides ${normalized || "route"} data as JSON.`;
if (method === "post") return `Processes the ${normalized || "route"} action and stores or applies submitted form data.`;
return `Handles ${normalized || "this route"}.`;
}
function describeAccess(source, routePath) {
const signals = [];
if (/\brequireOkf(Edit|Management)\b/.test(source)) {
signals.push("OKF editor or manager permission required");
}
if (/\brequireOkfReview\b|\brequireOkfImplement\b/.test(source)) {
signals.push("higher OKF review or implementation permission may be required");
}
if (/\brequireAdmin\b|\.isAdmin\b|hasAccess\([^)]*admin/.test(source) || /\/admin(\/|$)/.test(routePath)) {
signals.push("admin access expected");
}
if (/\brequireMod\b|\.isMod\b|hasAccess\([^)]*mod/.test(source) || /\/moderator(\/|$)/.test(routePath)) {
signals.push("moderator access may be required");
}
if (/\brequire(Login|Auth)\b|\breq\.session\.user\b|authRequired/.test(source)) {
signals.push("logged-in session required or used");
}
if (/\bcanAccess\b/.test(source)) {
signals.push("mount-level canAccess predicate applies");
}
if (!signals.length && /^\/api\//.test(routePath)) return "API route; access requirements were not fully detected by static analysis.";
if (!signals.length) return "No explicit access guard detected in the route handler; check surrounding router/mount middleware.";
return Array.from(new Set(signals)).join("; ");
}
function describeSideEffects(source, method, bodyFields) {
const effects = [];
if (/\bset[A-Z]|\bsetSetting\b|\bsave[A-Z]|\bupdate[A-Z]|\bcreate[A-Z]|\bdelete[A-Z]|\bremove[A-Z]|\binstall[A-Z]|\bapply[A-Z]|\brevert[A-Z]|\brestore[A-Z]|\bgrant[A-Z]|\brevoke[A-Z]/.test(source)) {
effects.push("writes or mutates server-side state");
}
if (/\bfs\.(writeFileSync|rmSync|mkdirSync|renameSync|copyFileSync)\b/.test(source)) effects.push("writes files");
if (/\bdb\.(prepare|exec)\b|\.run\(/.test(source)) effects.push("writes database state when the called service mutates data");
if (/\bpublishWebEvent\b|subscribeWebEvents/.test(source)) effects.push("publishes or streams live WebUI events");
if (/\brequestRestart\b|process\.exit/.test(source)) effects.push("may restart or stop runtime processes");
if (method === "get" && !effects.length) return "Usually read-only.";
if (method === "post" && !effects.length && bodyFields.length) return "Consumes submitted data; state mutation happens in called helpers if present.";
if (method === "post" && !effects.length) return "Action route; side effects were not detected statically.";
return effects.length ? Array.from(new Set(effects)).join("; ") : "No side effects detected statically.";
}
function describeRouteLimits(source, method, routePath) {
const notes = ["Generated from static route source analysis; confirm exact behavior in the handler before changing integrations."];
if (/\bcleanText\b|slice\(0,\s*\d+|limitFromBody|boundedInt|boundedNumber|clampInt/.test(source)) {
notes.push("Input length or numeric bounds are enforced by helper functions in the handler.");
}
if (/\btry\b[\s\S]*\bcatch\b/.test(source)) {
notes.push("Errors are caught and usually returned as a flash message, JSON error, or error page.");
}
if (method === "post" && !/^\/api\//.test(routePath)) {
notes.push("Most non-API POST routes are browser form submissions and usually redirect after completion.");
}
if (/\/api\//.test(routePath)) {
notes.push("API consumers should expect JSON unless the response format says otherwise.");
}
return notes.join(" ");
}
function extractCallExpression(source, startIndex) {
const start = source.indexOf("(", startIndex);
if (start === -1) return source.slice(startIndex, startIndex + 1200);
let depth = 0;
let quote = "";
let escaped = false;
for (let index = start; index < source.length; index += 1) {
const char = source[index];
if (quote) {
if (escaped) {
escaped = false;
} else if (char === "\\") {
escaped = true;
} else if (char === quote) {
quote = "";
}
continue;
}
if (char === "\"" || char === "'" || char === "`") {
quote = char;
continue;
}
if (char === "(") {
depth += 1;
} else if (char === ")") {
depth -= 1;
if (depth === 0) {
return source.slice(startIndex, index + 1);
}
}
}
return source.slice(startIndex, startIndex + 4000);
}
function frontmatter(values) {
const lines = ["---"];
for (const [key, value] of Object.entries(values)) {
if (value === "" || value === null || value === undefined) continue;
lines.push(`${key}: ${frontmatterValue(value)}`);
}
lines.push("---", "");
return lines.join("\n");
}
function frontmatterValue(value) {
if (typeof value === "boolean" || typeof value === "number") return String(value);
const text = String(value).replace(/\n/g, " ");
return /[:#[\]{}"'\\]|^\s|\s$/.test(text) ? JSON.stringify(text) : text;
}
function readJson(filePath) {
try {
return JSON.parse(fs.readFileSync(filePath, "utf8"));
} catch {
return {};
}
}
function readText(filePath) {
try {
return fs.readFileSync(filePath, "utf8");
} catch {
return "";
}
}
function excerpt(value, limit) {
return cleanText(value, limit).split("\n").filter((line) => line.trim()).slice(0, 12).join("\n");
}
function slugify(value) {
return cleanText(value, 180)
.toLowerCase()
.replace(/[^a-z0-9]+/g, "-")
.replace(/^-+|-+$/g, "") || "entry";
}
function cleanText(value, maximum = 4000) {
return String(value || "").replace(/\r\n?/g, "\n").trim().slice(0, maximum);
}
module.exports = {
buildCoreKnowledge,
buildPluginKnowledge,
discoverPlugins,
generateKnowledgeFiles
};