const fs = require("fs"); const path = require("path"); const { ensureKnowledgeDirs, knowledgeRoot } = require("./file_knowledge"); function generateKnowledgeFiles(rootDir = process.cwd()) { const root = ensureKnowledgeDirs(rootDir); const generatedAt = new Date().toISOString(); const written = []; const coreFile = path.join(root, "core", "lumi-core.md"); const coreMarkdown = buildCoreKnowledge(rootDir, generatedAt); if (writeGeneratedFile(coreFile, coreMarkdown)) { written.push(path.relative(rootDir, coreFile)); } for (const plugin of discoverPlugins(rootDir)) { const pluginFile = path.join(root, "plugins", `${slugify(plugin.id)}.md`); const pluginMarkdown = buildPluginKnowledge(plugin, generatedAt); if (writeGeneratedFile(pluginFile, pluginMarkdown)) { written.push(path.relative(rootDir, pluginFile)); } } return written; } function buildCoreKnowledge(rootDir, generatedAt) { const packageJson = readJson(path.join(rootDir, "package.json")); const readme = readText(path.join(rootDir, "README.md")); const routes = discoverCoreRoutes(rootDir); const routeReference = buildRouteReference(routes); return frontmatter({ id: "core.lumi", title: packageJson.name || "Lumi Core", scope: "core", status: "active", priority: 20, visibility: "user", category: "Core", tags: "core, routes, commands, settings", generated: true, editable: false, updated_at: generatedAt }) + [ `# ${packageJson.name || "Lumi Core"}`, "", packageJson.description || "Lumi is the core web UI and bot runtime.", "", "## Runtime", "", `Package: ${packageJson.name || "unknown"}`, packageJson.version ? `Version: ${packageJson.version}` : "", packageJson.main ? `Entry point: ${packageJson.main}` : "", "", "## Routes", "", routes.length ? routes.map((route) => `- ${route.method.toUpperCase()} ${route.path}`).join("\n") : "- No core routes detected.", "", "## Route Reference", "", routeReference || "No route details detected.", "", "## README Summary", "", excerpt(readme, 1800) || "No README summary available." ].filter((line) => line !== "").join("\n"); } function buildPluginKnowledge(plugin, generatedAt) { const routeLines = plugin.routes.length ? plugin.routes.map((route) => `- ${route.method ? `${route.method.toUpperCase()} ` : ""}${route.path}`).join("\n") : "- No plugin routes detected."; const routeReference = buildRouteReference(plugin.routes); const commandLines = plugin.commands.length ? plugin.commands.map((command) => `- ${command}`).join("\n") : "- No plugin command triggers detected."; return frontmatter({ id: `plugin.${plugin.id}`, title: plugin.name || plugin.id, scope: "plugins", status: "active", priority: 10, visibility: "user", category: "Plugin", tags: ["plugin", plugin.id, ...(plugin.keywords || [])].join(", "), generated: true, editable: false, updated_at: generatedAt }) + [ `# ${plugin.name || plugin.id}`, "", plugin.description || "No plugin description is available.", "", "## Metadata", "", `Plugin ID: ${plugin.id}`, plugin.version ? `Version: ${plugin.version}` : "", plugin.author ? `Author: ${plugin.author}` : "", plugin.enabled === false ? "Default state: disabled" : "Default state: enabled", "", "## Web Routes", "", routeLines, "", "## Route Reference", "", routeReference || "No route details detected.", "", "## Commands", "", commandLines, "", "## Source", "", `Plugin folder: plugins/${plugin.folder}` ].filter((line) => line !== "").join("\n"); } function discoverPlugins(rootDir) { const pluginsDir = path.join(rootDir, "plugins"); if (!fs.existsSync(pluginsDir)) return []; return fs.readdirSync(pluginsDir, { withFileTypes: true }) .filter((entry) => entry.isDirectory()) .map((entry) => { const dir = path.join(pluginsDir, entry.name); const manifest = readJson(path.join(dir, "plugin.json")); if (!manifest.id && !manifest.name) return null; const source = readText(path.join(dir, "index.js")); return { id: cleanText(manifest.id || entry.name, 120), folder: entry.name, name: cleanText(manifest.name || manifest.label || manifest.id || entry.name, 180), version: cleanText(manifest.version, 80), description: cleanText(manifest.description, 800), author: cleanText(manifest.author, 180), enabled: manifest.enabled, keywords: Array.isArray(manifest.keywords) ? manifest.keywords.map((item) => cleanText(item, 80)).filter(Boolean) : [], routes: discoverPluginRoutes(source, manifest.id || entry.name), commands: discoverCommandTriggers(source) }; }) .filter(Boolean) .sort((a, b) => a.id.localeCompare(b.id)); } function discoverCoreRoutes(rootDir) { const server = readText(path.join(rootDir, "src", "web", "server.js")); const routes = []; for (const match of server.matchAll(/\bapp\.(get|post|put|patch|delete)\(\s*["'`]([^"'`]+)["'`]/g)) { const source = extractCallExpression(server, match.index); routes.push({ method: match[1], path: match[2], details: analyzeRouteSource(source, match[1], match[2]) }); } return uniqueRoutes(routes).slice(0, 200); } function discoverPluginRoutes(source, pluginId) { const routes = []; const mountPaths = discoverMountPaths(source, pluginId); for (const match of source.matchAll(/\bweb\.mount\(\s*["'`]([^"'`]+)["'`]/g)) { routes.push({ method: "", path: normalizeTemplatePath(match[1], pluginId), details: { purpose: "Mounts the plugin router at this base WebUI path.", inputs: [], response: "Plugin router mount point.", access: "Access is controlled by the mount options and individual plugin routes.", sideEffects: "No direct route action; child routes handle requests.", limits: "Mount metadata is inferred from static source scanning." } }); } for (const match of source.matchAll(/\brouter\.(get|post|put|patch|delete)\(\s*["'`]([^"'`]+)["'`]/g)) { const sourceSnippet = extractCallExpression(source, match.index); const routePath = normalizeTemplatePath(match[2], pluginId); const bases = mountPaths.length ? mountPaths : [`/plugins/${pluginId}`]; for (const base of bases) { const fullPath = joinRoutePath(base, routePath); routes.push({ method: match[1], path: fullPath, details: analyzeRouteSource(sourceSnippet, match[1], fullPath) }); } } return uniqueRoutes(routes).slice(0, 100); } function discoverMountPaths(source, pluginId) { const paths = []; for (const match of source.matchAll(/\bweb\.mount\(\s*["'`]([^"'`]+)["'`]/g)) { paths.push(normalizeTemplatePath(match[1], pluginId)); } return Array.from(new Set(paths)).filter(Boolean); } function normalizeTemplatePath(value, pluginId) { return String(value || "").replace(/\$\{PLUGIN_ID\}/g, pluginId); } function joinRoutePath(base, child) { const normalizedBase = String(base || "").replace(/\/+$/, "") || "/"; const normalizedChild = String(child || "").replace(/^\/+/, ""); if (!normalizedChild) return normalizedBase; if (normalizedChild === "/") return normalizedBase; return `${normalizedBase}/${normalizedChild}`.replace(/\/+/g, "/"); } function discoverCommandTriggers(source) { const triggers = new Set(); for (const match of source.matchAll(/\btrigger\s*:\s*["'`]([^"'`]+)["'`]/g)) { triggers.add(match[1]); } for (const match of source.matchAll(/\bcommand\s*:\s*["'`]([^"'`]+)["'`]/g)) { triggers.add(match[1]); } return Array.from(triggers).sort().slice(0, 100); } function writeGeneratedFile(filePath, content) { if (!canOverwriteGeneratedFile(filePath)) return false; fs.mkdirSync(path.dirname(filePath), { recursive: true }); fs.writeFileSync(filePath, `${content.trim()}\n`); return true; } function canOverwriteGeneratedFile(filePath) { if (!fs.existsSync(filePath)) return true; const text = readText(filePath); if (/generated\s*:\s*false/i.test(text)) return false; if (/editable\s*:\s*true/i.test(text)) return false; return true; } function uniqueRoutes(routes) { const seen = new Set(); return routes.filter((route) => { const key = `${route.method}:${route.path}`; if (seen.has(key)) return false; seen.add(key); return true; }); } function buildRouteReference(routes) { if (!routes.length) return ""; return routes.map((route) => { const details = route.details || analyzeRouteSource("", route.method, route.path); return [ `### ${route.method ? route.method.toUpperCase() : "MOUNT"} ${route.path}`, "", `- Purpose: ${details.purpose}`, `- Inputs: ${details.inputs.length ? details.inputs.join("; ") : "No request parameters detected by static analysis."}`, `- Response format: ${details.response}`, `- Access: ${details.access}`, `- Side effects: ${details.sideEffects}`, `- Limits/notes: ${details.limits}` ].join("\n"); }).join("\n\n"); } function analyzeRouteSource(source, method, routePath) { const bodyFields = findRequestFields(source, "body"); const queryFields = findRequestFields(source, "query"); const paramFields = Array.from(new Set([ ...extractPathParams(routePath), ...findRequestFields(source, "params") ])).sort(); const inputs = []; if (paramFields.length) inputs.push(`path params: ${paramFields.map((field) => `\`${field}\``).join(", ")}`); if (queryFields.length) inputs.push(`query: ${queryFields.map((field) => `\`${field}\``).join(", ")}`); if (bodyFields.length) inputs.push(`body: ${bodyFields.map((field) => `\`${field}\``).join(", ")}`); if (/\breq\.file\b|\breq\.files\b|\bupload\.(single|array|fields)\(/.test(source)) { inputs.push("file upload: multipart form file data"); } if (!bodyFields.length && /\breq\.body\b/.test(source)) { inputs.push("body: full submitted body is passed to a helper; exact fields are defined by the matching form/service"); } if (!queryFields.length && /\breq\.query\b/.test(source)) { inputs.push("query: full query object is passed to a helper; exact fields are defined by the matching view/service"); } if (!paramFields.length && /\breq\.params\b/.test(source)) { inputs.push("path params: full params object is used; exact keys come from the route path"); } const response = describeResponseFormat(source, method, routePath); return { purpose: describeRoutePurpose(source, method, routePath, response), inputs, response, access: describeAccess(source, routePath), sideEffects: describeSideEffects(source, method, bodyFields), limits: describeRouteLimits(source, method, routePath) }; } function findRequestFields(source, bucket) { const fields = new Set(); const dotPattern = new RegExp(`\\breq\\.${bucket}\\.([A-Za-z_$][A-Za-z0-9_$]*)`, "g"); for (const match of source.matchAll(dotPattern)) { fields.add(match[1]); } const bracketPattern = new RegExp(`\\breq\\.${bucket}\\s*\\[\\s*["'\`]([^"'\`]+)["'\`]\\s*\\]`, "g"); for (const match of source.matchAll(bracketPattern)) { fields.add(match[1]); } return Array.from(fields).sort().slice(0, 40); } function extractPathParams(routePath) { const fields = new Set(); for (const match of String(routePath || "").matchAll(/:([A-Za-z0-9_]+)/g)) { fields.add(match[1]); } return Array.from(fields); } function describeResponseFormat(source, method, routePath) { const formats = []; if (/\bres\.render\(/.test(source)) formats.push("HTML page rendered from an EJS view"); if (/\brenderAdmin\(/.test(source)) formats.push("HTML page rendered by a helper"); if (/\bres\.json\(/.test(source)) formats.push("JSON response"); if (/\bres\.redirect\(/.test(source)) formats.push("HTTP redirect after handling the request"); if (/\bres\.download\(/.test(source)) formats.push("file download"); if (/\bres\.sendFile\(/.test(source)) formats.push("static file response"); if (/\bres\.write\(/.test(source) || /text\/event-stream|subscribeWebEvents/.test(source)) formats.push("streaming event response"); if (/\bres\.send\(/.test(source) && !formats.length) formats.push("plain or HTML response"); if (!formats.length && !method) return "Router mount; child routes provide concrete responses."; if (!formats.length && String(routePath).startsWith("/api/")) return "API response; exact schema was not detected statically."; if (!formats.length && method === "get") return "HTML or data response; exact format was not detected statically."; if (!formats.length) return "Form/action response; exact format was not detected statically."; return Array.from(new Set(formats)).join("; "); } function describeRoutePurpose(source, method, routePath, response) { const pathText = String(routePath || ""); const normalized = pathText.replace(/[:/._-]+/g, " ").trim(); if (!method) return "Registers a plugin route namespace."; if (/\/api\/events/.test(pathText)) return "Streams live WebUI event notifications to the browser."; const pluginAdminMatch = pathText.match(/^\/plugins\/([^/]+)\/admin(?:\/(.+))?$/); if (pluginAdminMatch) { const pluginId = pluginAdminMatch[1]; const action = pluginAdminMatch[2] ? pluginAdminMatch[2].replace(/[:/._-]+/g, " ").trim() : ""; if (method === "get") return `Renders the ${pluginId} plugin administration page${action ? ` for ${action}` : ""}.`; return `Processes the ${pluginId} plugin administration action${action ? ` for ${action}` : ""}.`; } const pluginPageMatch = pathText.match(/^\/plugins\/([^/]+)(?:\/(.+))?$/); if (pluginPageMatch) { const pluginId = pluginPageMatch[1]; const action = pluginPageMatch[2] ? pluginPageMatch[2].replace(/[:/._-]+/g, " ").trim() : ""; if (method === "get") return `Renders or serves the ${pluginId} plugin page${action ? ` for ${action}` : ""}.`; return `Processes the ${pluginId} plugin action${action ? ` for ${action}` : ""}.`; } if (/\/api\/users\/search/.test(pathText)) return "Searches known linked Lumi users for shared user-lookup fields."; if (/\/api\/placeholders\/catalog/.test(pathText)) return "Returns placeholder suggestions available to the current user and field policy."; if (/\/api\/placeholders\/preview/.test(pathText)) return "Previews placeholder rendering for a submitted template without saving it."; if (/\/feedback/.test(pathText) && method === "post") return "Creates, updates, comments on, exports, or manages feedback records depending on the action path."; if (/\/admin\/updates/.test(pathText)) return "Checks, applies, reverts, or reports update state for core or plugin updates."; if (/\/admin\/settings/.test(pathText)) return method === "get" ? "Renders the core settings page." : "Saves core settings from the admin settings form."; if (/\/auth\//.test(pathText)) return "Starts, completes, or cancels a platform authentication/linking flow."; if (/\/setup/.test(pathText)) return "Guides first-time platform setup and verification."; if (/\/commands/.test(pathText)) return method === "get" ? "Displays configured bot commands." : "Creates, updates, previews, toggles, or deletes custom commands."; if (/^\/admin\/plugins/.test(pathText)) return "Manages installed plugins, plugin uploads, installs, updates, or plugin enablement."; if (/\/plugins/.test(pathText)) return "Displays or handles plugin-specific WebUI functionality."; if (/\/health/.test(pathText)) return "Returns runtime health information."; if (/\/logs/.test(pathText)) return "Displays, downloads, or manages application logs."; if (/\/stats/.test(pathText)) return "Displays user or community statistics."; if (/\/leaderboards/.test(pathText)) return "Displays leaderboard data."; if (/\bres\.render\(/.test(source)) return `Renders the ${normalized || "requested"} WebUI page.`; if (/\bres\.json\(/.test(source)) return `Provides ${normalized || "route"} data as JSON.`; if (method === "post") return `Processes the ${normalized || "route"} action and stores or applies submitted form data.`; return `Handles ${normalized || "this route"}.`; } function describeAccess(source, routePath) { const signals = []; if (/\brequireOkf(Edit|Management)\b/.test(source)) { signals.push("OKF editor or manager permission required"); } if (/\brequireOkfReview\b|\brequireOkfImplement\b/.test(source)) { signals.push("higher OKF review or implementation permission may be required"); } if (/\brequireAdmin\b|\.isAdmin\b|hasAccess\([^)]*admin/.test(source) || /\/admin(\/|$)/.test(routePath)) { signals.push("admin access expected"); } if (/\brequireMod\b|\.isMod\b|hasAccess\([^)]*mod/.test(source) || /\/moderator(\/|$)/.test(routePath)) { signals.push("moderator access may be required"); } if (/\brequire(Login|Auth)\b|\breq\.session\.user\b|authRequired/.test(source)) { signals.push("logged-in session required or used"); } if (/\bcanAccess\b/.test(source)) { signals.push("mount-level canAccess predicate applies"); } if (!signals.length && /^\/api\//.test(routePath)) return "API route; access requirements were not fully detected by static analysis."; if (!signals.length) return "No explicit access guard detected in the route handler; check surrounding router/mount middleware."; return Array.from(new Set(signals)).join("; "); } function describeSideEffects(source, method, bodyFields) { const effects = []; if (/\bset[A-Z]|\bsetSetting\b|\bsave[A-Z]|\bupdate[A-Z]|\bcreate[A-Z]|\bdelete[A-Z]|\bremove[A-Z]|\binstall[A-Z]|\bapply[A-Z]|\brevert[A-Z]|\brestore[A-Z]|\bgrant[A-Z]|\brevoke[A-Z]/.test(source)) { effects.push("writes or mutates server-side state"); } if (/\bfs\.(writeFileSync|rmSync|mkdirSync|renameSync|copyFileSync)\b/.test(source)) effects.push("writes files"); if (/\bdb\.(prepare|exec)\b|\.run\(/.test(source)) effects.push("writes database state when the called service mutates data"); if (/\bpublishWebEvent\b|subscribeWebEvents/.test(source)) effects.push("publishes or streams live WebUI events"); if (/\brequestRestart\b|process\.exit/.test(source)) effects.push("may restart or stop runtime processes"); if (method === "get" && !effects.length) return "Usually read-only."; if (method === "post" && !effects.length && bodyFields.length) return "Consumes submitted data; state mutation happens in called helpers if present."; if (method === "post" && !effects.length) return "Action route; side effects were not detected statically."; return effects.length ? Array.from(new Set(effects)).join("; ") : "No side effects detected statically."; } function describeRouteLimits(source, method, routePath) { const notes = ["Generated from static route source analysis; confirm exact behavior in the handler before changing integrations."]; if (/\bcleanText\b|slice\(0,\s*\d+|limitFromBody|boundedInt|boundedNumber|clampInt/.test(source)) { notes.push("Input length or numeric bounds are enforced by helper functions in the handler."); } if (/\btry\b[\s\S]*\bcatch\b/.test(source)) { notes.push("Errors are caught and usually returned as a flash message, JSON error, or error page."); } if (method === "post" && !/^\/api\//.test(routePath)) { notes.push("Most non-API POST routes are browser form submissions and usually redirect after completion."); } if (/\/api\//.test(routePath)) { notes.push("API consumers should expect JSON unless the response format says otherwise."); } return notes.join(" "); } function extractCallExpression(source, startIndex) { const start = source.indexOf("(", startIndex); if (start === -1) return source.slice(startIndex, startIndex + 1200); let depth = 0; let quote = ""; let escaped = false; for (let index = start; index < source.length; index += 1) { const char = source[index]; if (quote) { if (escaped) { escaped = false; } else if (char === "\\") { escaped = true; } else if (char === quote) { quote = ""; } continue; } if (char === "\"" || char === "'" || char === "`") { quote = char; continue; } if (char === "(") { depth += 1; } else if (char === ")") { depth -= 1; if (depth === 0) { return source.slice(startIndex, index + 1); } } } return source.slice(startIndex, startIndex + 4000); } function frontmatter(values) { const lines = ["---"]; for (const [key, value] of Object.entries(values)) { if (value === "" || value === null || value === undefined) continue; lines.push(`${key}: ${frontmatterValue(value)}`); } lines.push("---", ""); return lines.join("\n"); } function frontmatterValue(value) { if (typeof value === "boolean" || typeof value === "number") return String(value); const text = String(value).replace(/\n/g, " "); return /[:#[\]{}"'\\]|^\s|\s$/.test(text) ? JSON.stringify(text) : text; } function readJson(filePath) { try { return JSON.parse(fs.readFileSync(filePath, "utf8")); } catch { return {}; } } function readText(filePath) { try { return fs.readFileSync(filePath, "utf8"); } catch { return ""; } } function excerpt(value, limit) { return cleanText(value, limit).split("\n").filter((line) => line.trim()).slice(0, 12).join("\n"); } function slugify(value) { return cleanText(value, 180) .toLowerCase() .replace(/[^a-z0-9]+/g, "-") .replace(/^-+|-+$/g, "") || "entry"; } function cleanText(value, maximum = 4000) { return String(value || "").replace(/\r\n?/g, "\n").trim().slice(0, maximum); } module.exports = { buildCoreKnowledge, buildPluginKnowledge, discoverPlugins, generateKnowledgeFiles };