diff --git a/plugins/lumi_ai/backend/ai_provider.js b/plugins/lumi_ai/backend/ai_provider.js index bf8608d..042a457 100644 --- a/plugins/lumi_ai/backend/ai_provider.js +++ b/plugins/lumi_ai/backend/ai_provider.js @@ -1,7 +1,7 @@ const crypto = require("crypto"); -const { buildPrompt } = require("./prompt_builder"); +const { buildPrompt, buildToolResultInstruction } = require("./prompt_builder"); const { roleOf } = require("./permissions"); -const { parseToolCall } = require("./tool_router"); +const { parseToolCallResult } = require("./tool_router"); const { normalizeScope } = require("./scope_manager"); const { classifyRequestType } = require("./gate_provider"); @@ -21,7 +21,9 @@ class AiProvider { allowDeterministicShortcut = null, history = [], signal = null, - onStage = () => {} + onStage = () => {}, + allowTools = true, + includePrompt = false }) { const requestId = crypto.randomUUID(); const role = roleOf(user); @@ -140,7 +142,22 @@ class AiProvider { origin: originContext?.origin || originContext?.platform || "webui", platform: originContext?.platform || originContext?.origin || "webui" }) || []; - const platformToolsAllowed = originContext?.permission_context?.webui_actions_allowed !== false; + const toolExposure = allowTools + ? this.tools.inspect({ role, user, context: originContext }) + : { considered: [], exposed: [] }; + this.metrics.record({ + kind: "tool_exposure", + status: allowTools ? "evaluated" : "disabled", + request_id: requestId, + user_id: user.id, + role, + origin: originContext?.origin || originContext?.platform || "other", + considered_tools: toolExposure.considered.map((decision) => decision.tool.tool_id), + exposed_tools: toolExposure.exposed.map((tool) => tool.tool_id), + rejected_tools: toolExposure.considered + .filter((decision) => !decision.exposed) + .map((decision) => ({ tool_id: decision.tool.tool_id, reason: decision.reason })) + }); const prompt = buildPrompt({ config: cfg, role, @@ -150,7 +167,7 @@ class AiProvider { correctionContext, repoContext, originContext, - tools: platformToolsAllowed ? this.tools.list(role) : [] + tools: toolExposure.exposed }); const conversation = normalizeHistory(history); const outputTokenLimit = resolveOutputBudget({ @@ -192,43 +209,133 @@ class AiProvider { clearTimeout(generatingTimer); } if (signal?.aborted) throw requestCancelledError(); - const text = result.choices?.[0]?.message?.content || ""; - const inference = normalizeInferenceDiagnostics(result, Date.now() - generateStarted); + const initialText = result.choices?.[0]?.message?.content || ""; + const initialInference = normalizeInferenceDiagnostics(result, Date.now() - generateStarted); onStage("generating", { route: "main_llm", queue_ms: queueWait, ...runtimeSettings, - ...inference + ...initialInference }); - const toolCall = platformToolsAllowed ? parseToolCall(text) : null; + const parsedToolCall = allowTools ? parseToolCallResult(initialText) : { status: "none", call: null }; + const toolCall = parsedToolCall.call; let confirmation = null; let toolResult = null; - if (toolCall) { - const prepared = this.tools.prepare({ tool: toolCall.tool, args: toolCall.arguments, user, role, sessionId, context: originContext }); - if (prepared.execute) toolResult = await this.tools.execute({ checked: prepared.checked, user, requestId, context: originContext }); - confirmation = prepared.confirmation; + let finalText = initialText; + let finalResult = null; + let finalInference = emptyInferenceDiagnostics(); + let selectedTool = null; + let rejectedReason = null; + let toolExecutionMs = 0; + if (parsedToolCall.status === "malformed") { + rejectedReason = "malformed_tool_call"; + finalText = "I could not validate the requested tool call. Please retry or clarify the request."; + this.metrics.record({ + kind: "tool_decision", + status: "rejected", + request_id: requestId, + user_id: user.id, + origin: originContext?.origin || originContext?.platform || "other", + rejected_reason: rejectedReason + }); + } else if (toolCall) { + selectedTool = toolCall.tool; + try { + const prepared = this.tools.prepare({ + tool: toolCall.tool, + args: toolCall.arguments, + user, + role, + sessionId, + context: originContext + }); + confirmation = prepared.confirmation; + if (prepared.execute) { + const executionStarted = Date.now(); + onStage("tool_running", { selected_tool: selectedTool }); + try { + toolResult = await this.tools.execute({ + checked: prepared.checked, + user, + requestId, + context: originContext + }); + } catch (error) { + toolResult = { + status: "failed", + error: "The selected tool failed to complete." + }; + rejectedReason = error.code || "execution_failed"; + } + toolExecutionMs = Date.now() - executionStarted; + if (prepared.checked.def.read_only) { + onStage("formatting", { selected_tool: selectedTool, tool_execution_ms: toolExecutionMs }); + const finalStarted = Date.now(); + try { + finalResult = await this.runtime.infer( + [ + { role: "system", content: prompt }, + ...conversation, + { role: "user", content: effectiveMessage }, + { role: "assistant", content: initialText }, + { + role: "user", + content: buildToolResultInstruction({ + tool: prepared.checked.def, + result: toolResult, + originContext + }) + } + ], + outputTokenLimit, + { + signal, + timeoutMs: cfg.hard_generation_timeout_ms + } + ); + finalInference = normalizeInferenceDiagnostics(finalResult, Date.now() - finalStarted); + const candidate = finalResult.choices?.[0]?.message?.content || ""; + const repeatedCall = parseToolCallResult(candidate); + finalText = repeatedCall.status === "none" + ? candidate + : fallbackToolMessage(toolResult); + } catch { + finalText = fallbackToolMessage(toolResult); + } + } else { + finalText = safeActionResult(toolResult); + } + } else { + finalText = `Please confirm: ${confirmation.display_name}.`; + } + } catch (error) { + rejectedReason = error.code || "tool_rejected"; + finalText = `I could not use ${toolCall.tool}: ${safeToolError(error)}`; + } } + const inference = combineInferenceDiagnostics(initialInference, finalInference); const out = { success: true, - text: confirmation ? `Please confirm: ${confirmation.display_name}.` - : toolResult?.user_message ? toolResult.user_message - : toolResult ? `Action completed: ${JSON.stringify(toolResult)}` : text, + text: finalText, links: [], - raw_response: cfg.logging.log_responses || includeRaw ? result : null, + raw_response: cfg.logging.log_responses || includeRaw + ? finalResult ? { initial: result, final: finalResult } : result + : null, + raw_prompt: includePrompt ? prompt : undefined, tool_call: toolCall, tool_result: toolResult, confirmation, model_id: cfg.selected_model_id, duration_ms: Date.now() - started, queue_wait_ms: queueWait, - finish_reason: result.choices?.[0]?.finish_reason || null, + finish_reason: (finalResult || result).choices?.[0]?.finish_reason || null, request_id: requestId, route_used: gateDecision ? "main_llm" : "llm", route_class: requestClass, max_output_tokens_used: outputTokenLimit, gate_decision: gateDecision, force_through_reason: gateDecision?.forced ? gateDecision.reason_code : null, - internal_generated_length: text.length, + internal_generated_length: initialText.length + String(finalText || "").length, stage_timings: { deterministic_ms: gateDecision?.deterministic_ms || 0, gate_ms: gateDecision?.gate_ms || 0, @@ -250,7 +357,13 @@ class AiProvider { this.metrics.record({ kind: "request", status: "success", request_id: requestId, user_id: user.id, role, scope, model: cfg.selected_model_id, duration_ms: out.duration_ms, queue_wait_ms: queueWait, - tool_requested: toolCall?.tool || null, tool_executed: false, + tool_requested: toolCall?.tool || null, + considered_tools: toolExposure.considered.map((decision) => decision.tool.tool_id), + exposed_tools: toolExposure.exposed.map((tool) => tool.tool_id), + selected_tool: selectedTool, + rejected_reason: rejectedReason, + execution_ms: toolExecutionMs, + tool_executed: Boolean(toolResult), route_used: gateDecision ? "main_llm" : "llm", route_class: requestClass, max_output_tokens_used: outputTokenLimit, @@ -266,7 +379,7 @@ class AiProvider { generation_ms: out.stage_timings.generation_ms, total_ms: out.stage_timings.total_ms, ...out.diagnostics, - internal_generated_length: text.length + internal_generated_length: out.internal_generated_length }); return out; }, { signal }); @@ -287,7 +400,21 @@ class AiProvider { }); } - async test({ message, user, max_tokens = 300, includeRaw = false }) { + async test({ message, user, max_tokens = 300, includeRaw = false, allowTools = false, originContext = null }) { + if (allowTools) { + const result = await this.generate({ + message, + user, + sessionId: `admin-test:${user.id}:${Date.now()}`, + scope: "model_test", + max_tokens, + includeRaw, + includePrompt: true, + originContext, + allowTools: true + }); + return { ...result, tools_notice: "Tools were enabled for this test." }; + } const requestId = crypto.randomUUID(); const role = roleOf(user); const started = Date.now(); @@ -305,7 +432,8 @@ class AiProvider { success: true, text, raw_response: includeRaw ? result : null, raw_prompt: prompt, tool_call: null, tool_result: null, confirmation: null, model_id: cfg.selected_model_id, duration_ms: Date.now() - started, queue_wait_ms: queueWait, - finish_reason: result.choices?.[0]?.finish_reason || null, request_id: requestId + finish_reason: result.choices?.[0]?.finish_reason || null, request_id: requestId, + tools_notice: "Tools were disabled for this test; this result does not exercise tool discovery or execution." }; this.metrics.record({ kind: "request", status: "success", request_id: requestId, user_id: user.id, role, @@ -316,6 +444,56 @@ class AiProvider { } } +function emptyInferenceDiagnostics() { + return { + prompt_tokens: 0, + generated_tokens: 0, + prompt_eval_ms: 0, + generation_ms: 0, + prompt_tps: 0, + generation_tps: 0 + }; +} + +function combineInferenceDiagnostics(initial, final) { + const promptTokens = Number(initial.prompt_tokens || 0) + Number(final.prompt_tokens || 0); + const generatedTokens = Number(initial.generated_tokens || 0) + Number(final.generated_tokens || 0); + const promptEvalMs = Number(initial.prompt_eval_ms || 0) + Number(final.prompt_eval_ms || 0); + const generationMs = Number(initial.generation_ms || 0) + Number(final.generation_ms || 0); + return { + prompt_tokens: promptTokens, + generated_tokens: generatedTokens, + prompt_eval_ms: promptEvalMs, + generation_ms: generationMs, + prompt_tps: ratePerSecond(promptTokens, promptEvalMs), + generation_tps: ratePerSecond(generatedTokens, generationMs) + }; +} + +function fallbackToolMessage(result) { + if (result?.user_message) return String(result.user_message); + if (result?.status === "blocked") return "The requested lookup was blocked by the configured tool policy."; + if (["failed", "unavailable"].includes(result?.status)) return "The requested tool is currently unavailable."; + if (result?.status === "no_results") return "The tool completed but found no usable results."; + return "The tool completed, but I could not format a final response."; +} + +function safeActionResult(result) { + if (result?.user_message) return String(result.user_message); + if (result?.status === "failed") return "The action failed."; + return "The action completed successfully."; +} + +function safeToolError(error) { + return ({ + not_registered: "the tool is not registered.", + permission_blocked: "permission was denied.", + origin_blocked: "the tool is unavailable for this origin.", + scope_blocked: "the tool is outside this request context.", + schema_invalid: "the tool arguments were invalid." + })[error?.code] || "the tool is unavailable."; +} + function isClearlyOutOfScope() { return false; } function isInScope() { return true; } function isIdentityQuery(message) { diff --git a/plugins/lumi_ai/backend/metrics.js b/plugins/lumi_ai/backend/metrics.js index 8fa4dac..0763eb3 100644 --- a/plugins/lumi_ai/backend/metrics.js +++ b/plugins/lumi_ai/backend/metrics.js @@ -5,7 +5,7 @@ const historyFile = () => resolveData("metrics", "history.jsonl"); const stateFile = () => resolveData("metrics", "summary.json"); function getSummary() { try { return JSON.parse(fs.readFileSync(stateFile(), "utf8")); } - catch { return { total_requests:0, successful:0, failed:0, refusals:0, gate_decisions:0, tool_suggestions:0, tool_executions:0, tool_denials:0, confirmation_cancellations:0, timeout_count:0, runtime_crash_count:0, runtime_self_test_total:0, runtime_self_test_failed_total:0, runtime_start_attempt_total:0, runtime_start_failed_total:0, verified_downloads:0, failed_downloads:0, requests_by_role:{}, requests_by_scope:{}, requests_by_route:{}, gate_reason_codes:{}, runtime_exit_code_counts:{}, stage_totals:{}, stage_samples:0, slow_requests:[], durations:[], queue_wait_total_ms:0 }; } + catch { return { total_requests:0, successful:0, failed:0, refusals:0, gate_decisions:0, tool_decisions:0, tool_suggestions:0, tool_executions:0, tool_denials:0, confirmation_cancellations:0, timeout_count:0, runtime_crash_count:0, runtime_self_test_total:0, runtime_self_test_failed_total:0, runtime_start_attempt_total:0, runtime_start_failed_total:0, verified_downloads:0, failed_downloads:0, requests_by_role:{}, requests_by_scope:{}, requests_by_route:{}, gate_reason_codes:{}, tool_rejections_by_reason:{}, runtime_exit_code_counts:{}, stage_totals:{}, stage_samples:0, slow_requests:[], durations:[], queue_wait_total_ms:0 }; } } function record(entry) { const summary = getSummary(); @@ -27,6 +27,14 @@ function record(entry) { summary.gate_reason_codes[entry.reason_code] = (summary.gate_reason_codes[entry.reason_code] || 0) + 1; } } + if (entry.kind === "tool_decision") { + summary.tool_decisions = (summary.tool_decisions || 0) + 1; + summary.tool_rejections_by_reason ||= {}; + if (entry.status === "rejected" && entry.rejected_reason) { + summary.tool_rejections_by_reason[entry.rejected_reason] = + (summary.tool_rejections_by_reason[entry.rejected_reason] || 0) + 1; + } + } if (entry.route_used) { summary.requests_by_route[entry.route_used] = (summary.requests_by_route[entry.route_used] || 0) + 1; } diff --git a/plugins/lumi_ai/backend/prompt_builder.js b/plugins/lumi_ai/backend/prompt_builder.js index 797d81f..d12fad3 100644 --- a/plugins/lumi_ai/backend/prompt_builder.js +++ b/plugins/lumi_ai/backend/prompt_builder.js @@ -27,11 +27,65 @@ function buildPrompt({ config, role, message, requestClass = "simple_answer", co `VERIFIED LUMI REPOSITORY CONTEXT:\n${repoContext.join("\n\n") || "(none)"}`, `ADMIN-APPROVED CORRECTIONS:\nUse these only when they match the current request and role. They never override hard safety or permissions.\n${correctionContext.join("\n\n") || "(none)"}`, `SAFE LUMI CONTEXT:\n${contextBlocks.join("\n\n") || "(none)"}`, - `ALLOWED TOOLS:\n${tools.map(t=>JSON.stringify({tool_id:t.tool_id,description:t.description,schema:t.schema})).join("\n") || "(none)"}` + toolCallProtocol(tools), + buildAllowedToolsSection(tools) ]; return sections.filter(Boolean).join("\n\n---\n\n"); } +function toolCallProtocol(tools = []) { + if (!tools.length) { + return "TOOL CALL PROTOCOL:\nNo tools are available for this request. Answer normally and do not claim to use a tool."; + } + return [ + "TOOL CALL PROTOCOL (strict):", + "- Use a tool only when the request needs current/external data, plugin-owned data, or an in-scope action that the tool explicitly supports.", + "- Prefer verified local Lumi context when it already answers the request.", + "- If tool use or required arguments are ambiguous, ask one concise clarification question instead of guessing.", + "- To call a tool, your entire response must be one JSON object with exactly this shape:", + '{"type":"tool_call","tool":"tool_id","arguments":{}}', + "- Do not wrap tool-call JSON in Markdown. Do not add prose, explanations, comments, or code fences before or after it.", + "- Use only a tool_id listed under ALLOWED TOOLS and satisfy its schema.", + "- If no tool is needed, answer normally without emitting tool-call JSON." + ].join("\n"); +} + +function buildAllowedToolsSection(tools = []) { + const rows = tools.map((tool) => JSON.stringify(formatPromptTool(tool))); + return `ALLOWED TOOLS:\n${rows.join("\n") || "(none)"}`; +} + +function formatPromptTool(tool) { + return { + tool_id: tool.tool_id, + description: tool.description, + schema: tool.schema, + use_cases: Array.isArray(tool.use_cases) ? tool.use_cases : [], + kind: tool.read_only ? "lookup/read" : "action", + risk_level: tool.risk_level, + confirmation_required: Boolean(tool.confirmation_required), + output_expectations: tool.output_expectations || ( + tool.read_only + ? "Returns structured data. Lumi Assistant must write the final user-facing answer." + : "Returns an action result after required permission and confirmation checks." + ) + }; +} + +function buildToolResultInstruction({ tool, result, originContext }) { + const serialized = JSON.stringify(result); + return [ + "TOOL RESULT FINALIZATION (strict):", + `The tool ${tool.tool_id} has completed. Use only the structured result below.`, + "Write a natural, concise final answer to the user's original request.", + "Do not emit another tool call. Do not dump or describe raw JSON.", + "If the result is blocked, unavailable, failed, or empty, say so plainly.", + "Use only source URLs present in the tool result. Do not invent sources or inaccessible links.", + `Respect this request origin and output limit: ${JSON.stringify(originContext || {})}`, + `TOOL RESULT:\n${serialized.slice(0, 24000)}` + ].join("\n"); +} + function requestClassPolicy(requestClass) { if (requestClass === "code_custom_command") { return [ @@ -52,4 +106,11 @@ function requestClassPolicy(requestClass) { return "Answer directly and concisely. Avoid unnecessary preambles, repetition, and broad background."; } -module.exports = { buildPrompt, requestClassPolicy }; +module.exports = { + buildAllowedToolsSection, + buildPrompt, + buildToolResultInstruction, + formatPromptTool, + requestClassPolicy, + toolCallProtocol +}; diff --git a/plugins/lumi_ai/backend/tool_loader.js b/plugins/lumi_ai/backend/tool_loader.js index cf8fb27..cae1cf6 100644 --- a/plugins/lumi_ai/backend/tool_loader.js +++ b/plugins/lumi_ai/backend/tool_loader.js @@ -16,13 +16,53 @@ class ToolLoader { } async loadEnabled() { + return this.reconcile(); + } + + async reconcile({ reload = false } = {}) { const state = this.readState(); - for (const local of this.installer.scanLocal()) { - if (state.enabled[local.tool_id] === true) { - try { await this.enable(local.tool_id, { persist: false }); } - catch (error) { this.setStatus(local.tool_id, "unavailable", error.message); } + const locals = this.installer.scanLocal(); + const localMap = new Map(locals.map((local) => [local.tool_id, local])); + for (const toolId of [...this.loaded.keys()]) { + if (!localMap.has(toolId) || state.enabled[toolId] !== true) { + await this.disable(toolId, { persist: false }); } } + for (const local of locals) { + if (!local.valid) { + this.registry.unregisterOwner(local.tool_id); + this.loaded.delete(local.tool_id); + this.setStatus(local.tool_id, "unavailable", local.error, { blocking: ["schema_invalid"], optional: [] }); + continue; + } + if (state.enabled[local.tool_id] !== true) { + this.setStatus(local.tool_id, "disabled", ""); + continue; + } + const dependencies = this.inspectDependencies(local.metadata, local.dir); + if (dependencies.blocking.length) { + await this.disable(local.tool_id, { persist: false }); + this.setStatus( + local.tool_id, + "unavailable", + `Unavailable: ${dependencies.blocking.join("; ")}`, + dependencies + ); + continue; + } + let signature; + try { + signature = sourceSignature(local); + } catch (error) { + await this.disable(local.tool_id, { persist: false }); + this.setStatus(local.tool_id, "unavailable", error.message, dependencies); + continue; + } + if (!reload && this.loaded.get(local.tool_id)?.source_signature === signature) continue; + try { await this.enable(local.tool_id, { persist: false }); } + catch (error) { this.setStatus(local.tool_id, "unavailable", error.message); } + } + return this.diagnostics(); } async enable(toolId, options = {}) { @@ -30,13 +70,21 @@ class ToolLoader { if (!local?.valid) throw new Error(local?.error || "Installed AI tool metadata is invalid."); const dependencies = this.inspectDependencies(local.metadata, local.dir); if (dependencies.blocking.length) { + await this.disable(toolId, { persist: false }); const message = `Unavailable: ${dependencies.blocking.join("; ")}`; this.setStatus(toolId, "unavailable", message, dependencies); if (options.persist !== false) this.setEnabled(toolId, true); return { loaded: false, unavailable: true, message, dependencies }; } await this.disable(toolId, { persist: false }); - const backend = backendEntrypoint(local.metadata, local.dir); + let backend; + try { + backend = backendEntrypoint(local.metadata, local.dir); + } catch (error) { + this.setStatus(toolId, "unavailable", error.message, dependencies); + if (options.persist !== false) this.setEnabled(toolId, true); + return { loaded: false, unavailable: true, message: error.message, dependencies }; + } const registered = []; let cleanup = null; if (backend) { @@ -62,7 +110,13 @@ class ToolLoader { if (typeof module.checkAvailability === "function") { const availability = await module.checkAvailability(context); if (availability?.available === false) { - this.loaded.set(toolId, { cleanup: null, registered, metadata: local.metadata, dir: local.dir }); + this.loaded.set(toolId, { + cleanup: null, + registered, + metadata: local.metadata, + dir: local.dir, + source_signature: sourceSignature(local) + }); this.setStatus(toolId, "unavailable", String(availability.message || "Tool configuration is incomplete."), dependencies); if (options.persist !== false) this.setEnabled(toolId, true); return { loaded: false, unavailable: true, message: availability.message, dependencies }; @@ -77,7 +131,13 @@ class ToolLoader { return { loaded: false, unavailable: true, message: error.message, dependencies }; } } - this.loaded.set(toolId, { cleanup, registered, metadata: local.metadata, dir: local.dir }); + this.loaded.set(toolId, { + cleanup, + registered, + metadata: local.metadata, + dir: local.dir, + source_signature: sourceSignature(local) + }); this.setStatus(toolId, "enabled", dependencies.optional.length ? `Enabled with limitations: ${dependencies.optional.join("; ")}` : "", dependencies); if (options.persist !== false) this.setEnabled(toolId, true); return { loaded: true, registered: registered.map((entry) => entry.id), dependencies }; @@ -109,6 +169,27 @@ class ToolLoader { return this.statuses.get(toolId) || { state: this.isEnabled(toolId) ? "pending" : "disabled", message: "" }; } + diagnostics() { + return this.installer.scanLocal().map((local) => { + const enabled = this.isEnabled(local.tool_id); + const status = this.status(local.tool_id); + return { + tool_id: local.tool_id, + installed: true, + enabled, + valid: local.valid, + state: status.state, + message: status.message || local.error || "", + dependencies: status.dependencies || (local.valid + ? this.inspectDependencies(local.metadata, local.dir) + : { blocking: ["schema_invalid"], optional: [] }), + registered_tools: [...this.registry.tools.values()] + .filter((definition) => definition.owning_plugin === local.tool_id) + .map((definition) => definition.tool_id) + }; + }); + } + inspectDependencies(metadata, toolDir) { const blocking = []; const optional = []; @@ -209,10 +290,19 @@ function compareVersions(left, right) { return 0; } +function sourceSignature(local) { + const entry = backendEntrypoint(local.metadata, local.dir); + const entryMtime = entry && fs.existsSync(entry) ? fs.statSync(entry).mtimeMs : 0; + const metadataFile = path.join(local.dir, "tool_info.json"); + const metadataMtime = fs.existsSync(metadataFile) ? fs.statSync(metadataFile).mtimeMs : 0; + return `${local.metadata.version}:${metadataMtime}:${entryMtime}`; +} + module.exports = { ToolLoader, assetRoots, backendEntrypoint, clearRequireCache, - compareVersions + compareVersions, + sourceSignature }; diff --git a/plugins/lumi_ai/backend/tool_manager.js b/plugins/lumi_ai/backend/tool_manager.js index 7bf695f..9520ede 100644 --- a/plugins/lumi_ai/backend/tool_manager.js +++ b/plugins/lumi_ai/backend/tool_manager.js @@ -11,6 +11,7 @@ class ToolManager { } async list({ force = false } = {}) { + await this.loader.reconcile({ reload: force }); const remoteResult = await this.repoClient.discover({ force }); const localRows = this.installer.scanLocal(); const remoteMap = new Map(remoteResult.tools.map((metadata) => [metadata.tool_id, metadata])); @@ -52,6 +53,9 @@ class ToolManager { remote_error: remote?.remote_error || null, runtime_state: runtime.state, runtime_message: runtime.message, + registered_tools: [...this.loader.registry.tools.values()] + .filter((definition) => definition.owning_plugin === toolId) + .map((definition) => definition.tool_id), dependency_status: dependencies, primary_type: metadata?.tool_type || "general", primary_scope: displayScope(metadata?.scope), @@ -128,6 +132,42 @@ class ToolManager { return saved; } + async diagnostics({ role, user, context }) { + await this.loader.reconcile(); + const exposure = this.loader.registry.inspect({ role, user, context }); + const decisionsByOwner = new Map(); + for (const decision of exposure.considered) { + const owner = decision.tool.owning_plugin; + if (!decisionsByOwner.has(owner)) decisionsByOwner.set(owner, []); + decisionsByOwner.get(owner).push(decision); + } + const plugins = this.loader.diagnostics().map((plugin) => { + const decisions = decisionsByOwner.get(plugin.tool_id) || []; + let hiddenReason = null; + if (!plugin.valid) hiddenReason = "schema_invalid"; + else if (!plugin.enabled) hiddenReason = "disabled"; + else if (plugin.dependencies.blocking.length) hiddenReason = "dependency_failed"; + else if (plugin.state === "unavailable") hiddenReason = "unavailable"; + else if (!decisions.some((decision) => decision.exposed)) { + hiddenReason = decisions[0]?.reason || "unavailable"; + } + return { + ...plugin, + prompt_exposed: decisions.some((decision) => decision.exposed), + hidden_reason: hiddenReason, + decisions + }; + }); + return { + role, + origin: context?.origin || context?.platform || "other", + considered_tools: exposure.considered.map((decision) => decision.tool.tool_id), + exposed_tools: exposure.exposed.map((tool) => tool.tool_id), + prompt_tools: exposure.exposed, + plugins + }; + } + async loadEnabled() { return this.loader.loadEnabled(); } diff --git a/plugins/lumi_ai/backend/tool_registry.js b/plugins/lumi_ai/backend/tool_registry.js index 24b1b0d..c8db12a 100644 --- a/plugins/lumi_ai/backend/tool_registry.js +++ b/plugins/lumi_ai/backend/tool_registry.js @@ -16,6 +16,7 @@ function registerManagedTool(registry, metadata, definition) { const backendRole = normalizeRole(definition.required_role); const requiredRole = stricterRole(metadataRole, backendRole); const backendPermissionCheck = definition.permission_check; + const promptPermissionCheck = definition.prompt_permission_check; const mutating = definition.mutating === true || ["sensitive", "high", "destructive"].includes(String(definition.risk_level || metadata.risk_level || "").toLowerCase()); return registry.register({ @@ -24,6 +25,9 @@ function registerManagedTool(registry, metadata, definition) { required_role: requiredRole, required_permission: String(definition.required_permission || `${metadata.tool_id}.use`), audit_category: String(definition.audit_category || metadata.tool_type || "ai_tool"), + read_only: definition.read_only === true, + use_cases: normalizeTextArray(definition.use_cases || metadata.capabilities), + output_expectations: String(definition.output_expectations || metadata.output_expectations || ""), confirmation_required: mutating || metadata.confirmation_required === true ? true : definition.confirmation_required !== false, @@ -31,10 +35,17 @@ function registerManagedTool(registry, metadata, definition) { const actualRole = input.user?.isAdmin ? "admin" : input.user?.isMod ? "mod" : "user"; if (!roleAllows(actualRole, requiredRole)) return false; return backendPermissionCheck(input) === true; - } + }, + prompt_permission_check: typeof promptPermissionCheck === "function" + ? (input) => promptPermissionCheck(input) === true + : undefined }); } +function normalizeTextArray(value) { + return Array.isArray(value) ? value.map(String).map((entry) => entry.trim()).filter(Boolean) : []; +} + function validateManagedDefinition(metadata, definition) { if (!definition || typeof definition !== "object") throw new Error("AI tool definition is required."); if (!String(definition.tool_id || "").startsWith(`${metadata.tool_id}.`)) { diff --git a/plugins/lumi_ai/backend/tool_router.js b/plugins/lumi_ai/backend/tool_router.js index 0e0888c..43d56cd 100644 --- a/plugins/lumi_ai/backend/tool_router.js +++ b/plugins/lumi_ai/backend/tool_router.js @@ -2,63 +2,445 @@ const crypto = require("crypto"); const { roleAllows } = require("./permissions"); class ToolRegistry { - constructor(audit){ this.tools=new Map(); this.confirmations=new Map(); this.audit=audit; } - register(def){ - if(!def?.tool_id || !def.display_name || !def.description || !def.owning_plugin || !def.required_permission || !def.audit_category || typeof def.workflow_handler!=="function" || typeof def.permission_check!=="function" || !def.schema) throw new Error("Invalid AI tool definition."); - if(this.tools.has(def.tool_id)) throw new Error(`AI tool ${def.tool_id} is already registered.`); - this.tools.set(def.tool_id,{required_role:"user",confirmation_required:true,risk_level:"sensitive",...def}); - return () => this.unregister(def.tool_id, def.owning_plugin); + constructor(audit) { + this.tools = new Map(); + this.confirmations = new Map(); + this.audit = typeof audit === "function" ? audit : () => {}; } - unregister(toolId, owner = null){ - const def=this.tools.get(toolId); if(!def || (owner && def.owning_plugin!==owner)) return false; + + register(definition) { + validateDefinition(definition); + if (this.tools.has(definition.tool_id)) { + throw new Error(`AI tool ${definition.tool_id} is already registered.`); + } + const normalized = { + required_role: "user", + confirmation_required: true, + risk_level: "sensitive", + read_only: false, + use_cases: [], + output_expectations: "", + ...definition + }; + this.tools.set(normalized.tool_id, normalized); + return () => this.unregister(normalized.tool_id, normalized.owning_plugin); + } + + unregister(toolId, owner = null) { + const definition = this.tools.get(toolId); + if (!definition || (owner && definition.owning_plugin !== owner)) return false; this.tools.delete(toolId); - for(const [id,pending] of this.confirmations){if(pending.def?.tool_id===toolId)this.confirmations.delete(id);} + for (const [id, pending] of this.confirmations) { + if (pending.def?.tool_id === toolId) this.confirmations.delete(id); + } return true; } - unregisterOwner(owner){ - let removed=0; - for(const [id,def] of this.tools){if(def.owning_plugin===owner && this.unregister(id,owner))removed+=1;} + + unregisterOwner(owner) { + let removed = 0; + for (const [id, definition] of this.tools) { + if (definition.owning_plugin === owner && this.unregister(id, owner)) removed += 1; + } return removed; } - list(role){ return [...this.tools.values()].filter(t=>roleAllows(role,t.required_role)).map(({workflow_handler,permission_check,...t})=>t); } - validate(tool,args,role){ - const def=this.tools.get(tool); if(!def) throw new Error("Tool is not registered."); - if(!roleAllows(role,def.required_role)) throw new Error("Permission denied for this tool."); - const schema=def.schema||{}; const clean={}; - for(const [key,spec] of Object.entries(schema)){ - const descriptor=typeof spec==="string"?{type:spec,required:true}:spec||{}; - const value=args?.[key]; - if(value==null && descriptor.required===false)continue; - if(descriptor.type==="integer" && !Number.isInteger(Number(value))) throw new Error(`${key} must be an integer.`); - if(descriptor.type==="string" && typeof value!=="string") throw new Error(`${key} must be a string.`); - if(Array.isArray(descriptor.enum) && !descriptor.enum.includes(value)) throw new Error(`${key} is invalid.`); - clean[key]=descriptor.type==="integer"?Number(value):value; + + has(toolId) { + return this.tools.has(toolId); + } + + list(role) { + return [...this.tools.values()] + .filter((tool) => roleAllows(role, tool.required_role)) + .map(publicDefinition); + } + + inspect({ role, user, context = null } = {}) { + const decisions = [...this.tools.values()].map((definition) => + exposureDecision(definition, { role, user, context }) + ); + return { + considered: decisions, + exposed: decisions.filter((decision) => decision.exposed).map((decision) => decision.tool) + }; + } + + validate(toolId, args, role) { + const definition = this.tools.get(toolId); + if (!definition) throw toolError("not_registered", "Tool is not registered."); + if (!roleAllows(role, definition.required_role)) { + throw toolError("permission_blocked", "Permission denied for this tool."); } - return {def,args:clean}; + const schema = definition.schema || {}; + const input = args && typeof args === "object" && !Array.isArray(args) ? args : {}; + const unknown = Object.keys(input).filter((key) => !Object.hasOwn(schema, key)); + if (unknown.length) throw toolError("schema_invalid", `Unknown tool argument: ${unknown[0]}.`); + const clean = {}; + for (const [key, spec] of Object.entries(schema)) { + const descriptor = typeof spec === "string" ? { type: spec, required: true } : spec || {}; + const value = input[key]; + if (value == null) { + if (descriptor.required === false) continue; + throw toolError("schema_invalid", `${key} is required.`); + } + clean[key] = normalizeArgument(key, value, descriptor); + } + return { def: definition, args: clean }; } - prepare({tool,args,user,role,sessionId,context=null}){ - const checked=this.validate(tool,args,role); - const allowed=checked.def.permission_check({user,arguments:checked.args,required_permission:checked.def.required_permission,context}); - if(allowed && typeof allowed.then==="function")throw new Error("AI tool permission checks must be synchronous."); - if(!allowed)throw new Error("The requesting user does not have permission for this action."); - if(!checked.def.confirmation_required) return {execute:true,checked,context}; - const id=crypto.randomUUID(); this.confirmations.set(id,{id,userId:user.id,sessionId,expiresAt:Date.now()+120000,context,...checked}); - return {execute:false,confirmation:{id,display_name:checked.def.display_name,arguments:checked.args,expires_at:Date.now()+120000}}; + + prepare({ tool, args, user, role, sessionId, context = null }) { + const definition = this.tools.get(tool); + if (!definition) { + this.recordDecision({ + user, + role, + context, + tool, + args, + decision: "rejected", + rejected_reason: "not_registered" + }); + throw toolError("not_registered", "Tool is not registered."); + } + const exposure = exposureDecision(definition, { role, user, context }); + if (!exposure.exposed) { + this.recordDecision({ + user, + role, + context, + tool, + args, + decision: "rejected", + rejected_reason: exposure.reason + }); + throw toolError(exposure.reason, exposure.message); + } + let checked; + try { + checked = this.validate(tool, args, role); + } catch (error) { + this.recordDecision({ + user, + role, + context, + tool, + args, + decision: "rejected", + rejected_reason: error.code || "schema_invalid" + }); + throw error; + } + let allowed; + try { + allowed = checked.def.permission_check({ + user, + arguments: checked.args, + required_permission: checked.def.required_permission, + context, + phase: "execute" + }); + } catch { + allowed = false; + } + if (allowed && typeof allowed.then === "function") { + this.recordDecision({ + user, + role, + context, + tool, + args: checked.args, + decision: "rejected", + rejected_reason: "permission_blocked" + }); + throw toolError("permission_blocked", "AI tool permission checks must be synchronous."); + } + if (!allowed) { + this.recordDecision({ + user, + role, + context, + tool, + args: checked.args, + decision: "rejected", + rejected_reason: "permission_blocked" + }); + throw toolError("permission_blocked", "The requesting user does not have permission for this action."); + } + this.recordDecision({ + user, + role, + context, + tool, + args: checked.args, + decision: "selected", + rejected_reason: null + }); + if (!checked.def.confirmation_required) return { execute: true, checked, context }; + const id = crypto.randomUUID(); + this.confirmations.set(id, { + id, + userId: user.id, + sessionId, + expiresAt: Date.now() + 120000, + context, + ...checked + }); + return { + execute: false, + checked, + context, + confirmation: { + id, + display_name: checked.def.display_name, + arguments: checked.args, + expires_at: Date.now() + 120000 + } + }; } - async execute({checked,user,requestId,context=null}){ - const result=await checked.def.workflow_handler({arguments:checked.args,user,ctx:context,initiated_via_ai:true,ai_request_id:requestId}); - this.audit({kind:"tool",status:"success",user_id:user.id,tool_requested:checked.def.tool_id,tool_executed:true}); - return result; + + async execute({ checked, user, requestId, context = null }) { + const started = Date.now(); + try { + const result = await checked.def.workflow_handler({ + arguments: checked.args, + user, + ctx: context, + initiated_via_ai: true, + ai_request_id: requestId + }); + this.audit({ + kind: "tool", + status: "success", + user_id: user.id, + actor: auditActor(user), + origin: contextOrigin(context), + tool_requested: checked.def.tool_id, + selected_tool: checked.def.tool_id, + tool_executed: true, + arguments_summary: summarizeArguments(checked.args), + execution_ms: Date.now() - started + }); + return result; + } catch (error) { + this.audit({ + kind: "tool", + status: "failed", + user_id: user.id, + actor: auditActor(user), + origin: contextOrigin(context), + tool_requested: checked.def.tool_id, + selected_tool: checked.def.tool_id, + tool_executed: false, + arguments_summary: summarizeArguments(checked.args), + execution_ms: Date.now() - started, + rejected_reason: error.code || "execution_failed" + }); + throw error; + } } - async confirm({id,user,sessionId}){ - const pending=this.confirmations.get(id); this.confirmations.delete(id); - if(!pending || pending.expiresAt keys.includes(key))) { + return { status: "malformed", call: null, error: "Tool-call JSON must contain only type, tool, and arguments." }; + } + if (typeof parsed.tool !== "string" || !parsed.tool || !parsed.arguments || + typeof parsed.arguments !== "object" || Array.isArray(parsed.arguments)) { + return { status: "malformed", call: null, error: "Tool-call JSON is missing a valid tool or arguments object." }; + } + return { + status: "valid", + call: { type: "tool_call", tool: parsed.tool, arguments: parsed.arguments }, + error: null + }; +} + +function publicDefinition(definition) { + const { + workflow_handler, + permission_check, + prompt_permission_check, + origin_check, + scope_check, + ...tool + } = definition; + return tool; +} + +function validateDefinition(definition) { + if (!definition?.tool_id || !definition.display_name || !definition.description || + !definition.owning_plugin || !definition.required_permission || !definition.audit_category || + typeof definition.workflow_handler !== "function" || + typeof definition.permission_check !== "function" || + !definition.schema || typeof definition.schema !== "object" || Array.isArray(definition.schema)) { + throw new Error("Invalid AI tool definition."); + } +} + +function normalizeArgument(key, value, descriptor) { + if (descriptor.type === "integer") { + if (!Number.isInteger(Number(value))) throw toolError("schema_invalid", `${key} must be an integer.`); + value = Number(value); + } else if (descriptor.type === "number") { + if (!Number.isFinite(Number(value))) throw toolError("schema_invalid", `${key} must be a number.`); + value = Number(value); + } else if (descriptor.type === "string") { + if (typeof value !== "string") throw toolError("schema_invalid", `${key} must be a string.`); + } else if (descriptor.type === "boolean") { + if (typeof value !== "boolean") throw toolError("schema_invalid", `${key} must be a boolean.`); + } else if (descriptor.type === "array") { + if (!Array.isArray(value)) throw toolError("schema_invalid", `${key} must be an array.`); + } else if (descriptor.type === "object") { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw toolError("schema_invalid", `${key} must be an object.`); + } + } + if (Array.isArray(descriptor.enum) && !descriptor.enum.includes(value)) { + throw toolError("schema_invalid", `${key} is invalid.`); + } + return value; +} + +function summarizeArguments(args) { + return Object.fromEntries(Object.entries(args || {}).map(([key, value]) => { + if (/token|secret|password|key|authorization|cookie/i.test(key)) return [key, "[REDACTED]"]; + if (typeof value === "string") return [key, { type: "string", length: value.length }]; + if (Array.isArray(value)) return [key, { type: "array", length: value.length }]; + return [key, { type: typeof value }]; + })); +} + +function contextOrigin(context) { + const value = String(context?.origin || context?.platform || "other").toLowerCase(); + return ["webui", "discord", "twitch", "youtube", "kick"].includes(value) ? value : "other"; +} + +function auditActor(user, role = null) { + return { + user_id: user?.id || null, + username: user?.username || null, + role: role || (user?.isAdmin ? "admin" : user?.isMod ? "mod" : "user") + }; +} + +function hidden(tool, reason, message) { + return { tool, exposed: false, reason, message }; +} + +function toolError(code, message) { + return Object.assign(new Error(message), { code }); +} + +module.exports = { + ToolRegistry, + contextOrigin, + exposureDecision, + parseToolCall, + parseToolCallResult, + publicDefinition, + summarizeArguments +}; diff --git a/plugins/lumi_ai/index.js b/plugins/lumi_ai/index.js index 5dab022..2b4e6d9 100644 --- a/plugins/lumi_ai/index.js +++ b/plugins/lumi_ai/index.js @@ -34,6 +34,7 @@ const { ToolInstaller } = require("./backend/tool_installer"); const { ToolLoader } = require("./backend/tool_loader"); const { ToolManager } = require("./backend/tool_manager"); const { ToolSettings } = require("./backend/tool_settings"); +const { buildAllowedToolsSection } = require("./backend/prompt_builder"); const storage = require("./backend/storage"); const { formatBytes, bytesFromMb, sanityCheckSize } = require("./backend/size_utils"); @@ -966,11 +967,17 @@ module.exports = { isAdmin: simulatedRole === "admin", isMod: simulatedRole === "mod" }; + const allowTools = req.body.allow_tools === true; + const testOrigin = ["webui", "discord", "twitch", "youtube", "kick", "other"].includes(req.body.origin) + ? req.body.origin + : "webui"; try { const result = await provider.test({ message, user: simulatedUser, - includeRaw: Boolean(req.body.show_raw_output) + includeRaw: Boolean(req.body.show_raw_output), + allowTools, + originContext: diagnosticOriginContext(simulatedUser, testOrigin) }); if (!req.body.show_raw_prompt) delete result.raw_prompt; res.json(result); @@ -1013,6 +1020,33 @@ module.exports = { } }); + router.get("/api/tools-diagnostics", async (req, res) => { + if (!req.session.user?.isAdmin) return res.status(403).json({ error: "Access denied." }); + const role = ["admin", "mod", "user"].includes(req.query.role) ? req.query.role : "admin"; + const origin = ["webui", "discord", "twitch", "youtube", "kick", "other"].includes(req.query.origin) + ? req.query.origin + : "webui"; + const simulatedUser = { + id: req.session.user.id, + username: req.session.user.username, + isAdmin: role === "admin", + isMod: role === "mod" + }; + try { + const diagnostics = await toolManager.diagnostics({ + role, + user: simulatedUser, + context: diagnosticOriginContext(simulatedUser, origin) + }); + return res.json({ + ...diagnostics, + prompt_preview: buildAllowedToolsSection(diagnostics.prompt_tools) + }); + } catch (error) { + return res.status(503).json({ error: error.message }); + } + }); + router.get("/api/tools/:id/readme", async (req, res) => { if (!req.session.user?.isAdmin) return res.status(403).json({ error: "Access denied." }); try { @@ -1685,6 +1719,37 @@ function webOriginContext(req) { }; } +function diagnosticOriginContext(user, origin = "webui") { + const role = roleOf(user); + const limits = { webui: 8000, discord: 1900, twitch: 450, youtube: 1800, kick: 450, other: 1000 }; + return { + origin, + platform: origin, + channel_id: "diagnostic-channel", + channel_name: "Diagnostic channel", + server_id: "diagnostic-server", + user_id: user.id, + username: user.username, + display_name: user.username, + role, + is_admin: role === "admin", + is_mod: role === "mod", + message_id: "diagnostic-message", + reply_mode: origin === "webui" ? "panel" : "same_channel", + format_capabilities: { + markdown: origin === "discord", + html: origin === "webui" + }, + max_message_length: limits[origin] || limits.other, + source_plugin: "lumi_ai", + source_command: "admin_diagnostic", + permission_context: { + identified_user: true, + webui_actions_allowed: origin === "webui" + } + }; +} + function finalizeAssistantResult(result, { role, config, baseUrl = "", maxLength = null, requestMessage = "" }) { const formatted = formatAssistantResponse({ text: normalizeCustomCommandReply(result.text, requestMessage), diff --git a/plugins/lumi_ai/plugin.json b/plugins/lumi_ai/plugin.json index 35a8817..f0d3684 100644 --- a/plugins/lumi_ai/plugin.json +++ b/plugins/lumi_ai/plugin.json @@ -1,7 +1,7 @@ { "id": "lumi_ai", "name": "Lumi AI", - "version": "0.7.1", + "version": "0.8.0", "description": "Managed local AI provider and scoped WebUI assistant for Lumi.", "main": "index.js" } diff --git a/plugins/lumi_ai/public/settings.css b/plugins/lumi_ai/public/settings.css index 977a3b5..34d9cc0 100644 --- a/plugins/lumi_ai/public/settings.css +++ b/plugins/lumi_ai/public/settings.css @@ -86,6 +86,14 @@ .ai-tools-dialog .modal-header p { margin: 4px 0 0; color: var(--ink-soft); } .ai-tools-source { margin: 10px 0; color: var(--ink-soft); font-size: 12px; overflow-wrap: anywhere; } .ai-tools-source.error { color: var(--rose); } +.ai-tool-diagnostics { margin: 10px 0; padding: 10px; border: 1px solid var(--border); border-radius: 7px; background: var(--surface-2); } +.ai-tool-diagnostic-controls { display: flex; flex-wrap: wrap; align-items: end; gap: 10px; margin: 10px 0; } +.ai-tool-diagnostic-controls label { display: grid; gap: 4px; font-weight: 800; } +.ai-tool-diagnostic-results { display: grid; gap: 6px; margin: 8px 0; } +.ai-tool-diagnostic-row { padding: 8px; border-left: 3px solid var(--border); background: var(--card); font-size: 12px; } +.ai-tool-diagnostic-row.exposed { border-left-color: var(--sea); } +.ai-tool-diagnostic-row.hidden { border-left-color: var(--rose); } +.ai-tool-diagnostics pre { max-height: 280px; overflow: auto; white-space: pre-wrap; } .ai-tools-list { display: grid; gap: 9px; } .ai-tool-row { border: 1px solid var(--border); border-radius: 7px; background: var(--card); } .ai-tool-summary { display: grid; grid-template-columns: minmax(170px, 1fr) minmax(280px, 1.5fr) minmax(130px, .7fr) auto; align-items: center; gap: 12px; padding: 11px; } diff --git a/plugins/lumi_ai/public/settings.js b/plugins/lumi_ai/public/settings.js index 20c3f54..3d8d16e 100644 --- a/plugins/lumi_ai/public/settings.js +++ b/plugins/lumi_ai/public/settings.js @@ -4,6 +4,7 @@ const downloadStatus = document.querySelector("[data-download-status]"); const testForm = document.querySelector("[data-ai-test-form]"); const testOutput = document.querySelector("[data-ai-test-output]"); + const testToolsNotice = document.querySelector("[data-ai-test-tools-notice]"); const gpuControl = document.querySelector("[data-gpu-control]"); const accessForm = document.querySelector("[data-ai-access-form]"); if (actions) { @@ -44,6 +45,15 @@ } catch {} }; if (testForm && testOutput) { + const updateTestToolsNotice = () => { + if (!testToolsNotice) return; + const enabled = testForm.elements.allow_tools?.checked === true; + testToolsNotice.textContent = enabled + ? "Tools are enabled. This test uses normal discovery, prompt exposure, permission checks, and execution." + : "Tools are disabled for this test; this result will not exercise tool discovery or execution."; + }; + testForm.elements.allow_tools?.addEventListener("change", updateTestToolsNotice); + updateTestToolsNotice(); testForm.addEventListener("submit", async (event) => { event.preventDefault(); const form = new FormData(testForm); @@ -55,7 +65,9 @@ headers: { "Content-Type": "application/json" }, body: JSON.stringify({ role: form.get("role"), + origin: form.get("origin"), message: form.get("message"), + allow_tools: form.get("allow_tools") === "on", show_raw_prompt: form.get("show_raw_prompt") === "on", show_raw_output: form.get("show_raw_output") === "on" }) diff --git a/plugins/lumi_ai/public/tool-manager.js b/plugins/lumi_ai/public/tool-manager.js index 8217a88..ba98649 100644 --- a/plugins/lumi_ai/public/tool-manager.js +++ b/plugins/lumi_ai/public/tool-manager.js @@ -12,8 +12,15 @@ const settingsForm = settingsModal?.querySelector("[data-ai-tool-settings-form]"); const settingsFields = settingsModal?.querySelector("[data-ai-tool-settings-fields]"); const settingsSave = settingsModal?.querySelector("[data-ai-tool-settings-save]"); + const diagnostics = modal?.querySelector("[data-ai-tool-diagnostics]"); + const diagnosticRole = diagnostics?.querySelector("[data-ai-tool-diagnostic-role]"); + const diagnosticOrigin = diagnostics?.querySelector("[data-ai-tool-diagnostic-origin]"); + const diagnosticRefresh = diagnostics?.querySelector("[data-ai-tool-diagnostic-refresh]"); + const diagnosticResults = diagnostics?.querySelector("[data-ai-tool-diagnostic-results]"); + const promptPreview = diagnostics?.querySelector("[data-ai-tool-prompt-preview]"); if (!openButton || !modal || !list || !source || !readmeModal || !readmeTitle || !readmeBody || - !settingsModal || !settingsTitle || !settingsForm || !settingsFields || !settingsSave) return; + !settingsModal || !settingsTitle || !settingsForm || !settingsFields || !settingsSave || + !diagnostics || !diagnosticRole || !diagnosticOrigin || !diagnosticResults || !promptPreview) return; let loading = false; let activeSettingsTool = null; @@ -126,11 +133,61 @@ detail("Dependency availability", dependencyStatus(tool.dependency_status)), detail("Risk / confirmation", `${tool.risk_level || "sensitive"} / ${tool.confirmation_required === false ? "not required" : "required"}`), detail("Runtime", `${tool.runtime_state || "unknown"}${tool.runtime_message ? `: ${tool.runtime_message}` : ""}`), + detail("Registered definitions", joinValue(tool.registered_tools)), detail("Install status", tool.local_error || tool.remote_error || (tool.remote_missing ? "Installed locally; missing from remote repository." : "Ready")) ); return details; }; + const loadDiagnostics = async () => { + diagnosticResults.replaceChildren(message("Evaluating registered tools...")); + promptPreview.textContent = "Loading..."; + diagnosticRefresh.disabled = true; + try { + const query = new URLSearchParams({ + role: diagnosticRole.value, + origin: diagnosticOrigin.value + }); + const response = await fetch(`/plugins/lumi_ai/api/tools-diagnostics?${query}`, { + cache: "no-store", + headers: { Accept: "application/json" } + }); + const payload = await response.json(); + if (!response.ok) throw new Error(payload.error || "Tool diagnostics are unavailable."); + renderDiagnostics(payload); + } catch (error) { + diagnosticResults.replaceChildren(message(error.message, true)); + promptPreview.textContent = "(unavailable)"; + } finally { + diagnosticRefresh.disabled = false; + } + }; + + const renderDiagnostics = (payload) => { + diagnosticResults.replaceChildren(); + if (!payload.plugins?.length) { + diagnosticResults.append(message("No installed AI tool plugins were discovered.")); + } + for (const plugin of payload.plugins || []) { + const row = document.createElement("div"); + row.className = `ai-tool-diagnostic-row ${plugin.prompt_exposed ? "exposed" : "hidden"}`; + const decisions = (plugin.decisions || []).map((decision) => + `${decision.tool.tool_id}: ${decision.exposed ? "exposed" : decision.reason}` + ).join(" | "); + row.textContent = [ + plugin.tool_id, + `state=${plugin.state}`, + `enabled=${plugin.enabled}`, + `registered=${(plugin.registered_tools || []).join(", ") || "none"}`, + plugin.prompt_exposed ? "prompt=exposed" : `prompt=hidden (${plugin.hidden_reason || "unknown"})`, + plugin.message || "", + decisions + ].filter(Boolean).join(" ยท "); + diagnosticResults.append(row); + } + promptPreview.textContent = payload.prompt_preview || "ALLOWED TOOLS:\n(none)"; + }; + const runAction = async (tool, action, control) => { control.disabled = true; const original = control.textContent; @@ -448,6 +505,12 @@ loadTools(false); }); refresh?.addEventListener("click", () => loadTools(true)); + diagnosticRefresh?.addEventListener("click", loadDiagnostics); + diagnostics.addEventListener("toggle", () => { + if (diagnostics.open) loadDiagnostics(); + }); + diagnosticRole.addEventListener("change", () => diagnostics.open && loadDiagnostics()); + diagnosticOrigin.addEventListener("change", () => diagnostics.open && loadDiagnostics()); modal.querySelectorAll("[data-ai-tools-close]").forEach((control) => control.addEventListener("click", () => setOpen(modal, false))); readmeModal.querySelectorAll("[data-ai-tool-readme-close]").forEach((control) => control.addEventListener("click", () => setOpen(readmeModal, false))); settingsModal.querySelectorAll("[data-ai-tool-settings-close]").forEach((control) => control.addEventListener("click", () => setOpen(settingsModal, false))); diff --git a/plugins/lumi_ai/tests/tool-loading.js b/plugins/lumi_ai/tests/tool-loading.js new file mode 100644 index 0000000..3644a86 --- /dev/null +++ b/plugins/lumi_ai/tests/tool-loading.js @@ -0,0 +1,321 @@ +const assert = require("assert"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +const { AiProvider } = require("../backend/ai_provider"); +const { buildPrompt } = require("../backend/prompt_builder"); +const { ToolInstaller } = require("../backend/tool_installer"); +const { ToolLoader } = require("../backend/tool_loader"); +const { ToolManager } = require("../backend/tool_manager"); +const { ToolRegistry, parseToolCallResult } = require("../backend/tool_router"); + +async function run() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "lumi-ai-loading-")); + try { + const pluginsDir = path.join(root, "plugins"); + const stateFile = path.join(root, "data", "tools", "enabled.json"); + fs.mkdirSync(pluginsDir, { recursive: true }); + fs.mkdirSync(path.join(pluginsDir, "lumi_ai"), { recursive: true }); + createTool(pluginsDir, "lumi_ai_web_search", { readOnly: true }); + createTool(pluginsDir, "lumi_ai_test_tool", { readOnly: true }); + createInvalidTool(pluginsDir, "lumi_ai_invalid"); + createMissingEntrypointTool(pluginsDir, "lumi_ai_missing_entrypoint"); + + const audit = []; + const registry = new ToolRegistry((entry) => audit.push(entry)); + const installer = new ToolInstaller({ + pluginsDir, + stagingRoot: path.join(root, "staging"), + repoClient: {} + }); + const loader = new ToolLoader({ + registry, + installer, + settings: { getSetting: (_key, fallback) => fallback }, + stateFile, + lumiAiVersion: "0.8.0", + lumiVersion: "0.1.0" + }); + const manager = new ToolManager({ + loader, + installer, + settings: { describe: () => ({}) }, + repoClient: { + async discover() { + return { + repository: "local", + branch: "main", + checked_at: new Date().toISOString(), + cached: false, + stale: false, + tools: [] + }; + } + } + }); + + assert.deepEqual( + installer.scanLocal().map((entry) => entry.tool_id).sort(), + ["lumi_ai_invalid", "lumi_ai_missing_entrypoint", "lumi_ai_test_tool", "lumi_ai_web_search"] + ); + + loader.setEnabled("lumi_ai_web_search", true); + loader.setEnabled("lumi_ai_test_tool", true); + loader.setEnabled("lumi_ai_invalid", true); + loader.setEnabled("lumi_ai_missing_entrypoint", true); + await loader.loadEnabled(); + assert.equal(registry.has("lumi_ai_web_search.lookup"), true); + assert.equal(registry.has("lumi_ai_test_tool.lookup"), true); + assert.equal(loader.status("lumi_ai_invalid").state, "unavailable"); + assert.equal(loader.status("lumi_ai_missing_entrypoint").state, "unavailable"); + + await loader.disable("lumi_ai_test_tool"); + assert.equal(registry.has("lumi_ai_test_tool.lookup"), false); + const disabledDiagnostics = await manager.diagnostics({ + role: "user", + user: actor("user"), + context: origin("webui") + }); + assert.equal( + disabledDiagnostics.plugins.find((plugin) => plugin.tool_id === "lumi_ai_test_tool").hidden_reason, + "disabled" + ); + assert.equal( + disabledDiagnostics.plugins.find((plugin) => plugin.tool_id === "lumi_ai_invalid").hidden_reason, + "schema_invalid" + ); + + const webui = registry.inspect({ + role: "user", + user: actor("user"), + context: origin("webui") + }); + assert(webui.exposed.some((tool) => tool.tool_id === "lumi_ai_web_search.lookup")); + const discord = registry.inspect({ + role: "user", + user: actor("user"), + context: origin("discord", false) + }); + assert(discord.exposed.some((tool) => tool.tool_id === "lumi_ai_web_search.lookup")); + + registry.register(actionDefinition()); + const platformActions = registry.inspect({ + role: "admin", + user: actor("admin"), + context: origin("discord", false) + }); + assert.equal( + platformActions.considered.find((entry) => entry.tool.tool_id === "test_admin.remove").reason, + "scope_blocked" + ); + + const prompt = buildPrompt({ + config: promptConfig(), + role: "user", + message: "Search for current Lumi information", + tools: webui.exposed, + originContext: origin("webui") + }); + assert(prompt.includes('{"type":"tool_call","tool":"tool_id","arguments":{}}')); + assert(prompt.includes('"tool_id":"lumi_ai_web_search.lookup"')); + assert(prompt.includes('"kind":"lookup/read"')); + + assert.equal(parseToolCallResult('{"type":"tool_call","tool":"x","arguments":{}}').status, "valid"); + assert.equal(parseToolCallResult('Use this: {"type":"tool_call","tool":"x","arguments":{}}').status, "malformed"); + assert.equal( + parseToolCallResult('{"type":"tool_call","tool":"x","arguments":{},"comment":"no"}').status, + "malformed" + ); + + await verifyReadOnlyFinalization(registry); + verifyConfirmation(registry); + assert(audit.some((entry) => entry.kind === "tool" && entry.status === "success")); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + console.log("Lumi AI tool loading verification passed."); +} + +async function verifyReadOnlyFinalization(registry) { + const outputs = [ + '{"type":"tool_call","tool":"lumi_ai_web_search.lookup","arguments":{"query":"current Lumi release"}}', + "The current result is Lumi 1.2.3 according to the returned source." + ]; + let inferCalls = 0; + const provider = new AiProvider({ + getConfig: () => ({ + selected_model_id: "test", + support_scope: {}, + instructions: {}, + logging: {}, + hard_generation_timeout_ms: 5000, + output_budgets: { simple_answer: 128 } + }), + runtime: { + infer: async () => ({ + choices: [{ message: { content: outputs[inferCalls++] }, finish_reason: "stop" }], + usage: { prompt_tokens: 10, completion_tokens: 5 } + }) + }, + queue: { + length: 0, + run: async (_userId, _role, callback) => callback(0) + }, + tools: registry, + metrics: { record() {} }, + getContext: () => [], + lookupRepo: () => null, + getRepoContext: () => [], + getCorrections: () => [] + }); + const result = await provider.generate({ + message: "What is the current Lumi release?", + user: actor("user"), + sessionId: "tool-test", + originContext: origin("webui"), + allowTools: true + }); + assert.equal(inferCalls, 2); + assert.match(result.text, /Lumi 1\.2\.3/); + assert.deepEqual(result.tool_result, { + status: "ok", + query: "current Lumi release", + results: [{ title: "Lumi 1.2.3", url: "https://example.test/lumi" }] + }); +} + +function verifyConfirmation(registry) { + const prepared = registry.prepare({ + tool: "test_admin.remove", + args: { target: "example" }, + user: actor("admin"), + role: "admin", + sessionId: "confirm-test", + context: origin("webui") + }); + assert.equal(prepared.execute, false); + assert(prepared.confirmation.id); +} + +function createTool(pluginsDir, toolId, { readOnly }) { + const directory = path.join(pluginsDir, toolId); + fs.mkdirSync(directory, { recursive: true }); + const metadata = { + tool_id: toolId, + display_name: toolId, + version: "1.0.0", + description: "Test lookup tool", + scope: "assistant", + permissions: { required_role: "user" }, + capabilities: ["Current information lookup"], + limitations: ["Test data only"], + tool_type: "lookup", + entrypoints: { backend: "index.js" }, + risk_level: "low", + confirmation_required: false + }; + fs.writeFileSync(path.join(directory, "tool_info.json"), `${JSON.stringify(metadata, null, 2)}\n`); + fs.writeFileSync(path.join(directory, "readme.md"), `# ${toolId}\n`); + fs.writeFileSync(path.join(directory, "index.js"), ` +module.exports.register = ({ registerTool }) => registerTool({ + tool_id: "${toolId}.lookup", + display_name: "Lookup", + description: "Looks up current test information.", + required_role: "user", + required_permission: "${toolId}.use", + audit_category: "lookup", + confirmation_required: false, + risk_level: "low", + read_only: ${readOnly}, + use_cases: ["Current information"], + output_expectations: "Structured test results.", + schema: { query: { type: "string", required: true } }, + permission_check: ({ user }) => Boolean(user && user.id), + workflow_handler: async ({ arguments: args }) => ({ + status: "ok", + query: args.query, + results: [{ title: "Lumi 1.2.3", url: "https://example.test/lumi" }] + }) +}); +`); +} + +function createInvalidTool(pluginsDir, toolId) { + const directory = path.join(pluginsDir, toolId); + fs.mkdirSync(directory, { recursive: true }); + fs.writeFileSync(path.join(directory, "tool_info.json"), "{invalid"); + fs.writeFileSync(path.join(directory, "readme.md"), "# Invalid\n"); +} + +function createMissingEntrypointTool(pluginsDir, toolId) { + const directory = path.join(pluginsDir, toolId); + fs.mkdirSync(directory, { recursive: true }); + const metadata = { + tool_id: toolId, + display_name: toolId, + version: "1.0.0", + description: "Missing entrypoint test", + scope: "assistant", + permissions: { required_role: "user" }, + capabilities: ["Test"], + limitations: ["Unavailable"], + entrypoints: { backend: "missing.js" } + }; + fs.writeFileSync(path.join(directory, "tool_info.json"), `${JSON.stringify(metadata, null, 2)}\n`); + fs.writeFileSync(path.join(directory, "readme.md"), "# Missing entrypoint\n"); +} + +function actionDefinition() { + return { + tool_id: "test_admin.remove", + display_name: "Remove test data", + description: "Mutates test data.", + owning_plugin: "test_admin", + required_role: "admin", + required_permission: "test.remove", + audit_category: "test", + confirmation_required: true, + risk_level: "sensitive", + read_only: false, + schema: { target: { type: "string", required: true } }, + permission_check: ({ user }) => user?.isAdmin === true, + workflow_handler: async () => ({ status: "ok" }) + }; +} + +function actor(role) { + return { + id: `${role}-1`, + username: role, + isAdmin: role === "admin", + isMod: role === "mod" + }; +} + +function origin(platform, webuiActionsAllowed = platform === "webui") { + return { + origin: platform, + platform, + max_message_length: platform === "twitch" ? 450 : 1900, + permission_context: { + identified_user: true, + webui_actions_allowed: webuiActionsAllowed + } + }; +} + +function promptConfig() { + return { + support_scope: {}, + instructions: { + roleplay_intensity: 0, + community_tone: "", + admin_custom: "" + } + }; +} + +run().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/plugins/lumi_ai/tests/verify-tools.js b/plugins/lumi_ai/tests/verify-tools.js index d3096d4..25ab915 100644 --- a/plugins/lumi_ai/tests/verify-tools.js +++ b/plugins/lumi_ai/tests/verify-tools.js @@ -58,7 +58,7 @@ async function run() { installer, settings, stateFile, - lumiAiVersion: "0.7.1", + lumiAiVersion: "0.8.0", lumiVersion: "0.1.0" }); const toolSettings = new ToolSettings({ installer }); diff --git a/plugins/lumi_ai/tests/verify.js b/plugins/lumi_ai/tests/verify.js index aad34d2..c9b873f 100644 --- a/plugins/lumi_ai/tests/verify.js +++ b/plugins/lumi_ai/tests/verify.js @@ -686,7 +686,7 @@ async function run() { await registry.confirm({ id: valid.confirmation.id, user: { id: "user-1" }, sessionId: "s1" }); assert.equal(calls[0].user.id, "user-1"); assert.equal(calls[0].initiated_via_ai, true); - assert.equal(audit[0].tool_executed, true); + assert.equal(audit.find((entry) => entry.kind === "tool")?.tool_executed, true); const queueConfig = { concurrency: 1, max_queue_length: 1, per_user_requests_per_minute: 20 }; const queue = new RequestQueue(() => queueConfig); diff --git a/plugins/lumi_ai/views/settings.ejs b/plugins/lumi_ai/views/settings.ejs index 0214192..9b0420f 100644 --- a/plugins/lumi_ai/views/settings.ejs +++ b/plugins/lumi_ai/views/settings.ejs @@ -487,11 +487,14 @@

Test console

Run a request as a simulated role without changing the logged-in actor.

+
+
+

Tools are disabled for this test unless explicitly enabled above.

diff --git a/plugins/lumi_ai/views/tool-modal.ejs b/plugins/lumi_ai/views/tool-modal.ejs index 409bea3..96b422a 100644 --- a/plugins/lumi_ai/views/tool-modal.ejs +++ b/plugins/lumi_ai/views/tool-modal.ejs @@ -8,6 +8,34 @@
Checking configured repository...
+
+ Prompt exposure diagnostics +
+ + + +
+
Open diagnostics to inspect registry exposure.
+
+ Current ALLOWED TOOLS prompt preview +
(not loaded)
+
+
Loading AI tool plugins...
diff --git a/plugins/lumi_ai_web_search/index.js b/plugins/lumi_ai_web_search/index.js index 8ba792c..cfb84ab 100644 --- a/plugins/lumi_ai_web_search/index.js +++ b/plugins/lumi_ai_web_search/index.js @@ -36,6 +36,13 @@ module.exports.register = ({ registerTool, paths }) => { audit_category: "web_search", confirmation_required: false, risk_level: "low", + read_only: true, + use_cases: [ + "Current facts or recent news", + "External documentation and troubleshooting sources", + "Public resources not present in verified Lumi context" + ], + output_expectations: "Returns policy-filtered structured search results. Lumi Assistant writes the final origin-limited answer and cites only returned URLs.", schema: { query: { type: "string", required: true }, reason: { @@ -53,6 +60,12 @@ module.exports.register = ({ registerTool, paths }) => { requested_depth: { type: "string", required: false, enum: ["search", "full_page"] }, freshness: { type: "string", required: false } }, + origin_check: ({ context }) => { + const settings = readSettings(paths.data); + const origin = normalizeOrigin(context?.origin || context?.platform || "other"); + return settings.enabled && settings.allowed_origins.includes(origin); + }, + prompt_permission_check: ({ user }) => Boolean(user?.id), permission_check: ({ user, context }) => { const settings = readSettings(paths.data); const origin = normalizeOrigin(context?.origin || context?.platform || "other"); diff --git a/plugins/lumi_ai_web_search/readme.md b/plugins/lumi_ai_web_search/readme.md index 4ed154c..4538d0d 100644 --- a/plugins/lumi_ai_web_search/readme.md +++ b/plugins/lumi_ai_web_search/readme.md @@ -5,7 +5,7 @@ ## Installation and enablement 1. Install this directory as `plugins/lumi_ai_web_search/`. -2. Install Lumi AI `0.7.1` or newer. +2. Install Lumi AI `0.8.0` or newer. 3. Open **Plugins -> Lumi AI -> Tools**. 4. Select **Settings** for Lumi AI Web Search. 5. Configure the provider and URL policy, turn on **Web search enabled**, and save. @@ -57,6 +57,8 @@ The registered tool ID is `lumi_ai_web_search.search`. It accepts: The assistant should use this tool only for current or external information that is not available in verified local Lumi context. +The tool registers as a read-only lookup. This allows permitted Discord, Twitch, YouTube, Kick, and other contexts to use it even though those contexts cannot run WebUI action tools. Lumi AI evaluates the configured origin allowlist before including the tool in the model prompt and again before execution. + Results are sanitized and returned as structured data rather than raw provider JSON. Each result contains a title, permitted URL or no URL when links are disabled, domain, condensed snippet, source type, date, and relevance score. Documentation and troubleshooting searches prioritize authoritative sources; recent searches prioritize dated sources. Optional full-page mode extracts bounded visible text only when the administrator enables it. It does not automate a browser, submit forms, execute scripts, or follow unrestricted links. diff --git a/plugins/lumi_ai_web_search/tests/verify.js b/plugins/lumi_ai_web_search/tests/verify.js index e6a4993..1372eb6 100644 --- a/plugins/lumi_ai_web_search/tests/verify.js +++ b/plugins/lumi_ai_web_search/tests/verify.js @@ -42,7 +42,7 @@ async function verifyLoaderLifecycle() { installer, settings: { getSetting: (_key, fallback) => fallback }, stateFile: path.join(root, "enabled.json"), - lumiAiVersion: "0.7.1", + lumiAiVersion: "0.8.0", lumiVersion: "0.1.0" }); const unavailable = await loader.enable("lumi_ai_web_search"); @@ -267,9 +267,13 @@ function verifyRegistrationAvailability() { }); assert.equal(definitions.length, 1); assert.equal(definitions[0].tool_id, "lumi_ai_web_search.search"); + assert.equal(definitions[0].read_only, true); + assert.equal(definitions[0].origin_check({ + context: { origin: "discord", permission_context: { webui_actions_allowed: false } } + }), true); assert.equal(definitions[0].permission_check({ user: { id: "user" }, - context: { origin: "webui" } + context: { origin: "discord", permission_context: { webui_actions_allowed: false } } }), true); fs.rmSync(root, { recursive: true, force: true }); } diff --git a/plugins/lumi_ai_web_search/tool_info.json b/plugins/lumi_ai_web_search/tool_info.json index c875385..9dfc41a 100644 --- a/plugins/lumi_ai_web_search/tool_info.json +++ b/plugins/lumi_ai_web_search/tool_info.json @@ -2,7 +2,7 @@ "tool_id": "lumi_ai_web_search", "name": "lumi_ai_web_search", "display_name": "Lumi AI Web Search", - "version": "1.0.0", + "version": "1.0.1", "description": "Controlled current-information search for Lumi Assistant with URL policy, origin budgets, and source normalization.", "scope": { "label": "Assistant web lookup", @@ -25,7 +25,7 @@ "Search quality and freshness depend on the configured provider" ], "tool_type": "web_search", - "owning_plugin": "lumi_ai", + "owning_plugin": "lumi_ai_web_search", "entrypoints": { "backend": "index.js" }, @@ -37,7 +37,7 @@ ], "dependencies": [], "minimum_lumi_version": "0.1.0", - "minimum_lumi_ai_version": "0.7.1", + "minimum_lumi_ai_version": "0.8.0", "required_plugins": [ "core", "lumi_ai" @@ -52,7 +52,7 @@ "preserve_on_update": [ "data" ], - "update_notes": "Initial controlled web-search provider with policy enforcement and per-origin output budgets.", + "update_notes": "Registers as a read-only lookup tool with context-aware prompt exposure for Lumi AI 0.8.0.", "author": "Lumi", "homepage": "https://git.rolfsvaag.no/Rolfsvaag_Datateknikk/Lumi", "repository_path": "plugins/lumi_ai_web_search",