const fs = require("fs"); const path = require("path"); const os = require("os"); const crypto = require("crypto"); let AdmZip = null; try { AdmZip = require("adm-zip"); } catch { AdmZip = null; } const { db } = require("./db"); const { createRecoveryMarker, updateRecoveryMarker, markRecoveryMarkerFailed, markRecoveryMarkerComplete } = require("./recovery-mode"); const repoRoot = path.join(__dirname, "..", ".."); const dataDir = path.join(repoRoot, "data"); const snapshotsDir = path.join(dataDir, "snapshots"); const indexPath = path.join(snapshotsDir, "index.json"); const maxSnapshots = 20; const PRESERVE_RELATIVE_PATHS = new Set([ ".git", "node_modules", "data", "config", "storage", "uploads", "logs", "database", "databases", "plugins", "knowledge/community", "knowledge/corrections", ".env", ".env.local", ".env.production", ".secrets", "codex-guidelines" ]); const GENERATED_RELATIVE_PATHS = new Set([ "dist", "build", "coverage", ".cache", ".parcel-cache", ".turbo", "tmp", "temp" ]); const SNAPSHOT_EXCLUDE_RELATIVE_PATHS = new Set([ ".git", "node_modules", "data/update-cache", "data/snapshots" ]); const HARDLINK_SNAPSHOT_MIN_BYTES = 64 * 1024 * 1024; function ensureSnapshotsDir() { fs.mkdirSync(snapshotsDir, { recursive: true }); } function loadIndex() { if (!fs.existsSync(indexPath)) { return []; } try { const raw = fs.readFileSync(indexPath, "utf8"); const data = JSON.parse(raw); return Array.isArray(data) ? data : []; } catch { return []; } } function saveIndex(entries) { ensureSnapshotsDir(); fs.writeFileSync(indexPath, JSON.stringify(entries, null, 2), "utf8"); } async function backupDatabase(targetPath) { if (typeof db.backup === "function") { await db.backup(targetPath); return; } const source = path.join(dataDir, "app.db"); if (fs.existsSync(source)) { fs.copyFileSync(source, targetPath); } } async function createSnapshot({ type, pluginId, metadata = {} }) { ensureSnapshotsDir(); const id = `${Date.now()}-${crypto.randomUUID()}`; const snapshotPath = path.join(snapshotsDir, id); fs.mkdirSync(snapshotPath, { recursive: true }); const dbPath = path.join(snapshotPath, "app.db"); await backupDatabase(dbPath); let pluginExisted = false; let pluginZip = null; let fullPath = null; if (type === "bot") { const coreZip = path.join(snapshotPath, "core.zip"); zipCore(coreZip); fullPath = path.join(snapshotPath, "full"); snapshotFullInstall(fullPath); } if (type === "plugin" && pluginId) { const pluginDir = path.join(repoRoot, "plugins", pluginId); pluginExisted = fs.existsSync(pluginDir); if (pluginExisted) { pluginZip = path.join(snapshotPath, "plugin.zip"); zipFolder(pluginDir, pluginZip, { base: pluginDir, ignore: new Set(["node_modules", "data"]) }); } } return { id, type, pluginId, pluginExisted, pluginZip, fullPath, snapshotPath, metadata }; } function finalizeSnapshot(snapshot) { const entries = loadIndex(); const record = { id: snapshot.id, type: snapshot.type, pluginId: snapshot.pluginId || null, pluginExisted: snapshot.pluginExisted || false, createdAt: Date.now(), status: "available", path: snapshot.snapshotPath, target_kind: snapshot.metadata?.target_kind || snapshot.type, target_id: snapshot.metadata?.target_id || snapshot.pluginId || null, from_version: snapshot.metadata?.from_version || null, to_version: snapshot.metadata?.to_version || null, source_branch: snapshot.metadata?.source_branch || null, update_method: snapshot.metadata?.update_method || null, rollback_safe: snapshot.metadata?.rollback_safe !== false, recovery_marker_id: snapshot.metadata?.recovery_marker_id || null, major_crossing: Boolean(snapshot.metadata?.major_crossing), migration_notes: snapshot.metadata?.migration_notes || "", danger_notes: snapshot.metadata?.danger_notes || [] }; entries.push(record); saveIndex(pruneEntries(entries)); return record; } function discardSnapshot(snapshot) { if (!snapshot?.snapshotPath) { return; } try { fs.rmSync(snapshot.snapshotPath, { recursive: true, force: true }); } catch { // Ignore cleanup failures. } } function pruneEntries(entries) { const available = entries .filter((entry) => entry.status === "available") .sort((a, b) => b.createdAt - a.createdAt); const keep = new Set(available.slice(0, maxSnapshots).map((entry) => entry.id)); const pruned = entries.filter((entry) => entry.status !== "available" || keep.has(entry.id)); for (const entry of entries) { if (entry.status === "available" && !keep.has(entry.id)) { try { fs.rmSync(entry.path, { recursive: true, force: true }); } catch { // Ignore cleanup failures. } } } return pruned; } function listSnapshots() { return loadIndex() .filter((entry) => entry.status === "available") .sort((a, b) => b.createdAt - a.createdAt); } function markSnapshotRolledBack(id) { const entries = loadIndex(); const entry = entries.find((item) => item.id === id); if (!entry) { return null; } entry.status = "rolled_back"; entry.rolledBackAt = Date.now(); saveIndex(entries); return entry; } function extractZip(zipPath, targetDir) { if (!AdmZip) { throw new Error("adm-zip is not installed. Run npm install."); } const zip = new AdmZip(zipPath); zip.extractAllTo(targetDir, true); } function resolveZipRoot(extractedDir) { const packagePath = path.join(extractedDir, "package.json"); if (fs.existsSync(packagePath)) { return extractedDir; } const entries = fs.readdirSync(extractedDir, { withFileTypes: true }); const dirs = entries.filter((entry) => entry.isDirectory()); if (dirs.length === 1) { const candidate = path.join(extractedDir, dirs[0].name); if (fs.existsSync(path.join(candidate, "package.json"))) { return candidate; } } return extractedDir; } function resolvePatchRoot(extractedDir) { // Patch archives are copied relative to the repository root. Do not infer a // nested root from a single top-level directory such as src/, or patches will // be applied to services/ and web/ instead of src/services/ and src/web/. return extractedDir; } function resolvePluginRoot(extractedDir) { const pluginPath = path.join(extractedDir, "plugin.json"); if (fs.existsSync(pluginPath)) { return extractedDir; } const entries = fs.readdirSync(extractedDir, { withFileTypes: true }); const dirs = entries.filter((entry) => entry.isDirectory()); if (dirs.length === 1) { const candidate = path.join(extractedDir, dirs[0].name); if (fs.existsSync(path.join(candidate, "plugin.json"))) { return candidate; } } return extractedDir; } function verifyBotPackage(rootPath) { const required = [ path.join(rootPath, "package.json"), path.join(rootPath, "safe-mode.js"), path.join(rootPath, "src", "main.js"), path.join(rootPath, "src", "web", "server.js") ]; for (const filePath of required) { if (!fs.existsSync(filePath)) { throw new Error(`Missing required file: ${path.relative(rootPath, filePath)}`); } } JSON.parse(fs.readFileSync(required[0], "utf8")); } function verifyPatchPackage(rootPath) { if (!hasAnyFiles(rootPath)) { throw new Error("Patch archive is empty."); } } function verifyPluginPackage(rootPath) { const pluginPath = path.join(rootPath, "plugin.json"); if (!fs.existsSync(pluginPath)) { throw new Error("plugin.json not found in plugin package."); } const manifest = JSON.parse(fs.readFileSync(pluginPath, "utf8")); if (!manifest.id) { throw new Error("plugin.json must include an id."); } const mainFile = manifest.main || "index.js"; const mainPath = path.join(rootPath, mainFile); if (!fs.existsSync(mainPath)) { throw new Error(`Plugin entry ${mainFile} not found.`); } return manifest; } function zipCore(destination) { if (!AdmZip) { throw new Error("adm-zip is not installed. Run npm install."); } const zip = new AdmZip(); addFolder(zip, repoRoot, repoRoot, new Set([".git", "node_modules", "data", "plugins"])); zip.writeZip(destination); } function zipFolder(source, destination, options) { if (!AdmZip) { throw new Error("adm-zip is not installed. Run npm install."); } const zip = new AdmZip(); const base = options?.base || source; addFolder(zip, source, base, options?.ignore || new Set(["node_modules"])); zip.writeZip(destination); } function snapshotFullInstall(destination) { fs.mkdirSync(destination, { recursive: true }); copySnapshotDirectory(repoRoot, destination, SNAPSHOT_EXCLUDE_RELATIVE_PATHS, { base: repoRoot }); } function addFolder(zip, folderPath, basePath, ignore) { const entries = fs.readdirSync(folderPath, { withFileTypes: true }); for (const entry of entries) { const fullPath = path.join(folderPath, entry.name); const relPath = normalizeRelative(path.relative(basePath, fullPath)); const topLevel = relPath.split("/")[0]; if (ignore.has(relPath) || ignore.has(topLevel)) { continue; } if (entry.isDirectory()) { addFolder(zip, fullPath, basePath, ignore); } else if (entry.isFile()) { zip.addLocalFile(fullPath, path.dirname(relPath)); } } } function resetCoreFiles() { const ignore = new Set([".git", "node_modules", "data", "plugins"]); const entries = fs.readdirSync(repoRoot, { withFileTypes: true }); for (const entry of entries) { if (ignore.has(entry.name)) { continue; } const fullPath = path.join(repoRoot, entry.name); fs.rmSync(fullPath, { recursive: true, force: true }); } } function normalizeRelative(value) { return String(value || "").split(path.sep).join("/"); } function isRelativePathIgnored(relativePath, ignore) { const rel = normalizeRelative(relativePath); if (!rel) return false; if (ignore.has(rel)) return true; const parts = rel.split("/"); for (let index = 1; index <= parts.length; index += 1) { if (ignore.has(parts.slice(0, index).join("/"))) { return true; } } return false; } function copyDirectory(source, target, ignore, options = {}) { const base = options.base || source; const entries = fs.readdirSync(source, { withFileTypes: true }); for (const entry of entries) { const srcPath = path.join(source, entry.name); const relPath = normalizeRelative(path.relative(base, srcPath)); if (isRelativePathIgnored(relPath, ignore)) { continue; } const destPath = path.join(target, entry.name); if (entry.isDirectory()) { fs.mkdirSync(destPath, { recursive: true }); copyDirectory(srcPath, destPath, ignore, { base }); } else if (entry.isFile()) { fs.mkdirSync(path.dirname(destPath), { recursive: true }); fs.copyFileSync(srcPath, destPath); } } } function copySnapshotDirectory(source, target, ignore, options = {}) { const base = options.base || source; const entries = fs.readdirSync(source, { withFileTypes: true }); for (const entry of entries) { const srcPath = path.join(source, entry.name); const relPath = normalizeRelative(path.relative(base, srcPath)); if (isRelativePathIgnored(relPath, ignore)) { continue; } const destPath = path.join(target, entry.name); if (entry.isDirectory()) { fs.mkdirSync(destPath, { recursive: true }); copySnapshotDirectory(srcPath, destPath, ignore, { base }); } else if (entry.isFile()) { fs.mkdirSync(path.dirname(destPath), { recursive: true }); copySnapshotFile(srcPath, destPath); } } } function copySnapshotFile(source, target) { const stat = fs.statSync(source); if (stat.size >= HARDLINK_SNAPSHOT_MIN_BYTES) { try { fs.linkSync(source, target); return; } catch { // Fall back to copying when hard links are unavailable across devices. } } fs.copyFileSync(source, target); } function removeGeneratedPaths() { for (const relativePath of GENERATED_RELATIVE_PATHS) { const target = path.join(repoRoot, relativePath); if (target.startsWith(path.join(repoRoot, "data", "snapshots"))) { continue; } fs.rmSync(target, { recursive: true, force: true }); } } function applyCoreFilesFromDirectory(rootPath) { removeGeneratedPaths(); copyDirectory( rootPath, repoRoot, PRESERVE_RELATIVE_PATHS, { base: rootPath } ); } function applyCorePatch(rootPath) { copyDirectory( rootPath, repoRoot, PRESERVE_RELATIVE_PATHS, { base: rootPath } ); } function hasAnyFiles(rootPath) { const entries = fs.readdirSync(rootPath, { withFileTypes: true }); for (const entry of entries) { if (entry.isFile()) { return true; } if (entry.isDirectory()) { if (hasAnyFiles(path.join(rootPath, entry.name))) { return true; } } } return false; } function resetPluginCode(targetDir) { if (!fs.existsSync(targetDir)) { return; } for (const entry of fs.readdirSync(targetDir, { withFileTypes: true })) { if (entry.name === "data") { continue; } fs.rmSync(path.join(targetDir, entry.name), { recursive: true, force: true }); } } function applyPluginFiles(rootPath, pluginId, options = {}) { const pluginsDir = path.join(repoRoot, "plugins"); const targetDir = path.join(pluginsDir, pluginId); fs.mkdirSync(pluginsDir, { recursive: true }); replacePluginDirectory(rootPath, targetDir, options); } function replacePluginDirectory(rootPath, targetDir, options = {}) { if (options.preserveData) { resetPluginCode(targetDir); } else { fs.rmSync(targetDir, { recursive: true, force: true }); } fs.mkdirSync(targetDir, { recursive: true }); copyDirectory( rootPath, targetDir, options.preserveData ? new Set(["node_modules", "data"]) : new Set(["node_modules"]) ); } async function applyBotUpdate(zipPath, options = {}) { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "lumi-update-")); try { extractZip(zipPath, tempDir); const mode = options.mode === "patch" ? "patch" : "full"; const rootPath = mode === "patch" ? resolvePatchRoot(tempDir) : resolveZipRoot(tempDir); if (mode === "patch") { verifyPatchPackage(rootPath); } else { verifyBotPackage(rootPath); } const marker = createRecoveryMarker({ target_kind: "core", target_id: "core", from_version: options.metadata?.from_version || null, to_version: options.metadata?.to_version || null, source_branch: options.metadata?.source_branch || "manual_zip", update_method: options.mode === "patch" ? "zip_patch" : "zip", rollback_safe: options.metadata?.rollback_safe !== false, major_crossing: Boolean(options.metadata?.major_crossing) }); updateRecoveryMarker({ status: "applying" }); const snapshot = await createSnapshot({ type: "bot", metadata: { target_kind: "core", update_method: options.mode === "patch" ? "zip_patch" : "zip", ...options.metadata, recovery_marker_id: marker.id } }); try { if (mode === "patch") { applyCorePatch(rootPath); } else { applyCoreFilesFromDirectory(rootPath); } const record = finalizeSnapshot(snapshot); markRecoveryMarkerComplete({ snapshot_id: record.id }); return record; } catch (error) { discardSnapshot(snapshot); markRecoveryMarkerFailed(error); throw error; } } finally { fs.rmSync(tempDir, { recursive: true, force: true }); } } async function applyPluginUpdate(zipPath, options = {}) { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "lumi-plugin-")); try { extractZip(zipPath, tempDir); const rootPath = resolvePluginRoot(tempDir); const manifest = verifyPluginPackage(rootPath); if (options.expectedPluginId && manifest.id !== options.expectedPluginId) { throw new Error(`Plugin ZIP id ${manifest.id} does not match ${options.expectedPluginId}.`); } const marker = createRecoveryMarker({ target_kind: "plugin", target_id: manifest.id, from_version: options.metadata?.from_version || null, to_version: manifest.version || options.metadata?.to_version || null, source_branch: options.metadata?.source_branch || "manual_zip", update_method: "zip", rollback_safe: options.metadata?.rollback_safe !== false, major_crossing: Boolean(options.metadata?.major_crossing) }); updateRecoveryMarker({ status: "applying" }); const snapshot = await createSnapshot({ type: "plugin", pluginId: manifest.id, metadata: { target_kind: "plugin", target_id: manifest.id, update_method: "zip", ...options.metadata, recovery_marker_id: marker.id } }); try { applyPluginFiles(rootPath, manifest.id, { preserveData: snapshot.pluginExisted }); const record = finalizeSnapshot(snapshot); markRecoveryMarkerComplete({ snapshot_id: record.id }); return record; } catch (error) { discardSnapshot(snapshot); markRecoveryMarkerFailed(error); throw error; } } finally { fs.rmSync(tempDir, { recursive: true, force: true }); } } function restoreDatabase(snapshotPath) { const source = path.join(snapshotPath, "app.db"); const target = path.join(dataDir, "app.db"); if (!fs.existsSync(source)) { throw new Error("Snapshot database not found."); } fs.copyFileSync(source, target); const wal = path.join(dataDir, "app.db-wal"); const shm = path.join(dataDir, "app.db-shm"); fs.rmSync(wal, { force: true }); fs.rmSync(shm, { force: true }); } function restoreSnapshot(id, options = {}) { const entries = loadIndex(); const entry = entries.find((item) => item.id === id); if (!entry) { throw new Error("Snapshot not found."); } if (entry.status !== "available") { throw new Error("Snapshot is no longer available."); } if (options.expectedType && entry.type !== options.expectedType) { throw new Error("Snapshot target type does not match this revert action."); } if (options.expectedPluginId && entry.pluginId !== options.expectedPluginId) { throw new Error("Snapshot target plugin does not match this revert action."); } if (entry.major_crossing && entry.rollback_safe === false && !options.allowUnsafeMajorRollback) { throw new Error("This snapshot crossed a major version and is not marked rollback safe."); } if (options.currentVersion && entry.to_version && entry.to_version !== options.currentVersion) { throw new Error("Only the previous version snapshot can be reverted from this action."); } if (entry.type === "bot") { const fullPath = path.join(entry.path, "full"); if (fs.existsSync(fullPath)) { applyCoreFilesFromDirectory(fullPath); restoreDatabase(entry.path); markSnapshotRolledBack(id); return entry; } const fullZip = path.join(entry.path, "full.zip"); const coreZip = path.join(entry.path, "core.zip"); const restoreZip = fs.existsSync(fullZip) ? fullZip : coreZip; if (!fs.existsSync(restoreZip)) { throw new Error("Snapshot core archive missing."); } const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "lumi-restore-")); extractZip(restoreZip, tempDir); const rootPath = resolveZipRoot(tempDir); applyCoreFilesFromDirectory(rootPath); fs.rmSync(tempDir, { recursive: true, force: true }); } if (entry.type === "plugin") { const pluginsDir = path.join(repoRoot, "plugins"); const targetDir = path.join(pluginsDir, entry.pluginId); if (entry.pluginExisted) { const pluginZip = path.join(entry.path, "plugin.zip"); if (!fs.existsSync(pluginZip)) { throw new Error("Snapshot plugin archive missing."); } const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "lumi-restore-")); extractZip(pluginZip, tempDir); const rootPath = resolvePluginRoot(tempDir); applyPluginFiles(rootPath, entry.pluginId, { preserveData: true }); fs.rmSync(tempDir, { recursive: true, force: true }); } else { fs.rmSync(targetDir, { recursive: true, force: true }); } } restoreDatabase(entry.path); markSnapshotRolledBack(id); return entry; } module.exports = { applyBotUpdate, applyPluginUpdate, createSnapshot, finalizeSnapshot, discardSnapshot, applyCoreFilesFromDirectory, applyPluginFiles, resetPluginCode, replacePluginDirectory, listSnapshots, restoreSnapshot };