1047 lines
34 KiB
JavaScript
1047 lines
34 KiB
JavaScript
const fs = require("fs");
|
|
const path = require("path");
|
|
const os = require("os");
|
|
const crypto = require("crypto");
|
|
const {
|
|
runFileOperationWithRetries,
|
|
writeJsonAtomicSync
|
|
} = require("./safe-files");
|
|
const { getSetting, setSetting } = require("./settings");
|
|
let AdmZip = null;
|
|
try {
|
|
AdmZip = require("adm-zip");
|
|
} catch {
|
|
AdmZip = null;
|
|
}
|
|
const { db } = require("./db");
|
|
const {
|
|
createRecoveryMarker,
|
|
updateRecoveryMarker,
|
|
markRecoveryMarkerFailed,
|
|
markRecoveryMarkerComplete
|
|
} = require("./recovery-mode");
|
|
|
|
const repoRoot = path.join(__dirname, "..", "..");
|
|
const dataDir = path.join(repoRoot, "data");
|
|
const snapshotsDir = path.join(dataDir, "snapshots");
|
|
const indexPath = path.join(snapshotsDir, "index.json");
|
|
const DEFAULT_SNAPSHOT_RETENTION_DAYS = 30;
|
|
const DEFAULT_SNAPSHOTS_PER_TARGET = 5;
|
|
const PRESERVE_RELATIVE_PATHS = new Set([
|
|
".git",
|
|
"node_modules",
|
|
"data",
|
|
"config",
|
|
"storage",
|
|
"uploads",
|
|
"logs",
|
|
"database",
|
|
"databases",
|
|
"plugins",
|
|
"updates",
|
|
".codex-local-backups",
|
|
"knowledge/community",
|
|
"knowledge/corrections",
|
|
"taskfile.txt",
|
|
"taskfile-export.json",
|
|
"security-audit-report.md",
|
|
"security-audit-findings.json",
|
|
".bot details.md",
|
|
"Discord profile banner.png",
|
|
"twitch-credentials-lumi.png",
|
|
".env",
|
|
".env.local",
|
|
".env.production",
|
|
".secrets",
|
|
"codex-guidelines"
|
|
]);
|
|
const GENERATED_RELATIVE_PATHS = new Set([
|
|
"dist",
|
|
"build",
|
|
"coverage",
|
|
".cache",
|
|
".parcel-cache",
|
|
".turbo",
|
|
"tmp",
|
|
"temp"
|
|
]);
|
|
const SNAPSHOT_EXCLUDE_RELATIVE_PATHS = new Set([
|
|
".git",
|
|
"node_modules",
|
|
"data",
|
|
"plugins",
|
|
"config",
|
|
"storage",
|
|
"uploads",
|
|
"logs",
|
|
"database",
|
|
"databases",
|
|
"updates",
|
|
".codex-local-backups",
|
|
"data/update-cache",
|
|
"data/snapshots"
|
|
]);
|
|
|
|
function ensureSnapshotsDir() {
|
|
fs.mkdirSync(snapshotsDir, { recursive: true });
|
|
}
|
|
|
|
function loadIndex() {
|
|
if (!fs.existsSync(indexPath)) {
|
|
return [];
|
|
}
|
|
try {
|
|
const raw = fs.readFileSync(indexPath, "utf8");
|
|
const data = JSON.parse(raw);
|
|
return Array.isArray(data) ? data : [];
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
function saveIndex(entries) {
|
|
ensureSnapshotsDir();
|
|
writeJsonAtomicSync(indexPath, entries);
|
|
}
|
|
|
|
async function backupDatabase(targetPath) {
|
|
if (typeof db.backup === "function") {
|
|
await db.backup(targetPath);
|
|
return;
|
|
}
|
|
const source = path.join(dataDir, "app.db");
|
|
if (fs.existsSync(source)) {
|
|
fs.copyFileSync(source, targetPath);
|
|
}
|
|
}
|
|
|
|
async function createSnapshot({ type, pluginId, metadata = {} }) {
|
|
ensureSnapshotsDir();
|
|
cleanupSnapshots();
|
|
const id = `${Date.now()}-${crypto.randomUUID()}`;
|
|
const snapshotPath = path.join(snapshotsDir, id);
|
|
fs.mkdirSync(snapshotPath, { recursive: true });
|
|
|
|
const dbPath = path.join(snapshotPath, "app.db");
|
|
await backupDatabase(dbPath);
|
|
const databaseZip = compressSnapshotFile(dbPath, path.join(snapshotPath, "database.zip"));
|
|
|
|
let pluginExisted = false;
|
|
let pluginZip = null;
|
|
if (type === "bot") {
|
|
const coreZip = path.join(snapshotPath, "core.zip");
|
|
zipCore(coreZip);
|
|
}
|
|
|
|
if (type === "plugin" && pluginId) {
|
|
const pluginDir = path.join(repoRoot, "plugins", pluginId);
|
|
pluginExisted = fs.existsSync(pluginDir);
|
|
if (pluginExisted) {
|
|
pluginZip = path.join(snapshotPath, "plugin.zip");
|
|
zipFolder(pluginDir, pluginZip, {
|
|
base: pluginDir,
|
|
ignore: new Set(["node_modules", "data"])
|
|
});
|
|
}
|
|
}
|
|
|
|
return { id, type, pluginId, pluginExisted, pluginZip, databaseZip, snapshotPath, metadata };
|
|
}
|
|
|
|
function finalizeSnapshot(snapshot) {
|
|
const entries = loadIndex();
|
|
const record = {
|
|
id: snapshot.id,
|
|
type: snapshot.type,
|
|
pluginId: snapshot.pluginId || null,
|
|
pluginExisted: snapshot.pluginExisted || false,
|
|
createdAt: Date.now(),
|
|
status: "available",
|
|
path: snapshot.snapshotPath,
|
|
target_kind: snapshot.metadata?.target_kind || snapshot.type,
|
|
target_id: snapshot.metadata?.target_id || snapshot.pluginId || null,
|
|
from_version: snapshot.metadata?.from_version || null,
|
|
to_version: snapshot.metadata?.to_version || null,
|
|
source_branch: snapshot.metadata?.source_branch || null,
|
|
update_method: snapshot.metadata?.update_method || null,
|
|
rollback_safe: snapshot.metadata?.rollback_safe !== false,
|
|
recovery_marker_id: snapshot.metadata?.recovery_marker_id || null,
|
|
major_crossing: Boolean(snapshot.metadata?.major_crossing),
|
|
migration_notes: snapshot.metadata?.migration_notes || "",
|
|
danger_notes: snapshot.metadata?.danger_notes || [],
|
|
compressed: true,
|
|
storage_bytes: directorySize(snapshot.snapshotPath)
|
|
};
|
|
entries.push(record);
|
|
saveIndex(pruneEntries(entries));
|
|
return record;
|
|
}
|
|
|
|
function discardSnapshot(snapshot) {
|
|
if (!snapshot?.snapshotPath) {
|
|
return;
|
|
}
|
|
try {
|
|
fs.rmSync(snapshot.snapshotPath, { recursive: true, force: true });
|
|
} catch {
|
|
// Ignore cleanup failures.
|
|
}
|
|
}
|
|
|
|
function pruneEntries(entries, options = {}) {
|
|
const now = Number(options.now) || Date.now();
|
|
const retention = options.retention || getSnapshotRetention();
|
|
const maximumAge = retention.max_age_days * 24 * 60 * 60 * 1000;
|
|
const available = entries
|
|
.filter((entry) => entry.status === "available")
|
|
.sort((a, b) => b.createdAt - a.createdAt);
|
|
const perTarget = new Map();
|
|
const keep = new Set();
|
|
for (const entry of available) {
|
|
const group = snapshotTargetKey(entry);
|
|
const count = perTarget.get(group) || 0;
|
|
const freshEnough = now - Number(entry.createdAt || 0) <= maximumAge;
|
|
if (freshEnough && count < retention.max_per_target && fs.existsSync(entry.path)) {
|
|
keep.add(entry.id);
|
|
perTarget.set(group, count + 1);
|
|
compactLegacySnapshot(entry);
|
|
}
|
|
}
|
|
const pruned = entries.filter((entry) => {
|
|
if (entry.status === "available") return keep.has(entry.id);
|
|
return now - Number(entry.createdAt || 0) <= maximumAge;
|
|
});
|
|
|
|
for (const entry of entries) {
|
|
if ((entry.status === "available" && !keep.has(entry.id)) ||
|
|
(entry.status !== "available" && !pruned.includes(entry))) {
|
|
try {
|
|
fs.rmSync(entry.path, { recursive: true, force: true });
|
|
} catch {
|
|
// Ignore cleanup failures.
|
|
}
|
|
}
|
|
}
|
|
|
|
return pruned;
|
|
}
|
|
|
|
function listSnapshots() {
|
|
return cleanupSnapshots()
|
|
.filter((entry) => entry.status === "available")
|
|
.sort((a, b) => b.createdAt - a.createdAt);
|
|
}
|
|
|
|
function cleanupSnapshots(options = {}) {
|
|
const entries = loadIndex();
|
|
const pruned = pruneEntries(entries, options);
|
|
saveIndex(pruned);
|
|
return pruned;
|
|
}
|
|
|
|
function getSnapshotRetention() {
|
|
return {
|
|
max_age_days: boundedInteger(
|
|
getSetting("update_snapshot_retention_days", DEFAULT_SNAPSHOT_RETENTION_DAYS),
|
|
1,
|
|
3650,
|
|
DEFAULT_SNAPSHOT_RETENTION_DAYS
|
|
),
|
|
max_per_target: boundedInteger(
|
|
getSetting("update_snapshot_retention_count", DEFAULT_SNAPSHOTS_PER_TARGET),
|
|
1,
|
|
50,
|
|
DEFAULT_SNAPSHOTS_PER_TARGET
|
|
)
|
|
};
|
|
}
|
|
|
|
function setSnapshotRetention(values = {}) {
|
|
const retention = {
|
|
max_age_days: boundedInteger(values.max_age_days, 1, 3650, DEFAULT_SNAPSHOT_RETENTION_DAYS),
|
|
max_per_target: boundedInteger(values.max_per_target, 1, 50, DEFAULT_SNAPSHOTS_PER_TARGET)
|
|
};
|
|
setSetting("update_snapshot_retention_days", retention.max_age_days);
|
|
setSetting("update_snapshot_retention_count", retention.max_per_target);
|
|
cleanupSnapshots({ retention, forceSave: true });
|
|
return retention;
|
|
}
|
|
|
|
function markSnapshotRolledBack(id) {
|
|
const entries = loadIndex();
|
|
const entry = entries.find((item) => item.id === id);
|
|
if (!entry) {
|
|
return null;
|
|
}
|
|
entry.status = "rolled_back";
|
|
entry.rolledBackAt = Date.now();
|
|
try {
|
|
fs.rmSync(entry.path, { recursive: true, force: true });
|
|
} catch {
|
|
// The restore already succeeded; stale payload cleanup can retry later.
|
|
}
|
|
saveIndex(entries);
|
|
return entry;
|
|
}
|
|
|
|
function extractZip(zipPath, targetDir) {
|
|
if (!AdmZip) {
|
|
throw new Error("adm-zip is not installed. Run npm install.");
|
|
}
|
|
const zip = new AdmZip(zipPath);
|
|
zip.extractAllTo(targetDir, true);
|
|
}
|
|
|
|
function resolveZipRoot(extractedDir) {
|
|
const packagePath = path.join(extractedDir, "package.json");
|
|
if (fs.existsSync(packagePath)) {
|
|
return extractedDir;
|
|
}
|
|
const entries = fs.readdirSync(extractedDir, { withFileTypes: true });
|
|
const dirs = entries.filter((entry) => entry.isDirectory());
|
|
if (dirs.length === 1) {
|
|
const candidate = path.join(extractedDir, dirs[0].name);
|
|
if (fs.existsSync(path.join(candidate, "package.json"))) {
|
|
return candidate;
|
|
}
|
|
}
|
|
return extractedDir;
|
|
}
|
|
|
|
function resolvePatchRoot(extractedDir) {
|
|
// Patch archives are copied relative to the repository root. Do not infer a
|
|
// nested root from a single top-level directory such as src/, or patches will
|
|
// be applied to services/ and web/ instead of src/services/ and src/web/.
|
|
return extractedDir;
|
|
}
|
|
|
|
function resolvePluginRoot(extractedDir) {
|
|
const pluginPath = path.join(extractedDir, "plugin.json");
|
|
if (fs.existsSync(pluginPath)) {
|
|
return extractedDir;
|
|
}
|
|
const entries = fs.readdirSync(extractedDir, { withFileTypes: true });
|
|
const dirs = entries.filter((entry) => entry.isDirectory());
|
|
if (dirs.length === 1) {
|
|
const candidate = path.join(extractedDir, dirs[0].name);
|
|
if (fs.existsSync(path.join(candidate, "plugin.json"))) {
|
|
return candidate;
|
|
}
|
|
}
|
|
return extractedDir;
|
|
}
|
|
|
|
function verifyBotPackage(rootPath) {
|
|
const required = [
|
|
path.join(rootPath, "package.json"),
|
|
path.join(rootPath, "safe-mode.js"),
|
|
path.join(rootPath, "src", "main.js"),
|
|
path.join(rootPath, "src", "web", "server.js")
|
|
];
|
|
for (const filePath of required) {
|
|
if (!fs.existsSync(filePath)) {
|
|
throw new Error(`Missing required file: ${path.relative(rootPath, filePath)}`);
|
|
}
|
|
}
|
|
JSON.parse(fs.readFileSync(required[0], "utf8"));
|
|
}
|
|
|
|
function verifyPatchPackage(rootPath) {
|
|
if (!hasAnyFiles(rootPath)) {
|
|
throw new Error("Patch archive is empty.");
|
|
}
|
|
const manifestPath = path.join(rootPath, "patch-manifest.json");
|
|
if (!fs.existsSync(manifestPath)) return null;
|
|
const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
|
|
if (manifest.target !== "core" || manifest.data_policy !== "preserve") {
|
|
throw new Error("Patch manifest must target core and preserve local data.");
|
|
}
|
|
const currentVersion = JSON.parse(fs.readFileSync(path.join(repoRoot, "package.json"), "utf8")).version;
|
|
if (Array.isArray(manifest.from_versions) && !manifest.from_versions.includes(currentVersion)) {
|
|
throw new Error(`This patch supports core ${manifest.from_versions.join(", ")}, not ${currentVersion}.`);
|
|
}
|
|
const files = manifest.files && typeof manifest.files === "object" ? manifest.files : {};
|
|
if (!Object.keys(files).length) throw new Error("Patch manifest does not list any files.");
|
|
const preservePaths = buildCorePreservePaths(repoRoot);
|
|
for (const [relativePath, expectedHash] of Object.entries(files)) {
|
|
const normalized = normalizeRelative(relativePath);
|
|
if (!normalized || normalized.startsWith("/") || normalized.includes("../") || isRelativePathIgnored(normalized, preservePaths)) {
|
|
throw new Error(`Patch manifest contains an unsafe or preserved path: ${relativePath}`);
|
|
}
|
|
const filePath = path.resolve(rootPath, normalized);
|
|
if (!filePath.startsWith(path.resolve(rootPath) + path.sep) || !fs.statSync(filePath, { throwIfNoEntry: false })?.isFile()) {
|
|
throw new Error(`Patch file is missing: ${relativePath}`);
|
|
}
|
|
const actualHash = crypto.createHash("sha256").update(fs.readFileSync(filePath)).digest("hex");
|
|
if (actualHash !== expectedHash) throw new Error(`Patch file checksum failed: ${relativePath}`);
|
|
}
|
|
const packageFile = path.join(rootPath, "package.json");
|
|
if (manifest.to_version && fs.existsSync(packageFile)) {
|
|
const packageVersion = JSON.parse(fs.readFileSync(packageFile, "utf8")).version;
|
|
if (packageVersion !== manifest.to_version) {
|
|
throw new Error(`Patch targets ${manifest.to_version}, but package.json contains ${packageVersion}.`);
|
|
}
|
|
}
|
|
return manifest;
|
|
}
|
|
|
|
function verifyPluginPackage(rootPath) {
|
|
const pluginPath = path.join(rootPath, "plugin.json");
|
|
if (!fs.existsSync(pluginPath)) {
|
|
throw new Error("plugin.json not found in plugin package.");
|
|
}
|
|
const manifest = JSON.parse(fs.readFileSync(pluginPath, "utf8"));
|
|
if (!manifest.id) {
|
|
throw new Error("plugin.json must include an id.");
|
|
}
|
|
const mainFile = manifest.main || "index.js";
|
|
const mainPath = path.join(rootPath, mainFile);
|
|
if (!fs.existsSync(mainPath)) {
|
|
throw new Error(`Plugin entry ${mainFile} not found.`);
|
|
}
|
|
return manifest;
|
|
}
|
|
|
|
function zipCore(destination) {
|
|
if (!AdmZip) {
|
|
throw new Error("adm-zip is not installed. Run npm install.");
|
|
}
|
|
const zip = new AdmZip();
|
|
addFolder(zip, repoRoot, repoRoot, new Set([
|
|
...buildCorePreservePaths(repoRoot),
|
|
...SNAPSHOT_EXCLUDE_RELATIVE_PATHS
|
|
]));
|
|
zip.writeZip(destination);
|
|
}
|
|
|
|
function zipFolder(source, destination, options) {
|
|
if (!AdmZip) {
|
|
throw new Error("adm-zip is not installed. Run npm install.");
|
|
}
|
|
const zip = new AdmZip();
|
|
const base = options?.base || source;
|
|
addFolder(zip, source, base, options?.ignore || new Set(["node_modules"]));
|
|
zip.writeZip(destination);
|
|
}
|
|
|
|
function compressSnapshotFile(source, destination) {
|
|
if (!fs.existsSync(source)) return null;
|
|
if (!AdmZip) throw new Error("adm-zip is not installed. Run npm install.");
|
|
const zip = new AdmZip();
|
|
zip.addLocalFile(source);
|
|
zip.writeZip(destination);
|
|
fs.rmSync(source, { force: true });
|
|
return destination;
|
|
}
|
|
|
|
function addFolder(zip, folderPath, basePath, ignore) {
|
|
const entries = fs.readdirSync(folderPath, { withFileTypes: true });
|
|
for (const entry of entries) {
|
|
const fullPath = path.join(folderPath, entry.name);
|
|
const relPath = normalizeRelative(path.relative(basePath, fullPath));
|
|
if (isRelativePathIgnored(relPath, ignore)) {
|
|
continue;
|
|
}
|
|
if (entry.isDirectory()) {
|
|
addFolder(zip, fullPath, basePath, ignore);
|
|
} else if (entry.isFile()) {
|
|
zip.addLocalFile(fullPath, path.dirname(relPath));
|
|
}
|
|
}
|
|
}
|
|
|
|
function compactLegacySnapshot(entry) {
|
|
if (!entry?.path || !fs.existsSync(entry.path)) return false;
|
|
let changed = false;
|
|
try {
|
|
const rawDatabase = path.join(entry.path, "app.db");
|
|
const databaseZip = path.join(entry.path, "database.zip");
|
|
if (fs.existsSync(rawDatabase) && !fs.existsSync(databaseZip)) {
|
|
compressSnapshotFile(rawDatabase, databaseZip);
|
|
changed = true;
|
|
}
|
|
if (entry.type === "bot") {
|
|
const fullPath = path.join(entry.path, "full");
|
|
const coreZip = path.join(entry.path, "core.zip");
|
|
if (fs.existsSync(fullPath)) {
|
|
if (!fs.existsSync(coreZip)) {
|
|
zipFolder(fullPath, coreZip, {
|
|
base: fullPath,
|
|
ignore: new Set([
|
|
...buildCorePreservePaths(fullPath),
|
|
...SNAPSHOT_EXCLUDE_RELATIVE_PATHS
|
|
])
|
|
});
|
|
}
|
|
fs.rmSync(fullPath, { recursive: true, force: true });
|
|
changed = true;
|
|
}
|
|
}
|
|
entry.compressed = true;
|
|
entry.storage_bytes = directorySize(entry.path);
|
|
} catch {
|
|
// Keep the original payload when migration/compaction is unavailable.
|
|
}
|
|
return changed;
|
|
}
|
|
|
|
function snapshotTargetKey(entry) {
|
|
const type = String(entry.target_kind || entry.type || "unknown");
|
|
const id = String(entry.target_id || entry.pluginId || "core");
|
|
return `${type}:${id}`;
|
|
}
|
|
|
|
function directorySize(directory) {
|
|
if (!directory || !fs.existsSync(directory)) return 0;
|
|
let total = 0;
|
|
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
|
|
const target = path.join(directory, entry.name);
|
|
if (entry.isDirectory()) total += directorySize(target);
|
|
else if (entry.isFile()) total += fs.statSync(target).size;
|
|
}
|
|
return total;
|
|
}
|
|
|
|
function boundedInteger(value, minimum, maximum, fallback) {
|
|
const number = Number.parseInt(value, 10);
|
|
return Number.isFinite(number) ? Math.max(minimum, Math.min(maximum, number)) : fallback;
|
|
}
|
|
|
|
function resetCoreFiles() {
|
|
resetDirectoryForFullUpdate(repoRoot, buildCorePreservePaths(repoRoot));
|
|
}
|
|
|
|
function resetDirectoryForFullUpdate(targetRoot, preservePaths = buildCorePreservePaths(targetRoot)) {
|
|
removeUnpreservedEntries(targetRoot, targetRoot, preservePaths);
|
|
}
|
|
|
|
function buildCorePreservePaths(targetRoot = repoRoot) {
|
|
const preserve = new Set(PRESERVE_RELATIVE_PATHS);
|
|
for (const scope of ["core", "plugins"]) {
|
|
const directory = path.join(targetRoot, "knowledge", scope);
|
|
collectLocalKnowledgePaths(directory, targetRoot, preserve);
|
|
}
|
|
return preserve;
|
|
}
|
|
|
|
function collectLocalKnowledgePaths(directory, base, preserve) {
|
|
if (!fs.existsSync(directory)) return;
|
|
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
|
|
const filePath = path.join(directory, entry.name);
|
|
if (entry.isDirectory()) {
|
|
collectLocalKnowledgePaths(filePath, base, preserve);
|
|
} else if (!entry.isFile() || isLocallyOwnedKnowledgeFile(filePath)) {
|
|
preserve.add(normalizeRelative(path.relative(base, filePath)));
|
|
}
|
|
}
|
|
}
|
|
|
|
function isLocallyOwnedKnowledgeFile(filePath) {
|
|
if (path.extname(filePath).toLowerCase() !== ".md") return true;
|
|
let source = "";
|
|
try {
|
|
source = fs.readFileSync(filePath, "utf8");
|
|
} catch {
|
|
return true;
|
|
}
|
|
const frontmatter = source.match(/^---\s*\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/)?.[1] || "";
|
|
const generated = /^generated:\s*true\s*$/im.test(frontmatter);
|
|
const readOnly = /^editable:\s*false\s*$/im.test(frontmatter);
|
|
return !(generated && readOnly);
|
|
}
|
|
|
|
function removeUnpreservedEntries(directory, base, preservePaths) {
|
|
const entries = fs.readdirSync(directory, { withFileTypes: true });
|
|
for (const entry of entries) {
|
|
const fullPath = path.join(directory, entry.name);
|
|
const relativePath = normalizeRelative(path.relative(base, fullPath));
|
|
if (isRelativePathIgnored(relativePath, preservePaths)) continue;
|
|
if (entry.isDirectory() && hasPreservedDescendant(relativePath, preservePaths)) {
|
|
removeUnpreservedEntries(fullPath, base, preservePaths);
|
|
continue;
|
|
}
|
|
fs.rmSync(fullPath, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
function hasPreservedDescendant(relativePath, preservePaths) {
|
|
const prefix = `${normalizeRelative(relativePath)}/`;
|
|
return [...preservePaths].some((candidate) => normalizeRelative(candidate).startsWith(prefix));
|
|
}
|
|
|
|
function normalizeRelative(value) {
|
|
return String(value || "").split(path.sep).join("/");
|
|
}
|
|
|
|
function isRelativePathIgnored(relativePath, ignore) {
|
|
const rel = normalizeRelative(relativePath);
|
|
if (!rel) return false;
|
|
if (ignore.has(rel)) return true;
|
|
const parts = rel.split("/");
|
|
for (let index = 1; index <= parts.length; index += 1) {
|
|
if (ignore.has(parts.slice(0, index).join("/"))) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
function copyDirectory(source, target, ignore, options = {}) {
|
|
const base = options.base || source;
|
|
const entries = fs.readdirSync(source, { withFileTypes: true });
|
|
for (const entry of entries) {
|
|
const srcPath = path.join(source, entry.name);
|
|
const relPath = normalizeRelative(path.relative(base, srcPath));
|
|
if (isRelativePathIgnored(relPath, ignore)) {
|
|
continue;
|
|
}
|
|
const destPath = path.join(target, entry.name);
|
|
if (entry.isDirectory()) {
|
|
fs.mkdirSync(destPath, { recursive: true });
|
|
copyDirectory(srcPath, destPath, ignore, { base });
|
|
} else if (entry.isFile()) {
|
|
fs.mkdirSync(path.dirname(destPath), { recursive: true });
|
|
fs.copyFileSync(srcPath, destPath);
|
|
}
|
|
}
|
|
}
|
|
|
|
function removeGeneratedPaths() {
|
|
for (const relativePath of GENERATED_RELATIVE_PATHS) {
|
|
const target = path.join(repoRoot, relativePath);
|
|
if (target.startsWith(path.join(repoRoot, "data", "snapshots"))) {
|
|
continue;
|
|
}
|
|
fs.rmSync(target, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
function applyCoreFilesFromDirectory(rootPath) {
|
|
removeGeneratedPaths();
|
|
const preservePaths = buildCorePreservePaths(repoRoot);
|
|
resetDirectoryForFullUpdate(repoRoot, preservePaths);
|
|
copyDirectory(
|
|
rootPath,
|
|
repoRoot,
|
|
preservePaths,
|
|
{ base: rootPath }
|
|
);
|
|
}
|
|
|
|
function applyCorePatch(rootPath) {
|
|
const preservePaths = buildCorePreservePaths(repoRoot);
|
|
preservePaths.add("patch-manifest.json");
|
|
copyDirectory(
|
|
rootPath,
|
|
repoRoot,
|
|
preservePaths,
|
|
{ base: rootPath }
|
|
);
|
|
}
|
|
|
|
function hasAnyFiles(rootPath) {
|
|
const entries = fs.readdirSync(rootPath, { withFileTypes: true });
|
|
for (const entry of entries) {
|
|
if (entry.isFile()) {
|
|
return true;
|
|
}
|
|
if (entry.isDirectory()) {
|
|
if (hasAnyFiles(path.join(rootPath, entry.name))) {
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
function resetPluginCode(targetDir) {
|
|
if (!fs.existsSync(targetDir)) {
|
|
return;
|
|
}
|
|
for (const entry of fs.readdirSync(targetDir, { withFileTypes: true })) {
|
|
if (entry.name === "data") {
|
|
continue;
|
|
}
|
|
fs.rmSync(path.join(targetDir, entry.name), {
|
|
recursive: true,
|
|
force: true
|
|
});
|
|
}
|
|
}
|
|
|
|
function applyPluginFiles(rootPath, pluginId, options = {}) {
|
|
const pluginsDir = path.join(repoRoot, "plugins");
|
|
const targetDir = path.join(pluginsDir, pluginId);
|
|
fs.mkdirSync(pluginsDir, { recursive: true });
|
|
replacePluginDirectory(rootPath, targetDir, options);
|
|
}
|
|
|
|
function replacePluginDirectory(rootPath, targetDir, options = {}) {
|
|
const parent = path.dirname(targetDir);
|
|
const name = path.basename(targetDir);
|
|
const nonce = `${process.pid}-${crypto.randomBytes(6).toString("hex")}`;
|
|
const staging = path.join(parent, `.${name}.update-${nonce}`);
|
|
const backup = path.join(parent, `.${name}.backup-${nonce}`);
|
|
const ignore = options.preserveData
|
|
? new Set(["node_modules", "data"])
|
|
: new Set(["node_modules"]);
|
|
let targetMoved = false;
|
|
let dataMoved = false;
|
|
let replacementInstalled = false;
|
|
let rollbackFailed = false;
|
|
fs.mkdirSync(parent, { recursive: true });
|
|
try {
|
|
fs.mkdirSync(staging, { recursive: true });
|
|
copyDirectory(rootPath, staging, ignore);
|
|
// A running plugin may legitimately keep files below data open. In
|
|
// particular, Lumi AI starts llama-server with data/runtime as its working
|
|
// directory. Windows and SMB then reject renaming the plugin root even
|
|
// though none of the code being updated is locked. Keep the data directory
|
|
// in place and transactionally replace only the plugin's code entries.
|
|
if (options.preserveData && fs.existsSync(targetDir)) {
|
|
replacePluginCodeInPlace(staging, targetDir, backup);
|
|
return;
|
|
}
|
|
if (fs.existsSync(targetDir)) {
|
|
moveDirectory(targetDir, backup, "backup_plugin");
|
|
targetMoved = true;
|
|
}
|
|
const backupData = path.join(backup, "data");
|
|
const stagedData = path.join(staging, "data");
|
|
if (options.preserveData && targetMoved && fs.existsSync(backupData)) {
|
|
moveDirectory(backupData, stagedData, "preserve_plugin_data");
|
|
dataMoved = true;
|
|
}
|
|
moveDirectory(staging, targetDir, "install_plugin");
|
|
replacementInstalled = true;
|
|
fs.rmSync(backup, { recursive: true, force: true });
|
|
} catch (error) {
|
|
try {
|
|
if (replacementInstalled && fs.existsSync(targetDir)) {
|
|
const installedData = path.join(targetDir, "data");
|
|
if (dataMoved && targetMoved && fs.existsSync(installedData) && fs.existsSync(backup)) {
|
|
moveDirectory(installedData, path.join(backup, "data"), "restore_plugin_data");
|
|
}
|
|
fs.rmSync(targetDir, { recursive: true, force: true });
|
|
} else if (dataMoved && targetMoved && fs.existsSync(path.join(staging, "data")) && fs.existsSync(backup)) {
|
|
moveDirectory(path.join(staging, "data"), path.join(backup, "data"), "restore_plugin_data");
|
|
}
|
|
if (targetMoved && fs.existsSync(backup) && !fs.existsSync(targetDir)) {
|
|
moveDirectory(backup, targetDir, "restore_plugin");
|
|
}
|
|
} catch (restoreError) {
|
|
rollbackFailed = true;
|
|
error.message = `${error.message} Plugin rollback also failed: ${restoreError.message}. Backup: ${backup}. Staging: ${staging}`;
|
|
}
|
|
throw error;
|
|
} finally {
|
|
if (!rollbackFailed) fs.rmSync(staging, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
function replacePluginCodeInPlace(staging, targetDir, backup) {
|
|
const movedExisting = [];
|
|
const installed = [];
|
|
let rollbackFailed = false;
|
|
fs.mkdirSync(backup, { recursive: true });
|
|
try {
|
|
for (const entry of fs.readdirSync(targetDir, { withFileTypes: true })) {
|
|
if (entry.name === "data") continue;
|
|
moveDirectory(
|
|
path.join(targetDir, entry.name),
|
|
path.join(backup, entry.name),
|
|
"backup_plugin_code"
|
|
);
|
|
movedExisting.push(entry.name);
|
|
}
|
|
for (const entry of fs.readdirSync(staging, { withFileTypes: true })) {
|
|
moveDirectory(
|
|
path.join(staging, entry.name),
|
|
path.join(targetDir, entry.name),
|
|
"install_plugin_code"
|
|
);
|
|
installed.push(entry.name);
|
|
}
|
|
fs.rmSync(backup, { recursive: true, force: true });
|
|
} catch (error) {
|
|
try {
|
|
for (const name of installed.reverse()) {
|
|
fs.rmSync(path.join(targetDir, name), { recursive: true, force: true });
|
|
}
|
|
for (const name of movedExisting) {
|
|
const saved = path.join(backup, name);
|
|
if (!fs.existsSync(saved)) continue;
|
|
const live = path.join(targetDir, name);
|
|
fs.rmSync(live, { recursive: true, force: true });
|
|
moveDirectory(saved, live, "restore_plugin_code");
|
|
}
|
|
} catch (restoreError) {
|
|
rollbackFailed = true;
|
|
error.message = `${error.message} Plugin code rollback also failed: ${restoreError.message}. Backup: ${backup}. Staging: ${staging}`;
|
|
}
|
|
throw error;
|
|
} finally {
|
|
if (!rollbackFailed) {
|
|
fs.rmSync(staging, { recursive: true, force: true });
|
|
fs.rmSync(backup, { recursive: true, force: true });
|
|
}
|
|
}
|
|
}
|
|
|
|
function moveDirectory(source, target, operation) {
|
|
runFileOperationWithRetries(() => fs.renameSync(source, target), {
|
|
target,
|
|
operation,
|
|
attempts: 8,
|
|
delayMs: 20
|
|
});
|
|
}
|
|
|
|
async function applyBotUpdate(zipPath, options = {}) {
|
|
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "lumi-update-"));
|
|
try {
|
|
extractZip(zipPath, tempDir);
|
|
const mode = options.mode === "patch" ? "patch" : "full";
|
|
const rootPath =
|
|
mode === "patch" ? resolvePatchRoot(tempDir) : resolveZipRoot(tempDir);
|
|
let patchManifest = null;
|
|
if (mode === "patch") {
|
|
patchManifest = verifyPatchPackage(rootPath);
|
|
} else {
|
|
verifyBotPackage(rootPath);
|
|
}
|
|
|
|
const marker = createRecoveryMarker({
|
|
target_kind: "core",
|
|
target_id: "core",
|
|
from_version: options.metadata?.from_version || (patchManifest ? JSON.parse(fs.readFileSync(path.join(repoRoot, "package.json"), "utf8")).version : null),
|
|
to_version: options.metadata?.to_version || patchManifest?.to_version || null,
|
|
source_branch: options.metadata?.source_branch || "manual_zip",
|
|
update_method: options.mode === "patch" ? "zip_patch" : "zip",
|
|
rollback_safe: options.metadata?.rollback_safe !== false,
|
|
major_crossing: Boolean(options.metadata?.major_crossing)
|
|
});
|
|
updateRecoveryMarker({ status: "applying" });
|
|
const snapshot = await createSnapshot({
|
|
type: "bot",
|
|
metadata: {
|
|
target_kind: "core",
|
|
update_method: options.mode === "patch" ? "zip_patch" : "zip",
|
|
...options.metadata,
|
|
from_version: options.metadata?.from_version || (patchManifest ? JSON.parse(fs.readFileSync(path.join(repoRoot, "package.json"), "utf8")).version : null),
|
|
to_version: options.metadata?.to_version || patchManifest?.to_version || null,
|
|
recovery_marker_id: marker.id
|
|
}
|
|
});
|
|
let record = null;
|
|
try {
|
|
record = finalizeSnapshot(snapshot);
|
|
if (mode === "patch") {
|
|
applyCorePatch(rootPath);
|
|
} else {
|
|
applyCoreFilesFromDirectory(rootPath);
|
|
}
|
|
markRecoveryMarkerComplete({ snapshot_id: record.id });
|
|
return record;
|
|
} catch (error) {
|
|
if (record) {
|
|
try {
|
|
restoreSnapshot(record.id, {
|
|
expectedType: "bot",
|
|
allowUnsafeMajorRollback: true
|
|
});
|
|
} catch (restoreError) {
|
|
error.message = `${error.message} Automatic rollback also failed: ${restoreError.message}`;
|
|
}
|
|
} else {
|
|
discardSnapshot(snapshot);
|
|
}
|
|
markRecoveryMarkerFailed(error);
|
|
throw error;
|
|
}
|
|
} finally {
|
|
fs.rmSync(tempDir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
async function applyPluginUpdate(zipPath, options = {}) {
|
|
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "lumi-plugin-"));
|
|
try {
|
|
extractZip(zipPath, tempDir);
|
|
const rootPath = resolvePluginRoot(tempDir);
|
|
const manifest = verifyPluginPackage(rootPath);
|
|
if (options.expectedPluginId && manifest.id !== options.expectedPluginId) {
|
|
throw new Error(`Plugin ZIP id ${manifest.id} does not match ${options.expectedPluginId}.`);
|
|
}
|
|
|
|
const marker = createRecoveryMarker({
|
|
target_kind: "plugin",
|
|
target_id: manifest.id,
|
|
from_version: options.metadata?.from_version || null,
|
|
to_version: manifest.version || options.metadata?.to_version || null,
|
|
source_branch: options.metadata?.source_branch || "manual_zip",
|
|
update_method: "zip",
|
|
rollback_safe: options.metadata?.rollback_safe !== false,
|
|
major_crossing: Boolean(options.metadata?.major_crossing)
|
|
});
|
|
updateRecoveryMarker({ status: "applying" });
|
|
const snapshot = await createSnapshot({
|
|
type: "plugin",
|
|
pluginId: manifest.id,
|
|
metadata: {
|
|
target_kind: "plugin",
|
|
target_id: manifest.id,
|
|
update_method: "zip",
|
|
...options.metadata,
|
|
recovery_marker_id: marker.id
|
|
}
|
|
});
|
|
let record = null;
|
|
try {
|
|
record = finalizeSnapshot(snapshot);
|
|
applyPluginFiles(rootPath, manifest.id, {
|
|
preserveData: snapshot.pluginExisted
|
|
});
|
|
markRecoveryMarkerComplete({ snapshot_id: record.id });
|
|
return record;
|
|
} catch (error) {
|
|
if (record) {
|
|
try {
|
|
restoreSnapshot(record.id, {
|
|
expectedType: "plugin",
|
|
expectedPluginId: manifest.id,
|
|
allowUnsafeMajorRollback: true
|
|
});
|
|
} catch (restoreError) {
|
|
error.message = `${error.message} Automatic rollback also failed: ${restoreError.message}`;
|
|
}
|
|
} else {
|
|
discardSnapshot(snapshot);
|
|
}
|
|
markRecoveryMarkerFailed(error);
|
|
throw error;
|
|
}
|
|
} finally {
|
|
fs.rmSync(tempDir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
function restoreDatabase(snapshotPath) {
|
|
const source = path.join(snapshotPath, "app.db");
|
|
const target = path.join(dataDir, "app.db");
|
|
const databaseZip = path.join(snapshotPath, "database.zip");
|
|
let restoreSource = source;
|
|
let temporary = null;
|
|
if (!fs.existsSync(restoreSource) && fs.existsSync(databaseZip)) {
|
|
temporary = fs.mkdtempSync(path.join(os.tmpdir(), "lumi-database-restore-"));
|
|
extractZip(databaseZip, temporary);
|
|
restoreSource = path.join(temporary, "app.db");
|
|
}
|
|
if (!fs.existsSync(restoreSource)) {
|
|
if (temporary) fs.rmSync(temporary, { recursive: true, force: true });
|
|
throw new Error("Snapshot database archive is missing or invalid.");
|
|
}
|
|
try {
|
|
fs.copyFileSync(restoreSource, target);
|
|
} finally {
|
|
if (temporary) fs.rmSync(temporary, { recursive: true, force: true });
|
|
}
|
|
const wal = path.join(dataDir, "app.db-wal");
|
|
const shm = path.join(dataDir, "app.db-shm");
|
|
fs.rmSync(wal, { force: true });
|
|
fs.rmSync(shm, { force: true });
|
|
}
|
|
|
|
function restoreSnapshot(id, options = {}) {
|
|
const entries = loadIndex();
|
|
const entry = entries.find((item) => item.id === id);
|
|
if (!entry) {
|
|
throw new Error("Snapshot not found.");
|
|
}
|
|
if (entry.status !== "available") {
|
|
throw new Error("Snapshot is no longer available.");
|
|
}
|
|
if (options.expectedType && entry.type !== options.expectedType) {
|
|
throw new Error("Snapshot target type does not match this revert action.");
|
|
}
|
|
if (options.expectedPluginId && entry.pluginId !== options.expectedPluginId) {
|
|
throw new Error("Snapshot target plugin does not match this revert action.");
|
|
}
|
|
if (entry.major_crossing && entry.rollback_safe === false && !options.allowUnsafeMajorRollback) {
|
|
throw new Error("This snapshot crossed a major version and is not marked rollback safe.");
|
|
}
|
|
if (options.currentVersion && entry.to_version && entry.to_version !== options.currentVersion) {
|
|
throw new Error("Only the previous version snapshot can be reverted from this action.");
|
|
}
|
|
|
|
if (entry.type === "bot") {
|
|
const fullPath = path.join(entry.path, "full");
|
|
if (fs.existsSync(fullPath)) {
|
|
applyCoreFilesFromDirectory(fullPath);
|
|
restoreDatabase(entry.path);
|
|
markSnapshotRolledBack(id);
|
|
return entry;
|
|
}
|
|
const fullZip = path.join(entry.path, "full.zip");
|
|
const coreZip = path.join(entry.path, "core.zip");
|
|
const restoreZip = fs.existsSync(fullZip) ? fullZip : coreZip;
|
|
if (!fs.existsSync(restoreZip)) {
|
|
throw new Error("Snapshot core archive missing.");
|
|
}
|
|
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "lumi-restore-"));
|
|
try {
|
|
extractZip(restoreZip, tempDir);
|
|
const rootPath = resolveZipRoot(tempDir);
|
|
applyCoreFilesFromDirectory(rootPath);
|
|
} finally {
|
|
fs.rmSync(tempDir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
if (entry.type === "plugin") {
|
|
const pluginsDir = path.join(repoRoot, "plugins");
|
|
const targetDir = path.join(pluginsDir, entry.pluginId);
|
|
if (entry.pluginExisted) {
|
|
const pluginZip = path.join(entry.path, "plugin.zip");
|
|
if (!fs.existsSync(pluginZip)) {
|
|
throw new Error("Snapshot plugin archive missing.");
|
|
}
|
|
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "lumi-restore-"));
|
|
try {
|
|
extractZip(pluginZip, tempDir);
|
|
const rootPath = resolvePluginRoot(tempDir);
|
|
applyPluginFiles(rootPath, entry.pluginId, { preserveData: true });
|
|
} finally {
|
|
fs.rmSync(tempDir, { recursive: true, force: true });
|
|
}
|
|
} else {
|
|
fs.rmSync(targetDir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
restoreDatabase(entry.path);
|
|
markSnapshotRolledBack(id);
|
|
return entry;
|
|
}
|
|
|
|
module.exports = {
|
|
applyBotUpdate,
|
|
applyPluginUpdate,
|
|
createSnapshot,
|
|
finalizeSnapshot,
|
|
discardSnapshot,
|
|
applyCoreFilesFromDirectory,
|
|
applyPluginFiles,
|
|
verifyPatchPackage,
|
|
resetPluginCode,
|
|
resetDirectoryForFullUpdate,
|
|
buildCorePreservePaths,
|
|
replacePluginDirectory,
|
|
cleanupSnapshots,
|
|
getSnapshotRetention,
|
|
setSnapshotRetention,
|
|
pruneEntries,
|
|
listSnapshots,
|
|
restoreSnapshot
|
|
};
|