From 782b93b3c1f742bfc784a4b0e8458a82f467b67e Mon Sep 17 00:00:00 2001 From: Franz Rolfsvaag Date: Wed, 17 Jun 2026 15:13:10 +0200 Subject: [PATCH] fix large core update snapshots --- TODO.md | 1 + package-lock.json | 4 +- package.json | 2 +- src/services/update-manager.js | 69 +++++++++++++++++++++++++++------- 4 files changed, 60 insertions(+), 16 deletions(-) diff --git a/TODO.md b/TODO.md index 056a6ee..4ef53b2 100644 --- a/TODO.md +++ b/TODO.md @@ -80,6 +80,7 @@ This file tracks larger Lumi work that cannot safely be completed in one pass. K ## Done +- 2026-06-17: Fixed core update snapshots for large ZIP-origin installs by replacing the full-install ZIP backup with a filesystem snapshot directory, avoiding the 2 GiB ZIP limit for large preserved files such as local AI models. - 2026-06-17: Completed `/admin/updates` UX pass: viewport-fixed dismissible notifications, auto-dismiss for non-critical results, async core/plugin check actions without page refresh or scroll jumps, in-place update-card data refresh, loading states, and collapsed advanced Manual ZIP fallback below repo update containers. - 2026-06-17: Completed homepage hero reliability pass: server-side hero validation before save, admin-visible validation errors, home-page fallback message for broken legacy heroes, automatic YouTube/Twitch/Discord embed derivation, correct Twitch parent host at render time, and image/embed conflict handling. - 2026-06-17: Completed homepage hero builder UX pass: friendlier labels, contextual help text, normal URL paste support, automatic embed filling, per-row readiness messages, and test preview support. diff --git a/package-lock.json b/package-lock.json index a0b4c3a..f47809f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "lumi-bot", - "version": "0.1.1", + "version": "0.1.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "lumi-bot", - "version": "0.1.1", + "version": "0.1.2", "dependencies": { "adm-zip": "^0.5.12", "better-sqlite3": "^11.5.0", diff --git a/package.json b/package.json index a741b48..898d773 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "lumi-bot", - "version": "0.1.1", + "version": "0.1.2", "private": true, "type": "commonjs", "scripts": { diff --git a/src/services/update-manager.js b/src/services/update-manager.js index 369f8ed..788c4d0 100644 --- a/src/services/update-manager.js +++ b/src/services/update-manager.js @@ -51,6 +51,13 @@ const GENERATED_RELATIVE_PATHS = new Set([ "temp", "data/update-cache" ]); +const SNAPSHOT_EXCLUDE_RELATIVE_PATHS = new Set([ + ".git", + "node_modules", + "data/update-cache", + "data/snapshots" +]); +const HARDLINK_SNAPSHOT_MIN_BYTES = 64 * 1024 * 1024; function ensureSnapshotsDir() { fs.mkdirSync(snapshotsDir, { recursive: true }); @@ -96,12 +103,12 @@ async function createSnapshot({ type, pluginId, metadata = {} }) { let pluginExisted = false; let pluginZip = null; - let fullZip = null; + let fullPath = null; if (type === "bot") { const coreZip = path.join(snapshotPath, "core.zip"); zipCore(coreZip); - fullZip = path.join(snapshotPath, "full.zip"); - zipFullInstall(fullZip); + fullPath = path.join(snapshotPath, "full"); + snapshotFullInstall(fullPath); } if (type === "plugin" && pluginId) { @@ -116,7 +123,7 @@ async function createSnapshot({ type, pluginId, metadata = {} }) { } } - return { id, type, pluginId, pluginExisted, pluginZip, fullZip, snapshotPath, metadata }; + return { id, type, pluginId, pluginExisted, pluginZip, fullPath, snapshotPath, metadata }; } function finalizeSnapshot(snapshot) { @@ -289,15 +296,6 @@ function zipCore(destination) { zip.writeZip(destination); } -function zipFullInstall(destination) { - if (!AdmZip) { - throw new Error("adm-zip is not installed. Run npm install."); - } - const zip = new AdmZip(); - addFolder(zip, repoRoot, repoRoot, new Set([".git", "node_modules", "data/update-cache", "data/snapshots"])); - zip.writeZip(destination); -} - function zipFolder(source, destination, options) { if (!AdmZip) { throw new Error("adm-zip is not installed. Run npm install."); @@ -308,6 +306,11 @@ function zipFolder(source, destination, options) { zip.writeZip(destination); } +function snapshotFullInstall(destination) { + fs.mkdirSync(destination, { recursive: true }); + copySnapshotDirectory(repoRoot, destination, SNAPSHOT_EXCLUDE_RELATIVE_PATHS, { base: repoRoot }); +} + function addFolder(zip, folderPath, basePath, ignore) { const entries = fs.readdirSync(folderPath, { withFileTypes: true }); for (const entry of entries) { @@ -374,6 +377,39 @@ function copyDirectory(source, target, ignore, options = {}) { } } +function copySnapshotDirectory(source, target, ignore, options = {}) { + const base = options.base || source; + const entries = fs.readdirSync(source, { withFileTypes: true }); + for (const entry of entries) { + const srcPath = path.join(source, entry.name); + const relPath = normalizeRelative(path.relative(base, srcPath)); + if (isRelativePathIgnored(relPath, ignore)) { + continue; + } + const destPath = path.join(target, entry.name); + if (entry.isDirectory()) { + fs.mkdirSync(destPath, { recursive: true }); + copySnapshotDirectory(srcPath, destPath, ignore, { base }); + } else if (entry.isFile()) { + fs.mkdirSync(path.dirname(destPath), { recursive: true }); + copySnapshotFile(srcPath, destPath); + } + } +} + +function copySnapshotFile(source, target) { + const stat = fs.statSync(source); + if (stat.size >= HARDLINK_SNAPSHOT_MIN_BYTES) { + try { + fs.linkSync(source, target); + return; + } catch { + // Fall back to copying when hard links are unavailable across devices. + } + } + fs.copyFileSync(source, target); +} + function removeGeneratedPaths() { for (const relativePath of GENERATED_RELATIVE_PATHS) { const target = path.join(repoRoot, relativePath); @@ -593,6 +629,13 @@ function restoreSnapshot(id, options = {}) { } if (entry.type === "bot") { + const fullPath = path.join(entry.path, "full"); + if (fs.existsSync(fullPath)) { + applyCoreFilesFromDirectory(fullPath); + restoreDatabase(entry.path); + markSnapshotRolledBack(id); + return entry; + } const fullZip = path.join(entry.path, "full.zip"); const coreZip = path.join(entry.path, "core.zip"); const restoreZip = fs.existsSync(fullZip) ? fullZip : coreZip;