From 34a458279178a6f30bee93949d4c3b09e6c37078 Mon Sep 17 00:00:00 2001 From: frarol96 Date: Sat, 16 Aug 2025 05:06:23 +0000 Subject: [PATCH] Update wrapper/wrapper.py Added playwright and chromium for data scraping, and simplified env designation --- wrapper/wrapper.py | 517 +++++++++++++++++++++++---------------------- 1 file changed, 262 insertions(+), 255 deletions(-) diff --git a/wrapper/wrapper.py b/wrapper/wrapper.py index 8b86f0e..6a6e3ac 100644 --- a/wrapper/wrapper.py +++ b/wrapper/wrapper.py @@ -1,255 +1,262 @@ -import os, sys, time, shutil, subprocess, signal, json, pathlib, re, datetime -from typing import Tuple - -# ---------- Config (env) ---------- -REPO_URL = os.getenv("REPO_URL", "").strip() # e.g. https://git.rolfsvaag.no/frarol96/shaiwatcher -REPO_BRANCH = os.getenv("REPO_BRANCH", "main").strip() -REPO_TOKEN = os.getenv("REPO_TOKEN", "").strip() # optional (for private), not used if empty -RECHECK_UTC = os.getenv("RECHECK_UTC", "03:00").strip() # HH:MM (UTC) -PIP_INSTALL = os.getenv("PIP_INSTALL_REQUIREMENTS", "1").strip() == "1" - -CACHE_DIR = pathlib.Path("/cache/app") # current code -TMP_DIR = pathlib.Path("/cache/tmp") # temp checkout -PREV_DIR = pathlib.Path("/cache/prev") # rollback -DATA_DIR = pathlib.Path("/data") # persistent data volume -RUN_TIMEOUT = int(os.getenv("WRAPPER_STOP_TIMEOUT", "25")) -ROLLBACK_MAX_FAILS = 3 - -# ---------- Helpers ---------- -def log(msg: str): - print(f"[wrapper] {msg}", flush=True) - -def run(*cmd, cwd=None, check=True) -> subprocess.CompletedProcess: - log(f"$ {' '.join(cmd)}") - return subprocess.run(cmd, cwd=cwd, check=check, text=True, capture_output=True) - -def ensure_git(): - try: - run("git","--version") - except subprocess.CalledProcessError as e: - log(f"git missing? {e.stderr}") - sys.exit(1) - -def utc_now() -> datetime.datetime: - return datetime.datetime.utcnow() - -def next_utc(hhmm: str) -> float: - hh, mm = map(int, hhmm.split(":")) - now = utc_now() - tgt = now.replace(hour=hh, minute=mm, second=0, microsecond=0) - if tgt <= now: - tgt = tgt + datetime.timedelta(days=1) - return (tgt - now).total_seconds() - -_VERSION_RE = re.compile(r'^\s*VERSION\s*=\s*[\'"]([^\'"]+)[\'"]', re.M) -def extract_version_from(path: pathlib.Path) -> str: - try: - txt = path.read_text(encoding="utf-8", errors="ignore") - m = _VERSION_RE.search(txt) - return m.group(1).strip() if m else "v0.0.0.0" - except Exception: - return "v0.0.0.0" - -def parse_version(ver: str) -> Tuple[int,int,int,int,bool]: - # Format: vMajor.Minor.Enh.Patch[-T...] - # Example: v1.2.3.4-T7 - test = "-T" in ver - core = ver.split("-T")[0].lstrip("v") - parts = [int(p or 0) for p in core.split(".")+["0","0","0","0"]][:4] - return parts[0], parts[1], parts[2], parts[3], test - -def should_update(old: str, new: str) -> bool: - """ - Update if the numeric tuple increases. - Ignore updates that change *only* the test suffix (e.g., v1.2.3.4-T1 -> v1.2.3.4-T2). - """ - oM,oE,oN,oP,ot = parse_version(old) - nM,nE,nN,nP,nt = parse_version(new) - if (oM,oE,oN,oP) != (nM,nE,nN,nP): - return True - # numeric same -> only test part differs → do NOT update - return False - -def clone_or_fetch(target: pathlib.Path): - if target.exists() and (target / ".git").exists(): - try: - run("git","fetch","--all","-p", cwd=target) - run("git","reset","--hard", f"origin/{REPO_BRANCH}", cwd=target) - return - except Exception as e: - log(f"fetch failed, recloning: {e}") - shutil.rmtree(target, ignore_errors=True) - - url = REPO_URL - # optional token (only if provided) - if REPO_TOKEN and REPO_URL.startswith("https://"): - url = REPO_URL.replace("https://", f"https://{REPO_TOKEN}@") - run("git","clone","--branch",REPO_BRANCH,"--depth","1", url, str(target)) - -def prime_tmp_then_decide(): - # Pull to TMP, compare versions (bot.py VERSION), decide if we swap CACHE - TMP_DIR.mkdir(parents=True, exist_ok=True) - shutil.rmtree(TMP_DIR, ignore_errors=True) - clone_or_fetch(TMP_DIR) - - new_ver = extract_version_from(TMP_DIR / "bot.py") - old_ver = extract_version_from(CACHE_DIR / "bot.py") if (CACHE_DIR / "bot.py").exists() else "v0.0.0.0" - log(f"cached version: {old_ver} / remote version: {new_ver}") - - if not (CACHE_DIR / "bot.py").exists(): - # First time - do_swap = True - reason = f"first fetch -> {new_ver}" - else: - do_swap = should_update(old_ver, new_ver) - reason = f"update allowed: {do_swap} (old={old_ver}, new={new_ver})" - - return do_swap, old_ver, new_ver, reason - -def swap_cache_to_prev(): - PREV_DIR.mkdir(parents=True, exist_ok=True) - shutil.rmtree(PREV_DIR, ignore_errors=True) - if CACHE_DIR.exists(): - shutil.move(str(CACHE_DIR), str(PREV_DIR)) - -def copy_tmp_to_cache(): - shutil.rmtree(CACHE_DIR, ignore_errors=True) - shutil.copytree(TMP_DIR, CACHE_DIR, dirs_exist_ok=False) - -def pip_install(cwd: pathlib.Path): - req = cwd / "requirements.txt" - if PIP_INSTALL and req.exists(): - try: - run(sys.executable, "-m", "pip", "install", "-r", str(req)) - except subprocess.CalledProcessError as e: - log("pip install failed (will continue anyway)") - log(e.stdout + "\n" + e.stderr) - -def set_boot_env(status: str, old_ver: str, new_ver: str, commit: str = "", subject: str = ""): - # Env passed to the bot; bot should read and post to modlog on_ready - os.environ["SHAI_BOOT_STATUS"] = status - os.environ["SHAI_BOOT_OLDVER"] = old_ver - os.environ["SHAI_BOOT_NEWVER"] = new_ver - os.environ["SHAI_BUILD_COMMIT"] = commit - os.environ["SHAI_BUILD_SUBJECT"]= subject - -def get_head_info(cwd: pathlib.Path) -> Tuple[str,str]: - try: - c1 = run("git","rev-parse","--short","HEAD", cwd=cwd, check=True) - sha = c1.stdout.strip() - c2 = run("git","log","-1","--pretty=%s", cwd=cwd, check=True) - subj = c2.stdout.strip() - return (sha, subj) - except Exception: - return ("", "") - -def start_bot(cwd: pathlib.Path) -> subprocess.Popen: - env = os.environ.copy() - # Make sure data dir exists (the bot should use SHAI_DATA or SHAI_DATA_FILE or config) - env.setdefault("SHAI_DATA", "/data/data.json") - # Run from the cached code directory - return subprocess.Popen([sys.executable, "-u", "bot.py"], cwd=cwd, env=env) - -def graceful_restart(proc: subprocess.Popen) -> bool: - try: - proc.terminate() - try: - proc.wait(timeout=RUN_TIMEOUT) - return True - except subprocess.TimeoutExpired: - proc.kill() - proc.wait(timeout=10) - return True - except Exception: - return False - -def run_loop(): - ensure_git() - - # initial fetch/decide - updated, old_ver, new_ver, reason = prime_tmp_then_decide() - if updated: - log(f"updating cache: {reason}") - swap_cache_to_prev() - copy_tmp_to_cache() - else: - log(f"no update: {reason}") - - # pip install if needed (requirements.txt inside repo) - pip_install(CACHE_DIR) - - # boot status env - sha, subj = get_head_info(CACHE_DIR) - if updated: - set_boot_env( - f"Successfully fetched, cached, and booted new version", - old_ver, new_ver, sha, subj - ) - else: - msg = "Successfully booted from cached version" - if sha or subj: - msg += " (repo reachable)" - set_boot_env(msg, old_ver, new_ver, sha, subj) - - # start bot - proc = start_bot(CACHE_DIR) - log(f"bot started pid={proc.pid}") - - consecutive_failures = 0 - - while True: - # sleep until next 03:00 UTC - delay = next_utc(RECHECK_UTC) - log(f"sleeping {int(delay)}s until {RECHECK_UTC} UTC for update check") - time.sleep(delay) - - # check for update - try: - upd, cur_ver, remote_ver, why = prime_tmp_then_decide() - log(f"nightly check: {why}") - if not upd: - # no update -> continue loop - continue - - # graceful restart into new version - log("updating to new version at nightly window") - ok = graceful_restart(proc) - if not ok: - log("warning: bot did not stop cleanly") - - # swap and boot - swap_cache_to_prev() - copy_tmp_to_cache() - pip_install(CACHE_DIR) - sha, subj = get_head_info(CACHE_DIR) - set_boot_env("Successfully fetched, cached, and booted new version", cur_ver, remote_ver, sha, subj) - proc = start_bot(CACHE_DIR) - log(f"bot restarted on new version pid={proc.pid}") - consecutive_failures = 0 - - except Exception as e: - log(f"nightly update failed: {e}") - consecutive_failures += 1 - if consecutive_failures < ROLLBACK_MAX_FAILS and PREV_DIR.exists() and (PREV_DIR / "bot.py").exists(): - log("attempting rollback to previous cached version") - if proc.poll() is None: - graceful_restart(proc) - shutil.rmtree(CACHE_DIR, ignore_errors=True) - shutil.copytree(PREV_DIR, CACHE_DIR, dirs_exist_ok=False) - pip_install(CACHE_DIR) - set_boot_env("Rolled back to last known working version", "-", extract_version_from(CACHE_DIR / "bot.py")) - proc = start_bot(CACHE_DIR) - elif consecutive_failures >= ROLLBACK_MAX_FAILS: - log("critical: failed 3 times to update/restart; entering freeze mode") - # Optional: DM owner could be done in a tiny fallback bot here if OWNER_ID provided. - # For now, just idle to allow SSH/exec into container. - try: - if proc.poll() is None: - graceful_restart(proc) - except Exception: - pass - while True: - time.sleep(3600) - -if __name__ == "__main__": - run_loop() +import os, sys, time, shutil, subprocess, signal, json, pathlib, re, datetime +from typing import Tuple + +# ---------- Config (env) ---------- +REPO_URL = os.getenv("REPO_URL", "").strip() +REPO_BRANCH = os.getenv("REPO_BRANCH", "main").strip() +REPO_TOKEN = os.getenv("REPO_TOKEN", "").strip() # optional + +# prefer CHECK_TIME_UTC; fall back to old RECHECK_UTC +CHECK_TIME_UTC = os.getenv("CHECK_TIME_UTC", os.getenv("RECHECK_UTC", "03:00")).strip() + +# 1 = ignore test-only bumps (e.g. v1.2.3.4-T3 -> v1.2.3.4-T4) +IGNORE_TEST_LEVEL = os.getenv("IGNORE_TEST_LEVEL", "1").strip() == "1" + +PIP_INSTALL = os.getenv("PIP_INSTALL_REQUIREMENTS", "1").strip() == "1" + +CACHE_DIR = pathlib.Path("/cache/app") +TMP_DIR = pathlib.Path("/cache/tmp") +PREV_DIR = pathlib.Path("/cache/prev") +DATA_DIR = pathlib.Path("/data") + +RUN_TIMEOUT = int(os.getenv("WRAPPER_STOP_TIMEOUT", "25")) +ROLLBACK_MAX_FAILS = 3 + +# ---------- Helpers ---------- +def log(msg: str): + print(f"[wrapper] {msg}", flush=True) + +def run(*cmd, cwd=None, check=True) -> subprocess.CompletedProcess: + log(f"$ {' '.join(cmd)}") + return subprocess.run(cmd, cwd=cwd, check=check, text=True, capture_output=True) + +def ensure_git(): + try: + run("git","--version") + except subprocess.CalledProcessError as e: + log(f"git missing? {e.stderr}") + sys.exit(1) + +def utc_now() -> datetime.datetime: + return datetime.datetime.utcnow() + +def next_utc(hhmm: str) -> float: + hh, mm = map(int, hhmm.split(":")) + now = utc_now() + tgt = now.replace(hour=hh, minute=mm, second=0, microsecond=0) + if tgt <= now: + tgt = tgt + datetime.timedelta(days=1) + return (tgt - now).total_seconds() + +_VERSION_RE = re.compile(r'^\s*VERSION\s*=\s*[\'"]([^\'"]+)[\'"]', re.M) +def extract_version_from(path: pathlib.Path) -> str: + try: + txt = path.read_text(encoding="utf-8", errors="ignore") + m = _VERSION_RE.search(txt) + return m.group(1).strip() if m else "v0.0.0.0" + except Exception: + return "v0.0.0.0" + +def parse_version(ver: str) -> Tuple[int,int,int,int,bool]: + # Format: vMajor.Minor.Enh.Patch[-T...] + # Example: v1.2.3.4-T7 + test = "-T" in ver + core = ver.split("-T")[0].lstrip("v") + parts = [int(p or 0) for p in core.split(".")+["0","0","0","0"]][:4] + return parts[0], parts[1], parts[2], parts[3], test + +def should_update(old: str, new: str) -> bool: + """ + Update if the numeric tuple increases. + Ignore updates that change *only* the test suffix (e.g., v1.2.3.4-T1 -> v1.2.3.4-T2). + """ + oM,oE,oN,oP,ot = parse_version(old) + nM,nE,nN,nP,nt = parse_version(new) + if (oM,oE,oN,oP) != (nM,nE,nN,nP): + return True + # numeric parts same → only test suffix changed + return not IGNORE_TEST_LEVEL and (ot != nt) + +def clone_or_fetch(target: pathlib.Path): + if target.exists() and (target / ".git").exists(): + try: + run("git","fetch","--all","-p", cwd=target) + run("git","reset","--hard", f"origin/{REPO_BRANCH}", cwd=target) + return + except Exception as e: + log(f"fetch failed, recloning: {e}") + shutil.rmtree(target, ignore_errors=True) + + url = REPO_URL + # optional token (only if provided) + if REPO_TOKEN and REPO_URL.startswith("https://"): + url = REPO_URL.replace("https://", f"https://{REPO_TOKEN}@") + run("git","clone","--branch",REPO_BRANCH,"--depth","1", url, str(target)) + +def prime_tmp_then_decide(): + # Pull to TMP, compare versions (bot.py VERSION), decide if we swap CACHE + TMP_DIR.mkdir(parents=True, exist_ok=True) + shutil.rmtree(TMP_DIR, ignore_errors=True) + clone_or_fetch(TMP_DIR) + + new_ver = extract_version_from(TMP_DIR / "bot.py") + old_ver = extract_version_from(CACHE_DIR / "bot.py") if (CACHE_DIR / "bot.py").exists() else "v0.0.0.0" + log(f"cached version: {old_ver} / remote version: {new_ver}") + + if not (CACHE_DIR / "bot.py").exists(): + # First time + do_swap = True + reason = f"first fetch -> {new_ver}" + else: + do_swap = should_update(old_ver, new_ver) + reason = f"update allowed: {do_swap} (old={old_ver}, new={new_ver})" + + return do_swap, old_ver, new_ver, reason + +def swap_cache_to_prev(): + PREV_DIR.mkdir(parents=True, exist_ok=True) + shutil.rmtree(PREV_DIR, ignore_errors=True) + if CACHE_DIR.exists(): + shutil.move(str(CACHE_DIR), str(PREV_DIR)) + +def copy_tmp_to_cache(): + shutil.rmtree(CACHE_DIR, ignore_errors=True) + shutil.copytree(TMP_DIR, CACHE_DIR, dirs_exist_ok=False) + +def pip_install(cwd: pathlib.Path): + req = cwd / "requirements.txt" + if PIP_INSTALL and req.exists(): + try: + run(sys.executable, "-m", "pip", "install", "-r", str(req)) + except subprocess.CalledProcessError as e: + log("pip install failed (will continue anyway)") + log(e.stdout + "\n" + e.stderr) + +def set_boot_env(status: str, old_ver: str, new_ver: str, commit: str = "", subject: str = ""): + # Env passed to the bot; bot should read and post to modlog on_ready + os.environ["SHAI_BOOT_STATUS"] = status + os.environ["SHAI_BOOT_OLDVER"] = old_ver + os.environ["SHAI_BOOT_NEWVER"] = new_ver + os.environ["SHAI_BUILD_COMMIT"] = commit + os.environ["SHAI_BUILD_SUBJECT"]= subject + +def get_head_info(cwd: pathlib.Path) -> Tuple[str,str]: + try: + c1 = run("git","rev-parse","--short","HEAD", cwd=cwd, check=True) + sha = c1.stdout.strip() + c2 = run("git","log","-1","--pretty=%s", cwd=cwd, check=True) + subj = c2.stdout.strip() + return (sha, subj) + except Exception: + return ("", "") + +def start_bot(cwd: pathlib.Path) -> subprocess.Popen: + env = os.environ.copy() + # Make sure data dir exists (the bot should use SHAI_DATA or SHAI_DATA_FILE or config) + env.setdefault("SHAI_DATA", "/data/data.json") + # Run from the cached code directory + return subprocess.Popen([sys.executable, "-u", "bot.py"], cwd=cwd, env=env) + +def graceful_restart(proc: subprocess.Popen) -> bool: + try: + proc.terminate() + try: + proc.wait(timeout=RUN_TIMEOUT) + return True + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=10) + return True + except Exception: + return False + +def run_loop(): + ensure_git() + + # initial fetch/decide + updated, old_ver, new_ver, reason = prime_tmp_then_decide() + if updated: + log(f"updating cache: {reason}") + swap_cache_to_prev() + copy_tmp_to_cache() + else: + log(f"no update: {reason}") + + # pip install if needed (requirements.txt inside repo) + pip_install(CACHE_DIR) + + # boot status env + sha, subj = get_head_info(CACHE_DIR) + if updated: + set_boot_env( + f"Successfully fetched, cached, and booted new version", + old_ver, new_ver, sha, subj + ) + else: + msg = "Successfully booted from cached version" + if sha or subj: + msg += " (repo reachable)" + set_boot_env(msg, old_ver, new_ver, sha, subj) + + # start bot + proc = start_bot(CACHE_DIR) + log(f"bot started pid={proc.pid}") + + consecutive_failures = 0 + + while True: + # sleep until next check (03:00 UTC by default) + delay = next_utc(CHECK_TIME_UTC) + log(f"sleeping {int(delay)}s until {RECHECK_UTC} UTC for update check") + time.sleep(delay) + + # check for update + try: + upd, cur_ver, remote_ver, why = prime_tmp_then_decide() + log(f"nightly check: {why}") + if not upd: + # no update -> continue loop + continue + + # graceful restart into new version + log("updating to new version at nightly window") + ok = graceful_restart(proc) + if not ok: + log("warning: bot did not stop cleanly") + + # swap and boot + swap_cache_to_prev() + copy_tmp_to_cache() + pip_install(CACHE_DIR) + sha, subj = get_head_info(CACHE_DIR) + set_boot_env("Successfully fetched, cached, and booted new version", cur_ver, remote_ver, sha, subj) + proc = start_bot(CACHE_DIR) + log(f"bot restarted on new version pid={proc.pid}") + consecutive_failures = 0 + + except Exception as e: + log(f"nightly update failed: {e}") + consecutive_failures += 1 + if consecutive_failures < ROLLBACK_MAX_FAILS and PREV_DIR.exists() and (PREV_DIR / "bot.py").exists(): + log("attempting rollback to previous cached version") + if proc.poll() is None: + graceful_restart(proc) + shutil.rmtree(CACHE_DIR, ignore_errors=True) + shutil.copytree(PREV_DIR, CACHE_DIR, dirs_exist_ok=False) + pip_install(CACHE_DIR) + set_boot_env("Rolled back to last known working version", "-", extract_version_from(CACHE_DIR / "bot.py")) + proc = start_bot(CACHE_DIR) + elif consecutive_failures >= ROLLBACK_MAX_FAILS: + log("critical: failed 3 times to update/restart; entering freeze mode") + # Optional: DM owner could be done in a tiny fallback bot here if OWNER_ID provided. + # For now, just idle to allow SSH/exec into container. + try: + if proc.poll() is None: + graceful_restart(proc) + except Exception: + pass + while True: + time.sleep(3600) + +if __name__ == "__main__": + run_loop()