Update wrapper/wrapper.py
This commit is contained in:
parent
b9ceb71781
commit
e7beee2462
@ -1,18 +1,12 @@
|
|||||||
import os, sys, time, shutil, subprocess, signal, json, pathlib, re, datetime
|
import os, sys, time, shutil, subprocess, json, pathlib, re, datetime
|
||||||
from typing import Tuple
|
from typing import Tuple
|
||||||
|
|
||||||
# ---------- Config (env) ----------
|
|
||||||
# Provide sane defaults so a plain run works without envs.
|
|
||||||
REPO_URL = os.getenv("REPO_URL", "https://git.rolfsvaag.no/frarol96/shaiwatcher.git").strip()
|
REPO_URL = os.getenv("REPO_URL", "https://git.rolfsvaag.no/frarol96/shaiwatcher.git").strip()
|
||||||
REPO_BRANCH = os.getenv("REPO_BRANCH", "main").strip()
|
REPO_BRANCH = os.getenv("REPO_BRANCH", "main").strip()
|
||||||
REPO_TOKEN = os.getenv("REPO_AHTOKEN", os.getenv("REPO_TOKEN", "")).strip() # optional
|
REPO_TOKEN = os.getenv("REPO_AHTOKEN", os.getenv("REPO_TOKEN", "")).strip()
|
||||||
|
|
||||||
# prefer CHECK_TIME_UTC; fall back to old RECHECK_UTC (kept for compatibility with your wrapper envs only)
|
CHECK_TIME_UTC = os.getenv("CHECK_TIME_UTC", "03:00").strip()
|
||||||
CHECK_TIME_UTC = os.getenv("CHECK_TIME_UTC", os.getenv("RECHECK_UTC", "03:00")).strip()
|
|
||||||
|
|
||||||
# 1 = ignore test-only bumps (e.g. v1.2.3.4-T3 -> v1.2.3.4-T4)
|
|
||||||
IGNORE_TEST_LEVEL = os.getenv("IGNORE_TEST_LEVEL", "1").strip() == "1"
|
IGNORE_TEST_LEVEL = os.getenv("IGNORE_TEST_LEVEL", "1").strip() == "1"
|
||||||
|
|
||||||
PIP_INSTALL = os.getenv("PIP_INSTALL_REQUIREMENTS", "1").strip() == "1"
|
PIP_INSTALL = os.getenv("PIP_INSTALL_REQUIREMENTS", "1").strip() == "1"
|
||||||
|
|
||||||
CACHE_DIR = pathlib.Path("/cache/app")
|
CACHE_DIR = pathlib.Path("/cache/app")
|
||||||
@ -23,59 +17,44 @@ DATA_DIR = pathlib.Path("/data")
|
|||||||
RUN_TIMEOUT = int(os.getenv("WRAPPER_STOP_TIMEOUT", "25"))
|
RUN_TIMEOUT = int(os.getenv("WRAPPER_STOP_TIMEOUT", "25"))
|
||||||
ROLLBACK_MAX_FAILS = 3
|
ROLLBACK_MAX_FAILS = 3
|
||||||
|
|
||||||
# ---------- Helpers ----------
|
def log(m): print(f"[wrapper] {m}", flush=True)
|
||||||
def log(msg: str):
|
def run(*cmd, cwd=None, check=True):
|
||||||
print(f"[wrapper] {msg}", flush=True)
|
|
||||||
|
|
||||||
def run(*cmd, cwd=None, check=True) -> subprocess.CompletedProcess:
|
|
||||||
log(f"$ {' '.join(cmd)}")
|
log(f"$ {' '.join(cmd)}")
|
||||||
return subprocess.run(cmd, cwd=cwd, check=check, text=True, capture_output=True)
|
return subprocess.run(cmd, cwd=cwd, check=check, text=True, capture_output=True)
|
||||||
|
|
||||||
def ensure_git():
|
def ensure_git():
|
||||||
try:
|
try: run("git","--version")
|
||||||
run("git","--version")
|
|
||||||
except subprocess.CalledProcessError as e:
|
except subprocess.CalledProcessError as e:
|
||||||
log(f"git missing? {e.stderr}")
|
log(f"git missing? {e.stderr}"); sys.exit(1)
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
def utc_now() -> datetime.datetime:
|
|
||||||
return datetime.datetime.utcnow()
|
|
||||||
|
|
||||||
|
def utc_now(): return datetime.datetime.utcnow()
|
||||||
def next_utc(hhmm: str) -> float:
|
def next_utc(hhmm: str) -> float:
|
||||||
hh, mm = map(int, hhmm.split(":"))
|
hh, mm = map(int, hhmm.split(":"))
|
||||||
now = utc_now()
|
now = utc_now()
|
||||||
tgt = now.replace(hour=hh, minute=mm, second=0, microsecond=0)
|
tgt = now.replace(hour=hh, minute=mm, second=0, microsecond=0)
|
||||||
if tgt <= now:
|
if tgt <= now: tgt += datetime.timedelta(days=1)
|
||||||
tgt = tgt + datetime.timedelta(days=1)
|
|
||||||
return (tgt - now).total_seconds()
|
return (tgt - now).total_seconds()
|
||||||
|
|
||||||
_VERSION_RE = re.compile(r'^\s*VERSION\s*=\s*[\'"]([^\'"]+)[\'"]', re.M)
|
_VER_RE = re.compile(r'^\s*VERSION\s*=\s*[\'"]([^\'"]+)[\'"]', re.M)
|
||||||
def extract_version_from(path: pathlib.Path) -> str:
|
def extract_version_from(path: pathlib.Path) -> str:
|
||||||
try:
|
try:
|
||||||
txt = path.read_text(encoding="utf-8", errors="ignore")
|
txt = path.read_text(encoding="utf-8", errors="ignore")
|
||||||
m = _VERSION_RE.search(txt)
|
m = _VER_RE.search(txt)
|
||||||
return m.group(1).strip() if m else "v0.0.0.0"
|
return m.group(1).strip() if m else "v0.0.0.0"
|
||||||
except Exception:
|
except Exception:
|
||||||
return "v0.0.0.0"
|
return "v0.0.0.0"
|
||||||
|
|
||||||
def parse_version(ver: str) -> Tuple[int,int,int,int,bool]:
|
def parse_version(ver: str):
|
||||||
# Format: vMajor.Minor.Enh.Patch[-T...]
|
|
||||||
# Example: v1.2.3.4-T7
|
|
||||||
test = "-T" in ver
|
test = "-T" in ver
|
||||||
core = ver.split("-T")[0].lstrip("v")
|
core = ver.split("-T")[0].lstrip("v")
|
||||||
parts = [int(p or 0) for p in core.split(".")+["0","0","0","0"]][:4]
|
parts = [int(p or 0) for p in core.split(".")+["0","0","0","0"]][:4]
|
||||||
return parts[0], parts[1], parts[2], parts[3], test
|
return (*parts, test)
|
||||||
|
|
||||||
def should_update(old: str, new: str) -> bool:
|
def should_update(old, new):
|
||||||
"""
|
|
||||||
Update if the numeric tuple increases.
|
|
||||||
Ignore updates that change *only* the test suffix (e.g., v1.2.3.4-T1 -> v1.2.3.4-T2).
|
|
||||||
"""
|
|
||||||
oM,oE,oN,oP,ot = parse_version(old)
|
oM,oE,oN,oP,ot = parse_version(old)
|
||||||
nM,nE,nN,nP,nt = parse_version(new)
|
nM,nE,nN,nP,nt = parse_version(new)
|
||||||
if (oM,oE,oN,oP) != (nM,nE,nN,nP):
|
if (oM,oE,oN,oP) != (nM,nE,nN,nP):
|
||||||
return True
|
return True
|
||||||
# numeric parts same → only test suffix changed
|
|
||||||
return not IGNORE_TEST_LEVEL and (ot != nt)
|
return not IGNORE_TEST_LEVEL and (ot != nt)
|
||||||
|
|
||||||
def clone_or_fetch(target: pathlib.Path):
|
def clone_or_fetch(target: pathlib.Path):
|
||||||
@ -89,30 +68,20 @@ def clone_or_fetch(target: pathlib.Path):
|
|||||||
shutil.rmtree(target, ignore_errors=True)
|
shutil.rmtree(target, ignore_errors=True)
|
||||||
|
|
||||||
url = REPO_URL
|
url = REPO_URL
|
||||||
# optional token (only if provided)
|
|
||||||
if REPO_TOKEN and REPO_URL.startswith("https://"):
|
if REPO_TOKEN and REPO_URL.startswith("https://"):
|
||||||
url = REPO_URL.replace("https://", f"https://{REPO_TOKEN}@")
|
url = REPO_URL.replace("https://", f"https://{REPO_TOKEN}@")
|
||||||
run("git","clone","--branch", REPO_BRANCH, "--depth","1", url, str(target))
|
run("git","clone","--branch", REPO_BRANCH, "--depth","1", url, str(target))
|
||||||
|
|
||||||
def prime_tmp_then_decide():
|
def prime_tmp_then_decide():
|
||||||
# Pull to TMP, compare versions (bot.py VERSION), decide if we swap CACHE
|
|
||||||
TMP_DIR.mkdir(parents=True, exist_ok=True)
|
TMP_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
shutil.rmtree(TMP_DIR, ignore_errors=True)
|
shutil.rmtree(TMP_DIR, ignore_errors=True)
|
||||||
clone_or_fetch(TMP_DIR)
|
clone_or_fetch(TMP_DIR)
|
||||||
|
|
||||||
new_ver = extract_version_from(TMP_DIR / "bot.py")
|
new_ver = extract_version_from(TMP_DIR / "bot.py")
|
||||||
old_ver = extract_version_from(CACHE_DIR / "bot.py") if (CACHE_DIR / "bot.py").exists() else "v0.0.0.0"
|
old_ver = extract_version_from(CACHE_DIR / "bot.py") if (CACHE_DIR / "bot.py").exists() else "v0.0.0.0"
|
||||||
log(f"cached version: {old_ver} / remote version: {new_ver}")
|
do_swap = not (CACHE_DIR / "bot.py").exists() or should_update(old_ver, new_ver)
|
||||||
|
why = "first fetch" if not (CACHE_DIR / "bot.py").exists() else f"update allowed: {do_swap} (old={old_ver}, new={new_ver})"
|
||||||
if not (CACHE_DIR / "bot.py").exists():
|
return do_swap, old_ver, new_ver, why
|
||||||
# First time
|
|
||||||
do_swap = True
|
|
||||||
reason = f"first fetch -> {new_ver}"
|
|
||||||
else:
|
|
||||||
do_swap = should_update(old_ver, new_ver)
|
|
||||||
reason = f"update allowed: {do_swap} (old={old_ver}, new={new_ver})"
|
|
||||||
|
|
||||||
return do_swap, old_ver, new_ver, reason
|
|
||||||
|
|
||||||
def swap_cache_to_prev():
|
def swap_cache_to_prev():
|
||||||
PREV_DIR.mkdir(parents=True, exist_ok=True)
|
PREV_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
@ -127,106 +96,69 @@ def copy_tmp_to_cache():
|
|||||||
def pip_install(cwd: pathlib.Path):
|
def pip_install(cwd: pathlib.Path):
|
||||||
req = cwd / "requirements.txt"
|
req = cwd / "requirements.txt"
|
||||||
if PIP_INSTALL and req.exists():
|
if PIP_INSTALL and req.exists():
|
||||||
try:
|
try: run(sys.executable, "-m", "pip", "install", "-r", str(req))
|
||||||
run(sys.executable, "-m", "pip", "install", "-r", str(req))
|
|
||||||
except subprocess.CalledProcessError as e:
|
except subprocess.CalledProcessError as e:
|
||||||
log("pip install failed (will continue anyway)")
|
log("pip install failed (continuing)"); log(e.stdout + "\n" + e.stderr)
|
||||||
log(e.stdout + "\n" + e.stderr)
|
|
||||||
|
|
||||||
def set_boot_env(status: str, old_ver: str, new_ver: str, commit: str = "", subject: str = ""):
|
def set_boot_env(status: str, old_ver: str, new_ver: str, commit: str = "", subject: str = ""):
|
||||||
# Env passed to the bot; if you later choose to use them, they're non-SHAI now.
|
|
||||||
os.environ["BOOT_STATUS"] = status
|
os.environ["BOOT_STATUS"] = status
|
||||||
os.environ["BOOT_OLDVER"] = old_ver
|
os.environ["BOOT_OLDVER"] = old_ver
|
||||||
os.environ["BOOT_NEWVER"] = new_ver
|
os.environ["BOOT_NEWVER"] = new_ver
|
||||||
os.environ["BUILD_COMMIT"] = commit
|
os.environ["BUILD_COMMIT"] = commit
|
||||||
os.environ["BUILD_SUBJECT"] = subject
|
os.environ["BUILD_SUBJECT"] = subject
|
||||||
|
|
||||||
def get_head_info(cwd: pathlib.Path) -> Tuple[str,str]:
|
def get_head_info(cwd: pathlib.Path):
|
||||||
try:
|
try:
|
||||||
c1 = run("git","rev-parse","--short","HEAD", cwd=cwd, check=True)
|
sha = run("git","rev-parse","--short","HEAD", cwd=cwd).stdout.strip()
|
||||||
sha = c1.stdout.strip()
|
subj = run("git","log","-1","--pretty=%s", cwd=cwd).stdout.strip()
|
||||||
c2 = run("git","log","-1","--pretty=%s", cwd=cwd, check=True)
|
|
||||||
subj = c2.stdout.strip()
|
|
||||||
return (sha, subj)
|
return (sha, subj)
|
||||||
except Exception:
|
except Exception:
|
||||||
return ("","")
|
return ("","")
|
||||||
|
|
||||||
def start_bot(cwd: pathlib.Path) -> subprocess.Popen:
|
def start_bot(cwd: pathlib.Path) -> subprocess.Popen:
|
||||||
env = os.environ.copy()
|
env = os.environ.copy()
|
||||||
# Make sure data file env is present under the new name (no SHAI_).
|
env.setdefault("DATA_FILE", "/data/data.json") # new name
|
||||||
env.setdefault("DATA_FILE", "/data/data.json")
|
|
||||||
# Run from the cached code directory
|
|
||||||
return subprocess.Popen([sys.executable, "-u", "bot.py"], cwd=cwd, env=env)
|
return subprocess.Popen([sys.executable, "-u", "bot.py"], cwd=cwd, env=env)
|
||||||
|
|
||||||
def graceful_restart(proc: subprocess.Popen) -> bool:
|
def graceful_stop(proc: subprocess.Popen) -> bool:
|
||||||
try:
|
try:
|
||||||
proc.terminate()
|
proc.terminate()
|
||||||
try:
|
try:
|
||||||
proc.wait(timeout=RUN_TIMEOUT)
|
proc.wait(timeout=RUN_TIMEOUT); return True
|
||||||
return True
|
|
||||||
except subprocess.TimeoutExpired:
|
except subprocess.TimeoutExpired:
|
||||||
proc.kill()
|
proc.kill(); proc.wait(timeout=10); return True
|
||||||
proc.wait(timeout=10)
|
|
||||||
return True
|
|
||||||
except Exception:
|
except Exception:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def run_loop():
|
def run_loop():
|
||||||
ensure_git()
|
ensure_git()
|
||||||
|
updated, old_ver, new_ver, why = prime_tmp_then_decide()
|
||||||
|
if updated: log(f"updating cache: {why}"); swap_cache_to_prev(); copy_tmp_to_cache()
|
||||||
|
else: log(f"no update: {why}")
|
||||||
|
|
||||||
# initial fetch/decide
|
|
||||||
updated, old_ver, new_ver, reason = prime_tmp_then_decide()
|
|
||||||
if updated:
|
|
||||||
log(f"updating cache: {reason}")
|
|
||||||
swap_cache_to_prev()
|
|
||||||
copy_tmp_to_cache()
|
|
||||||
else:
|
|
||||||
log(f"no update: {reason}")
|
|
||||||
|
|
||||||
# pip install if needed (requirements.txt inside repo)
|
|
||||||
pip_install(CACHE_DIR)
|
pip_install(CACHE_DIR)
|
||||||
|
|
||||||
# boot status env
|
|
||||||
sha, subj = get_head_info(CACHE_DIR)
|
sha, subj = get_head_info(CACHE_DIR)
|
||||||
if updated:
|
|
||||||
set_boot_env(
|
set_boot_env(
|
||||||
f"Successfully fetched, cached, and booted new version",
|
"Successfully fetched, cached, and booted new version" if updated else "Successfully booted from cached version",
|
||||||
old_ver, new_ver, sha, subj
|
old_ver, new_ver, sha, subj
|
||||||
)
|
)
|
||||||
else:
|
|
||||||
msg = "Successfully booted from cached version"
|
|
||||||
if sha or subj:
|
|
||||||
msg += " (repo reachable)"
|
|
||||||
set_boot_env(msg, old_ver, new_ver, sha, subj)
|
|
||||||
|
|
||||||
# start bot
|
|
||||||
proc = start_bot(CACHE_DIR)
|
proc = start_bot(CACHE_DIR)
|
||||||
log(f"bot started pid={proc.pid}")
|
log(f"bot started pid={proc.pid}")
|
||||||
|
|
||||||
consecutive_failures = 0
|
consecutive_failures = 0
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
# sleep until next check (03:00 UTC by default)
|
|
||||||
delay = next_utc(CHECK_TIME_UTC)
|
delay = next_utc(CHECK_TIME_UTC)
|
||||||
# FIX: use CHECK_TIME_UTC here (your original printed RECHECK_UTC)
|
|
||||||
log(f"sleeping {int(delay)}s until {CHECK_TIME_UTC} UTC for update check")
|
log(f"sleeping {int(delay)}s until {CHECK_TIME_UTC} UTC for update check")
|
||||||
time.sleep(delay)
|
time.sleep(delay)
|
||||||
|
|
||||||
# check for update
|
|
||||||
try:
|
try:
|
||||||
upd, cur_ver, remote_ver, why = prime_tmp_then_decide()
|
upd, cur_ver, remote_ver, why = prime_tmp_then_decide()
|
||||||
log(f"nightly check: {why}")
|
log(f"nightly check: {why}")
|
||||||
if not upd:
|
if not upd:
|
||||||
# no update -> continue loop
|
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# graceful restart into new version
|
|
||||||
log("updating to new version at nightly window")
|
log("updating to new version at nightly window")
|
||||||
ok = graceful_restart(proc)
|
graceful_stop(proc)
|
||||||
if not ok:
|
|
||||||
log("warning: bot did not stop cleanly")
|
|
||||||
|
|
||||||
# swap and boot
|
|
||||||
swap_cache_to_prev()
|
swap_cache_to_prev()
|
||||||
copy_tmp_to_cache()
|
copy_tmp_to_cache()
|
||||||
pip_install(CACHE_DIR)
|
pip_install(CACHE_DIR)
|
||||||
@ -241,8 +173,7 @@ def run_loop():
|
|||||||
consecutive_failures += 1
|
consecutive_failures += 1
|
||||||
if consecutive_failures < ROLLBACK_MAX_FAILS and PREV_DIR.exists() and (PREV_DIR / "bot.py").exists():
|
if consecutive_failures < ROLLBACK_MAX_FAILS and PREV_DIR.exists() and (PREV_DIR / "bot.py").exists():
|
||||||
log("attempting rollback to previous cached version")
|
log("attempting rollback to previous cached version")
|
||||||
if proc.poll() is None:
|
if proc.poll() is None: graceful_stop(proc)
|
||||||
graceful_restart(proc)
|
|
||||||
shutil.rmtree(CACHE_DIR, ignore_errors=True)
|
shutil.rmtree(CACHE_DIR, ignore_errors=True)
|
||||||
shutil.copytree(PREV_DIR, CACHE_DIR, dirs_exist_ok=False)
|
shutil.copytree(PREV_DIR, CACHE_DIR, dirs_exist_ok=False)
|
||||||
pip_install(CACHE_DIR)
|
pip_install(CACHE_DIR)
|
||||||
@ -250,15 +181,10 @@ def run_loop():
|
|||||||
proc = start_bot(CACHE_DIR)
|
proc = start_bot(CACHE_DIR)
|
||||||
elif consecutive_failures >= ROLLBACK_MAX_FAILS:
|
elif consecutive_failures >= ROLLBACK_MAX_FAILS:
|
||||||
log("critical: failed 3 times to update/restart; entering freeze mode")
|
log("critical: failed 3 times to update/restart; entering freeze mode")
|
||||||
# Optional: DM owner could be done in a tiny fallback bot here if OWNER_ID provided.
|
|
||||||
# For now, just idle to allow SSH/exec into container.
|
|
||||||
try:
|
try:
|
||||||
if proc.poll() is None:
|
if proc.poll() is None: graceful_stop(proc)
|
||||||
graceful_restart(proc)
|
except Exception: pass
|
||||||
except Exception:
|
while True: time.sleep(3600)
|
||||||
pass
|
|
||||||
while True:
|
|
||||||
time.sleep(3600)
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
run_loop()
|
run_loop()
|
||||||
|
Loading…
Reference in New Issue
Block a user