From ebbebbacf7056f75a77dff93f1cccaa68c6b8412 Mon Sep 17 00:00:00 2001 From: Franz Rolfsvaag Date: Sat, 16 Aug 2025 07:31:13 +0200 Subject: [PATCH] 0.4.2.1.a1 - Added a new owner-only `/data [download/upload]` command for datafile backup and restoration *This is required as v0.4.2 requires a rebuild of the stack, and the existing data should be backed up in case of data loss* --- .env.example | 127 +++++++++++++++++ .gitignore | 1 + bot.py | 2 +- example/.env.example | 5 - modules/data_admin/__init__.py | 0 modules/data_admin/data_admin.py | 237 +++++++++++++++++++++++++++++++ 6 files changed, 366 insertions(+), 6 deletions(-) create mode 100644 .env.example delete mode 100644 example/.env.example create mode 100644 modules/data_admin/__init__.py create mode 100644 modules/data_admin/data_admin.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..58e6e6e --- /dev/null +++ b/.env.example @@ -0,0 +1,127 @@ +# ───────────────────────────────────────────────────────────── +# Required +# ───────────────────────────────────────────────────────────── +DISCORD_TOKEN= + +# Git repo to run (wrapper clones nightly and on boot) +REPO_URL= +REPO_BRANCH=main +# If your repo is private, you can use a token; leave empty if not used +REPO_TOKEN= + +# Wrapper update time (UTC) & test bump policy +CHECK_TIME_UTC=03:00 +IGNORE_TEST_LEVEL=1 # 1 = ignore updates that change only the -T test suffix + +# Optional commit subject feed used by your boot notice +SHAI_REPO_RSS= + +# Where the bot stores data inside the container (don’t change) +SHAI_DATA_FILE=/data/data.json + +# ───────────────────────────────────────────────────────────── +# Volumes (external, never wiped by stack re-deploys) +# Create once: docker volume create shaiwatcher_data && docker volume create shaiwatcher_cache +# ───────────────────────────────────────────────────────────── +SHAI_VOL_DATA=shaiwatcher_data +SHAI_VOL_CACHE=shaiwatcher_cache + +# ───────────────────────────────────────────────────────────── +# Playwright / Headless browser (for DD scraping) +# Build with: docker compose build --build-arg WITH_PLAYWRIGHT=1 +# ───────────────────────────────────────────────────────────── +WITH_PLAYWRIGHT=1 # 1 to bake Chromium+Playwright into the image +SHAI_DD_CHANNEL_ID= +SHAI_DD_FETCHER=playwright # playwright|aiohttp (playwright recommended) +SHAI_DD_PW_TIMEOUT_MS=60000 # nav timeout +SHAI_DD_PW_WAIT= # domcontentloaded|load|networkidle (empty = default domcontentloaded) + +# ───────────────────────────────────────────────────────────── +# Slash command scope (your bot already honors this) +# ───────────────────────────────────────────────────────────── +SHAI_SLASH_GUILD_ONLY=true +SHAI_HOME_GUILD_ID= + +# ───────────────────────────────────────────────────────────── +# Channels +# ───────────────────────────────────────────────────────────── +SHAI_MOD_CHANNEL_ID= +SHAI_MODLOG_CHANNEL_ID= +SHAI_USERSLIST_CHANNEL_ID= +SHAI_REPORT_CHANNEL_ID= +SHAI_PIRATES_LIST_CHANNEL_ID= +SHAI_TRIGGER_CHANNEL_ID= # Auto-VC +SHAI_AUTO_VC_CATEGORY_ID= + +# ───────────────────────────────────────────────────────────── +# Reaction messages +# ───────────────────────────────────────────────────────────── +SHAI_RULES_MESSAGE_ID= +SHAI_ENGAGEMENT_MESSAGE_ID= +SHAI_NICKNAME_MESSAGE_ID= + +# ───────────────────────────────────────────────────────────── +# Roles +# ───────────────────────────────────────────────────────────── +SHAI_RULES_ROLE_ID= +SHAI_ENGAGEMENT_ROLE_ID= +SHAI_FULL_ACCESS_ROLE_ID= +SHAI_ADMIN_ROLE_ID= +SHAI_FIELD_MOD_ROLE_ID= +SHAI_INTEL_MOD_ROLE_ID= +SHAI_MODERATOR_ROLE_ID= + +# ───────────────────────────────────────────────────────────── +# Auto-VC +# ───────────────────────────────────────────────────────────── +SHAI_VC_NAME_PREFIX=DD Crew +SHAI_AUTO_VC_CLEANUP_DELAY=30 + +# ───────────────────────────────────────────────────────────── +# Threat weights +# ───────────────────────────────────────────────────────────── +SHAI_THREAT_W_KILL=0.30 +SHAI_THREAT_W_DESTRUCTION=0.40 +SHAI_THREAT_W_GROUP=0.20 +SHAI_THREAT_W_SKILL=0.10 +SHAI_THREAT_GROUP_THRESHOLD=3 +SHAI_THREAT_MIN_SAMPLES_FOR_STATS=3 + +# ───────────────────────────────────────────────────────────── +# Toggles +# ───────────────────────────────────────────────────────────── +SHAI_NICK_NUDGE_LOOP_ENABLED=false +SHAI_USER_CARDS_CRON_ENABLED=true + +# ───────────────────────────────────────────────────────────── +# SpicePay defaults +# ───────────────────────────────────────────────────────────── +SHAI_SPICEPAY_LSR_CUT_PERCENT=10 +SHAI_SPICEPAY_BASE_WEIGHT=25 +SHAI_SPICEPAY_CARRIER_BONUS=12.5 +SHAI_SPICEPAY_CRAWLER_BONUS=12.5 + +# ───────────────────────────────────────────────────────────── +# Emojis (IDs) +# ───────────────────────────────────────────────────────────── +SHAI_EMOJI_MELANGE_ID= +SHAI_EMOJI_SAND_ID= +SHAI_EMOJI_CARRIER_CRAWLER_ID= + +# ───────────────────────────────────────────────────────────── +# Docs site (optional) +# ───────────────────────────────────────────────────────────── +SHAI_DOCS_HOST=0.0.0.0 +SHAI_DOCS_PORT=8910 +SHAI_DOCS_TITLE=ShaiWatcher Commands +SHAI_DOCS_SUPPORT_URL= +SHAI_DOCS_SUPPORT_LABEL=Buy me a ☕ + +# ───────────────────────────────────────────────────────────── +# Wrapper knobs (optional) +# ───────────────────────────────────────────────────────────── +PIP_INSTALL_REQUIREMENTS=1 +WRAPPER_STOP_TIMEOUT=25 + +# Keep Docker base locale happy +LANG=C.UTF-8 diff --git a/.gitignore b/.gitignore index 4c4dcce..c233395 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ settings*.conf NOTES.md sanity/ .offline_data.json +dev/.env.production # Tools wrapper/ diff --git a/bot.py b/bot.py index cab6675..b5f87cd 100644 --- a/bot.py +++ b/bot.py @@ -9,7 +9,7 @@ from modules.common.boot_notice import post_boot_notice # Version consists of: # Major.Enhancement.Minor.Patch.Test (Test is alphanumeric; doesn’t trigger auto update) -VERSION = "0.4.2.0.a1" +VERSION = "0.4.2.1.a1" # ---------- Env loading ---------- diff --git a/example/.env.example b/example/.env.example deleted file mode 100644 index efa6482..0000000 --- a/example/.env.example +++ /dev/null @@ -1,5 +0,0 @@ -DISCORD_TOKEN={DISCORD_BOT_TOKEN} -DISCORD_APPLICATION_ID= -DISCORD_PUBLIC_KEY= -DISCORD_CLIENT_ID= -DISCORD_CLIENT_SECRET= \ No newline at end of file diff --git a/modules/data_admin/__init__.py b/modules/data_admin/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/modules/data_admin/data_admin.py b/modules/data_admin/data_admin.py new file mode 100644 index 0000000..a01e5ff --- /dev/null +++ b/modules/data_admin/data_admin.py @@ -0,0 +1,237 @@ +from __future__ import annotations + +import io +import os +import json +import time +import shutil +import asyncio +from typing import Optional, Literal + +import aiohttp +import discord +from discord import app_commands +from discord.ext import commands + + +MAX_UPLOAD_BYTES = 8 * 1024 * 1024 # 8 MiB hard cap for safety +OWNER_HINT = "This command is restricted to the **server owner** (or bot owner)." + +def _now_stamp() -> str: + return time.strftime("%Y%m%d-%H%M%S", time.gmtime()) + +class DataAdmin(commands.Cog): + """ + [ADMIN] Backup/restore the bot data file. + Owner-only: guild owner or application (bot) owner. + """ + def __init__(self, bot: commands.Bot): + self.bot = bot + self._app_owner_id: Optional[int] = None + + # --- permission helper --- + async def _is_owner(self, interaction: discord.Interaction) -> bool: + uid = interaction.user.id + # cache application owner id + if self._app_owner_id is None: + try: + info = await self.bot.application_info() + if info and info.owner: + self._app_owner_id = info.owner.id + except Exception: + self._app_owner_id = None + + guild_owner_id = getattr(getattr(interaction, "guild", None), "owner_id", None) + if guild_owner_id and uid == guild_owner_id: + return True + if self._app_owner_id and uid == self._app_owner_id: + return True + return False + + # --- helpers --- + def _dm_path(self) -> str: + dm = getattr(self.bot, "data_manager", None) + if not dm or not getattr(dm, "json_path", None): + raise RuntimeError("DataManager/json_path unavailable") + return dm.json_path + + def _merge_with_defaults(self, incoming: dict) -> dict: + """ + Ensure required keys exist; preserve unknown keys. + """ + dm = getattr(self.bot, "data_manager", None) + if not dm: + raise RuntimeError("DataManager unavailable") + + # Create a minimal default schema by calling _default_payload if present, + # otherwise fall back to a thin set. + try: + defaults = dm._default_payload() # type: ignore[attr-defined] + except Exception: + defaults = { + "_counters": {}, + "_events_seen": {}, + "_counter_last_ts": {}, + } + + merged = dict(incoming) + for k, v in defaults.items(): + merged.setdefault(k, v if not isinstance(v, list) else list(v)) + return merged + + async def _download_attachment_bytes(self, att: discord.Attachment) -> bytes: + if att.size > MAX_UPLOAD_BYTES: + raise ValueError(f"Attachment too large ({att.size} bytes)") + return await att.read() + + async def _download_url_bytes(self, url: str) -> bytes: + timeout = aiohttp.ClientTimeout(total=25, sock_connect=10, sock_read=15) + headers = { + "User-Agent": "ShaiWatcher/backup-restore (+https://example.invalid)" + } + async with aiohttp.ClientSession(timeout=timeout) as sess: + async with sess.get(url, headers=headers, allow_redirects=True) as resp: + if resp.status >= 400: + raise RuntimeError(f"HTTP {resp.status}") + data = await resp.read() + if len(data) > MAX_UPLOAD_BYTES: + raise ValueError(f"Downloaded file too large ({len(data)} bytes)") + return data + + def _atomic_replace(self, new_payload: dict) -> None: + """ + Replace DataManager payload atomically, with a timestamped backup. + """ + dm = getattr(self.bot, "data_manager", None) + if not dm: + raise RuntimeError("DataManager unavailable") + + src_path = self._dm_path() + bak_path = f"{src_path}.manual.{_now_stamp()}.bak" + + with dm.lock: + # backup current file if exists + try: + if os.path.exists(src_path): + shutil.copy2(src_path, bak_path) + except Exception: + pass + + # write new file and update in-memory view + dm._data = self._merge_with_defaults(new_payload) # type: ignore[attr-defined] + dm._save(dm._data) # type: ignore[attr-defined] + + # --- slash command --- + @app_commands.command( + name="data", + description="[ADMIN] Download or upload the bot data file (owner-only)" + ) + @app_commands.describe( + action="Choose 'download' to get the current file, or 'upload' to restore from JSON", + attachment="Optional JSON attachment (used for 'upload')", + url="Optional direct URL to a JSON file (used for 'upload')" + ) + async def data_cmd( + self, + interaction: discord.Interaction, + action: Literal["download", "upload"], + attachment: Optional[discord.Attachment] = None, + url: Optional[str] = None, + ): + # perms + if not await self._is_owner(interaction): + return await interaction.response.send_message(OWNER_HINT, ephemeral=True) + + # ensure dm available + try: + dm_path = self._dm_path() + except Exception as e: + return await interaction.response.send_message( + f"DataManager unavailable: {e}", ephemeral=True + ) + + # dispatch + if action == "download": + await interaction.response.defer(ephemeral=True, thinking=False) + try: + # Read raw file bytes to guarantee exact copy + with open(dm_path, "rb") as f: + data = f.read() + file = discord.File(io.BytesIO(data), filename="data.json") + await interaction.followup.send( + content="Here is the current data file.", + file=file, + ephemeral=True, + ) + except Exception as e: + await interaction.followup.send( + f"Failed to read data file: {e}", ephemeral=True + ) + return + + # upload + # must provide exactly one source + sources = [s for s in (attachment, url) if s] + if len(sources) != 1: + return await interaction.response.send_message( + "For `upload`, provide **exactly one** of: `attachment` **or** `url`.", + ephemeral=True, + ) + + await interaction.response.defer(ephemeral=True, thinking=True) + + try: + if attachment: + raw = await self._download_attachment_bytes(attachment) + else: + assert url is not None + raw = await self._download_url_bytes(url) + + # decode → JSON + try: + text = raw.decode("utf-8") + except UnicodeDecodeError: + return await interaction.followup.send( + "The file/URL is not valid UTF-8 text.", ephemeral=True + ) + + try: + payload = json.loads(text) + except json.JSONDecodeError as e: + return await interaction.followup.send( + f"Invalid JSON: {e}", ephemeral=True + ) + + if not isinstance(payload, dict): + return await interaction.followup.send( + "Top-level JSON must be an **object** (not an array/string).", + ephemeral=True, + ) + + # final size sanity (after parse) + encoded_size = len(json.dumps(payload, ensure_ascii=False).encode("utf-8")) + if encoded_size > MAX_UPLOAD_BYTES: + return await interaction.followup.send( + f"Refusing to import unusually large JSON ({encoded_size} bytes).", + ephemeral=True, + ) + + # write & backup + self._atomic_replace(payload) + + # tiny summary + top_keys = sorted(list(payload.keys())) + shown = ", ".join(top_keys[:12]) + ("…" if len(top_keys) > 12 else "") + await interaction.followup.send( + f"✅ Imported data and wrote a timestamped backup of the previous file.\n" + f"Path: `{dm_path}`\n" + f"Top-level keys ({len(top_keys)}): {shown}", + ephemeral=True, + ) + + except Exception as e: + await interaction.followup.send(f"Import failed: {e}", ephemeral=True) + + +async def setup(bot: commands.Bot): + await bot.add_cog(DataAdmin(bot))