0.4.2.1.a1

- Added a new owner-only `/data [download/upload]` command for datafile backup and restoration
*This is required as v0.4.2 requires a rebuild of the stack, and the existing data should be backed up in case of data loss*

.env.example

@@ -0,0 +1,127 @@
+# ─────────────────────────────────────────────────────────────
+# Required
+# ─────────────────────────────────────────────────────────────
+DISCORD_TOKEN=<paste your bot token>
+
+# Git repo to run (wrapper clones nightly and on boot)
+REPO_URL=<https://git.example.com/you/shaiwatcher.git>
+REPO_BRANCH=main
+# If your repo is private, you can use a token; leave empty if not used
+REPO_TOKEN=
+
+# Wrapper update time (UTC) & test bump policy
+CHECK_TIME_UTC=03:00
+IGNORE_TEST_LEVEL=1           # 1 = ignore updates that change only the -T test suffix
+
+# Optional commit subject feed used by your boot notice
+SHAI_REPO_RSS=<https://git.example.com/you/shaiwatcher.rss>
+
+# Where the bot stores data inside the container (don’t change)
+SHAI_DATA_FILE=/data/data.json
+
+# ─────────────────────────────────────────────────────────────
+# Volumes (external, never wiped by stack re-deploys)
+# Create once:  docker volume create shaiwatcher_data && docker volume create shaiwatcher_cache
+# ─────────────────────────────────────────────────────────────
+SHAI_VOL_DATA=shaiwatcher_data
+SHAI_VOL_CACHE=shaiwatcher_cache
+
+# ─────────────────────────────────────────────────────────────
+# Playwright / Headless browser (for DD scraping)
+# Build with: docker compose build --build-arg WITH_PLAYWRIGHT=1
+# ─────────────────────────────────────────────────────────────
+WITH_PLAYWRIGHT=1             # 1 to bake Chromium+Playwright into the image
+SHAI_DD_CHANNEL_ID=<channel id for DD message>
+SHAI_DD_FETCHER=playwright    # playwright|aiohttp  (playwright recommended)
+SHAI_DD_PW_TIMEOUT_MS=60000   # nav timeout
+SHAI_DD_PW_WAIT=              # domcontentloaded|load|networkidle (empty = default domcontentloaded)
+
+# ─────────────────────────────────────────────────────────────
+# Slash command scope (your bot already honors this)
+# ─────────────────────────────────────────────────────────────
+SHAI_SLASH_GUILD_ONLY=true
+SHAI_HOME_GUILD_ID=<home guild id>
+
+# ─────────────────────────────────────────────────────────────
+# Channels
+# ─────────────────────────────────────────────────────────────
+SHAI_MOD_CHANNEL_ID=<id>
+SHAI_MODLOG_CHANNEL_ID=<id>
+SHAI_USERSLIST_CHANNEL_ID=<id>
+SHAI_REPORT_CHANNEL_ID=<id>
+SHAI_PIRATES_LIST_CHANNEL_ID=<id>
+SHAI_TRIGGER_CHANNEL_ID=<id>          # Auto-VC
+SHAI_AUTO_VC_CATEGORY_ID=<id>
+
+# ─────────────────────────────────────────────────────────────
+# Reaction messages
+# ─────────────────────────────────────────────────────────────
+SHAI_RULES_MESSAGE_ID=<id>
+SHAI_ENGAGEMENT_MESSAGE_ID=<id>
+SHAI_NICKNAME_MESSAGE_ID=<id>
+
+# ─────────────────────────────────────────────────────────────
+# Roles
+# ─────────────────────────────────────────────────────────────
+SHAI_RULES_ROLE_ID=<id>
+SHAI_ENGAGEMENT_ROLE_ID=<id>
+SHAI_FULL_ACCESS_ROLE_ID=<id>
+SHAI_ADMIN_ROLE_ID=<id>
+SHAI_FIELD_MOD_ROLE_ID=<id>
+SHAI_INTEL_MOD_ROLE_ID=<id>
+SHAI_MODERATOR_ROLE_ID=<id>
+
+# ─────────────────────────────────────────────────────────────
+# Auto-VC
+# ─────────────────────────────────────────────────────────────
+SHAI_VC_NAME_PREFIX=DD Crew
+SHAI_AUTO_VC_CLEANUP_DELAY=30
+
+# ─────────────────────────────────────────────────────────────
+# Threat weights
+# ─────────────────────────────────────────────────────────────
+SHAI_THREAT_W_KILL=0.30
+SHAI_THREAT_W_DESTRUCTION=0.40
+SHAI_THREAT_W_GROUP=0.20
+SHAI_THREAT_W_SKILL=0.10
+SHAI_THREAT_GROUP_THRESHOLD=3
+SHAI_THREAT_MIN_SAMPLES_FOR_STATS=3
+
+# ─────────────────────────────────────────────────────────────
+# Toggles
+# ─────────────────────────────────────────────────────────────
+SHAI_NICK_NUDGE_LOOP_ENABLED=false
+SHAI_USER_CARDS_CRON_ENABLED=true
+
+# ─────────────────────────────────────────────────────────────
+# SpicePay defaults
+# ─────────────────────────────────────────────────────────────
+SHAI_SPICEPAY_LSR_CUT_PERCENT=10
+SHAI_SPICEPAY_BASE_WEIGHT=25
+SHAI_SPICEPAY_CARRIER_BONUS=12.5
+SHAI_SPICEPAY_CRAWLER_BONUS=12.5
+
+# ─────────────────────────────────────────────────────────────
+# Emojis (IDs)
+# ─────────────────────────────────────────────────────────────
+SHAI_EMOJI_MELANGE_ID=<id>
+SHAI_EMOJI_SAND_ID=<id>
+SHAI_EMOJI_CARRIER_CRAWLER_ID=<id>
+
+# ─────────────────────────────────────────────────────────────
+# Docs site (optional)
+# ─────────────────────────────────────────────────────────────
+SHAI_DOCS_HOST=0.0.0.0
+SHAI_DOCS_PORT=8910
+SHAI_DOCS_TITLE=ShaiWatcher Commands
+SHAI_DOCS_SUPPORT_URL=
+SHAI_DOCS_SUPPORT_LABEL=Buy me a ☕
+
+# ─────────────────────────────────────────────────────────────
+# Wrapper knobs (optional)
+# ─────────────────────────────────────────────────────────────
+PIP_INSTALL_REQUIREMENTS=1
+WRAPPER_STOP_TIMEOUT=25
+
+# Keep Docker base locale happy
+LANG=C.UTF-8

.gitignore

@@ -13,6 +13,7 @@ settings*.conf
 NOTES.md
 sanity/
 .offline_data.json
+dev/.env.production
 
 # Tools
 wrapper/

bot.py

@@ -9,7 +9,7 @@ from modules.common.boot_notice import post_boot_notice
 
 # Version consists of:
 # Major.Enhancement.Minor.Patch.Test  (Test is alphanumeric; doesn’t trigger auto update)
-VERSION = "0.4.2.0.a1"
+VERSION = "0.4.2.1.a1"
 
 # ---------- Env loading ----------
 

example/.env.example

@@ -1,5 +0,0 @@
-DISCORD_TOKEN={DISCORD_BOT_TOKEN}
-DISCORD_APPLICATION_ID=
-DISCORD_PUBLIC_KEY=
-DISCORD_CLIENT_ID=
-DISCORD_CLIENT_SECRET=

modules/data_admin/data_admin.py

@@ -0,0 +1,237 @@
+from __future__ import annotations
+
+import io
+import os
+import json
+import time
+import shutil
+import asyncio
+from typing import Optional, Literal
+
+import aiohttp
+import discord
+from discord import app_commands
+from discord.ext import commands
+
+
+MAX_UPLOAD_BYTES = 8 * 1024 * 1024  # 8 MiB hard cap for safety
+OWNER_HINT = "This command is restricted to the **server owner** (or bot owner)."
+
+def _now_stamp() -> str:
+    return time.strftime("%Y%m%d-%H%M%S", time.gmtime())
+
+class DataAdmin(commands.Cog):
+    """
+    [ADMIN] Backup/restore the bot data file.
+    Owner-only: guild owner or application (bot) owner.
+    """
+    def __init__(self, bot: commands.Bot):
+        self.bot = bot
+        self._app_owner_id: Optional[int] = None
+
+    # --- permission helper ---
+    async def _is_owner(self, interaction: discord.Interaction) -> bool:
+        uid = interaction.user.id
+        # cache application owner id
+        if self._app_owner_id is None:
+            try:
+                info = await self.bot.application_info()
+                if info and info.owner:
+                    self._app_owner_id = info.owner.id
+            except Exception:
+                self._app_owner_id = None
+
+        guild_owner_id = getattr(getattr(interaction, "guild", None), "owner_id", None)
+        if guild_owner_id and uid == guild_owner_id:
+            return True
+        if self._app_owner_id and uid == self._app_owner_id:
+            return True
+        return False
+
+    # --- helpers ---
+    def _dm_path(self) -> str:
+        dm = getattr(self.bot, "data_manager", None)
+        if not dm or not getattr(dm, "json_path", None):
+            raise RuntimeError("DataManager/json_path unavailable")
+        return dm.json_path
+
+    def _merge_with_defaults(self, incoming: dict) -> dict:
+        """
+        Ensure required keys exist; preserve unknown keys.
+        """
+        dm = getattr(self.bot, "data_manager", None)
+        if not dm:
+            raise RuntimeError("DataManager unavailable")
+
+        # Create a minimal default schema by calling _default_payload if present,
+        # otherwise fall back to a thin set.
+        try:
+            defaults = dm._default_payload()  # type: ignore[attr-defined]
+        except Exception:
+            defaults = {
+                "_counters": {},
+                "_events_seen": {},
+                "_counter_last_ts": {},
+            }
+
+        merged = dict(incoming)
+        for k, v in defaults.items():
+            merged.setdefault(k, v if not isinstance(v, list) else list(v))
+        return merged
+
+    async def _download_attachment_bytes(self, att: discord.Attachment) -> bytes:
+        if att.size > MAX_UPLOAD_BYTES:
+            raise ValueError(f"Attachment too large ({att.size} bytes)")
+        return await att.read()
+
+    async def _download_url_bytes(self, url: str) -> bytes:
+        timeout = aiohttp.ClientTimeout(total=25, sock_connect=10, sock_read=15)
+        headers = {
+            "User-Agent": "ShaiWatcher/backup-restore (+https://example.invalid)"
+        }
+        async with aiohttp.ClientSession(timeout=timeout) as sess:
+            async with sess.get(url, headers=headers, allow_redirects=True) as resp:
+                if resp.status >= 400:
+                    raise RuntimeError(f"HTTP {resp.status}")
+                data = await resp.read()
+                if len(data) > MAX_UPLOAD_BYTES:
+                    raise ValueError(f"Downloaded file too large ({len(data)} bytes)")
+                return data
+
+    def _atomic_replace(self, new_payload: dict) -> None:
+        """
+        Replace DataManager payload atomically, with a timestamped backup.
+        """
+        dm = getattr(self.bot, "data_manager", None)
+        if not dm:
+            raise RuntimeError("DataManager unavailable")
+
+        src_path = self._dm_path()
+        bak_path = f"{src_path}.manual.{_now_stamp()}.bak"
+
+        with dm.lock:
+            # backup current file if exists
+            try:
+                if os.path.exists(src_path):
+                    shutil.copy2(src_path, bak_path)
+            except Exception:
+                pass
+
+            # write new file and update in-memory view
+            dm._data = self._merge_with_defaults(new_payload)  # type: ignore[attr-defined]
+            dm._save(dm._data)  # type: ignore[attr-defined]
+
+    # --- slash command ---
+    @app_commands.command(
+        name="data",
+        description="[ADMIN] Download or upload the bot data file (owner-only)"
+    )
+    @app_commands.describe(
+        action="Choose 'download' to get the current file, or 'upload' to restore from JSON",
+        attachment="Optional JSON attachment (used for 'upload')",
+        url="Optional direct URL to a JSON file (used for 'upload')"
+    )
+    async def data_cmd(
+        self,
+        interaction: discord.Interaction,
+        action: Literal["download", "upload"],
+        attachment: Optional[discord.Attachment] = None,
+        url: Optional[str] = None,
+    ):
+        # perms
+        if not await self._is_owner(interaction):
+            return await interaction.response.send_message(OWNER_HINT, ephemeral=True)
+
+        # ensure dm available
+        try:
+            dm_path = self._dm_path()
+        except Exception as e:
+            return await interaction.response.send_message(
+                f"DataManager unavailable: {e}", ephemeral=True
+            )
+
+        # dispatch
+        if action == "download":
+            await interaction.response.defer(ephemeral=True, thinking=False)
+            try:
+                # Read raw file bytes to guarantee exact copy
+                with open(dm_path, "rb") as f:
+                    data = f.read()
+                file = discord.File(io.BytesIO(data), filename="data.json")
+                await interaction.followup.send(
+                    content="Here is the current data file.",
+                    file=file,
+                    ephemeral=True,
+                )
+            except Exception as e:
+                await interaction.followup.send(
+                    f"Failed to read data file: {e}", ephemeral=True
+                )
+            return
+
+        # upload
+        # must provide exactly one source
+        sources = [s for s in (attachment, url) if s]
+        if len(sources) != 1:
+            return await interaction.response.send_message(
+                "For `upload`, provide **exactly one** of: `attachment` **or** `url`.",
+                ephemeral=True,
+            )
+
+        await interaction.response.defer(ephemeral=True, thinking=True)
+
+        try:
+            if attachment:
+                raw = await self._download_attachment_bytes(attachment)
+            else:
+                assert url is not None
+                raw = await self._download_url_bytes(url)
+
+            # decode → JSON
+            try:
+                text = raw.decode("utf-8")
+            except UnicodeDecodeError:
+                return await interaction.followup.send(
+                    "The file/URL is not valid UTF-8 text.", ephemeral=True
+                )
+
+            try:
+                payload = json.loads(text)
+            except json.JSONDecodeError as e:
+                return await interaction.followup.send(
+                    f"Invalid JSON: {e}", ephemeral=True
+                )
+
+            if not isinstance(payload, dict):
+                return await interaction.followup.send(
+                    "Top-level JSON must be an **object** (not an array/string).",
+                    ephemeral=True,
+                )
+
+            # final size sanity (after parse)
+            encoded_size = len(json.dumps(payload, ensure_ascii=False).encode("utf-8"))
+            if encoded_size > MAX_UPLOAD_BYTES:
+                return await interaction.followup.send(
+                    f"Refusing to import unusually large JSON ({encoded_size} bytes).",
+                    ephemeral=True,
+                )
+
+            # write & backup
+            self._atomic_replace(payload)
+
+            # tiny summary
+            top_keys = sorted(list(payload.keys()))
+            shown = ", ".join(top_keys[:12]) + ("…" if len(top_keys) > 12 else "")
+            await interaction.followup.send(
+                f"✅ Imported data and wrote a timestamped backup of the previous file.\n"
+                f"Path: `{dm_path}`\n"
+                f"Top-level keys ({len(top_keys)}): {shown}",
+                ephemeral=True,
+            )
+
+        except Exception as e:
+            await interaction.followup.send(f"Import failed: {e}", ephemeral=True)
+
+
+async def setup(bot: commands.Bot):
+    await bot.add_cog(DataAdmin(bot))