diff --git a/CHANGELOG.md b/CHANGELOG.md index ef47655..72e3d39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Lumi changelog +## 0.3.10 + +- Added an extensible UTC interval-history engine for Twitch follows, subscriptions, moderators, editors, and VIPs; YouTube channel memberships and moderators; and Discord membership and server boosts. +- Added idempotent live-event handling, complete-snapshot reconciliation that never treats API failures as empty state, authoritative timestamp backfill for open intervals, and preserved history across restarts and account changes. +- Added contextual `current` and `total` age placeholders with deterministic years/months, selected-unit remainder flow, singular/plural output, and the shared embedded `{format.YY-MM-DD-HH-mm-ss}` grammar. +- Added Platform tenure summaries to Stats and scope-separated Platform tenure boards to Leaderboards, plus admin interval diagnostics with source, precision, and authoritative/first-observed provenance. +- Preserved all existing settings, identities, statistics, interval history, plugins, pairing records, databases, uploads, models, and secrets through additive migrations and the normal updater. + ## 0.3.9 - Restored animated Twitch/BTTV and Discord emotes plus Discord GIF media across the existing normalized OBS and native Companion overlay paths, and rendered real Twitch badge artwork with safe fallbacks. diff --git a/docs/placeholder-system.md b/docs/placeholder-system.md index f280e1a..37b6cae 100644 --- a/docs/placeholder-system.md +++ b/docs/placeholder-system.md @@ -11,6 +11,10 @@ The frontend must not declare placeholder permissions, allowed plugins, or sensitivity. Editable fields reference a trusted `field_id`, and the server uses that field policy to decide which placeholders are available. +Platform membership and role duration placeholders, including their embedded +`{format.YY-MM-DD}` grammar, are documented in +[platform-tenure.md](platform-tenure.md). + Core and plugins can register: - placeholder definitions with metadata and a resolver function diff --git a/docs/platform-tenure.md b/docs/platform-tenure.md new file mode 100644 index 0000000..514b9b1 --- /dev/null +++ b/docs/platform-tenure.md @@ -0,0 +1,44 @@ +# Platform tenure + +Lumi records platform membership and role periods as immutable UTC intervals. The same history powers custom-command placeholders, the **Platform tenure** cards on `/stats`, the **Platform tenure** section on `/leaderboards`, and the admin diagnostics page at `/admin/platform-tenure`. + +## Supported statistics + +| Platform | Statistics | Start timestamp | +| --- | --- | --- | +| Twitch | Follow, subscription, moderator, editor, VIP | Platform timestamp where Twitch supplies one; otherwise first reliable observation | +| YouTube | Paid channel membership and live-chat moderator | Membership event time where available; otherwise first reliable live-chat observation | +| Discord | Server membership and server boost | Discord `joinedTimestamp` and `premiumSinceTimestamp` where available | + +YouTube does not expose a reliable timestamp for a viewer's ordinary public channel subscription through the existing live-chat integration. `youtube.user.subscriber_age` therefore represents paid channel membership (sponsor/member state), not a fabricated public-subscription age. YouTube roles are observed only when the live-chat API reports them. + +Temporary API failures do not close intervals. Lumi closes missing states only after a complete successful platform snapshot or a reliable end event. Unlinking a platform identity does not delete interval history. + +## Placeholders + +The current uninterrupted interval and cumulative recorded history are exposed as `current` and `total`: + +```text +{{twitch.user.follow_age.current}} +{{twitch.user.subscriber_age.total}} +{{twitch.user.mod_age.current}} +{{twitch.user.editor_age.total}} +{{twitch.user.vip_age.current}} +{{youtube.user.subscriber_age.current}} +{{youtube.user.mod_age.total}} +{{discord.user.member_age.current}} +{{discord.user.nitro_age.total}} +``` + +The current command context selects the platform user and channel or guild scope. Lumi never combines values across Twitch channels, YouTube channels, or Discord servers. + +Append an embedded format to select duration units: + +```text +{{twitch.user.follow_age.total}.{format.YY-MM-DD-HH-mm-ss}} +{{discord.user.member_age.current}.{format.DD-HH-mm}} +``` + +Units are case-sensitive: `Y`, `YY`, or `YYYY` means years; `M` or `MM` months; `D` or `DD` days; `H` or `HH` hours; `m` or `mm` minutes; and `s` or `ss` seconds. Repetition identifies the unit and does not add zero-padding. Units must be ordered largest to smallest and may appear once. + +Lumi uses deterministic cumulative-duration units: one year is 365 days and one month is 30 days. Omitted time flows into the next smaller selected unit, so a format containing only `mm` returns total complete minutes. Time below the smallest selected unit is truncated. Zero units are omitted; an all-zero result is empty. diff --git a/knowledge/core/lumi-core.md b/knowledge/core/lumi-core.md index 38b311b..87eec8f 100644 --- a/knowledge/core/lumi-core.md +++ b/knowledge/core/lumi-core.md @@ -14,7 +14,7 @@ editable: false Lumi is the core web UI and bot runtime. ## Runtime Package: lumi-bot -Version: 0.3.9 +Version: 0.3.10 ## Routes - POST /api/diagnostics/v1/run - GET /api/events diff --git a/package-lock.json b/package-lock.json index 0c8ffc7..3c43410 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "lumi-bot", - "version": "0.3.9", + "version": "0.3.10", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "lumi-bot", - "version": "0.3.9", + "version": "0.3.10", "dependencies": { "adm-zip": "^0.6.0", "better-sqlite3": "^11.5.0", diff --git a/package.json b/package.json index 2263edb..9070194 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "lumi-bot", - "version": "0.3.9", + "version": "0.3.10", "private": true, "type": "commonjs", "scripts": { @@ -26,7 +26,8 @@ "verify:transcription": "node plugins/lumi_transcription/tests/verify.js", "verify:companion-overlay": "node plugins/lumi_overlay/tests/verify.js", "verify:auto-vc": "node plugins/auto-vc/tests/verify.js", - "verify:dev-updates": "node scripts/verify-local-development-updates.js" + "verify:dev-updates": "node scripts/verify-local-development-updates.js", + "verify:user-age": "node scripts/verify-user-age-statistics.js" }, "engines": { "node": ">=18" diff --git a/release-index.json b/release-index.json index 3661445..719e54f 100644 --- a/release-index.json +++ b/release-index.json @@ -2,6 +2,39 @@ "schema_version": 1, "channel": "stable", "releases": [ + { + "version": "0.3.10", + "ref": "refs/tags/v0.3.10", + "released_at": "2026-07-27", + "installable": true, + "rollback_safe": true, + "replaces_versions": [ + "1.2.0" + ], + "data_policy": "preserve", + "dependency_policy": "sync_on_restart", + "migration_notes": "Adds reusable platform-tenure interval history for Twitch, YouTube, and Discord; contextual current and total duration placeholders with deterministic formatting; safe platform reconciliation; admin diagnostics; and Platform tenure sections on Stats and Leaderboards. Existing settings, identities, statistics, intervals, plugins, pairing records, databases, uploads, models, and secrets remain preserved.", + "plugins": { + "auto-vc": "0.1.7", + "birthday": "0.1.3", + "economy-framework": "0.2.10", + "economy-games": "0.1.7", + "expression-interaction": "0.2.1", + "lumi_ai": "0.8.5", + "lumi_overlay": "0.1.1", + "lumi_transcription": "0.2.7", + "moderation": "0.1.5", + "now_playing": "0.1.3", + "okf": "0.1.2", + "quotes": "0.1.2", + "sample-plugin": "0.1.0", + "throne_wishlist": "0.1.2", + "welcome_messages": "0.1.1" + }, + "tools": { + "lumi_ai_web_search": "0.1.1" + } + }, { "version": "0.3.9", "ref": "refs/tags/v0.3.9", diff --git a/scripts/verify-all.js b/scripts/verify-all.js index c68ac23..552e706 100644 --- a/scripts/verify-all.js +++ b/scripts/verify-all.js @@ -11,6 +11,7 @@ const checks = [ "scripts/verify-feedback-system.js", "scripts/verify-logging.js", "scripts/verify-placeholders.js", + "scripts/verify-user-age-statistics.js", "scripts/verify-release-metadata.js", "scripts/verify-update-system.js", "scripts/verify-local-development-updates.js", diff --git a/scripts/verify-release-metadata.js b/scripts/verify-release-metadata.js index 741ecd1..01da638 100644 --- a/scripts/verify-release-metadata.js +++ b/scripts/verify-release-metadata.js @@ -4,9 +4,9 @@ const path = require("path"); const { findSafeTarget } = require("../src/services/versioning"); const root = path.join(__dirname, ".."); -const releaseVersion = "0.3.9"; -const previousStableVersion = "0.3.8"; -const priorStableVersion = "0.3.7"; +const releaseVersion = "0.3.10"; +const previousStableVersion = "0.3.9"; +const priorStableVersion = "0.3.8"; const earliestCompatibleCoreVersion = "0.1.9"; const introducedPlugins = { "auto-vc": { version: "0.1.7", knowledge: "auto-vc", compatibleFrom: "0.1.5" }, @@ -84,4 +84,4 @@ assert.equal(webSearch.minimum_lumi_version, "0.2.0"); assert.equal(webSearch.minimum_lumi_ai_version, "0.8.2"); assert.equal(hasVersionHeading(readText("plugins/lumi_ai_web_search/CHANGELOG.md"), webSearch.version), true); -console.log("Release metadata verification passed: stable core 0.3.9 after 0.3.8 with synchronized Companion and plugin metadata."); +console.log("Release metadata verification passed: stable core 0.3.10 after 0.3.9 with synchronized Companion and plugin metadata."); diff --git a/scripts/verify-update-system.js b/scripts/verify-update-system.js index 011bdfd..0bf69c2 100644 --- a/scripts/verify-update-system.js +++ b/scripts/verify-update-system.js @@ -24,7 +24,7 @@ function readJson(relativePath) { const releaseIndex = readJson("release-index.json"); const releaseVersions = releaseIndex.releases.map((release) => release.version); -assert.deepEqual(releaseVersions, ["0.3.9", "0.3.8", "0.3.7", "0.3.6", "0.3.5", "0.3.4", "0.3.3", "0.3.2", "0.3.1", "0.3.0", "0.2.27", "0.2.26", "0.2.25", "0.2.24", "0.2.23", "0.2.22", "0.2.21", "0.2.20", "0.2.19", "0.2.18", "0.2.17", "0.2.16", "0.2.15", "0.2.14", "0.2.13", "0.2.12", "0.2.11", "0.2.10", "0.2.9", "0.2.8", "0.2.7", "0.2.6", "0.2.5", "0.2.4", "0.2.3", "0.2.2", "0.2.1", "0.2.0", "0.1.9"]); +assert.deepEqual(releaseVersions, ["0.3.10", "0.3.9", "0.3.8", "0.3.7", "0.3.6", "0.3.5", "0.3.4", "0.3.3", "0.3.2", "0.3.1", "0.3.0", "0.2.27", "0.2.26", "0.2.25", "0.2.24", "0.2.23", "0.2.22", "0.2.21", "0.2.20", "0.2.19", "0.2.18", "0.2.17", "0.2.16", "0.2.15", "0.2.14", "0.2.13", "0.2.12", "0.2.11", "0.2.10", "0.2.9", "0.2.8", "0.2.7", "0.2.6", "0.2.5", "0.2.4", "0.2.3", "0.2.2", "0.2.1", "0.2.0", "0.1.9"]); assert.equal(new Set(releaseVersions).size, releaseVersions.length, "release versions must be unique"); for (const release of releaseIndex.releases) { assert.equal(normalizeRepositoryRef(release.ref), release.ref); @@ -37,8 +37,8 @@ const packageVersion = readJson("package.json").version; const coreManifest = readJson("update-manifest.json"); assert.equal(packageVersion, coreManifest.version); assert.equal(coreManifest.channel, "stable"); -assert.equal(packageVersion, "0.3.9"); -assert.equal(currentRelease.version, "0.3.9"); +assert.equal(packageVersion, "0.3.10"); +assert.equal(currentRelease.version, "0.3.10"); assert.deepEqual(currentRelease.replaces_versions, ["1.2.0"]); for (const [pluginId, version] of Object.entries(currentRelease.plugins)) { assert.equal(readJson(`plugins/${pluginId}/plugin.json`).version, version, `${pluginId} release catalog version`); @@ -50,6 +50,7 @@ for (const [toolId, version] of Object.entries(currentRelease.tools)) { const baseTarget = { current_version: "0.2.4", available_versions: [ + { version: "0.3.10", ref: "refs/tags/v0.3.10", rollback_safe: true }, { version: "0.3.9", ref: "refs/tags/v0.3.9", rollback_safe: true }, { version: "0.3.8", ref: "refs/tags/v0.3.8", rollback_safe: true }, { version: "0.3.7", ref: "refs/tags/v0.3.7", rollback_safe: true }, @@ -158,7 +159,7 @@ const corrected = buildStatus({ channel: "stable" }); assert.equal(corrected.version_correction, true); -assert.equal(corrected.safe_target_version, "0.3.9"); +assert.equal(corrected.safe_target_version, "0.3.10"); assert.equal(corrected.update_available, true); assert.equal(corrected.blocked, false); diff --git a/scripts/verify-user-age-statistics.js b/scripts/verify-user-age-statistics.js new file mode 100644 index 0000000..9756aab --- /dev/null +++ b/scripts/verify-user-age-statistics.js @@ -0,0 +1,175 @@ +const assert = require("assert"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +const Database = require("better-sqlite3"); +const isolatedDataDir = fs.mkdtempSync(path.join(os.tmpdir(), "lumi-user-age-")); +process.env.LUMI_DATA_DIR = isolatedDataDir; +const { formatDuration, parseDurationFormat } = require("../src/services/duration-format"); +const { UserAgeStatistics } = require("../src/services/user-age-statistics"); +const placeholders = require("../src/services/placeholders"); +const coreDatabase = require("../src/services/db"); +coreDatabase.migrate(); + +const hour = 60 * 60 * 1000; +const day = 24 * hour; +let now = 100 * day; +const database = new Database(":memory:"); +database.exec(` + CREATE TABLE user_profiles ( + id TEXT PRIMARY KEY, + internal_username TEXT NOT NULL + ); + CREATE TABLE user_identities ( + user_id TEXT NOT NULL, + provider TEXT NOT NULL, + provider_user_id TEXT NOT NULL + ); +`); +const service = new UserAgeStatistics(database, { now: () => now }); +database.prepare("INSERT INTO user_profiles (id, internal_username) VALUES (?, ?)").run("profile-1", "ViewerOne"); +database.prepare("INSERT INTO user_identities (user_id, provider, provider_user_id) VALUES (?, ?, ?)").run("profile-1", "twitch", "viewer-1"); + +const base = { + platform: "twitch", + userId: "viewer-1", + scopeId: "channel-1", + type: "follow", + source: "test-event", + authoritative: true +}; + +service.setState({ ...base, active: true, startAt: now - 10 * day, occurredAt: now, eventId: "follow-1" }); +let rows = service.listDiagnostics({ platform: "twitch", userId: "viewer-1", type: "follow" }); +assert.strictEqual(rows.length, 1, "new follow should create one interval"); +assert.strictEqual(rows[0].end_at, null, "new follow interval should remain open"); + +now += 2 * day; +service.setState({ ...base, active: false, occurredAt: now, eventId: "unfollow-1" }); +rows = service.listDiagnostics({ platform: "twitch", userId: "viewer-1", type: "follow" }); +assert.strictEqual(rows[0].end_at, now, "unfollow should close the open interval"); +const firstCompleted = { ...rows[0] }; + +now += day; +service.setState({ ...base, active: true, startAt: now, occurredAt: now, eventId: "follow-2" }); +service.setState({ ...base, active: true, startAt: now, occurredAt: now, eventId: "follow-2" }); +rows = service.listDiagnostics({ platform: "twitch", userId: "viewer-1", type: "follow" }); +assert.strictEqual(rows.length, 2, "refollow should create exactly one second interval"); +assert.deepStrictEqual( + Object.fromEntries(Object.entries(rows.find((row) => row.id === firstCompleted.id)).filter(([key]) => !["updated_at"].includes(key))), + Object.fromEntries(Object.entries(firstCompleted).filter(([key]) => !["updated_at"].includes(key))), + "completed intervals must not be modified" +); + +now += 3 * day; +let durations = service.getDurations(base); +assert.strictEqual(durations.current, 3 * day, "current should include only the open interval"); +assert.strictEqual(durations.total, 15 * day, "total should combine completed and active intervals"); +const leaders = service.getLeaderboard("follow", { platform: "twitch", limit: 10 }); +assert.strictEqual(leaders[0].username, "ViewerOne", "leaderboards should resolve linked Lumi profiles"); +assert.ok(leaders.some((entry) => entry.label.includes("channel-1")), "leaderboards should keep channel scopes visible and separate"); + +service.setState({ ...base, active: false, occurredAt: now, eventId: "unfollow-2" }); +durations = service.getDurations(base); +assert.strictEqual(durations.current, null, "inactive state should have no current duration"); + +service.setState({ ...base, scopeId: "channel-2", active: true, occurredAt: now, eventId: "channel-2-follow" }); +assert.strictEqual(service.getDurations({ ...base, scopeId: "channel-1" }).current, null, "channels must remain isolated"); +assert.strictEqual(service.getDurations({ ...base, scopeId: "channel-2" }).current, 0, "second channel should have its own interval"); + +service.setState({ + platform: "discord", + userId: "viewer-1", + scopeId: "guild-1", + type: "member", + active: true, + occurredAt: now, + eventId: "guild-1-member", + source: "test" +}); +assert.strictEqual(service.getDurations({ + platform: "discord", + userId: "viewer-1", + scopeId: "guild-2", + type: "member" +}).total, null, "Discord guilds must remain isolated"); + +const combined = 2 * 30 * day + 3 * day + 4 * hour + 47 * 60 * 1000; +assert.strictEqual(formatDuration(combined, "MM-DD-mm"), "2 months, 3 days, 287 minutes", "omitted hours should flow into minutes"); +assert.strictEqual(formatDuration(3 * day + 2 * hour, "HH-mm"), "74 hours", "larger omitted units should flow into the largest selected unit"); +assert.strictEqual(formatDuration(287 * 60 * 1000, "mm"), "287 minutes", "minutes-only format should contain total minutes"); +assert.strictEqual(formatDuration(day + hour + 60 * 1000, "DD-HH-mm"), "1 day, 1 hour, 1 minute", "singular labels should be correct"); +assert.strictEqual(formatDuration(2 * day + 2 * hour + 2 * 60 * 1000, "D-H-m"), "2 days, 2 hours, 2 minutes", "plural labels should be correct"); +assert.strictEqual(formatDuration(500, "s"), "", "all-zero selected units should return an empty string"); +assert.throws(() => parseDurationFormat("DD-YY"), /largest to smallest/, "out-of-order units should fail deterministically"); + +const restarted = new UserAgeStatistics(database, { now: () => now }); +const beforeRestart = restarted.listDiagnostics({ platform: "twitch", scopeId: "channel-2", type: "follow" }); +restarted.reconcileSnapshot({ + platform: "twitch", + scopeId: "channel-2", + type: "follow", + activeRecords: [{ userId: "viewer-1" }], + observedAt: now, + source: "restart-test", + complete: true +}); +const afterRestart = restarted.listDiagnostics({ platform: "twitch", scopeId: "channel-2", type: "follow" }); +assert.strictEqual(afterRestart.length, beforeRestart.length, "restart reconciliation must not duplicate valid intervals"); +assert.strictEqual(afterRestart[0].end_at, null, "restart reconciliation must not close valid intervals"); + +restarted.setState({ + platform: "youtube", + userId: "viewer-2", + scopeId: "youtube-channel", + type: "subscriber", + active: true, + occurredAt: now, + eventId: "youtube-observed", + source: "youtube-first-observed", + authoritative: false +}); +const observed = restarted.listDiagnostics({ platform: "youtube", userId: "viewer-2" })[0]; +assert.strictEqual(observed.start_at, now, "unavailable historical starts must begin at first observation"); +assert.strictEqual(observed.start_authoritative, 0, "first-observed timestamps must not be marked authoritative"); + +placeholders.registerCorePlaceholders({ userAgeStatistics: restarted }); +const invalidFormat = placeholders.validateTemplate({ + fieldId: "core.custom_commands.static_response", + template: "{{twitch.user.follow_age.current}.{format.YYY-DD}}", + outputAudience: "user", + runtimeContext: { runtime: true } +}); +assert.strictEqual(invalidFormat.ok, false, "unsupported duration tokens should fail shared placeholder validation"); +const existing = placeholders.renderTemplate({ + fieldId: "core.custom_commands.static_response", + template: "Hello {{user.public.display_name}}", + outputAudience: "user", + runtimeContext: { runtime: true, user: { displayName: "Lumi Friend" } } +}); +const formatted = placeholders.renderTemplate({ + fieldId: "core.custom_commands.static_response", + template: "{{twitch.user.follow_age.current}.{format.DD-HH-mm}}", + outputAudience: "user", + runtimeContext: { + runtime: true, + platform: "twitch", + user: { platformId: "viewer-1" }, + ctx: { meta: { tags: { "room-id": "channel-2" } } } + } +}); + +Promise.all([existing, formatted]).then(([existingResult, formattedResult]) => { + assert.strictEqual(existingResult.rendered, "Hello Lumi Friend", "existing placeholders must remain unchanged"); + assert.strictEqual(formattedResult.rendered, "", "a zero-duration current interval should render empty"); + database.close(); + coreDatabase.db.close(); + fs.rmSync(isolatedDataDir, { recursive: true, force: true }); + console.log("User-age statistics verification passed."); +}).catch((error) => { + database.close(); + coreDatabase.db.close(); + fs.rmSync(isolatedDataDir, { recursive: true, force: true }); + console.error(error); + process.exit(1); +}); diff --git a/src/main.js b/src/main.js index f5bce69..3463af4 100644 --- a/src/main.js +++ b/src/main.js @@ -28,6 +28,7 @@ const { overlayConnectorManager } = require("./services/overlay-connectors"); const { cleanupSnapshots } = require("./services/update-manager"); const { twitchEventSubManager } = require("./services/twitch-eventsub"); const { streamTestingService } = require("./services/stream-testing"); +const { userAgeStatistics } = require("./services/user-age-statistics"); const { isSafeModeRequested, markStartupVerification @@ -56,7 +57,8 @@ async function main() { plugin_count: completedPluginSync.plugin_ids.length }, { event: "bundled_plugin_sync_completed" }); } - registerCorePlaceholders(); + userAgeStatistics.start(); + registerCorePlaceholders({ userAgeStatistics }); const logCleanup = logger.cleanupLogs({ maxAgeDays: getSetting("log_retention_days", logger.DEFAULT_MAX_AGE_DAYS), maxEntries: getSetting("log_retention_max_entries", logger.DEFAULT_MAX_ENTRIES) @@ -205,6 +207,7 @@ async function main() { const closeWebServer = new Promise((resolve) => webServer.close(resolve)); for (const service of [ { name: "stream_testing", stop: () => streamTestingService.close() }, + { name: "user_age_statistics", stop: () => userAgeStatistics.stop() }, { name: "overlay_connectors", stop: () => overlayConnectorManager.stop() }, { name: "twitch_eventsub", stop: () => twitchEventSubManager.stop() }, { name: "plugins", stop: () => stopPlugins() }, diff --git a/src/services/auth.js b/src/services/auth.js index 6817265..363db23 100644 --- a/src/services/auth.js +++ b/src/services/auth.js @@ -92,7 +92,7 @@ function buildTwitchEventAuthUrl(state, redirectOverride) { client_id: clientId || "", redirect_uri: redirectUri || "", response_type: "code", - scope: "moderator:read:followers channel:read:subscriptions bits:read channel:read:redemptions", + scope: "moderator:read:followers moderation:read channel:read:subscriptions channel:read:editors channel:read:vips bits:read channel:read:redemptions", state, force_verify: "true" }); diff --git a/src/services/db.js b/src/services/db.js index f0eb353..556bb87 100644 --- a/src/services/db.js +++ b/src/services/db.js @@ -103,6 +103,39 @@ function migrate() { updated_at INTEGER NOT NULL ); + CREATE TABLE IF NOT EXISTS user_age_intervals ( + id TEXT PRIMARY KEY, + platform TEXT NOT NULL, + platform_user_id TEXT NOT NULL, + scope_id TEXT NOT NULL, + statistic_type TEXT NOT NULL, + start_at INTEGER NOT NULL, + end_at INTEGER, + start_source TEXT NOT NULL, + start_precision TEXT NOT NULL DEFAULT 'millisecond', + start_authoritative INTEGER NOT NULL DEFAULT 0, + end_source TEXT, + end_precision TEXT, + end_authoritative INTEGER, + observed_at INTEGER NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + + CREATE UNIQUE INDEX IF NOT EXISTS user_age_intervals_one_open_idx + ON user_age_intervals (platform, platform_user_id, scope_id, statistic_type) + WHERE end_at IS NULL; + + CREATE INDEX IF NOT EXISTS user_age_intervals_lookup_idx + ON user_age_intervals (platform, scope_id, statistic_type, platform_user_id, start_at); + + CREATE TABLE IF NOT EXISTS user_age_events ( + platform TEXT NOT NULL, + event_id TEXT NOT NULL, + processed_at INTEGER NOT NULL, + PRIMARY KEY (platform, event_id) + ); + CREATE TABLE IF NOT EXISTS custom_pages ( id INTEGER PRIMARY KEY AUTOINCREMENT, slug TEXT NOT NULL UNIQUE, diff --git a/src/services/discord.js b/src/services/discord.js index f240d87..a901073 100644 --- a/src/services/discord.js +++ b/src/services/discord.js @@ -11,6 +11,7 @@ const { createLogger } = require("./logger"); const { publishOverlayChatMessage } = require("./overlay-chat"); const { discordOverlayContent } = require("./discord-chat-content"); const { emitLumiEvent } = require("./lumi-events"); +const { userAgeStatistics } = require("./user-age-statistics"); const discordLog = createLogger("platform:discord", { category: "integration" }); @@ -49,12 +50,17 @@ async function startBot({ commandRouter } = {}) { client = new Client(options); - client.on("ready", () => { + client.on("ready", () => { discordLog.info("Discord bot ready", { account: client.user?.tag || null }, { event: "platform_ready" }); const avatarUrl = getBotAvatarUrl(client.user); - if (avatarUrl) { - setSetting("bot_avatar_url", avatarUrl); - } + if (avatarUrl) { + setSetting("bot_avatar_url", avatarUrl); + } + reconcileDiscordTenure(client).catch((error) => { + discordLog.warn("Discord platform tenure could not be reconciled", { + error + }, { event: "user_age_reconciliation_failed" }); + }); }); client.on("messageCreate", async (message) => { @@ -76,6 +82,9 @@ async function startBot({ commandRouter } = {}) { displayName, avatar: avatarUrl }); + if (!message.author.bot && message.member) { + observeDiscordMember(message.member, message.createdTimestamp || Date.now(), `discord-message:${message.id}`); + } publishOverlayChatMessage({ id: message.id, platform: "discord", @@ -140,6 +149,29 @@ async function startBot({ commandRouter } = {}) { source: "discord-gateway", occurredAt: member.joinedTimestamp || Date.now() }); + if (member.premiumSinceTimestamp) { + emitDiscordBoostEvent(member, true, member.premiumSinceTimestamp); + } + }); + + client.on("guildMemberRemove", (member) => { + if (!member?.user || member.user.bot || !isConfiguredGuild(member.guild?.id)) return; + const now = Date.now(); + emitLumiEvent("discord.member_leave", discordMemberPayload(member), { + id: `${member.guild?.id || "guild"}:${member.user.id}:leave:${now}`, + source: "discord-gateway", + occurredAt: now + }); + if (member.premiumSinceTimestamp) emitDiscordBoostEvent(member, false, now); + }); + + client.on("guildMemberUpdate", (previous, current) => { + if (!current?.user || current.user.bot || !isConfiguredGuild(current.guild?.id)) return; + const wasBoosting = Boolean(previous?.premiumSinceTimestamp); + const isBoosting = Boolean(current.premiumSinceTimestamp); + if (wasBoosting !== isBoosting) { + emitDiscordBoostEvent(current, isBoosting, isBoosting ? current.premiumSinceTimestamp : Date.now()); + } }); await client.login(token); @@ -176,6 +208,98 @@ function resolveIntent(key, legacyKey) { return null; } +function isConfiguredGuild(guildId) { + const configuredGuildId = String(getSetting("discord_guild_id", "") || ""); + return !configuredGuildId || configuredGuildId === String(guildId || ""); +} + +function discordMemberPayload(member) { + return { + guild_id: member.guild?.id || null, + guild_name: member.guild?.name || null, + user_id: member.user?.id || member.id || null, + user_name: member.user?.globalName || member.user?.username || member.user?.tag || null, + joined_at: member.joinedTimestamp || null, + boosted_at: member.premiumSinceTimestamp || null + }; +} + +function emitDiscordBoostEvent(member, active, occurredAt) { + emitLumiEvent(active ? "discord.boost_start" : "discord.boost_end", discordMemberPayload(member), { + id: `${member.guild?.id || "guild"}:${member.user?.id || member.id}:boost:${active ? "start" : "end"}:${occurredAt}`, + source: "discord-gateway", + occurredAt + }); +} + +function observeDiscordMember(member, observedAt, eventPrefix) { + if (!member?.user || member.user.bot || !isConfiguredGuild(member.guild?.id)) return; + userAgeStatistics.setState({ + platform: "discord", + userId: member.user.id, + scopeId: member.guild.id, + type: "member", + active: true, + startAt: member.joinedTimestamp || undefined, + occurredAt: observedAt, + eventId: `${eventPrefix}:member`, + source: member.joinedTimestamp ? "discord-guild-member" : "discord-first-observed", + authoritative: Boolean(member.joinedTimestamp) + }); + userAgeStatistics.setState({ + platform: "discord", + userId: member.user.id, + scopeId: member.guild.id, + type: "nitro", + active: Boolean(member.premiumSinceTimestamp), + startAt: member.premiumSinceTimestamp || undefined, + occurredAt: observedAt, + eventId: `${eventPrefix}:nitro`, + source: member.premiumSinceTimestamp ? "discord-premium-since" : "discord-guild-member", + authoritative: Boolean(member.premiumSinceTimestamp) + }); +} + +async function reconcileDiscordTenure(discordClient) { + const configuredGuildId = String(getSetting("discord_guild_id", "") || ""); + const guilds = configuredGuildId + ? [discordClient.guilds.cache.get(configuredGuildId)].filter(Boolean) + : Array.from(discordClient.guilds.cache.values()); + for (const guild of guilds) { + const members = await guild.members.fetch(); + const humans = Array.from(members.values()).filter((member) => !member.user?.bot); + const observedAt = Date.now(); + userAgeStatistics.reconcileSnapshot({ + platform: "discord", + scopeId: guild.id, + type: "member", + activeRecords: humans.map((member) => ({ + userId: member.user.id, + startAt: member.joinedTimestamp, + source: member.joinedTimestamp ? "discord-guild-member" : "discord-first-observed", + authoritative: Boolean(member.joinedTimestamp) + })), + observedAt, + source: "discord-guild-reconciliation", + complete: true + }); + userAgeStatistics.reconcileSnapshot({ + platform: "discord", + scopeId: guild.id, + type: "nitro", + activeRecords: humans.filter((member) => member.premiumSinceTimestamp).map((member) => ({ + userId: member.user.id, + startAt: member.premiumSinceTimestamp, + source: "discord-premium-since", + authoritative: true + })), + observedAt, + source: "discord-guild-reconciliation", + complete: true + }); + } +} + function resolvePartial(key, legacyKey) { if (Partials?.[key] !== undefined) return Partials[key]; return legacyKey; diff --git a/src/services/duration-format.js b/src/services/duration-format.js new file mode 100644 index 0000000..b147ccd --- /dev/null +++ b/src/services/duration-format.js @@ -0,0 +1,54 @@ +const UNIT_DEFINITIONS = Object.freeze({ + Y: { key: "years", milliseconds: 365 * 24 * 60 * 60 * 1000, singular: "year", plural: "years" }, + M: { key: "months", milliseconds: 30 * 24 * 60 * 60 * 1000, singular: "month", plural: "months" }, + D: { key: "days", milliseconds: 24 * 60 * 60 * 1000, singular: "day", plural: "days" }, + H: { key: "hours", milliseconds: 60 * 60 * 1000, singular: "hour", plural: "hours" }, + m: { key: "minutes", milliseconds: 60 * 1000, singular: "minute", plural: "minutes" }, + s: { key: "seconds", milliseconds: 1000, singular: "second", plural: "seconds" } +}); + +const DEFAULT_DURATION_FORMAT = "Y-M-D-H-m-s"; +const TOKEN_PATTERN = /^(?:Y|YY|YYYY|M|MM|D|DD|H|HH|m|mm|s|ss)$/; +const UNIT_ORDER = ["Y", "M", "D", "H", "m", "s"]; + +function parseDurationFormat(value = DEFAULT_DURATION_FORMAT) { + const source = String(value || DEFAULT_DURATION_FORMAT).trim(); + const tokens = source.split("-").filter(Boolean); + if (!tokens.length || tokens.some((token) => !TOKEN_PATTERN.test(token))) { + throw new Error("Invalid duration format."); + } + const units = tokens.map((token) => token[0]); + if (new Set(units).size !== units.length) { + throw new Error("A duration format may include each unit only once."); + } + for (let index = 1; index < units.length; index += 1) { + if (UNIT_ORDER.indexOf(units[index]) <= UNIT_ORDER.indexOf(units[index - 1])) { + throw new Error("Duration units must be ordered from largest to smallest."); + } + } + return units.map((unit, index) => ({ + ...UNIT_DEFINITIONS[unit], + unit, + token: tokens[index] + })); +} + +function formatDuration(milliseconds, format = DEFAULT_DURATION_FORMAT) { + let remaining = Math.max(0, Math.floor(Number(milliseconds) || 0)); + if (!remaining) return ""; + const units = parseDurationFormat(format); + const parts = []; + for (const unit of units) { + const value = Math.floor(remaining / unit.milliseconds); + remaining -= value * unit.milliseconds; + if (value) parts.push(`${value} ${value === 1 ? unit.singular : unit.plural}`); + } + return parts.join(", "); +} + +module.exports = { + DEFAULT_DURATION_FORMAT, + UNIT_DEFINITIONS, + formatDuration, + parseDurationFormat +}; diff --git a/src/services/lumi-events.js b/src/services/lumi-events.js index 3dac1ad..c8c1df2 100644 --- a/src/services/lumi-events.js +++ b/src/services/lumi-events.js @@ -62,8 +62,13 @@ function emitLumiEvent(type, payload = {}, metadata = {}) { for (const definition of [ { id: "twitch.follow", label: "Twitch follow", description: "A viewer follows a configured Twitch channel.", platform: "twitch" }, + { id: "twitch.unsubscribe", label: "Twitch subscription ended", description: "A viewer's subscription ends in a configured Twitch channel.", platform: "twitch" }, { id: "twitch.raid", label: "Twitch raid received", description: "A configured Twitch channel receives a raid that appears in chat.", platform: "twitch" }, { id: "twitch.subscribe", label: "Twitch subscription", description: "A configured Twitch channel receives a new subscription.", platform: "twitch", supportsTier: true }, + { id: "twitch.moderator_add", label: "Twitch moderator added", description: "A viewer becomes a channel moderator.", platform: "twitch" }, + { id: "twitch.moderator_remove", label: "Twitch moderator removed", description: "A viewer stops being a channel moderator.", platform: "twitch" }, + { id: "twitch.vip_add", label: "Twitch VIP added", description: "A viewer becomes a channel VIP.", platform: "twitch" }, + { id: "twitch.vip_remove", label: "Twitch VIP removed", description: "A viewer stops being a channel VIP.", platform: "twitch" }, { id: "twitch.subscription_gift", label: "Twitch gifted subscriptions", description: "A viewer gifts one or more subscriptions in a configured Twitch channel.", platform: "twitch", supportsTier: true }, { id: "twitch.cheer", label: "Twitch cheer", description: "A viewer cheers Bits in a configured Twitch channel.", platform: "twitch" }, { id: "twitch.channel_points", label: "Twitch channel-points redemption", description: "A viewer redeems a channel-points reward.", platform: "twitch" }, @@ -71,7 +76,10 @@ for (const definition of [ { id: "youtube.membership_gift", label: "YouTube gifted membership", description: "A YouTube viewer gifts channel memberships.", platform: "youtube" }, { id: "youtube.super_chat", label: "YouTube Super Chat", description: "A viewer sends a Super Chat.", platform: "youtube" }, { id: "youtube.super_sticker", label: "YouTube Super Sticker", description: "A viewer sends a Super Sticker.", platform: "youtube" }, - { id: "discord.member_join", label: "Discord member joined", description: "A member joins the configured Discord server.", platform: "discord" } + { id: "discord.member_join", label: "Discord member joined", description: "A member joins the configured Discord server.", platform: "discord" }, + { id: "discord.member_leave", label: "Discord member left", description: "A member leaves the configured Discord server.", platform: "discord" }, + { id: "discord.boost_start", label: "Discord boost started", description: "A member starts boosting the configured Discord server.", platform: "discord" }, + { id: "discord.boost_end", label: "Discord boost ended", description: "A member stops boosting the configured Discord server.", platform: "discord" } ]) registerEventType(definition); module.exports = { emitLumiEvent, listEventTypes, onLumiEvent, registerEventType }; diff --git a/src/services/placeholders.js b/src/services/placeholders.js index 6543ca8..ab33762 100644 --- a/src/services/placeholders.js +++ b/src/services/placeholders.js @@ -1,6 +1,7 @@ const { db } = require("./db"); const { getSetting } = require("./settings"); const { hasAccess } = require("./rbac"); +const { parseDurationFormat } = require("./duration-format"); const ROLE_LEVELS = Object.freeze({ public: 0, @@ -23,6 +24,7 @@ const SENSITIVITY_LEVELS = Object.freeze({ const VALUE_TYPES = new Set(["string", "number", "boolean", "url", "json", "date"]); const placeholders = new Map(); const fieldPolicies = new Map(); +const PLACEHOLDER_PATTERN = /\{\{\s*([A-Za-z0-9_.-]+)\s*\}\.\{\s*format\.([A-Za-z-]+)\s*\}\}|\{\{\s*([A-Za-z0-9_.-]+)\s*\}\}/g; function normalizeRole(value, fallback = "user") { const role = String(value || fallback).trim().toLowerCase(); @@ -100,7 +102,8 @@ function normalizeDefinition(definition = {}) { example: definition.example === undefined ? null : String(definition.example), plugin_id: definition.plugin_id ? String(definition.plugin_id).trim() : null, resolver: typeof definition.resolver === "function" ? definition.resolver : () => "", - available: typeof definition.available === "function" ? definition.available : null + available: typeof definition.available === "function" ? definition.available : null, + supports_duration_format: Boolean(definition.supports_duration_format) }; } @@ -246,13 +249,14 @@ function findDefinition(token) { function parsePlaceholders(template) { const found = []; - const matcher = /\{\{\s*([A-Za-z0-9_.-]+)\s*\}\}/g; + const matcher = new RegExp(PLACEHOLDER_PATTERN.source, PLACEHOLDER_PATTERN.flags); let match = null; while ((match = matcher.exec(String(template || "")))) { found.push({ token: match[0], - id: normalizeId(match[1]), - index: match.index + id: normalizeId(match[1] || match[3]), + index: match.index, + format: match[2] || null }); } return found; @@ -277,6 +281,7 @@ function catalog({ fieldId, user, outputAudience, runtimeContext } = {}) { value_type: definition.value_type, sensitivity: definition.sensitivity, group: definition.group, + supports_duration_format: definition.supports_duration_format, example: definition.sensitivity === "public_safe" ? definition.example : null })) .sort((a, b) => a.token.localeCompare(b.token)); @@ -302,6 +307,19 @@ function validateTemplate({ fieldId, template, user, outputAudience, runtimeCont } for (const token of parsePlaceholders(template)) { const definition = findDefinition(token.id); + if (token.format) { + try { + if (!definition?.supports_duration_format) throw new Error("unsupported"); + parseDurationFormat(token.format); + } catch { + errors.push({ + token: token.token, + id: token.id, + reason: definition?.supports_duration_format ? "invalid_duration_format" : "duration_format_unsupported" + }); + continue; + } + } const access = checkPlaceholderAccess(definition, policy, { user, outputAudience, @@ -328,8 +346,8 @@ async function renderTemplate({ fieldId, template, user, outputAudience, runtime }; } const errors = []; - const rendered = await replaceAsync(String(template || ""), /\{\{\s*([A-Za-z0-9_.-]+)\s*\}\}/g, async (match, rawId) => { - const id = normalizeId(rawId); + const rendered = await replaceAsync(String(template || ""), PLACEHOLDER_PATTERN, async (match, formattedId, format, regularId) => { + const id = normalizeId(formattedId || regularId); const definition = findDefinition(id); const access = checkPlaceholderAccess(definition, policy, { user, @@ -340,6 +358,16 @@ async function renderTemplate({ fieldId, template, user, outputAudience, runtime errors.push({ token: match, id, reason: access.reason }); return fallback; } + if (format) { + try { + if (!definition.supports_duration_format) throw new Error("unsupported"); + parseDurationFormat(format); + } catch { + const reason = definition.supports_duration_format ? "invalid_duration_format" : "duration_format_unsupported"; + errors.push({ token: match, id, reason }); + return fallback; + } + } try { const value = await withTimeout(Promise.resolve(definition.resolver({ user, @@ -347,7 +375,8 @@ async function renderTemplate({ fieldId, template, user, outputAudience, runtime outputAudience: normalizeRole(outputAudience || policy.output_audience, policy.output_audience), runtimeContext, token: match, - id + id, + format: format || null })), runtimeContext?.placeholder_timeout_ms); return stringifyResolvedValue(value); } catch (error) { @@ -393,14 +422,22 @@ function withTimeout(promise, requestedMs) { return Promise.race([promise, timeout]).finally(() => clearTimeout(timer)); } -function registerCorePlaceholders() { +function registerCorePlaceholders({ userAgeStatistics } = {}) { registerFieldPolicy({ field_id: "core.custom_commands.static_response", label: "Custom command response", field_type: "command_response", output_audience: "user", min_editor_role: "mod", - allowed_namespaces: ["core.main", "core.command", "custom", "user.public"], + allowed_namespaces: [ + "core.main", + "core.command", + "custom", + "user.public", + "twitch.user", + "youtube.user", + "discord.user" + ], max_sensitivity: "public_safe" }); registerPlaceholders([ @@ -461,6 +498,9 @@ function registerCorePlaceholders() { "" } ]); + if (userAgeStatistics?.placeholderDefinitions) { + registerPlaceholders(userAgeStatistics.placeholderDefinitions()); + } registerCustomPlaceholders(getSetting("custom_placeholders", [])); } diff --git a/src/services/top.js b/src/services/top.js index d132f3a..cb8c669 100644 --- a/src/services/top.js +++ b/src/services/top.js @@ -5,6 +5,7 @@ const { getSetting } = require("./settings"); const { getEnabledPlatformIds } = require("./platforms"); const { getPluginLeaderboards } = require("./plugin-stats"); const { getPlugins, pluginsDir } = require("./plugins"); +const { userAgeStatistics } = require("./user-age-statistics"); const coreProviders = new Map(); const coreOrder = []; @@ -97,17 +98,31 @@ function ensureCoreProviders() { description: "Most popular commands across platforms.", getRows: ({ limit }) => buildCommandUsageRows(limit) }); - registerTopProvider({ - id: "followage", - label: "Top followage", - section: "Platforms", - valueLabel: "Days", - description: "Longest follower durations.", - getRows: () => ({ - rows: [], - emptyMessage: "Followage tracking is not configured yet." - }) - }); + for (const definition of [ + { id: "followage", platform: "twitch", type: "follow", label: "Longest Twitch follows" }, + { id: "twitch_subscriber_age", platform: "twitch", type: "subscriber", label: "Longest Twitch subscriptions" }, + { id: "twitch_mod_age", platform: "twitch", type: "moderator", label: "Longest Twitch moderator tenure" }, + { id: "twitch_editor_age", platform: "twitch", type: "editor", label: "Longest Twitch editor tenure" }, + { id: "twitch_vip_age", platform: "twitch", type: "vip", label: "Longest Twitch VIP tenure" }, + { id: "youtube_member_age", platform: "youtube", type: "subscriber", label: "Longest YouTube memberships" }, + { id: "youtube_mod_age", platform: "youtube", type: "moderator", label: "Longest YouTube moderator tenure" }, + { id: "discord_member_age", platform: "discord", type: "member", label: "Longest Discord membership" }, + { id: "discord_nitro_age", platform: "discord", type: "nitro", label: "Longest Discord boosts" } + ]) { + registerTopProvider({ + ...definition, + section: "Platform tenure", + valueLabel: "Recorded tenure", + description: `${definition.label} across linked Lumi profiles.`, + getRows: ({ limit }) => ({ + rows: userAgeStatistics.getLeaderboard(definition.type, { + platform: definition.platform, + limit + }), + emptyMessage: "No recorded tenure is available yet." + }) + }); + } registerTopProvider({ id: "watchtime", label: "Top watchtime", diff --git a/src/services/twitch-eventsub.js b/src/services/twitch-eventsub.js index 8108469..dd3bf28 100644 --- a/src/services/twitch-eventsub.js +++ b/src/services/twitch-eventsub.js @@ -3,6 +3,7 @@ const { getSetting, setSetting } = require("./settings"); const { emitLumiEvent } = require("./lumi-events"); const { createLogger } = require("./logger"); const { setPlatformLiveState } = require("./platform-live-state"); +const { reconcileTwitchUserAges } = require("./twitch-user-age"); const EVENTSUB_URL = "wss://eventsub.wss.twitch.tv/ws?keepalive_timeout_seconds=30"; const eventLog = createLogger("platform:twitch:eventsub", { category: "integration" }); @@ -70,6 +71,8 @@ class TwitchEventSubManager { this.running = false; this.reconnectTimer = null; this.reconnectAttempt = 0; + this.reconcileTimer = null; + this.reconcileContext = null; this.generation = 0; this.status = { state: "disconnected", detail: "Not connected", subscriptions: 0 }; } @@ -98,7 +101,10 @@ class TwitchEventSubManager { this.running = false; this.generation += 1; clearTimeout(this.reconnectTimer); + clearInterval(this.reconcileTimer); this.reconnectTimer = null; + this.reconcileTimer = null; + this.reconcileContext = null; const socket = this.socket; this.socket = null; if (socket && socket.readyState < WebSocket.CLOSING) socket.close(1000, "Lumi shutdown"); @@ -204,8 +210,17 @@ class TwitchEventSubManager { } if (scopes.has("channel:read:subscriptions") && broadcaster.id === credentials.validation.user_id) { definitions.push({ type: "channel.subscribe", version: "1", condition: { broadcaster_user_id: broadcaster.id } }); + definitions.push({ type: "channel.subscription.end", version: "1", condition: { broadcaster_user_id: broadcaster.id } }); definitions.push({ type: "channel.subscription.gift", version: "1", condition: { broadcaster_user_id: broadcaster.id } }); } + if (scopes.has("moderation:read") && broadcaster.id === credentials.validation.user_id) { + definitions.push({ type: "channel.moderator.add", version: "1", condition: { broadcaster_user_id: broadcaster.id } }); + definitions.push({ type: "channel.moderator.remove", version: "1", condition: { broadcaster_user_id: broadcaster.id } }); + } + if (broadcaster.id === credentials.validation.user_id && (scopes.has("channel:read:vips") || scopes.has("channel:manage:vips"))) { + definitions.push({ type: "channel.vip.add", version: "1", condition: { broadcaster_user_id: broadcaster.id } }); + definitions.push({ type: "channel.vip.remove", version: "1", condition: { broadcaster_user_id: broadcaster.id } }); + } if (scopes.has("bits:read") && broadcaster.id === credentials.validation.user_id) definitions.push({ type: "channel.cheer", version: "1", condition: { broadcaster_user_id: broadcaster.id } }); if ((scopes.has("channel:read:redemptions") || scopes.has("channel:manage:redemptions")) && broadcaster.id === credentials.validation.user_id) @@ -235,6 +250,15 @@ class TwitchEventSubManager { } } if (generation !== this.generation) return; + this.reconcileContext = { ...credentials, broadcasters: users.data || [] }; + await reconcileTwitchUserAges(this.reconcileContext); + clearInterval(this.reconcileTimer); + this.reconcileTimer = setInterval(() => { + if (!this.running || generation !== this.generation || !this.reconcileContext) return; + reconcileTwitchUserAges(this.reconcileContext).catch((error) => { + eventLog.warn("Scheduled Twitch tenure reconciliation failed", { error }, { event: "user_age_reconciliation_failed" }); + }); + }, 10 * 60 * 1000); this.status = { state: "connected", detail: active ? `Receiving ${active} Twitch event feed${active === 1 ? "" : "s"}.` : "Connected, but no configured channel events could be subscribed.", @@ -243,7 +267,10 @@ class TwitchEventSubManager { cheers: scopes.has("bits:read") ? "available" : "missing bits:read", channel_points: scopes.has("channel:read:redemptions") || scopes.has("channel:manage:redemptions") ? "available" : "missing channel:read:redemptions", subscriptions: scopes.has("channel:read:subscriptions") ? "available" : "missing channel:read:subscriptions", - follows: scopes.has("moderator:read:followers") ? "available" : "missing moderator:read:followers" + follows: scopes.has("moderator:read:followers") ? "available" : "missing moderator:read:followers", + moderators: scopes.has("moderation:read") ? "available" : "missing moderation:read", + vips: scopes.has("channel:read:vips") || scopes.has("channel:manage:vips") ? "available" : "missing channel:read:vips", + editors: scopes.has("channel:read:editors") ? "available" : "missing channel:read:editors" } }; eventLog.info("Twitch EventSub connected", { channels: users.data?.length || 0, subscriptions: active }, { event: "eventsub_ready" }); @@ -265,6 +292,11 @@ class TwitchEventSubManager { if (subscription.type === "channel.follow") emitLumiEvent("twitch.follow", { ...common, followed_at: event.followed_at }, metadata); if (subscription.type === "channel.raid") emitLumiEvent("twitch.raid", { ...common, viewers: Number(event.viewers || 0) }, metadata); if (subscription.type === "channel.subscribe") emitLumiEvent("twitch.subscribe", { ...common, tier: event.tier, gifted: Boolean(event.is_gift) }, metadata); + if (subscription.type === "channel.subscription.end") emitLumiEvent("twitch.unsubscribe", { ...common, tier: event.tier }, metadata); + if (subscription.type === "channel.moderator.add") emitLumiEvent("twitch.moderator_add", common, metadata); + if (subscription.type === "channel.moderator.remove") emitLumiEvent("twitch.moderator_remove", common, metadata); + if (subscription.type === "channel.vip.add") emitLumiEvent("twitch.vip_add", common, metadata); + if (subscription.type === "channel.vip.remove") emitLumiEvent("twitch.vip_remove", common, metadata); if (subscription.type === "channel.subscription.gift") emitLumiEvent("twitch.subscription_gift", { ...common, tier: event.tier, total: Number(event.total || 0), anonymous: Boolean(event.is_anonymous) }, metadata); if (subscription.type === "channel.cheer") emitLumiEvent("twitch.cheer", { ...common, bits: Number(event.bits || 0), message: event.message || "", anonymous: Boolean(event.is_anonymous) }, metadata); if (subscription.type === "channel.channel_points_custom_reward_redemption.add") emitLumiEvent("twitch.channel_points", { diff --git a/src/services/twitch-user-age.js b/src/services/twitch-user-age.js new file mode 100644 index 0000000..a0e0a22 --- /dev/null +++ b/src/services/twitch-user-age.js @@ -0,0 +1,122 @@ +const { userAgeStatistics } = require("./user-age-statistics"); +const { createLogger } = require("./logger"); + +const log = createLogger("platform:twitch:user-age", { category: "integration" }); + +async function fetchAll(url, headers) { + const records = []; + let cursor = ""; + do { + const target = new URL(url); + target.searchParams.set("first", "100"); + if (cursor) target.searchParams.set("after", cursor); + const response = await fetch(target, { headers }); + if (!response.ok) { + const detail = await response.text().catch(() => ""); + throw new Error(`Twitch reconciliation failed (${response.status})${detail ? `: ${detail.slice(0, 160)}` : ""}`); + } + const body = await response.json(); + records.push(...(Array.isArray(body.data) ? body.data : [])); + cursor = String(body.pagination?.cursor || ""); + } while (cursor); + return records; +} + +async function reconcileTwitchUserAges({ accessToken, clientId, validation, broadcasters = [] } = {}) { + if (!accessToken || !clientId) return; + const headers = { + "Client-Id": clientId, + Authorization: `Bearer ${accessToken}` + }; + const scopes = new Set(validation?.scopes || []); + const moderatorId = validation?.user_id; + for (const broadcaster of broadcasters) { + const jobs = []; + if (scopes.has("moderator:read:followers")) { + jobs.push(reconcileList({ + platform: "twitch", + scopeId: broadcaster.id, + type: "follow", + url: `https://api.twitch.tv/helix/channels/followers?broadcaster_id=${encodeURIComponent(broadcaster.id)}&moderator_id=${encodeURIComponent(moderatorId)}`, + headers, + startField: "followed_at", + source: "twitch-followers-api" + })); + } + if (broadcaster.id === moderatorId && scopes.has("channel:read:subscriptions")) { + jobs.push(reconcileList({ + platform: "twitch", + scopeId: broadcaster.id, + type: "subscriber", + url: `https://api.twitch.tv/helix/subscriptions?broadcaster_id=${encodeURIComponent(broadcaster.id)}`, + headers, + source: "twitch-subscriptions-api" + })); + } + if (broadcaster.id === moderatorId && scopes.has("moderation:read")) { + jobs.push(reconcileList({ + platform: "twitch", + scopeId: broadcaster.id, + type: "moderator", + url: `https://api.twitch.tv/helix/moderation/moderators?broadcaster_id=${encodeURIComponent(broadcaster.id)}`, + headers, + source: "twitch-moderators-api" + })); + } + if (broadcaster.id === moderatorId && (scopes.has("channel:read:vips") || scopes.has("channel:manage:vips"))) { + jobs.push(reconcileList({ + platform: "twitch", + scopeId: broadcaster.id, + type: "vip", + url: `https://api.twitch.tv/helix/channels/vips?broadcaster_id=${encodeURIComponent(broadcaster.id)}`, + headers, + source: "twitch-vips-api" + })); + } + if (broadcaster.id === moderatorId && scopes.has("channel:read:editors")) { + jobs.push(reconcileList({ + platform: "twitch", + scopeId: broadcaster.id, + type: "editor", + url: `https://api.twitch.tv/helix/channels/editors?broadcaster_id=${encodeURIComponent(broadcaster.id)}`, + headers, + startField: "created_at", + source: "twitch-editors-api" + })); + } + await Promise.allSettled(jobs); + } +} + +async function reconcileList({ platform, scopeId, type, url, headers, startField = null, source }) { + try { + const rows = await fetchAll(url, headers); + const observedAt = Date.now(); + userAgeStatistics.reconcileSnapshot({ + platform, + scopeId, + type, + activeRecords: rows.map((row) => ({ + userId: row.user_id, + startAt: startField ? row[startField] : undefined, + source: startField && row[startField] ? source : `${source}:first-observed`, + authoritative: Boolean(startField && row[startField]) + })), + observedAt, + source, + complete: true + }); + } catch (error) { + // A failed or partial API read must never be interpreted as an empty state. + log.warn("Twitch platform tenure reconciliation was skipped", { + statistic_type: type, + scope_id: scopeId, + error + }, { event: "user_age_reconciliation_failed" }); + } +} + +module.exports = { + fetchAll, + reconcileTwitchUserAges +}; diff --git a/src/services/twitch.js b/src/services/twitch.js index bf728e9..d771322 100644 --- a/src/services/twitch.js +++ b/src/services/twitch.js @@ -5,6 +5,7 @@ const { ensureUserForIdentity } = require("./users"); const { createLogger } = require("./logger"); const { publishOverlayChatMessage } = require("./overlay-chat"); const { resolveBetterTtvEmotes, resolveTwitchAvatar, resolveTwitchBadges } = require("./twitch-chat-assets"); +const { userAgeStatistics } = require("./user-age-statistics"); const twitchLog = createLogger("platform:twitch", { category: "integration" }); @@ -57,6 +58,28 @@ async function startTwitchBot({ commandRouter } = {}) { displayName, avatar }); + if (!self && tags["room-id"]) { + const observedAt = Number(tags["tmi-sent-ts"]) || Date.now(); + const badgeNames = new Set(Object.keys(tags.badges || {})); + for (const [type, active] of [ + ["subscriber", badgeNames.has("subscriber") || badgeNames.has("founder") || Boolean(tags.subscriber)], + ["moderator", badgeNames.has("moderator") || tags.mod === true || tags.mod === "1"], + ["vip", badgeNames.has("vip")] + ]) { + userAgeStatistics.setState({ + platform: "twitch", + userId, + scopeId: tags["room-id"], + type, + active, + occurredAt: observedAt, + eventId: `twitch-chat:${tags.id || `${tags["room-id"]}:${userId}:${observedAt}`}:${type}`, + source: "twitch-chat-tags", + precision: "millisecond", + authoritative: false + }); + } + } publishOverlayChatMessage({ id: tags.id, platform: "twitch", diff --git a/src/services/user-age-statistics.js b/src/services/user-age-statistics.js new file mode 100644 index 0000000..062e775 --- /dev/null +++ b/src/services/user-age-statistics.js @@ -0,0 +1,423 @@ +const crypto = require("crypto"); +const { db } = require("./db"); +const { formatDuration } = require("./duration-format"); +const { createLogger } = require("./logger"); + +const log = createLogger("core:user-age-statistics", { category: "integration" }); + +const DEFAULT_STATISTICS = Object.freeze([ + { platform: "twitch", type: "follow", placeholder: "follow_age", label: "Follow age" }, + { platform: "twitch", type: "subscriber", placeholder: "subscriber_age", label: "Subscriber age" }, + { platform: "twitch", type: "moderator", placeholder: "mod_age", label: "Moderator age" }, + { platform: "twitch", type: "editor", placeholder: "editor_age", label: "Editor age" }, + { platform: "twitch", type: "vip", placeholder: "vip_age", label: "VIP age" }, + { platform: "youtube", type: "subscriber", placeholder: "subscriber_age", label: "Channel membership age" }, + { platform: "youtube", type: "moderator", placeholder: "mod_age", label: "Moderator age" }, + { platform: "discord", type: "member", placeholder: "member_age", label: "Server member age" }, + { platform: "discord", type: "nitro", placeholder: "nitro_age", label: "Server boost age" } +]); + +function normalizeText(value) { + return String(value ?? "").trim(); +} + +function timestamp(value, fallback = null) { + if (value instanceof Date) return value.getTime(); + const numeric = Number(value); + if (Number.isFinite(numeric) && numeric > 0) return Math.floor(numeric); + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed : fallback; +} + +class UserAgeStatistics { + constructor(database = db, { now = () => Date.now(), registerDefaults = true } = {}) { + this.db = database; + this.now = now; + this.statistics = new Map(); + this.stopListening = null; + this.ensureTables(); + if (registerDefaults) DEFAULT_STATISTICS.forEach((definition) => this.registerStatistic(definition)); + } + + ensureTables() { + this.db.exec(` + CREATE TABLE IF NOT EXISTS user_age_intervals ( + id TEXT PRIMARY KEY, + platform TEXT NOT NULL, + platform_user_id TEXT NOT NULL, + scope_id TEXT NOT NULL, + statistic_type TEXT NOT NULL, + start_at INTEGER NOT NULL, + end_at INTEGER, + start_source TEXT NOT NULL, + start_precision TEXT NOT NULL DEFAULT 'millisecond', + start_authoritative INTEGER NOT NULL DEFAULT 0, + end_source TEXT, + end_precision TEXT, + end_authoritative INTEGER, + observed_at INTEGER NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE UNIQUE INDEX IF NOT EXISTS user_age_intervals_one_open_idx + ON user_age_intervals (platform, platform_user_id, scope_id, statistic_type) + WHERE end_at IS NULL; + CREATE INDEX IF NOT EXISTS user_age_intervals_lookup_idx + ON user_age_intervals (platform, scope_id, statistic_type, platform_user_id, start_at); + CREATE TABLE IF NOT EXISTS user_age_events ( + platform TEXT NOT NULL, + event_id TEXT NOT NULL, + processed_at INTEGER NOT NULL, + PRIMARY KEY (platform, event_id) + ); + `); + } + + registerStatistic(definition = {}) { + const platform = normalizeText(definition.platform).toLowerCase(); + const type = normalizeText(definition.type).toLowerCase(); + const placeholder = normalizeText(definition.placeholder).toLowerCase(); + if (!platform || !type || !placeholder) throw new Error("Platform, type, and placeholder are required."); + const normalized = Object.freeze({ + platform, + type, + placeholder, + label: normalizeText(definition.label) || placeholder + }); + this.statistics.set(`${platform}:${type}`, normalized); + return normalized; + } + + listStatistics() { + return Array.from(this.statistics.values()); + } + + start() { + if (this.stopListening) return; + const { onLumiEvent } = require("./lumi-events"); + const mappings = { + "twitch.follow": ["follow", true, "followed_at"], + "twitch.subscribe": ["subscriber", true, null], + "twitch.unsubscribe": ["subscriber", false, null], + "twitch.moderator_add": ["moderator", true, null], + "twitch.moderator_remove": ["moderator", false, null], + "twitch.vip_add": ["vip", true, null], + "twitch.vip_remove": ["vip", false, null], + "discord.member_join": ["member", true, "joined_at"], + "discord.member_leave": ["member", false, null], + "discord.boost_start": ["nitro", true, "boosted_at"], + "discord.boost_end": ["nitro", false, null] + }; + this.stopListening = onLumiEvent((event) => { + const mapping = mappings[event.type]; + if (!mapping) return; + const [type, active, startField] = mapping; + const payload = event.payload || {}; + const platform = event.type.split(".")[0]; + const userId = payload.user_id; + const scopeId = platform === "discord" ? payload.guild_id : payload.broadcaster_id; + if (!userId || !scopeId) return; + try { + this.setState({ + platform, + type, + userId, + scopeId, + active, + startAt: startField ? payload[startField] : event.occurredAt, + occurredAt: event.occurredAt, + eventId: event.id, + source: event.source, + precision: "millisecond", + authoritative: Boolean(startField && payload[startField]) || active === false + }); + } catch (error) { + log.error("Platform tenure event could not be recorded", { + event_type: event.type, + event_id: event.id, + error + }, { event: "user_age_event_failed" }); + } + }); + } + + stop() { + this.stopListening?.(); + this.stopListening = null; + } + + setState(input = {}) { + return this.db.transaction(() => this._setState(input, true))(); + } + + _setState(input, recordEvent) { + const platform = normalizeText(input.platform).toLowerCase(); + const userId = normalizeText(input.userId || input.platformUserId); + const scopeId = normalizeText(input.scopeId); + const type = normalizeText(input.type).toLowerCase(); + const definition = this.statistics.get(`${platform}:${type}`); + if (!definition || !userId || !scopeId) throw new Error("Unknown or incomplete user-age statistic."); + const now = this.now(); + const occurredAt = timestamp(input.occurredAt, now); + const eventId = normalizeText(input.eventId); + if (recordEvent && eventId) { + const result = this.db.prepare( + "INSERT OR IGNORE INTO user_age_events (platform, event_id, processed_at) VALUES (?, ?, ?)" + ).run(platform, eventId, now); + if (!result.changes) return { changed: false, duplicate: true }; + } + + const open = this.db.prepare( + `SELECT * FROM user_age_intervals + WHERE platform = ? AND platform_user_id = ? AND scope_id = ? AND statistic_type = ? AND end_at IS NULL` + ).get(platform, userId, scopeId, type); + const active = Boolean(input.active); + const authoritative = Boolean(input.authoritative); + const source = normalizeText(input.source) || "first-observed"; + const precision = normalizeText(input.precision) || "millisecond"; + + if (active) { + const proposedStart = Math.min(timestamp(input.startAt, occurredAt), occurredAt); + if (open) { + if (authoritative && !open.start_authoritative && proposedStart <= open.start_at) { + this.db.prepare( + `UPDATE user_age_intervals + SET start_at = ?, start_source = ?, start_precision = ?, start_authoritative = 1, + observed_at = ?, updated_at = ? + WHERE id = ? AND end_at IS NULL` + ).run(proposedStart, source, precision, occurredAt, now, open.id); + return { changed: true, intervalId: open.id, backfilled: true }; + } + return { changed: false, intervalId: open.id }; + } + const id = crypto.randomUUID(); + this.db.prepare( + `INSERT INTO user_age_intervals + (id, platform, platform_user_id, scope_id, statistic_type, start_at, end_at, + start_source, start_precision, start_authoritative, observed_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?)` + ).run(id, platform, userId, scopeId, type, proposedStart, source, precision, authoritative ? 1 : 0, occurredAt, now, now); + return { changed: true, intervalId: id, opened: true }; + } + + if (!open) return { changed: false }; + const endAt = Math.max(open.start_at, occurredAt); + this.db.prepare( + `UPDATE user_age_intervals + SET end_at = ?, end_source = ?, end_precision = ?, end_authoritative = ?, + observed_at = ?, updated_at = ? + WHERE id = ? AND end_at IS NULL` + ).run(endAt, source, precision, authoritative ? 1 : 0, occurredAt, now, open.id); + return { changed: true, intervalId: open.id, closed: true }; + } + + reconcileSnapshot(input = {}) { + const platform = normalizeText(input.platform).toLowerCase(); + const scopeId = normalizeText(input.scopeId); + const type = normalizeText(input.type).toLowerCase(); + if (!input.complete) return { changed: false, skipped: true }; + const records = Array.isArray(input.activeRecords) ? input.activeRecords : []; + const observedAt = timestamp(input.observedAt, this.now()); + return this.db.transaction(() => { + const activeUsers = new Set(); + let changes = 0; + for (const record of records) { + const userId = normalizeText(record.userId || record.platformUserId); + if (!userId) continue; + activeUsers.add(userId); + const result = this._setState({ + platform, + scopeId, + type, + userId, + active: true, + startAt: record.startAt, + occurredAt: observedAt, + source: record.source || input.source || "platform-reconciliation", + precision: record.precision || input.precision || "millisecond", + authoritative: record.authoritative ?? input.authoritative + }, false); + if (result.changed) changes += 1; + } + const openRows = this.db.prepare( + `SELECT platform_user_id FROM user_age_intervals + WHERE platform = ? AND scope_id = ? AND statistic_type = ? AND end_at IS NULL` + ).all(platform, scopeId, type); + for (const row of openRows) { + if (activeUsers.has(row.platform_user_id)) continue; + const result = this._setState({ + platform, + scopeId, + type, + userId: row.platform_user_id, + active: false, + occurredAt: observedAt, + source: input.source || "platform-reconciliation", + precision: input.precision || "millisecond", + authoritative: Boolean(input.authoritative) + }, false); + if (result.changed) changes += 1; + } + return { changed: Boolean(changes), changes }; + })(); + } + + getDurations({ platform, userId, scopeId, type, now = this.now() } = {}) { + const rows = this.db.prepare( + `SELECT start_at, end_at FROM user_age_intervals + WHERE platform = ? AND platform_user_id = ? AND scope_id = ? AND statistic_type = ? + ORDER BY start_at ASC` + ).all(normalizeText(platform).toLowerCase(), normalizeText(userId), normalizeText(scopeId), normalizeText(type).toLowerCase()); + if (!rows.length) return { current: null, total: null, intervals: 0 }; + let current = null; + let total = 0; + for (const row of rows) { + const end = row.end_at === null ? now : row.end_at; + const duration = Math.max(0, end - row.start_at); + total += duration; + if (row.end_at === null) current = duration; + } + return { current, total, intervals: rows.length }; + } + + getProfileStatistics(userId, { now = this.now() } = {}) { + const identities = this.db.prepare( + "SELECT provider, provider_user_id FROM user_identities WHERE user_id = ?" + ).all(userId); + const output = []; + for (const identity of identities) { + const definitions = this.listStatistics().filter((entry) => entry.platform === identity.provider); + for (const definition of definitions) { + const rows = this.db.prepare( + `SELECT scope_id, start_at, end_at FROM user_age_intervals + WHERE platform = ? AND platform_user_id = ? AND statistic_type = ?` + ).all(identity.provider, identity.provider_user_id, definition.type); + if (!rows.length) continue; + const byScope = new Map(); + for (const row of rows) { + const scoped = byScope.get(row.scope_id) || []; + scoped.push(row); + byScope.set(row.scope_id, scoped); + } + for (const [scopeId, scopedRows] of byScope.entries()) { + const total = scopedRows.reduce((sum, row) => sum + Math.max(0, (row.end_at ?? now) - row.start_at), 0); + const open = scopedRows.find((row) => row.end_at === null); + output.push({ + platform: definition.platform, + type: definition.type, + scopeId, + label: `${capitalize(definition.platform)} ${definition.label} · ${scopeId}`, + current: open ? formatDuration(now - open.start_at) : "", + total: formatDuration(total), + value: formatDuration(total) + }); + } + } + } + return output; + } + + getLeaderboard(type, { platform = null, limit = 25, now = this.now() } = {}) { + const conditions = ["i.statistic_type = ?"]; + const params = [now, normalizeText(type).toLowerCase()]; + if (platform) { + conditions.push("i.platform = ?"); + params.push(normalizeText(platform).toLowerCase()); + } + params.push(Math.max(1, Math.min(Number(limit) || 25, 100))); + return this.db.prepare( + `SELECT p.internal_username AS username, i.scope_id, + SUM(MAX(0, COALESCE(i.end_at, ?) - i.start_at)) AS duration_ms + FROM user_age_intervals i + JOIN user_identities identity + ON identity.provider = i.platform AND identity.provider_user_id = i.platform_user_id + JOIN user_profiles p ON p.id = identity.user_id + WHERE ${conditions.join(" AND ")} + GROUP BY p.id, p.internal_username, i.scope_id + ORDER BY duration_ms DESC, p.internal_username ASC + LIMIT ?` + ).all(...params).map((row) => ({ + username: row.username, + label: `${row.username} · ${row.scope_id}`, + value: formatDuration(row.duration_ms, "D-H-m"), + numericValue: row.duration_ms + })); + } + + listDiagnostics({ platform = "", scopeId = "", userId = "", type = "", limit = 500 } = {}) { + const conditions = []; + const params = []; + if (platform) { conditions.push("platform = ?"); params.push(normalizeText(platform).toLowerCase()); } + if (scopeId) { conditions.push("scope_id = ?"); params.push(normalizeText(scopeId)); } + if (userId) { conditions.push("platform_user_id = ?"); params.push(normalizeText(userId)); } + if (type) { conditions.push("statistic_type = ?"); params.push(normalizeText(type).toLowerCase()); } + params.push(Math.max(1, Math.min(Number(limit) || 500, 1000))); + return this.db.prepare( + `SELECT * FROM user_age_intervals + ${conditions.length ? `WHERE ${conditions.join(" AND ")}` : ""} + ORDER BY end_at IS NULL DESC, start_at DESC LIMIT ?` + ).all(...params); + } + + placeholderDefinitions() { + return this.listStatistics().flatMap((definition) => ["current", "total"].map((mode) => ({ + id: `${definition.platform}.user.${definition.placeholder}.${mode}`, + namespace: `${definition.platform}.user`, + group: "Platform tenure", + label: `${definition.label} (${mode})`, + description: mode === "current" + ? `Current uninterrupted ${definition.label.toLowerCase()} in this platform context.` + : `Combined recorded ${definition.label.toLowerCase()} in this platform context.`, + value_type: "string", + sensitivity: "public_safe", + min_editor_role: "user", + min_viewer_role: "user", + allowed_field_types: ["command_response", "chat_message", "admin_template", "okf_markdown"], + example: "2 years, 3 months, 4 days", + supports_duration_format: true, + resolver: ({ runtimeContext, format }) => { + const identity = contextIdentity(definition.platform, runtimeContext); + if (!identity) return ""; + const durations = this.getDurations({ + platform: definition.platform, + userId: identity.userId, + scopeId: identity.scopeId, + type: definition.type + }); + const value = durations[mode]; + return value === null ? "" : formatDuration(value, format); + } + }))); + } +} + +function contextIdentity(platform, runtimeContext = {}) { + const ctx = runtimeContext.ctx || {}; + const activePlatform = normalizeText(runtimeContext.platform || ctx.platform).toLowerCase(); + if (activePlatform !== platform) return null; + const userId = normalizeText( + runtimeContext.platformUser?.id || + runtimeContext.user?.platformId || + ctx.platformUser?.id || + ctx.user?.platformId + ); + const meta = runtimeContext.meta || ctx.meta || {}; + let scopeId = ""; + if (platform === "twitch") scopeId = normalizeText(meta.tags?.["room-id"] || meta.broadcasterId); + if (platform === "youtube") scopeId = normalizeText(meta.broadcasterChannelId); + if (platform === "discord") scopeId = normalizeText(meta.message?.guildId || meta.message?.guild?.id || meta.guildId); + return userId && scopeId ? { userId, scopeId } : null; +} + +function capitalize(value) { + return value ? `${value[0].toUpperCase()}${value.slice(1)}` : ""; +} + +const userAgeStatistics = new UserAgeStatistics(); + +module.exports = { + DEFAULT_STATISTICS, + UserAgeStatistics, + contextIdentity, + userAgeStatistics +}; diff --git a/src/services/youtube.js b/src/services/youtube.js index 4713ec0..44440e7 100644 --- a/src/services/youtube.js +++ b/src/services/youtube.js @@ -5,6 +5,7 @@ const { createLogger } = require("./logger"); const { publishOverlayChatMessage } = require("./overlay-chat"); const { emitLumiEvent } = require("./lumi-events"); const { setPlatformLiveState } = require("./platform-live-state"); +const { userAgeStatistics } = require("./user-age-statistics"); const youtubeLog = createLogger("platform:youtube", { category: "integration" }); @@ -119,14 +120,41 @@ async function handleChatItem(state, liveChatId, item) { if (!snippet || !author) { return; } - const messageText = snippet.displayMessage; - if (!messageText) { - return; - } + const messageText = snippet.displayMessage || ""; const displayName = author.displayName || "YouTube User"; const avatar = author.profileImageUrl || null; const isSelf = Boolean(state.channelId && author.channelId === state.channelId); emitYouTubeEvent(item, state, liveChatId, displayName); + if (!isSelf && state.channelId && author.channelId) { + const observedAt = Date.parse(snippet.publishedAt) || Date.now(); + const membershipStart = ["newSponsorEvent", "giftMembershipReceivedEvent"].includes(snippet.type); + userAgeStatistics.setState({ + platform: "youtube", + userId: author.channelId, + scopeId: state.channelId, + type: "subscriber", + active: Boolean(author.isChatSponsor || membershipStart), + startAt: membershipStart ? observedAt : undefined, + occurredAt: observedAt, + eventId: `youtube-chat:${item.id}:subscriber`, + source: membershipStart ? "youtube-membership-event" : "youtube-live-chat-author", + precision: "millisecond", + authoritative: membershipStart + }); + userAgeStatistics.setState({ + platform: "youtube", + userId: author.channelId, + scopeId: state.channelId, + type: "moderator", + active: Boolean(author.isChatModerator), + occurredAt: observedAt, + eventId: `youtube-chat:${item.id}:moderator`, + source: "youtube-live-chat-author", + precision: "millisecond", + authoritative: false + }); + } + if (!messageText) return; const profile = isSelf ? null : ensureUserForIdentity({ provider: "youtube", providerUserId: author.channelId, diff --git a/src/web/server.js b/src/web/server.js index 708ecdb..4c7b3d8 100644 --- a/src/web/server.js +++ b/src/web/server.js @@ -80,6 +80,7 @@ const { getClient: getTwitchClient } = require("../services/twitch"); const { twitchEventSubManager } = require("../services/twitch-eventsub"); const { eventHooksApi } = require("../services/overlay-event-hooks"); const { streamTestingService } = require("../services/stream-testing"); +const { userAgeStatistics } = require("../services/user-age-statistics"); const { getReverseProxyIngestSettings, saveReverseProxyIngestSettings @@ -379,7 +380,7 @@ function getExpressionUserSummary(userId) { function buildUserStatsPayload(userId) { if (!userId) { - return { stats: null, expression: null, pluginStats: [] }; + return { stats: null, expression: null, tenureStats: [], pluginStats: [] }; } const stats = db .prepare("SELECT * FROM stats WHERE user_id = ?") @@ -387,6 +388,7 @@ function buildUserStatsPayload(userId) { return { stats, expression: getExpressionUserSummary(userId), + tenureStats: userAgeStatistics.getProfileStatistics(userId), pluginStats: getPluginProfileStats(userId) }; } @@ -425,6 +427,14 @@ function buildCompareRows(leftStats, rightStats) { ]; pushSection("Community Interaction", leftCommunity, rightCommunity); + if (leftStats.tenureStats?.length || rightStats.tenureStats?.length) { + pushSection( + "Platform tenure", + (leftStats.tenureStats || []).map((entry) => ({ label: entry.label, value: entry.total })), + (rightStats.tenureStats || []).map((entry) => ({ label: entry.label, value: entry.total })) + ); + } + if (leftStats.expression || rightStats.expression) { const leftExpression = leftStats.expression ? [ @@ -3113,7 +3123,7 @@ function createWebServer({ loadPlugins, discordClient, commandRouter }) { // network. Public clients remain unable to forge forwarding headers. app.set("trust proxy", isTrustedProxyAddress); const webhooks = createWebhookService(); - placeholders.registerCorePlaceholders(); + placeholders.registerCorePlaceholders({ userAgeStatistics }); placeholders.registerPlatformPlaceholders({ discordClient, getTwitchClient, @@ -5280,6 +5290,7 @@ function createWebServer({ loadPlugins, discordClient, commandRouter }) { title: "Your stats", stats: payload.stats, expression: payload.expression, + tenureStats: payload.tenureStats, pluginStats: payload.pluginStats, statsOwner: { username: req.session.user.username, @@ -5324,6 +5335,7 @@ function createWebServer({ loadPlugins, discordClient, commandRouter }) { title: `${profile.internal_username}'s stats`, stats: payload.stats, expression: payload.expression, + tenureStats: payload.tenureStats, pluginStats: payload.pluginStats, statsOwner: { username: profile.internal_username, @@ -6212,6 +6224,21 @@ function createWebServer({ loadPlugins, discordClient, commandRouter }) { renderDiagnosticsAdmin(req, res); }); + app.get("/admin/platform-tenure", requireRole("admin"), (req, res) => { + const filters = { + platform: String(req.query.platform || "").trim().toLowerCase(), + scopeId: String(req.query.scope || "").trim(), + userId: String(req.query.user || "").trim(), + type: String(req.query.type || "").trim().toLowerCase() + }; + res.render("admin-platform-tenure", { + title: "Platform tenure diagnostics", + intervals: userAgeStatistics.listDiagnostics(filters), + statistics: userAgeStatistics.listStatistics(), + filters + }); + }); + app.get("/admin/stream-testing", requireRole("admin"), (req, res) => { res.set("Cache-Control", "no-store"); res.render("admin-stream-testing", { @@ -7969,6 +7996,7 @@ function collectNavItems(user, pluginNav, currentPath) { }, { label: "Feedback review", path: "/admin/feedback", role: "admin", section: "admin" }, { label: "Stream testing", path: "/admin/stream-testing", role: "admin", section: "admin" }, + { label: "Platform tenure", path: "/admin/platform-tenure", role: "admin", section: "admin" }, { label: "Diagnostics", path: "/admin/diagnostics", role: "admin", section: "admin" }, { label: "Logs", path: "/admin/logs", role: "admin", section: "admin" }, { label: "Updates", path: "/admin/updates", role: "admin", section: "admin" }, @@ -8237,6 +8265,7 @@ function getDefaultNavIcon(item) { if (pathName === "/admin/theming") return "theming"; if (pathName === "/admin/privileges") return "privileges"; if (pathName === "/admin/diagnostics") return "admin"; + if (pathName === "/admin/platform-tenure") return "admin"; if (pathName === "/admin/logs") return "logs"; if (pathName === "/admin/updates") return "updates"; if (pathName === "/admin/commands") return "commands"; diff --git a/src/web/views/admin-platform-tenure.ejs b/src/web/views/admin-platform-tenure.ejs new file mode 100644 index 0000000..39a3209 --- /dev/null +++ b/src/web/views/admin-platform-tenure.ejs @@ -0,0 +1,98 @@ +<%- include("partials/layout-top", { title }) %> +
+ <%- include("partials/page-header", { + eyebrow: "Diagnostics", + pageTitle: "Platform tenure", + description: "Inspect the authoritative and first-observed intervals behind age placeholders, profile stats, and leaderboards." + }) %> +
+ +
+
+ Filter intervals +
+ + + + +
+ + Clear +
+
+
+
+ +
+
+

Recorded intervals

+ <%= intervals.length %> shown +
+ <% if (!intervals.length) { %> +

No matching intervals have been recorded.

+ <% } else { %> +
+ + + + + + + + + + + + + <% intervals.forEach((interval) => { %> + + + + + + + + + <% }) %> + +
StatePlatform / statisticUser / scopeStartEndTimestamp provenance
"><%= interval.end_at === null ? "Current" : "Previous" %><%= interval.platform %>
<%= interval.statistic_type %>		<%= interval.platform_user_id %>
Scope <%= interval.scope_id %>		 + <% if (interval.end_at === null) { %> + Active + <% } else { %> + + <% } %> + + <%= interval.start_source %> +
+ <%= interval.start_authoritative ? "Authoritative" : "First observed" %> · <%= interval.start_precision %> + <% if (interval.end_at !== null) { %> +
+ Ended by <%= interval.end_source || "unknown source" %> · <%= interval.end_authoritative ? "authoritative" : "observed" %> + <% } %> +
+
+ <% } %> +
+<%- include("partials/layout-bottom") %> diff --git a/src/web/views/leaderboards.ejs b/src/web/views/leaderboards.ejs index f3ce962..59ff039 100644 --- a/src/web/views/leaderboards.ejs +++ b/src/web/views/leaderboards.ejs @@ -40,7 +40,7 @@ <% if (rowType === "user" && entry.username) { %> - <%= entry.username %> + <%= entryLabel %> <% } else if (rowType === "command") { %> <% if (entry.href) { %> <%= entryLabel %> diff --git a/src/web/views/stats.ejs b/src/web/views/stats.ejs index 9b0ce33..5e38308 100644 --- a/src/web/views/stats.ejs +++ b/src/web/views/stats.ejs @@ -35,6 +35,25 @@ <% } %> +
+

Platform tenure

+ <% if (!tenureStats || !tenureStats.length) { %> +

Follow, membership, role, and server tenure will appear here as Lumi observes it.

+ <% } else { %> +
+ <% tenureStats.forEach((stat) => { %> +
+ <%= stat.label %> + <%= stat.total %> + <% if (stat.current) { %> + Current period: <%= stat.current %> + <% } %> +
+ <% }) %> +
+ <% } %> +
+

Expression Interaction

<% if (!expression) { %> diff --git a/update-manifest.json b/update-manifest.json index 857f25e..656cb6e 100644 --- a/update-manifest.json +++ b/update-manifest.json @@ -1,14 +1,14 @@ { "name": "Lumi Core", - "version": "0.3.9", + "version": "0.3.10", "channel": "stable", - "released_at": "2026-07-26", + "released_at": "2026-07-27", "compatible_from": "0.1.9", "migration_kind": "patch", "replaces_versions": [ "1.2.0" ], - "migration_notes": "Restores animated emotes, Twitch badge artwork, and Discord GIF media through the existing OBS and native Companion overlay paths; adds persistent Auto VC welcome controls and timed ownership claiming; and releases Companion 0.2.7. Existing settings, lobbies, rooms, owners, permissions, pairing identities, plugin data, OBS configuration, databases, uploads, models, and secrets remain preserved.", + "migration_notes": "Adds reusable platform-tenure interval history for Twitch, YouTube, and Discord; contextual current and total duration placeholders with deterministic formatting; safe platform reconciliation; admin diagnostics; and Platform tenure sections on Stats and Leaderboards. Existing settings, identities, statistics, intervals, plugins, pairing records, databases, uploads, models, and secrets remain preserved.", "rollback_safe": true, "requirements": [ "Node.js 18 or newer" @@ -457,6 +457,18 @@ ], "rollback_safe": true, "migration_notes": "Adds the native Windows Lumi Overlay through the existing paired-device and provider pipelines, preserves active and revoked device semantics, and releases Companion 0.2.6 with draft-safe settings, direct single-page plugin navigation, and clearer overlay controls. Existing settings, pairing identities, plugin data, OBS configuration, databases, uploads, models, and secrets remain preserved." + }, + { + "version": "0.3.9", + "channel": "stable", + "released_at": "2026-07-26", + "compatible_from": "0.1.9", + "migration_kind": "patch", + "replaces_versions": [ + "1.2.0" + ], + "rollback_safe": true, + "migration_notes": "Restores animated emotes, Twitch badge artwork, and Discord GIF media through the existing OBS and native Companion overlay paths; adds persistent Auto VC welcome controls and timed ownership claiming; and releases Companion 0.2.7. Existing settings, lobbies, rooms, owners, permissions, pairing identities, plugin data, OBS configuration, databases, uploads, models, and secrets remain preserved." } ] }