From 2d8dc86b2050811e70516f2c6f81b0d7e634d646 Mon Sep 17 00:00:00 2001 From: Franz Rolfsvaag Date: Sun, 26 Jul 2026 17:22:10 +0200 Subject: [PATCH] feat: complete companion streaming and logging upgrades --- .env.example | 6 +- companion/README.md | 18 + companion/native/obs-bridge/src/plugin.cpp | 14 +- .../Providers/SpotifyWindowsMediaProvider.cs | 14 +- .../SongOverlayRuntime.cs | 43 +- .../Spotify/SpotifyWebApiEnricher.cs | 6 +- .../CompanionLogSanitizer.cs | 53 +++ companion/src/Lumi.Companion.App/App.axaml | 7 +- .../Lumi.Companion.App/CompanionRuntime.cs | 429 +++++++++++++++--- .../CompanionSettingsStore.cs | 1 + .../src/Lumi.Companion.App/MainWindow.axaml | 10 + .../Lumi.Companion.App/MainWindow.axaml.cs | 130 +++++- .../TranscriptionPluginContribution.cs | 20 +- .../Lumi.Companion.Core.Tests/Program.cs | 13 +- docs/LOGGING_STANDARD.md | 56 ++- docs/lumi-companion-transcription.md | 11 +- docs/stream-testing.md | 57 ++- package-lock.json | 374 +++++++++++++++ package.json | 1 + plugins/auto-vc/index.js | 18 +- plugins/birthday/index.js | 10 +- plugins/economy-framework/index.js | 10 +- plugins/expression-interaction/index.js | 6 +- plugins/lumi_ai/index.js | 36 +- .../backend/companion/gateway.js | 6 + .../backend/companion/protocol.js | 2 +- .../backend/logs/jsonl_log.js | 17 +- .../backend/sessions/session_coordinator.js | 8 +- plugins/lumi_transcription/tests/verify.js | 23 +- plugins/now_playing/index.js | 13 +- plugins/throne_wishlist/index.js | 47 +- plugins/welcome_messages/index.js | 144 +++--- scripts/verify-local-development-updates.js | 2 + scripts/verify-logging.js | 163 ++++++- scripts/verify-stream-testing.js | 230 +++++++++- src/main.js | 83 ++-- src/services/logger.js | 7 +- src/services/lumi-events.js | 12 +- src/services/overlay-routes.js | 2 +- src/services/plugin-stats.js | 23 +- src/services/production-diagnostics.js | 4 +- src/services/stream-test-certificates.js | 223 +++++++++ src/services/stream-testing.js | 157 ++++++- src/services/twitch-eventsub.js | 8 +- src/services/updater.js | 6 +- src/services/webhooks.js | 18 +- src/web/public/stream-testing.css | 40 ++ src/web/public/stream-testing.js | 180 +++++++- src/web/server.js | 73 ++- src/web/views/admin-stream-testing.ejs | 3 +- 50 files changed, 2449 insertions(+), 388 deletions(-) create mode 100644 companion/src/Lumi.Companion.Abstractions/CompanionLogSanitizer.cs create mode 100644 src/services/stream-test-certificates.js diff --git a/.env.example b/.env.example index 90232bf..b562c12 100644 --- a/.env.example +++ b/.env.example @@ -55,9 +55,11 @@ LUMI_OPERATOR_PRIVACY_URL= # LUMI_STREAM_TEST_INGEST_HOST=lumi.example.com # LUMI_STREAM_TEST_INGEST_PORT=19350 # LUMI_STREAM_TEST_PUBLIC_PORT=19350 -# LUMI_STREAM_TEST_TRANSPORT=rtmps +# Production always uses RTMPS and Lumi manages its certificate automatically. +# Advanced certificate override only; set both or neither. # LUMI_STREAM_TEST_TLS_CERT=/absolute/path/to/fullchain.pem # LUMI_STREAM_TEST_TLS_KEY=/absolute/path/to/private-key.pem -# LUMI_STREAM_TEST_ALLOW_INSECURE_REMOTE=false +# Advanced ACME directory override only. +# LUMI_STREAM_TEST_ACME_DIRECTORY=https://acme-v02.api.letsencrypt.org/directory # Advanced/manual runtime override only; normal installations do not need this. # LUMI_MEDIAMTX_PATH=/absolute/path/to/mediamtx diff --git a/companion/README.md b/companion/README.md index 0bbc8ba..442b8a8 100644 --- a/companion/README.md +++ b/companion/README.md @@ -48,10 +48,25 @@ recording. Applying an update replaces the installed executable, bundled components, and legal bundle, then restarts Companion. Pairing credentials, settings, and plugin state remain in the per-user data directory. +If the paired Lumi host is temporarily unavailable, Companion remains paired and +reconnects quietly in the background with bounded backoff. The Overview and +Connection pages show the offline state without repeated dialogs; **Retry now** +remains available for an immediate manual attempt. + Localhost development builds can also receive checksum-addressed same-version updates without publishing a release. See [Localhost development updates](../docs/local-development-updates.md). +## Transcription control + +The Transcription page has a persistent **Generate and include captions** switch, +also available from the tray menu. Turning it off stops the active transcription +session, excludes captions from live and private-test output, and stops sending +microphone audio to Lumi for speech recognition. Private Stream Testing remains +available as a video-only test. Turning the switch back on restores automatic +caption startup with OBS and can start captions while a private test is already +running. + ## Private Stream Testing The core **Stream Testing** page asks the paired Lumi host for an expiring @@ -61,6 +76,9 @@ encoded video and audio to the existing paired Lumi hostname on the configured RTMP/RTMPS port; Companion does not need a second login, destination editor, or local media runtime. +While a private test is active, Companion exposes **Open stream viewer**, which +opens the paired Lumi host directly at **Admin > Stream testing**. + Lumi owns and verifies the MediaMTX receiver. MediaMTX remuxes the source stream for same-origin Admin playback without transcoding or recording it. In v0.3.3, `Source` is therefore the normal and only quality choice unless a future diff --git a/companion/native/obs-bridge/src/plugin.cpp b/companion/native/obs-bridge/src/plugin.cpp index 28385f8..a3e408d 100644 --- a/companion/native/obs-bridge/src/plugin.cpp +++ b/companion/native/obs-bridge/src/plugin.cpp @@ -598,7 +598,7 @@ static std::optional handle_command(const json &message) if (type == "select_sources") { const auto uuid = message.value("primary_source_uuid", ""); const auto installed = select_source(uuid); - blog(installed ? LOG_INFO : LOG_WARNING, "[Lumi Companion] %s selected OBS audio source %s", + blog(installed ? LOG_INFO : LOG_WARNING, "[Lumi Companion] event=audio_source_selection_changed %s selected OBS audio source %s", installed ? "Attached" : "Could not attach", uuid.c_str()); return json{{"type", "selection_state"}, {"protocol_version", protocol_version}, {"source_uuid", uuid}, {"attached", installed}}; } else if (type == "caption" && message.contains("payload")) { @@ -682,7 +682,7 @@ static bool read_available_command(HANDLE pipe) const auto response = handle_command(json::parse(body.begin(), body.end())); if (response && !write_json(pipe, *response)) return false; } - catch (const std::exception &error) { blog(LOG_WARNING, "[Lumi Companion] Ignored invalid IPC command: %s", error.what()); } + catch (const std::exception &error) { blog(LOG_WARNING, "[Lumi Companion] event=ipc_command_rejected Ignored invalid IPC command: %s", error.what()); } return true; } @@ -703,7 +703,7 @@ static void run_pipe_worker() worker_signal.wait_for(lock, std::chrono::seconds(2)); continue; } - blog(LOG_INFO, "[Lumi Companion] connected to the same-user Companion IPC endpoint"); + blog(LOG_INFO, "[Lumi Companion] event=ipc_connected connected to the same-user Companion IPC endpoint"); bool connected = write_json(pipe, {{"type", "hello"}, {"protocol_version", protocol_version}, {"obs_version", obs_get_version_string()}, {"bridge_version", bridge_version}, {"obs_process_id", static_cast(GetCurrentProcessId())}}); @@ -736,7 +736,7 @@ static void run_pipe_worker() worker_signal.wait_for(lock, std::chrono::milliseconds(10)); } CloseHandle(pipe); - blog(LOG_INFO, "[Lumi Companion] disconnected from Companion IPC; retrying safely"); + blog(LOG_INFO, "[Lumi Companion] event=ipc_disconnected disconnected from Companion IPC; retrying safely"); } } @@ -772,13 +772,13 @@ bool obs_module_load(void) const char *version = obs_get_version_string(); const int major = version ? std::atoi(version) : 0; if (major < 31) { - blog(LOG_ERROR, "[Lumi Companion] OBS %s is unsupported; version 31 or newer is required", version ? version : "unknown"); + blog(LOG_ERROR, "[Lumi Companion] event=obs_version_unsupported OBS %s is unsupported; version 31 or newer is required", version ? version : "unknown"); return false; } stopping.store(false, std::memory_order_release); obs_frontend_add_event_callback(frontend_event, nullptr); worker_thread = std::thread(run_pipe_worker); - blog(LOG_INFO, "[Lumi Companion] bridge %s loaded", bridge_version); + blog(LOG_INFO, "[Lumi Companion] event=bridge_loaded bridge %s loaded", bridge_version); return true; } @@ -798,5 +798,5 @@ void obs_module_unload(void) stopping.store(true, std::memory_order_release); worker_signal.notify_all(); if (worker_thread.joinable()) worker_thread.join(); - blog(LOG_INFO, "[Lumi Companion] bridge unloaded"); + blog(LOG_INFO, "[Lumi Companion] event=bridge_unloaded bridge unloaded"); } diff --git a/companion/plugins/Lumi.Companion.SongOverlay/Providers/SpotifyWindowsMediaProvider.cs b/companion/plugins/Lumi.Companion.SongOverlay/Providers/SpotifyWindowsMediaProvider.cs index c1165ed..0d120e3 100644 --- a/companion/plugins/Lumi.Companion.SongOverlay/Providers/SpotifyWindowsMediaProvider.cs +++ b/companion/plugins/Lumi.Companion.SongOverlay/Providers/SpotifyWindowsMediaProvider.cs @@ -13,7 +13,7 @@ internal sealed class SpotifyWindowsMediaProvider : IMediaProvider private const int MaxCoverOutputBytes = 500 * 1024; private readonly Func _settings; private readonly Func> _enrich; - private readonly Action _log; + private readonly Action _log; private GlobalSystemMediaTransportControlsSessionManager? _manager; private GlobalSystemMediaTransportControlsSession? _session; private MediaSnapshot? _last; @@ -21,7 +21,7 @@ internal sealed class SpotifyWindowsMediaProvider : IMediaProvider private bool _started; private readonly SemaphoreSlim _refreshLock = new(1, 1); - public SpotifyWindowsMediaProvider(Func settings, Func> enrich, Action log) + public SpotifyWindowsMediaProvider(Func settings, Func> enrich, Action log) { _settings = settings; _enrich = enrich; @@ -62,7 +62,7 @@ internal sealed class SpotifyWindowsMediaProvider : IMediaProvider private async void OnSessionCollectionChanged(GlobalSystemMediaTransportControlsSessionManager sender, object args) { try { await SelectSessionAsync(CancellationToken.None).ConfigureAwait(false); } - catch (Exception error) { _log("Could not refresh Spotify media sessions", error); } + catch (Exception error) { _log("spotify_session_refresh_failed", "Could not refresh Spotify media sessions", error); } } private async Task SelectSessionAsync(CancellationToken cancellationToken) @@ -105,19 +105,19 @@ internal sealed class SpotifyWindowsMediaProvider : IMediaProvider private async void OnMediaPropertiesChanged(GlobalSystemMediaTransportControlsSession sender, MediaPropertiesChangedEventArgs args) { try { await RefreshAndPublishAsync("media", true, CancellationToken.None).ConfigureAwait(false); } - catch (Exception error) { _log("Spotify media-property update failed", error); } + catch (Exception error) { _log("spotify_media_property_update_failed", "Spotify media-property update failed", error); } } private async void OnPlaybackInfoChanged(GlobalSystemMediaTransportControlsSession sender, PlaybackInfoChangedEventArgs args) { try { await RefreshAndPublishAsync("playback", false, CancellationToken.None).ConfigureAwait(false); } - catch (Exception error) { _log("Spotify playback update failed", error); } + catch (Exception error) { _log("spotify_playback_update_failed", "Spotify playback update failed", error); } } private async void OnTimelinePropertiesChanged(GlobalSystemMediaTransportControlsSession sender, TimelinePropertiesChangedEventArgs args) { try { await RefreshAndPublishAsync("timeline", false, CancellationToken.None).ConfigureAwait(false); } - catch (Exception error) { _log("Spotify timeline update failed", error); } + catch (Exception error) { _log("spotify_timeline_update_failed", "Spotify timeline update failed", error); } } private async Task RefreshAndPublishAsync(string reason, bool enrichTrack, CancellationToken cancellationToken) @@ -206,7 +206,7 @@ internal sealed class SpotifyWindowsMediaProvider : IMediaProvider finally { _refreshLock.Release(); } await RefreshAndPublishAsync("metadata", false, CancellationToken.None).ConfigureAwait(false); } - catch (Exception error) { _log("Spotify metadata enrichment update failed", error); } + catch (Exception error) { _log("spotify_metadata_update_failed", "Spotify metadata enrichment update failed", error); } } private static bool IsSpotifySession(GlobalSystemMediaTransportControlsSession session) diff --git a/companion/plugins/Lumi.Companion.SongOverlay/SongOverlayRuntime.cs b/companion/plugins/Lumi.Companion.SongOverlay/SongOverlayRuntime.cs index 28dfd50..c6635e5 100644 --- a/companion/plugins/Lumi.Companion.SongOverlay/SongOverlayRuntime.cs +++ b/companion/plugins/Lumi.Companion.SongOverlay/SongOverlayRuntime.cs @@ -1,6 +1,7 @@ using Lumi.Companion.Abstractions; using Lumi.Companion.SongOverlay.Providers; using Lumi.Companion.SongOverlay.Spotify; +using System.Text.Json; namespace Lumi.Companion.SongOverlay; @@ -192,7 +193,7 @@ public sealed class SongOverlayRuntime : ICompanionPluginContribution, IAsyncDis } } catch (OperationCanceledException) { } - catch (Exception error) { Log("Song Overlay heartbeat failed", error); } + catch (Exception error) { Log("heartbeat_failed", "Song Overlay heartbeat failed", error); } }, token); } @@ -274,7 +275,7 @@ public sealed class SongOverlayRuntime : ICompanionPluginContribution, IAsyncDis catch (OperationCanceledException) { } catch (Exception error) { - Log("Could not process a Song Overlay event", error); + Log("event_processing_failed", "Could not process a Song Overlay event", error); SetStatus(CompanionPluginHealth.Warning, "Delivery delayed", "Playback was detected, but Lumi could not be updated: " + error.Message); } finally { _eventLock.Release(); } @@ -358,17 +359,49 @@ public sealed class SongOverlayRuntime : ICompanionPluginContribution, IAsyncDis } private void RaiseChanged() => Changed?.Invoke(); - private void Log(string message, Exception? error = null) + private void Log(string eventId, string message, Exception? error = null) { try { Directory.CreateDirectory(_logDirectory); - File.AppendAllText(Path.Combine(_logDirectory, $"song-overlay-{DateTime.UtcNow:yyyyMMdd}.log"), - $"{DateTimeOffset.Now:O}\t{message}{(error is null ? "" : "\t" + error)}{Environment.NewLine}"); + PruneLogs(); + var normalizedEvent = CompanionLogSanitizer.NormalizeEvent(eventId); + var line = JsonSerializer.Serialize(new + { + timestamp = DateTimeOffset.UtcNow, + level = CompanionLogSanitizer.LevelForEvent(normalizedEvent), + source = $"plugin:{PluginId}", + category = "plugin", + @event = normalizedEvent, + message = CompanionLogSanitizer.Sanitize(message), + error = error is null ? null : CompanionLogSanitizer.Sanitize(error.ToString()) + }); + File.AppendAllText( + Path.Combine(_logDirectory, $"song-overlay-{DateTime.UtcNow:yyyy-MM-dd}.jsonl"), + line + Environment.NewLine + ); } catch { } } + private void PruneLogs() + { + var files = new DirectoryInfo(_logDirectory) + .GetFiles("song-overlay-*.*") + .Where(file => file.Extension is ".jsonl" or ".log") + .OrderBy(file => file.CreationTimeUtc) + .ToList(); + foreach (var file in files.Where(file => file.CreationTimeUtc < DateTime.UtcNow.AddDays(-7))) file.Delete(); + const long cap = 32L * 1024 * 1024; + var total = files.Where(file => file.Exists).Sum(file => file.Length); + foreach (var file in files.Where(file => file.Exists)) + { + if (total <= cap) break; + total -= file.Length; + file.Delete(); + } + } + private static string Describe(MediaSnapshot snapshot) => snapshot.Track is null ? "Connected; no active song." : $"{snapshot.Status}: {snapshot.Track.Title} — {snapshot.Track.Artist}"; diff --git a/companion/plugins/Lumi.Companion.SongOverlay/Spotify/SpotifyWebApiEnricher.cs b/companion/plugins/Lumi.Companion.SongOverlay/Spotify/SpotifyWebApiEnricher.cs index a369496..af21711 100644 --- a/companion/plugins/Lumi.Companion.SongOverlay/Spotify/SpotifyWebApiEnricher.cs +++ b/companion/plugins/Lumi.Companion.SongOverlay/Spotify/SpotifyWebApiEnricher.cs @@ -16,12 +16,12 @@ internal sealed class SpotifyWebApiEnricher : IDisposable private readonly HttpClient _http = new() { Timeout = TimeSpan.FromSeconds(15) }; private readonly SongOverlaySecretProtector _secrets; private readonly Action _save; - private readonly Action _log; + private readonly Action _log; private readonly SongOverlaySettings _settings; private string _accessToken = ""; private DateTimeOffset _accessTokenExpiresAt = DateTimeOffset.MinValue; - public SpotifyWebApiEnricher(SongOverlaySettings settings, SongOverlaySecretProtector secrets, Action save, Action log) + public SpotifyWebApiEnricher(SongOverlaySettings settings, SongOverlaySecretProtector secrets, Action save, Action log) { _settings = settings; _secrets = secrets; @@ -130,7 +130,7 @@ internal sealed class SpotifyWebApiEnricher : IDisposable } catch (Exception error) { - _log("Spotify metadata enrichment failed", error); + _log("spotify_metadata_enrichment_failed", "Spotify metadata enrichment failed", error); return null; } } diff --git a/companion/src/Lumi.Companion.Abstractions/CompanionLogSanitizer.cs b/companion/src/Lumi.Companion.Abstractions/CompanionLogSanitizer.cs new file mode 100644 index 0000000..e097fd0 --- /dev/null +++ b/companion/src/Lumi.Companion.Abstractions/CompanionLogSanitizer.cs @@ -0,0 +1,53 @@ +using System.Text.RegularExpressions; + +namespace Lumi.Companion.Abstractions; + +public static partial class CompanionLogSanitizer +{ + public static string Sanitize(string? value, int maxLength = 4000) + { + var safeMaxLength = Math.Clamp(maxLength, 64, 65536); + var text = value ?? ""; + text = AuthorizationPattern().Replace(text, "$1 [redacted]"); + text = CookiePattern().Replace(text, "$1[redacted]"); + text = QuerySecretPattern().Replace(text, "$1[redacted]"); + text = AssignedSecretPattern().Replace(text, "$1[redacted]"); + return text.Length <= safeMaxLength ? text : text[..safeMaxLength] + "\n[truncated]"; + } + + public static string NormalizeEvent(string? value) + { + var normalized = InvalidEventCharacters().Replace((value ?? "").Trim().ToLowerInvariant(), "_"); + normalized = RepeatedUnderscores().Replace(normalized, "_").Trim('_'); + return string.IsNullOrWhiteSpace(normalized) ? "unknown_event" : normalized[..Math.Min(80, normalized.Length)]; + } + + public static string LevelForEvent(string eventId) + { + var normalized = NormalizeEvent(eventId); + if (normalized.EndsWith("_failed", StringComparison.Ordinal) || + normalized.EndsWith("_error", StringComparison.Ordinal)) + { + return "error"; + } + return normalized is "disconnected" or "audio_dropped" ? "warn" : "info"; + } + + [GeneratedRegex(@"\b(Bearer|Basic|LumiDevice)\s+[A-Za-z0-9._~+/=-]+", RegexOptions.IgnoreCase)] + private static partial Regex AuthorizationPattern(); + + [GeneratedRegex(@"\b((?:set-)?cookie\s*:\s*)[^\r\n]+", RegexOptions.IgnoreCase)] + private static partial Regex CookiePattern(); + + [GeneratedRegex(@"([?&](?:token|key|secret|password|authorization)=)[^&#\s]+", RegexOptions.IgnoreCase)] + private static partial Regex QuerySecretPattern(); + + [GeneratedRegex(@"\b((?:api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|password|passwd|secret|signature|authorization)\s*[:=]\s*)[^\s,;}]+", RegexOptions.IgnoreCase)] + private static partial Regex AssignedSecretPattern(); + + [GeneratedRegex(@"[^a-z0-9]+")] + private static partial Regex InvalidEventCharacters(); + + [GeneratedRegex(@"_+")] + private static partial Regex RepeatedUnderscores(); +} diff --git a/companion/src/Lumi.Companion.App/App.axaml b/companion/src/Lumi.Companion.App/App.axaml index 1121586..f9f53a1 100644 --- a/companion/src/Lumi.Companion.App/App.axaml +++ b/companion/src/Lumi.Companion.App/App.axaml @@ -73,10 +73,6 @@ - +