From 16bd442cb2d308a969330b1808695f0b2527ecd3 Mon Sep 17 00:00:00 2001 From: mibac138 <5672750+mibac138@users.noreply.github.com> Date: Sat, 28 Mar 2026 23:40:05 +0100 Subject: [PATCH 1/4] Add mod compatibility database caching --- Source/Client/ModCompatibilityManager.cs | 92 ++++++++++++++++++++++-- Source/Client/Multiplayer.cs | 1 + 2 files changed, 89 insertions(+), 4 deletions(-) diff --git a/Source/Client/ModCompatibilityManager.cs b/Source/Client/ModCompatibilityManager.cs index 8d5f1fd69..77a6178a7 100644 --- a/Source/Client/ModCompatibilityManager.cs +++ b/Source/Client/ModCompatibilityManager.cs @@ -1,7 +1,13 @@ +#nullable enable using System; using System.Collections.Generic; +using System.Diagnostics; +using System.IO; using System.Linq; +using System.Net; using System.Threading.Tasks; +using LudeonTK; +using Multiplayer.Common; using RestSharp; using Steamworks; using Verse; @@ -16,19 +22,97 @@ public static class ModCompatibilityManager private static Dictionary workshopLookup = new(); public static Dictionary nameLookup = new(); + private const string CacheFileName = "compat.json"; + private static readonly string CacheFilePath = Path.Combine(Multiplayer.CacheDir, CacheFileName); + private class CacheRoot + { + public string? CachedDate { get; set; } + public string? CachedETag { get; set; } + public List Mods { get; set; } + } + + private static async Task TryLoadCachedDb() + { + if (!File.Exists(CacheFilePath)) return null; + var data = await File.ReadAllTextAsync(CacheFilePath); + try + { + return SimpleJson.DeserializeObject(data); + } + catch (Exception e) + { + Log.Warning($"MP: Failed to deserialize {CacheFileName}:\n{e}"); + return null; + } + } + private static async Task TrySaveCachedDb(CacheRoot cache) + { + try + { + var data = SimpleJson.SerializeObject(cache); + await File.WriteAllTextAsync(CacheFilePath, data); + } + catch (Exception e) + { + Log.Warning($"MP: Failed to save cache {CacheFileName}:\n{e}"); + } + } + + [DebugAction(category = "Multiplayer", allowedGameStates = AllowedGameStates.Entry)] + private static void ClearModCompatCache() => File.Delete(CacheFilePath); + + [DebugAction(category = "Multiplayer", allowedGameStates = AllowedGameStates.Entry)] private static void UpdateModCompatibilityDb() { startedLazyFetch = true; - Task.Run(() => { + Task.Run(async () => + { var client = new RestClient("https://bot.rimworldmultiplayer.com/"); + client.AddDefaultHeader("X-Multiplayer-Version", MpVersion.Version); try { + var cached = await TryLoadCachedDb(); var req = new RestRequest("mod-compatibility?version=1.1&format=metadata") { RequestFormat = DataFormat.Json }; - var response = client.Get>(req); - var modCompatibilities = response.Data; - Log.Message($"MP: successfully fetched {modCompatibilities.Count} mods compatibility info"); + if (cached?.CachedDate is { } date) + { + req.AddHeader("If-Modified-Since", date); + } + if (cached?.CachedETag is { } etag) + { + req.AddHeader("If-None-Match", etag); + } + + var stopwatch = Stopwatch.StartNew(); + var resp = client.Get(req); + if (resp.ErrorException != null) throw resp.ErrorException; + List modCompatibilities; + if (resp.StatusCode == HttpStatusCode.NotModified) + { + modCompatibilities = cached!.Mods; + } + else + { + if (resp.StatusCode != HttpStatusCode.OK) + { + Log.Warning($"MP: received unexpected status code {resp.StatusCode} when fetching mod compatibility. Headers: {resp.Headers}"); + } + modCompatibilities = SimpleJson.DeserializeObject>(resp.Content); + var cacheRoot = new CacheRoot + { + CachedDate = resp.Headers + .FirstOrDefault(header => header.Name.EqualsIgnoreCase("Last-Modified")) + ?.Value?.ToString(), + CachedETag = resp.Headers + .FirstOrDefault(header => header.Name.EqualsIgnoreCase("ETag")) + ?.Value?.ToString(), + Mods = modCompatibilities + }; + _ = Task.Run(async () => await TrySaveCachedDb(cacheRoot)); + } + var elapsed = stopwatch.Elapsed; + Log.Message($"MP: successfully fetched {modCompatibilities.Count} mods compatibility info in {elapsed}"); workshopLookup = modCompatibilities .Where(mod => mod.workshopId != 0) diff --git a/Source/Client/Multiplayer.cs b/Source/Client/Multiplayer.cs index 0da465d9d..6e10790dd 100644 --- a/Source/Client/Multiplayer.cs +++ b/Source/Client/Multiplayer.cs @@ -75,6 +75,7 @@ public static class Multiplayer public static string ReplaysDir => GenFilePaths.FolderUnderSaveData("MpReplays"); public static string DesyncsDir => GenFilePaths.FolderUnderSaveData("MpDesyncs"); public static string LogsDir => GenFilePaths.FolderUnderSaveData("MpLogs"); + public static string CacheDir => GenFilePaths.FolderUnderSaveData("MpCache"); public static Stopwatch clock = Stopwatch.StartNew(); From 2696ff5970c7a457cf240253e7da4348eea24310 Mon Sep 17 00:00:00 2001 From: mibac138 <5672750+mibac138@users.noreply.github.com> Date: Sun, 29 Mar 2026 16:57:47 +0200 Subject: [PATCH 2/4] Automatically initialize ModCompatibilityManager on class load --- Source/Client/ModCompatibilityManager.cs | 138 +++++++++++------------ 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/Source/Client/ModCompatibilityManager.cs b/Source/Client/ModCompatibilityManager.cs index 77a6178a7..cbbbdbd85 100644 --- a/Source/Client/ModCompatibilityManager.cs +++ b/Source/Client/ModCompatibilityManager.cs @@ -16,7 +16,11 @@ namespace Multiplayer.Client { public static class ModCompatibilityManager { - private static bool startedLazyFetch; + static ModCompatibilityManager() + { + Task.Run(UpdateModCompatibilityDb); + } + public static bool? fetchSuccess; private static Dictionary workshopLookup = new(); @@ -60,76 +64,80 @@ private static async Task TrySaveCachedDb(CacheRoot cache) [DebugAction(category = "Multiplayer", allowedGameStates = AllowedGameStates.Entry)] private static void ClearModCompatCache() => File.Delete(CacheFilePath); - [DebugAction(category = "Multiplayer", allowedGameStates = AllowedGameStates.Entry)] - private static void UpdateModCompatibilityDb() { - startedLazyFetch = true; + private static void UpdateModCompatCache() => Task.Run(UpdateModCompatibilityDb); - Task.Run(async () => + private static async Task UpdateModCompatibilityDb() + { + var client = new RestClient("https://bot.rimworldmultiplayer.com/"); + client.AddDefaultHeader("X-Multiplayer-Version", MpVersion.Version); + try { - var client = new RestClient("https://bot.rimworldmultiplayer.com/"); - client.AddDefaultHeader("X-Multiplayer-Version", MpVersion.Version); - try { - var cached = await TryLoadCachedDb(); - var req = new RestRequest("mod-compatibility?version=1.1&format=metadata") - { - RequestFormat = DataFormat.Json - }; - if (cached?.CachedDate is { } date) - { - req.AddHeader("If-Modified-Since", date); - } - if (cached?.CachedETag is { } etag) - { - req.AddHeader("If-None-Match", etag); - } + var cached = await TryLoadCachedDb(); + var req = new RestRequest("mod-compatibility?version=1.1&format=metadata") + { + RequestFormat = DataFormat.Json + }; + if (cached?.CachedDate is { } date) + { + req.AddHeader("If-Modified-Since", date); + } + + if (cached?.CachedETag is { } etag) + { + req.AddHeader("If-None-Match", etag); + } - var stopwatch = Stopwatch.StartNew(); - var resp = client.Get(req); - if (resp.ErrorException != null) throw resp.ErrorException; - List modCompatibilities; - if (resp.StatusCode == HttpStatusCode.NotModified) + var stopwatch = Stopwatch.StartNew(); + var resp = client.Get(req); + if (resp.ErrorException != null) throw resp.ErrorException; + List modCompatibilities; + if (resp.StatusCode == HttpStatusCode.NotModified) + { + modCompatibilities = cached!.Mods; + } + else + { + if (resp.StatusCode != HttpStatusCode.OK) { - modCompatibilities = cached!.Mods; + Log.Warning( + $"MP: received unexpected status code {resp.StatusCode} when fetching mod compatibility. Headers: {resp.Headers}"); } - else + + modCompatibilities = SimpleJson.DeserializeObject>(resp.Content); + var cacheRoot = new CacheRoot { - if (resp.StatusCode != HttpStatusCode.OK) - { - Log.Warning($"MP: received unexpected status code {resp.StatusCode} when fetching mod compatibility. Headers: {resp.Headers}"); - } - modCompatibilities = SimpleJson.DeserializeObject>(resp.Content); - var cacheRoot = new CacheRoot - { - CachedDate = resp.Headers - .FirstOrDefault(header => header.Name.EqualsIgnoreCase("Last-Modified")) - ?.Value?.ToString(), - CachedETag = resp.Headers - .FirstOrDefault(header => header.Name.EqualsIgnoreCase("ETag")) - ?.Value?.ToString(), - Mods = modCompatibilities - }; - _ = Task.Run(async () => await TrySaveCachedDb(cacheRoot)); - } - var elapsed = stopwatch.Elapsed; - Log.Message($"MP: successfully fetched {modCompatibilities.Count} mods compatibility info in {elapsed}"); + CachedDate = resp.Headers + .FirstOrDefault(header => header.Name.EqualsIgnoreCase("Last-Modified")) + ?.Value?.ToString(), + CachedETag = resp.Headers + .FirstOrDefault(header => header.Name.EqualsIgnoreCase("ETag")) + ?.Value?.ToString(), + Mods = modCompatibilities + }; + _ = Task.Run(async () => await TrySaveCachedDb(cacheRoot)); + } - workshopLookup = modCompatibilities - .Where(mod => mod.workshopId != 0) - .GroupBy(mod => mod.workshopId) - .ToDictionary(grouping => grouping.Key, grouping => grouping.First()); + var elapsed = stopwatch.Elapsed; + Log.Message( + $"MP: successfully fetched {modCompatibilities.Count} mods compatibility info in {elapsed}"); - nameLookup = modCompatibilities - .GroupBy(mod => mod.name.ToLower()) - .ToDictionary(grouping => grouping.Key, grouping => grouping.First()); + workshopLookup = modCompatibilities + .Where(mod => mod.workshopId != 0) + .GroupBy(mod => mod.workshopId) + .ToDictionary(grouping => grouping.Key, grouping => grouping.First()); - fetchSuccess = true; - } - catch (Exception e) { - Log.Warning($"MP: updating mod compatibility list failed {e.Message} {e.StackTrace}"); - fetchSuccess = false; - } - }); + nameLookup = modCompatibilities + .GroupBy(mod => mod.name.ToLower()) + .ToDictionary(grouping => grouping.Key, grouping => grouping.First()); + + fetchSuccess = true; + } + catch (Exception e) + { + Log.Warning($"MP: updating mod compatibility list failed {e.Message} {e.StackTrace}"); + fetchSuccess = false; + } } public static ModCompatibility LookupByWorkshopId(PublishedFileId_t workshopId) { @@ -137,18 +145,10 @@ public static ModCompatibility LookupByWorkshopId(PublishedFileId_t workshopId) } public static ModCompatibility LookupByWorkshopId(ulong workshopId) { - if (!startedLazyFetch) { - UpdateModCompatibilityDb(); - } - return workshopLookup.TryGetValue((long) workshopId); } public static ModCompatibility LookupByName(string name) { - if (!startedLazyFetch) { - UpdateModCompatibilityDb(); - } - return nameLookup.TryGetValue(name.ToLower()); } } From 1ebc60fb032427178eecf020253c4e4191085a1c Mon Sep 17 00:00:00 2001 From: mibac138 <5672750+mibac138@users.noreply.github.com> Date: Sun, 29 Mar 2026 17:16:15 +0200 Subject: [PATCH 3/4] Show mod compatibility data from cache until the HTTP request finishes --- Source/Client/ModCompatibilityManager.cs | 57 ++++++++++++++++-------- 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/Source/Client/ModCompatibilityManager.cs b/Source/Client/ModCompatibilityManager.cs index cbbbdbd85..1c93df161 100644 --- a/Source/Client/ModCompatibilityManager.cs +++ b/Source/Client/ModCompatibilityManager.cs @@ -32,7 +32,7 @@ private class CacheRoot { public string? CachedDate { get; set; } public string? CachedETag { get; set; } - public List Mods { get; set; } + public List? Mods { get; set; } } private static async Task TryLoadCachedDb() @@ -63,10 +63,21 @@ private static async Task TrySaveCachedDb(CacheRoot cache) } [DebugAction(category = "Multiplayer", allowedGameStates = AllowedGameStates.Entry)] - private static void ClearModCompatCache() => File.Delete(CacheFilePath); + private static void ClearCompatCacheFile() => File.Delete(CacheFilePath); + [DebugAction(category = "Multiplayer", allowedGameStates = AllowedGameStates.Entry)] + private static void ClearLoadedCompatData() { + workshopLookup.Clear(); + nameLookup.Clear(); + fetchSuccess = null; + } + [DebugAction(category = "Multiplayer", allowedGameStates = AllowedGameStates.Entry)] private static void UpdateModCompatCache() => Task.Run(UpdateModCompatibilityDb); + // Requires clearing loaded compat data to work (because of the static constructor which runs before it's + // possible to switch this value). + [TweakValue(category: "Multiplayer")] private static bool simulateOffline = false; + private static async Task UpdateModCompatibilityDb() { var client = new RestClient("https://bot.rimworldmultiplayer.com/"); @@ -74,6 +85,14 @@ private static async Task UpdateModCompatibilityDb() try { var cached = await TryLoadCachedDb(); + if (cached?.Mods != null) + { + ServerLog.Log("MP: displaying cached mod compat while updating..."); + SetupFrom(cached.Mods); + } + + if (simulateOffline) throw new Exception("Simulating offline state"); + var req = new RestRequest("mod-compatibility?version=1.1&format=metadata") { RequestFormat = DataFormat.Json @@ -82,7 +101,6 @@ private static async Task UpdateModCompatibilityDb() { req.AddHeader("If-Modified-Since", date); } - if (cached?.CachedETag is { } etag) { req.AddHeader("If-None-Match", etag); @@ -91,10 +109,10 @@ private static async Task UpdateModCompatibilityDb() var stopwatch = Stopwatch.StartNew(); var resp = client.Get(req); if (resp.ErrorException != null) throw resp.ErrorException; - List modCompatibilities; + List? modCompatibilities; if (resp.StatusCode == HttpStatusCode.NotModified) { - modCompatibilities = cached!.Mods; + modCompatibilities = cached!.Mods!; } else { @@ -118,28 +136,31 @@ private static async Task UpdateModCompatibilityDb() _ = Task.Run(async () => await TrySaveCachedDb(cacheRoot)); } - var elapsed = stopwatch.Elapsed; - Log.Message( - $"MP: successfully fetched {modCompatibilities.Count} mods compatibility info in {elapsed}"); - - workshopLookup = modCompatibilities - .Where(mod => mod.workshopId != 0) - .GroupBy(mod => mod.workshopId) - .ToDictionary(grouping => grouping.Key, grouping => grouping.First()); - - nameLookup = modCompatibilities - .GroupBy(mod => mod.name.ToLower()) - .ToDictionary(grouping => grouping.Key, grouping => grouping.First()); + Log.Message($"MP: successfully fetched {modCompatibilities.Count} mods compatibility info " + + $"in {stopwatch.Elapsed}"); + SetupFrom(modCompatibilities); fetchSuccess = true; } catch (Exception e) { - Log.Warning($"MP: updating mod compatibility list failed {e.Message} {e.StackTrace}"); + Log.Warning($"MP: updating mod compatibility list failed:\n{e}"); fetchSuccess = false; } } + private static void SetupFrom(List mods) + { + workshopLookup = mods + .Where(mod => mod.workshopId != 0) + .GroupBy(mod => mod.workshopId) + .ToDictionary(grouping => grouping.Key, grouping => grouping.First()); + + nameLookup = mods + .GroupBy(mod => mod.name.ToLower()) + .ToDictionary(grouping => grouping.Key, grouping => grouping.First()); + } + public static ModCompatibility LookupByWorkshopId(PublishedFileId_t workshopId) { return LookupByWorkshopId(workshopId.m_PublishedFileId); } From ea397707493573ad12cb2a19744fc669e109051a Mon Sep 17 00:00:00 2001 From: mibac138 <5672750+mibac138@users.noreply.github.com> Date: Sun, 29 Mar 2026 17:19:05 +0200 Subject: [PATCH 4/4] Small cleanup --- Source/Client/ModCompatibilityManager.cs | 28 ++++++++++++++---------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/Source/Client/ModCompatibilityManager.cs b/Source/Client/ModCompatibilityManager.cs index 1c93df161..b9a7125a1 100644 --- a/Source/Client/ModCompatibilityManager.cs +++ b/Source/Client/ModCompatibilityManager.cs @@ -24,7 +24,7 @@ static ModCompatibilityManager() public static bool? fetchSuccess; private static Dictionary workshopLookup = new(); - public static Dictionary nameLookup = new(); + private static Dictionary nameLookup = new(); private const string CacheFileName = "compat.json"; private static readonly string CacheFilePath = Path.Combine(Multiplayer.CacheDir, CacheFileName); @@ -113,13 +113,15 @@ private static async Task UpdateModCompatibilityDb() if (resp.StatusCode == HttpStatusCode.NotModified) { modCompatibilities = cached!.Mods!; + Log.Message($"MP: successfully validated {modCompatibilities.Count} mods compatibility info " + + $"in {stopwatch.Elapsed}"); } else { if (resp.StatusCode != HttpStatusCode.OK) { Log.Warning( - $"MP: received unexpected status code {resp.StatusCode} when fetching mod compatibility. Headers: {resp.Headers}"); + $"MP: received unexpected status code {resp.StatusCode} when fetching mod compatibility. Headers: {resp.Headers.Join(", ")}"); } modCompatibilities = SimpleJson.DeserializeObject>(resp.Content); @@ -134,11 +136,10 @@ private static async Task UpdateModCompatibilityDb() Mods = modCompatibilities }; _ = Task.Run(async () => await TrySaveCachedDb(cacheRoot)); + Log.Message($"MP: successfully fetched {modCompatibilities.Count} mods compatibility info " + + $"in {stopwatch.Elapsed}"); } - Log.Message($"MP: successfully fetched {modCompatibilities.Count} mods compatibility info " + - $"in {stopwatch.Elapsed}"); - SetupFrom(modCompatibilities); fetchSuccess = true; } @@ -161,16 +162,19 @@ private static void SetupFrom(List mods) .ToDictionary(grouping => grouping.Key, grouping => grouping.First()); } - public static ModCompatibility LookupByWorkshopId(PublishedFileId_t workshopId) { - return LookupByWorkshopId(workshopId.m_PublishedFileId); - } + public static ModCompatibility? LookupByWorkshopId(PublishedFileId_t workshopId) => + LookupByWorkshopId(workshopId.m_PublishedFileId); - public static ModCompatibility LookupByWorkshopId(ulong workshopId) { - return workshopLookup.TryGetValue((long) workshopId); + public static ModCompatibility? LookupByWorkshopId(ulong workshopId) + { + workshopLookup.TryGetValue((long)workshopId, out var compat); + return compat; } - public static ModCompatibility LookupByName(string name) { - return nameLookup.TryGetValue(name.ToLower()); + public static ModCompatibility? LookupByName(string name) + { + nameLookup.TryGetValue(name.ToLower(), out var compat); + return compat; } }