Add mod compatibility database caching

Source/Client/ModCompatibilityManager.cs

@@ -1,7 +1,13 @@
+#nullable enable
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
 using System.Linq;
+using System.Net;
 using System.Threading.Tasks;
+using LudeonTK;
+using Multiplayer.Common;
 using RestSharp;
 using Steamworks;
 using Verse;
@@ -10,62 +16,165 @@ namespace Multiplayer.Client
 {
     public static class ModCompatibilityManager
     {
-        private static bool startedLazyFetch;
+        static ModCompatibilityManager()
+        {
+            Task.Run(UpdateModCompatibilityDb);
+        }
+
         public static bool? fetchSuccess;
 
         private static Dictionary<long, ModCompatibility> workshopLookup = new();
-        public static Dictionary<string, ModCompatibility> nameLookup = new();
+        private static Dictionary<string, ModCompatibility> nameLookup = new();
 
-        private static void UpdateModCompatibilityDb() {
-            startedLazyFetch = true;
+        private const string CacheFileName = "compat.json";
+        private static readonly string CacheFilePath = Path.Combine(Multiplayer.CacheDir, CacheFileName);
+        private class CacheRoot
+        {
+            public string? CachedDate { get; set; }
+            public string? CachedETag { get; set; }
+            public List<ModCompatibility>? Mods { get; set; }
+        }
 
-            Task.Run(() => {
-                var client = new RestClient("https://bot.rimworldmultiplayer.com/");
-                try {
-                    var req = new RestRequest("mod-compatibility?version=1.1&format=metadata")
-                    {
-                        RequestFormat = DataFormat.Json
-                    };
-                    var response = client.Get<List<ModCompatibility>>(req);
-                    var modCompatibilities = response.Data;
-                    Log.Message($"MP: successfully fetched {modCompatibilities.Count} mods compatibility info");
+        private static async Task<CacheRoot?> TryLoadCachedDb()
+        {
+            if (!File.Exists(CacheFilePath)) return null;
+            var data = await File.ReadAllTextAsync(CacheFilePath);
+            try
+            {
+                return SimpleJson.DeserializeObject<CacheRoot>(data);
+            }
+            catch (Exception e)
+            {
+                Log.Warning($"MP: Failed to deserialize {CacheFileName}:\n{e}");
+                return null;
+            }
+        }
+        private static async Task TrySaveCachedDb(CacheRoot cache)
+        {
+            try
+            {
+                var data = SimpleJson.SerializeObject(cache);
+                await File.WriteAllTextAsync(CacheFilePath, data);
+            }
+            catch (Exception e)
+            {
+                Log.Warning($"MP: Failed to save cache {CacheFileName}:\n{e}");
+            }
+        }
+
+        [DebugAction(category = "Multiplayer", allowedGameStates = AllowedGameStates.Entry)]
+        private static void ClearCompatCacheFile() => File.Delete(CacheFilePath);
+        [DebugAction(category = "Multiplayer", allowedGameStates = AllowedGameStates.Entry)]
+        private static void ClearLoadedCompatData() {
+            workshopLookup.Clear();
+            nameLookup.Clear();
+            fetchSuccess = null;
+        }
+
+        [DebugAction(category = "Multiplayer", allowedGameStates = AllowedGameStates.Entry)]
+        private static void UpdateModCompatCache() => Task.Run(UpdateModCompatibilityDb);
+
+        // Requires clearing loaded compat data to work (because of the static constructor which runs before it's
+        // possible to switch this value).
+        [TweakValue(category: "Multiplayer")] private static bool simulateOffline = false;
 
-                    workshopLookup = modCompatibilities
-                        .Where(mod => mod.workshopId != 0)
-                        .GroupBy(mod => mod.workshopId)
-                        .ToDictionary(grouping => grouping.Key, grouping => grouping.First());
+        private static async Task UpdateModCompatibilityDb()
+        {
+            var client = new RestClient("https://bot.rimworldmultiplayer.com/");
+            client.AddDefaultHeader("X-Multiplayer-Version", MpVersion.Version);
+            try
+            {
+                var cached = await TryLoadCachedDb();
+                if (cached?.Mods != null)
+                {
+                    ServerLog.Log("MP: displaying cached mod compat while updating...");
+                    SetupFrom(cached.Mods);
+                }
 
-                    nameLookup = modCompatibilities
-                        .GroupBy(mod => mod.name.ToLower())
-                        .ToDictionary(grouping => grouping.Key, grouping => grouping.First());
+                if (simulateOffline) throw new Exception("Simulating offline state");
 
-                    fetchSuccess = true;
+                var req = new RestRequest("mod-compatibility?version=1.1&format=metadata")
+                {
+                    RequestFormat = DataFormat.Json
+                };
+                if (cached?.CachedDate is { } date)
+                {
+                    req.AddHeader("If-Modified-Since", date);
                 }
-                catch (Exception e) {
-                    Log.Warning($"MP: updating mod compatibility list failed {e.Message} {e.StackTrace}");
-                    fetchSuccess = false;
+                if (cached?.CachedETag is { } etag)
+                {
+                    req.AddHeader("If-None-Match", etag);
                 }
-            });
-        }
 
-        public static ModCompatibility LookupByWorkshopId(PublishedFileId_t workshopId) {
-            return LookupByWorkshopId(workshopId.m_PublishedFileId);
-        }
+                var stopwatch = Stopwatch.StartNew();
+                var resp = client.Get(req);
+                if (resp.ErrorException != null) throw resp.ErrorException;
+                List<ModCompatibility>? modCompatibilities;
+                if (resp.StatusCode == HttpStatusCode.NotModified)
+                {
+                    modCompatibilities = cached!.Mods!;
+                    Log.Message($"MP: successfully validated {modCompatibilities.Count} mods compatibility info " +
+                                $"in {stopwatch.Elapsed}");
+                }
+                else
+                {
+                    if (resp.StatusCode != HttpStatusCode.OK)
+                    {
+                        Log.Warning(
+                            $"MP: received unexpected status code {resp.StatusCode} when fetching mod compatibility. Headers: {resp.Headers.Join(", ")}");
+                    }
 
-        public static ModCompatibility LookupByWorkshopId(ulong workshopId) {
-            if (!startedLazyFetch) {
-                UpdateModCompatibilityDb();
+                    modCompatibilities = SimpleJson.DeserializeObject<List<ModCompatibility>>(resp.Content);
+                    var cacheRoot = new CacheRoot
+                    {
+                        CachedDate = resp.Headers
+                            .FirstOrDefault(header => header.Name.EqualsIgnoreCase("Last-Modified"))
+                            ?.Value?.ToString(),
+                        CachedETag = resp.Headers
+                            .FirstOrDefault(header => header.Name.EqualsIgnoreCase("ETag"))
+                            ?.Value?.ToString(),
+                        Mods = modCompatibilities
+                    };
+                    _ = Task.Run(async () => await TrySaveCachedDb(cacheRoot));
+                    Log.Message($"MP: successfully fetched {modCompatibilities.Count} mods compatibility info " +
+                                $"in {stopwatch.Elapsed}");
+                }
+
+                SetupFrom(modCompatibilities);
+                fetchSuccess = true;
+            }
+            catch (Exception e)
+            {
+                Log.Warning($"MP: updating mod compatibility list failed:\n{e}");
+                fetchSuccess = false;
             }
+        }
+
+        private static void SetupFrom(List<ModCompatibility> mods)
+        {
+            workshopLookup = mods
+                .Where(mod => mod.workshopId != 0)
+                .GroupBy(mod => mod.workshopId)
+                .ToDictionary(grouping => grouping.Key, grouping => grouping.First());
 
-            return workshopLookup.TryGetValue((long) workshopId);
+            nameLookup = mods
+                .GroupBy(mod => mod.name.ToLower())
+                .ToDictionary(grouping => grouping.Key, grouping => grouping.First());
         }
 
-        public static ModCompatibility LookupByName(string name) {
-            if (!startedLazyFetch) {
-                UpdateModCompatibilityDb();
-            }
+        public static ModCompatibility? LookupByWorkshopId(PublishedFileId_t workshopId) =>
+            LookupByWorkshopId(workshopId.m_PublishedFileId);
+
+        public static ModCompatibility? LookupByWorkshopId(ulong workshopId)
+        {
+            workshopLookup.TryGetValue((long)workshopId, out var compat);
+            return compat;
+        }
 
-            return nameLookup.TryGetValue(name.ToLower());
+        public static ModCompatibility? LookupByName(string name)
+        {
+            nameLookup.TryGetValue(name.ToLower(), out var compat);
+            return compat;
         }
     }
 

Source/Client/Multiplayer.cs

@@ -75,6 +75,7 @@ public static class Multiplayer
         public static string ReplaysDir => GenFilePaths.FolderUnderSaveData("MpReplays");
         public static string DesyncsDir => GenFilePaths.FolderUnderSaveData("MpDesyncs");
         public static string LogsDir => GenFilePaths.FolderUnderSaveData("MpLogs");
+        public static string CacheDir => GenFilePaths.FolderUnderSaveData("MpCache");
 
         public static Stopwatch clock = Stopwatch.StartNew();