From db1be8b9b13bec2a72d420b0a2cd010d61b069ea Mon Sep 17 00:00:00 2001 From: Phantomical Date: Mon, 17 Nov 2025 22:41:59 -0800 Subject: [PATCH 1/2] fix: Make ConfigNode parsing thread-safe The config node uses a couple of mutable global statics in order to avoid allocations. This means that if multiple threads attempt to load a config node at the same time then they will stomp all over each other. This commit fixes this issue by making the relevant statics `[ThreadStatic]`, so that each thread gets its own version. --- KSPCommunityFixes/Performance/ConfigNodePerf.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/KSPCommunityFixes/Performance/ConfigNodePerf.cs b/KSPCommunityFixes/Performance/ConfigNodePerf.cs index e3bea853..f793694b 100644 --- a/KSPCommunityFixes/Performance/ConfigNodePerf.cs +++ b/KSPCommunityFixes/Performance/ConfigNodePerf.cs @@ -27,10 +27,12 @@ private enum ParseMode const int _SaveBufferSize = 64 * 1024; const int _ReadBufferSize = 1024 * 1024; - private static readonly char[] _charBuf = new char[_ReadBufferSize]; + [ThreadStatic] + private static char[] _charBuf; static readonly UTF8Encoding _UTF8NoBOM = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); static readonly string _Newline = Environment.NewLine; - static readonly Stack _nodeStack = new Stack(128); + [ThreadStatic] + static Stack _nodeStack; public static bool _doClean = true; // so it is accessible from ModUpgradePipeline public static bool _AllowSkipIndent = false; // so it is accessible from other things if needed // This large-size stringbuilder is used for writing ConfigNodes to string. @@ -738,7 +740,7 @@ private static unsafe ConfigNode ReadFile(string path) if (fLength > _ReadBufferSize) chars = new char[fLength]; else - chars = _charBuf; + chars = _charBuf ??= new char[_ReadBufferSize]; numChars = reader.Read(chars, 0, chars.Length); } @@ -776,6 +778,7 @@ public static unsafe ConfigNode ParseConfigNode(char* pBase, int numChars) int pos = 0; string savedName = string.Empty; ParseMode mode = ParseMode.SkipToKey; + _nodeStack ??= new Stack(128); _nodeStack.Push(node); From 0471f35748beaaf769ad091687c6d97779d0c386 Mon Sep 17 00:00:00 2001 From: Phantomical Date: Tue, 18 Nov 2025 23:34:03 -0800 Subject: [PATCH 2/2] perf: Reparse the game database in parallel During startup FastLoader reloads the whole game database. This seems to take about 2s on my machine, though I have also see in vary quite a bit in other people's profiles. This PR parallelizes the directory traversal and config node parsing. The final result is that it now takes about 900ms to do. Tuning the parallelism for stock has actually been rather difficult. What I have here is that it recurses 3 directories deep, so: `GameData\ModName\InnerFolder`. This is probably undertuned for stock, since most of the work is in `GameData\Squad\Parts` but going deeper would likely result in too many tasks flying around (and, tbh, I expect that to already be the case on a large modded install already). --- KSPCommunityFixes/Performance/FastLoader.cs | 142 +++++++++++++++++++- 1 file changed, 141 insertions(+), 1 deletion(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index c744a53f..554e8093 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -31,6 +31,8 @@ using Debug = UnityEngine.Debug; using UnityEngine.Profiling; using System.Threading.Tasks; +using System.Runtime.Serialization; +using Unity.Profiling; namespace KSPCommunityFixes.Performance { @@ -239,6 +241,12 @@ private void Awake() MethodInfo m_DragCubeSystem_RenderDragCubes_MoveNext_Transpiler = AccessTools.Method(typeof(KSPCFFastLoader), nameof(DragCubeSystem_RenderDragCubes_MoveNext_Transpiler)); assetAndPartLoaderHarmony.Patch(m_DragCubeSystem_RenderDragCubes_MoveNext, null, null, new HarmonyMethod(m_DragCubeSystem_RenderDragCubes_MoveNext_Transpiler)); + // ApplicationRootPath isn't thread-safe so we patch it to instead load from a static variable + ApplicationRootPath = UrlDir.ApplicationRootPath; + MethodInfo m_UrlDir_ApplicationRootPath = AccessTools.PropertyGetter(typeof(UrlDir), nameof(UrlDir.ApplicationRootPath)); + MethodInfo o_UrlDir_ApplicationRootPath = AccessTools.Method(typeof(KSPCFFastLoader), nameof(UrlDir_ApplicationRootPath_Transpiler)); + assetAndPartLoaderHarmony.Patch(m_UrlDir_ApplicationRootPath, transpiler: new HarmonyMethod(o_UrlDir_ApplicationRootPath)); + expansionsLoaderHarmony = new Harmony(ExpansionsLoaderHarmonyID); MethodInfo m_ExpansionsLoader_StartLoad = AccessTools.Method(typeof(ExpansionsLoader), nameof(PartLoader.StartLoad)); MethodInfo p_ExpansionsLoader_StartLoad = AccessTools.Method(typeof(KSPCFFastLoader), nameof(ExpansionsLoader_StartLoad_Prefix)); @@ -449,7 +457,7 @@ static IEnumerator FastAssetLoader(List configFileTypes) // the fly from Awake() in a Startup.Instantly KSPAddon and have it being loaded. I've found // at least 2 mods doing that, so unfortunately this can't really be optimized... KSPCFFastLoaderReport.wSecondConfigLoad.Restart(); - gdb._root = new UrlDir(gdb.urlConfig.ToArray(), configFileTypes.ToArray()); + gdb._root = ConstructUrlDir(gdb.urlConfig, configFileTypes); KSPCFFastLoaderReport.wSecondConfigLoad.Stop(); // Optimized version of GameDatabase.translateLoadedNodes() @@ -2639,6 +2647,138 @@ private static bool GetPngCacheSize(string path, out int cacheSize, out bool isN #endregion + #region Parallel UrlDir Construction + private static string ApplicationRootPath; + + // It isn't safe to call UrlDir.ApplicationRootPath on other threads. + // This version uses a cached copy of the path. + private static IEnumerable UrlDir_ApplicationRootPath_Transpiler(IEnumerable _) + { + var field = typeof(KSPCFFastLoader) + .GetField(nameof(ApplicationRootPath), BindingFlags.Static | BindingFlags.NonPublic); + + return new CodeInstruction[] + { + new CodeInstruction(OpCodes.Ldsfld, field), + new CodeInstruction(OpCodes.Ret) + }; + } + + static readonly ProfilerMarker ConstructUrlDirMarker = new ProfilerMarker("KSPCFFastLoader.ConstructUrlDir"); + + private static UrlDir ConstructUrlDir(List dirConfig, List fileConfig) + { + using var guard = ConstructUrlDirMarker.Auto(); + + var root = (UrlDir)FormatterServices.GetUninitializedObject(typeof(UrlDir)); + root._files = new List(); + root._parent = null; + root._root = root; + root._name = "root"; + root._type = DirectoryType.GameData; + + var children = new List>(); + foreach (var dir in dirConfig) + children.Add(ConstructUrlDir(root, dir)); + + var configDict = new Dictionary(); + foreach (var config in fileConfig) + { + foreach (var extension in config.extensions) + configDict.TryAdd(extension, config.type); + } + + root._children = new List(Task.WhenAll(children).Result); + + foreach (var file in root.AllFiles) + ConfigureUrlFile(file, configDict); + + return root; + } + + private static Task ConstructUrlDir(UrlDir parent, ConfigDirectory rootInfo) + { + var urldir = (UrlDir)FormatterServices.GetUninitializedObject(typeof(UrlDir)); + var info = Directory.CreateDirectory(CreateApplicationPath(rootInfo.directory)); + urldir._name = rootInfo.urlRoot; + urldir._type = rootInfo.type; + + return Task.Run(() => UrlDirCreate(urldir, parent, info, 1)); + } + + private static Task ConstructUrlDir(UrlDir parent, DirectoryInfo info, int depth) + { + var urldir = (UrlDir)FormatterServices.GetUninitializedObject(typeof(UrlDir)); + urldir._name = info.Name; + urldir._type = parent.type; + + if (depth < 3) + return UrlDirCreate(urldir, parent, info, depth + 1); + + urldir.Create(parent, info); + return Task.FromResult(urldir); + } + +#if ENABLE_PROFILER + static readonly int UrlDirPrefixLength = AppDomain.CurrentDomain.BaseDirectory.Length + 1; +#endif + private static Task UrlDirCreate(UrlDir urldir, UrlDir parent, DirectoryInfo info, int depth) + { + urldir._path = info.FullName; + urldir._parent = parent; + urldir._root = parent.root; + + return Task.Factory.StartNew( + () => + { +#if ENABLE_PROFILER + // Show a profile sample as UrlDirCreate: + var path = urldir._path.Length > UrlDirPrefixLength + ? urldir._path.Substring(UrlDirPrefixLength) + : urldir._path; + Profiler.BeginSample($"UrlDirCreate: {path}"); +#endif + + var directories = info.GetDirectories(); + var tasks = new List>(directories.Length); + foreach (var dir in directories) + { + if (dir.Name == ".svn" || dir.Name == "PluginData" || dir.Name == "zDeprecated") + continue; + tasks.Add(ConstructUrlDir(urldir, dir, depth)); + } + + var files = info.GetFiles(); + urldir._files = new List(files.Length); + foreach (var file in files) + urldir._files.Add(new UrlFile(urldir, file)); + + Task.WhenAll(tasks) + .ContinueWith( + children => urldir._children = new List(children.Result), + TaskContinuationOptions.AttachedToParent + ); + +#if ENABLE_PROFILER + Profiler.EndSample(); +#endif + + return urldir; + }, + TaskCreationOptions.AttachedToParent + ); + } + + private static void ConfigureUrlFile(UrlFile file, Dictionary configDict) + { + if (file._fileType != 0) + return; + + if (configDict.TryGetValue(file.fileExtension, out var type)) + file._fileType = type; + } + #endregion + #region Utility private static int GetDefaultMipMapCount(int height, int width)