From 82fb55bdb3b4089b3b6b614a8697a3756061788c Mon Sep 17 00:00:00 2001 From: Sean Lynch Date: Wed, 29 Apr 2026 01:14:42 -0700 Subject: [PATCH 01/15] Initial texture loading overhaul prototype --- KSPCommunityFixes/Performance/FastLoader.cs | 1393 ++++++++++++------- 1 file changed, 893 insertions(+), 500 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index c744a53..893257a 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -22,6 +22,9 @@ using KSPCommunityFixes.Library; using TMPro; using Unity.Collections; +using Unity.Collections.LowLevel.Unsafe; +using Unity.IO.LowLevel.Unsafe; +using Unity.Profiling; using UnityEngine; using UnityEngine.Experimental.Rendering; using UnityEngine.Networking; @@ -151,6 +154,11 @@ internal class KSPCFFastLoader : MonoBehaviour // min amount of files to try to keep in memory, regardless of maxBufferSize private const int minFileRead = 10; + // max concurrent per-texture coroutines spawned by TextureDriverCoroutine. + // Each in-flight request holds at most one of {ReadHandle, UnityWebRequest, background Task}, + // so a single semaphore covers all resource bounds. + private const int MaxConcurrentTextures = 512; + private static Harmony persistentHarmony; private static string PersistentHarmonyID => typeof(KSPCFFastLoader).FullName; @@ -473,7 +481,7 @@ static IEnumerator FastAssetLoader(List configFileTypes) // Files loaded by our custom loaders List audioFiles = new List(1000); - List textureAssets = new List(10000); + List textureRequests = new List(10000); List modelAssets = new List(5000); // Files loaded by mod-defined loaders (ex : Shabby *.shab files) @@ -519,23 +527,23 @@ static IEnumerator FastAssetLoader(List configFileTypes) switch (file.fileExtension) { case "dds": - textureAssets.Add(new RawAsset(file, RawAsset.AssetType.TextureDDS)); + textureRequests.Add(new TextureLoadRequest(file, RawAsset.AssetType.TextureDDS)); break; case "jpg": case "jpeg": - textureAssets.Add(new RawAsset(file, RawAsset.AssetType.TextureJPG)); + textureRequests.Add(new TextureLoadRequest(file, RawAsset.AssetType.TextureJPG)); break; case "mbm": - textureAssets.Add(new RawAsset(file, RawAsset.AssetType.TextureMBM)); + textureRequests.Add(new TextureLoadRequest(file, RawAsset.AssetType.TextureMBM)); break; case "png": - textureAssets.Add(new RawAsset(file, RawAsset.AssetType.TexturePNG)); + textureRequests.Add(new TextureLoadRequest(file, RawAsset.AssetType.TexturePNG)); break; case "tga": - textureAssets.Add(new RawAsset(file, RawAsset.AssetType.TextureTGA)); + textureRequests.Add(new TextureLoadRequest(file, RawAsset.AssetType.TextureTGA)); break; case "truecolor": - textureAssets.Add(new RawAsset(file, RawAsset.AssetType.TextureTRUECOLOR)); + textureRequests.Add(new TextureLoadRequest(file, RawAsset.AssetType.TextureTRUECOLOR)); break; default: unsupportedTextureFiles.Add(file); @@ -567,16 +575,8 @@ static IEnumerator FastAssetLoader(List configFileTypes) } } - Thread textureCacheReaderThread; - if (textureCacheEnabled) - { - textureCacheReaderThread = new Thread(() => SetupTextureCacheThread(textureAssets)); - textureCacheReaderThread.Start(); - } - else - { - textureCacheReaderThread = null; - } + // PNG/DDS cache temporarily disabled — opt-in popup and settings are still + // wired up, but no cache I/O is performed during loading. gdb.progressTitle = "Loading sound assets..."; KSPCFFastLoaderReport.wAudioLoading.Restart(); @@ -668,6 +668,7 @@ static IEnumerator FastAssetLoader(List configFileTypes) // start texture loading gdb.progressFraction = 0.25f; KSPCFFastLoaderReport.wAudioLoading.Stop(); + SupportedFormatCache.Build(); KSPCFFastLoaderReport.wTextureLoading.Restart(); gdb.progressTitle = "Loading texture assets..."; yield return null; @@ -716,19 +717,9 @@ static IEnumerator FastAssetLoader(List configFileTypes) } } - if (textureCacheReaderThread != null) - { - while (textureCacheReaderThread.IsAlive) - yield return null; - } - // call our custom loader - yield return gdb.StartCoroutine(FilesLoader(textureAssets, allTextureFiles, "Loading texture asset")); - - // write texture cache json to disk - Thread writeTextureCacheThread = new Thread(() => loader.WriteTextureCache()); - writeTextureCacheThread.Start(); + yield return gdb.StartCoroutine(TextureDriverCoroutine(textureRequests, allTextureFiles)); // start model loading gdb.progressFraction = 0.75f; @@ -1057,13 +1048,10 @@ public enum Result } private UrlFile file; - private CachedTextureInfo cachedTextureInfo; private AssetType assetType; private bool useRentedBuffer; private byte[] buffer; private int dataLength; - private MemoryStream memoryStream; - private BinaryReader binaryReader; private Result result; private string resultMessage; @@ -1112,9 +1100,6 @@ public void ReadFromDiskWorkerThread() { switch (assetType) { - case AssetType.TextureDDS: - case AssetType.TextureMBM: - case AssetType.TextureTGA: case AssetType.ModelMU: case AssetType.ModelDAE: useRentedBuffer = true; @@ -1123,7 +1108,7 @@ public void ReadFromDiskWorkerThread() try { - string path = assetType == AssetType.TexturePNGCached ? cachedTextureInfo.FilePath : file.fullPath; + string path = file.fullPath; using (FileStream fileStream = System.IO.File.OpenRead(path)) { @@ -1184,54 +1169,7 @@ public void LoadAndDisposeMainThread() if (result == Result.Failed) return; - if (file.fileType == FileType.Texture) - { - TextureInfo textureInfo; - switch (assetType) - { - case AssetType.TextureDDS: - textureInfo = LoadDDS(); - break; - case AssetType.TextureJPG: - textureInfo = LoadJPG(); - break; - case AssetType.TextureMBM: - textureInfo = LoadMBM(); - break; - case AssetType.TexturePNG: - textureInfo = LoadPNG(); - break; - case AssetType.TexturePNGCached: - textureInfo = LoadPNGCached(); - break; - case AssetType.TextureTGA: - textureInfo = LoadTGA(); - break; - case AssetType.TextureTRUECOLOR: - textureInfo = LoadTRUECOLOR(); - break; - default: - SetError("Unknown texture format"); - return; - } - - if (result == Result.Failed || textureInfo == null || textureInfo.texture.IsNullOrDestroyed()) - { - result = Result.Failed; - if (string.IsNullOrEmpty(resultMessage)) - resultMessage = $"{TypeName} load error"; - } - else - { - textureInfo.name = file.url; - textureInfo.texture.name = file.url; - Instance.databaseTexture.Add(textureInfo); - texturesByUrl[file.url] = textureInfo; - KSPCFFastLoaderReport.texturesBytesLoaded += dataLength; - KSPCFFastLoaderReport.texturesLoaded++; - } - } - else if (file.fileType == FileType.Model) + if (file.fileType == FileType.Model) { GameObject model; switch (assetType) @@ -1284,516 +1222,969 @@ public void LoadAndDisposeMainThread() public void Dispose() { - if (binaryReader != null) - binaryReader.Dispose(); - - if (memoryStream != null) - memoryStream.Dispose(); - if (useRentedBuffer) arrayPool.Return(buffer); } - public void CheckTextureCache() + private GameObject LoadMU() { - CachedTextureInfo cachedTextureInfo = GetCachedTextureInfo(file); + return MuParser.Parse(file.parent.url, buffer, dataLength); + } - if (cachedTextureInfo == null) - return; + private GameObject LoadDAE() + { + // given that this is a quite obsolete thing and that it's mess to reimplement, just call the stock + // stuff and re-load the file - assetType = AssetType.TexturePNGCached; - this.cachedTextureInfo = cachedTextureInfo; - } - - // see https://learn.microsoft.com/en-us/windows/win32/direct3ddds/dx-graphics-dds-pguide - private enum DDSFourCC : uint - { - DXT1 = 0x31545844, // "DXT1" - DXT2 = 0x32545844, // "DXT2" - DXT3 = 0x33545844, // "DXT3" - DXT4 = 0x34545844, // "DXT4" - DXT5 = 0x35545844, // "DXT5" - BC4U_ATI = 0x31495441, // "ATI1" (actually BC4U) - BC4U = 0x55344342, // "BC4U" - BC4S = 0x53344342, // "BC4S" - BC5U_ATI = 0x32495441, // "ATI2" (actually BC5U) - BC5U = 0x55354342, // "BC5U" - BC5S = 0x53354342, // "BC5S" - RGBG = 0x47424752, // "RGBG" - GRGB = 0x42475247, // "GRGB" - UYVY = 0x59565955, // "UYVY" - YUY2 = 0x32595559, // "YUY2" - DX10 = 0x30315844, // "DX10", actual DXGI format specified in DX10 header - R16G16B16A16_UNORM = 36, - R16G16B16A16_SNORM = 110, - R16_FLOAT = 111, - R16G16_FLOAT = 112, - R16G16B16A16_FLOAT = 113, - R32_FLOAT = 114, - R32G32_FLOAT = 115, - R32G32B32A32_FLOAT = 116, - CxV8U8 = 117, - } - - private TextureInfo LoadDDS() - { - memoryStream = new MemoryStream(buffer, 0, dataLength); - binaryReader = new BinaryReader(memoryStream); - - if (binaryReader.ReadUInt32() != DDSValues.uintMagic) - { - SetError("DDS: File is not a DDS format file!"); - return null; - } - DDSHeader dDSHeader = new DDSHeader(binaryReader); - bool mipChain = (dDSHeader.dwCaps & DDSPixelFormatCaps.MIPMAP) != 0; - bool isNormalMap = (dDSHeader.ddspf.dwFlags & 0x80000u) != 0 || (dDSHeader.ddspf.dwFlags & 0x80000000u) != 0; - - DDSFourCC ddsFourCC = (DDSFourCC)dDSHeader.ddspf.dwFourCC; - Texture2D texture2D = null; - GraphicsFormat graphicsFormat = GraphicsFormat.None; - - switch (ddsFourCC) - { - case DDSFourCC.DXT1: - graphicsFormat = GraphicsFormatUtility.GetGraphicsFormat(TextureFormat.DXT1, true); - break; - case DDSFourCC.DXT5: - graphicsFormat = GraphicsFormatUtility.GetGraphicsFormat(TextureFormat.DXT5, true); - break; - case DDSFourCC.BC4U_ATI: - case DDSFourCC.BC4U: - graphicsFormat = GraphicsFormat.R_BC4_UNorm; - break; - case DDSFourCC.BC4S: - graphicsFormat = GraphicsFormat.R_BC4_SNorm; - break; - case DDSFourCC.BC5U_ATI: - case DDSFourCC.BC5U: - graphicsFormat = GraphicsFormat.RG_BC5_UNorm; - break; - case DDSFourCC.BC5S: - graphicsFormat = GraphicsFormat.RG_BC5_SNorm; - break; - case DDSFourCC.R16G16B16A16_UNORM: - graphicsFormat = GraphicsFormat.R16G16B16A16_UNorm; - break; - case DDSFourCC.R16G16B16A16_SNORM: - graphicsFormat = GraphicsFormat.R16G16B16A16_SNorm; - break; - case DDSFourCC.R16_FLOAT: - graphicsFormat = GraphicsFormat.R16_SFloat; - break; - case DDSFourCC.R16G16_FLOAT: - graphicsFormat = GraphicsFormat.R16G16_SFloat; - break; - case DDSFourCC.R16G16B16A16_FLOAT: - graphicsFormat = GraphicsFormat.R16G16B16A16_SFloat; - break; - case DDSFourCC.R32_FLOAT: - graphicsFormat = GraphicsFormat.R32_SFloat; - break; - case DDSFourCC.R32G32_FLOAT: - graphicsFormat = GraphicsFormat.R32G32_SFloat; - break; - case DDSFourCC.R32G32B32A32_FLOAT: - graphicsFormat = GraphicsFormat.R32G32B32A32_SFloat; - break; - case DDSFourCC.DX10: - DDSHeaderDX10 dx10Header = new DDSHeaderDX10(binaryReader); - switch (dx10Header.dxgiFormat) + GameObject gameObject = new DatabaseLoaderModel_DAE.DAE().Load(file, new FileInfo(file.fullPath)); + if (gameObject.IsNotNullOrDestroyed()) + { + MeshFilter[] componentsInChildren = gameObject.GetComponentsInChildren(); + foreach (MeshFilter meshFilter in componentsInChildren) + { + if (meshFilter.gameObject.name == "node_collider") { - case DXGI_FORMAT.DXGI_FORMAT_BC1_UNORM: - graphicsFormat = GraphicsFormat.RGBA_DXT1_UNorm; - break; - case DXGI_FORMAT.DXGI_FORMAT_BC1_UNORM_SRGB: - graphicsFormat = GraphicsFormat.RGBA_DXT1_SRGB; - break; - case DXGI_FORMAT.DXGI_FORMAT_BC3_UNORM: - graphicsFormat = GraphicsFormat.RGBA_DXT5_UNorm; - break; - case DXGI_FORMAT.DXGI_FORMAT_BC3_UNORM_SRGB: - graphicsFormat = GraphicsFormat.RGBA_DXT5_SRGB; - break; - case DXGI_FORMAT.DXGI_FORMAT_BC4_SNORM: - graphicsFormat = GraphicsFormat.R_BC4_SNorm; - break; - case DXGI_FORMAT.DXGI_FORMAT_BC4_UNORM: - graphicsFormat = GraphicsFormat.R_BC4_UNorm; - break; - case DXGI_FORMAT.DXGI_FORMAT_BC5_SNORM: - graphicsFormat = GraphicsFormat.RG_BC5_SNorm; - break; - case DXGI_FORMAT.DXGI_FORMAT_BC5_UNORM: - graphicsFormat = GraphicsFormat.RG_BC5_UNorm; - break; - case DXGI_FORMAT.DXGI_FORMAT_BC7_UNORM: - graphicsFormat = GraphicsFormat.RGBA_BC7_UNorm; - break; - case DXGI_FORMAT.DXGI_FORMAT_BC7_UNORM_SRGB: - graphicsFormat = GraphicsFormat.RGBA_BC7_SRGB; - break; - case DXGI_FORMAT.DXGI_FORMAT_BC6H_SF16: - graphicsFormat = GraphicsFormat.RGB_BC6H_SFloat; - break; - case DXGI_FORMAT.DXGI_FORMAT_BC6H_UF16: - graphicsFormat = GraphicsFormat.RGB_BC6H_UFloat; - break; - case DXGI_FORMAT.DXGI_FORMAT_R16G16B16A16_UNORM: - graphicsFormat = GraphicsFormat.R16G16B16A16_UNorm; - break; - case DXGI_FORMAT.DXGI_FORMAT_R16G16B16A16_SNORM: - graphicsFormat = GraphicsFormat.R16G16B16A16_SNorm; - break; - case DXGI_FORMAT.DXGI_FORMAT_R16_FLOAT: - graphicsFormat = GraphicsFormat.R16_SFloat; - break; - case DXGI_FORMAT.DXGI_FORMAT_R16G16_FLOAT: - graphicsFormat = GraphicsFormat.R16G16_SFloat; - break; - case DXGI_FORMAT.DXGI_FORMAT_R16G16B16A16_FLOAT: - graphicsFormat = GraphicsFormat.R16G16B16A16_SFloat; - break; - case DXGI_FORMAT.DXGI_FORMAT_R32_FLOAT: - graphicsFormat = GraphicsFormat.R32_SFloat; - break; - case DXGI_FORMAT.DXGI_FORMAT_R32G32_FLOAT: - graphicsFormat = GraphicsFormat.R32G32_SFloat; - break; - case DXGI_FORMAT.DXGI_FORMAT_R32G32B32A32_FLOAT: - graphicsFormat = GraphicsFormat.R32G32B32A32_SFloat; - break; - default: - SetError($"DDS: The '{dx10Header.dxgiFormat}' DXT10 format isn't supported"); - break; + meshFilter.gameObject.AddComponent().sharedMesh = meshFilter.mesh; + MeshRenderer component = meshFilter.gameObject.GetComponent(); + UnityEngine.Object.Destroy(meshFilter); + UnityEngine.Object.Destroy(component); } - break; - case DDSFourCC.DXT2: - case DDSFourCC.DXT3: - case DDSFourCC.DXT4: - case DDSFourCC.RGBG: - case DDSFourCC.GRGB: - case DDSFourCC.UYVY: - case DDSFourCC.YUY2: - case DDSFourCC.CxV8U8: - SetError($"DDS: The '{ddsFourCC}' format isn't supported, use DXT1 for RGB textures or DXT5 for RGBA textures"); - break; - default: - SetError($"DDS: Unknown dwFourCC format '0x{ddsFourCC:X}'"); - break; + } } - if (graphicsFormat != GraphicsFormat.None) - { - if (!SystemInfo.IsFormatSupported(graphicsFormat, FormatUsage.Sample)) + return gameObject; + } + + } + + #endregion + + #region Per-texture coroutine loader + + // Profiling markers for the work scheduled on background threads via Task.Run. + // Each marker.Auto() scope is opened inside the Task lambda so the timing + // appears under that thread in the Unity profiler. + private static readonly ProfilerMarker s_pmParseDDSHeader = new ProfilerMarker("KSPCF.Tex.ParseDDSHeader"); + private static readonly ProfilerMarker s_pmSwizzleNormalMap = new ProfilerMarker("KSPCF.Tex.SwizzleNormalMap"); + private static readonly ProfilerMarker s_pmCopyLevel0 = new ProfilerMarker("KSPCF.Tex.CopyLevel0"); + private static readonly ProfilerMarker s_pmFileSize = new ProfilerMarker("KSPCF.Tex.FileSize"); + private static readonly ProfilerMarker s_pmReadAllBytes = new ProfilerMarker("KSPCF.Tex.ReadAllBytes"); + + // Result/error carrier for each texture file. Replaces RawAsset for textures. + private sealed class TextureLoadRequest + { + public enum State : byte { Pending, Ready, Failed } + + public UrlFile File; + public RawAsset.AssetType AssetType; + public long FileLength; + public CachedTextureInfo CachedInfo; + public bool CameFromCache; + public volatile State Status; + public TextureInfo Result; + public string ErrorMessage; + public Exception Exception; + + public TextureLoadRequest(UrlFile file, RawAsset.AssetType assetType) + { + File = file; + AssetType = assetType; + Status = State.Pending; + } + + // Called from the texture cache reader thread before the driver runs. + public void CheckTextureCache() + { + CachedTextureInfo info = GetCachedTextureInfo(File); + if (info == null) + return; + AssetType = RawAsset.AssetType.TexturePNGCached; + CachedInfo = info; + CameFromCache = true; + } + } + + // Result of background DDS header parsing. + private struct DDSPreparedHeader + { + public int Width; + public int Height; + public bool MipChain; + public bool IsNormalMap; + public GraphicsFormat Format; + public long DataOffset; + public long FileLength; + } + + // Probes which GraphicsFormats are actually usable on the running GPU. + // Built once on the main thread before texture loading starts so that the + // background DDS header parser can produce a format and we can verify it + // against this set without needing main-thread access. + private static class SupportedFormatCache + { + private static HashSet supported; + + public static void Build() + { + supported = new HashSet(); + GraphicsFormat[] candidates = new[] + { + GraphicsFormat.RGBA_DXT1_UNorm, + GraphicsFormat.RGBA_DXT1_SRGB, + GraphicsFormat.RGBA_DXT5_UNorm, + GraphicsFormat.RGBA_DXT5_SRGB, + GraphicsFormat.R_BC4_UNorm, + GraphicsFormat.R_BC4_SNorm, + GraphicsFormat.RG_BC5_UNorm, + GraphicsFormat.RG_BC5_SNorm, + GraphicsFormat.RGBA_BC7_UNorm, + GraphicsFormat.RGBA_BC7_SRGB, + GraphicsFormat.RGB_BC6H_SFloat, + GraphicsFormat.RGB_BC6H_UFloat, + GraphicsFormat.R16G16B16A16_UNorm, + GraphicsFormat.R16G16B16A16_SNorm, + GraphicsFormat.R16G16B16A16_SFloat, + GraphicsFormat.R16_SFloat, + GraphicsFormat.R16G16_SFloat, + GraphicsFormat.R32_SFloat, + GraphicsFormat.R32G32_SFloat, + GraphicsFormat.R32G32B32A32_SFloat, + }; + foreach (GraphicsFormat fmt in candidates) + if (SystemInfo.IsFormatSupported(fmt, FormatUsage.Sample)) + supported.Add(fmt); + } + + public static bool IsSupported(GraphicsFormat fmt) => supported != null && supported.Contains(fmt); + } + + // Block-compressed format check used to ignore mipChain on NPOT textures + // (Unity rejects DXT5 + mipmaps on non-power-of-two sources). + private static bool IsBlockCompressed(GraphicsFormat fmt) + { + switch (fmt) + { + case GraphicsFormat.RGBA_DXT1_UNorm: + case GraphicsFormat.RGBA_DXT1_SRGB: + case GraphicsFormat.RGBA_DXT5_UNorm: + case GraphicsFormat.RGBA_DXT5_SRGB: + case GraphicsFormat.R_BC4_UNorm: + case GraphicsFormat.R_BC4_SNorm: + case GraphicsFormat.RG_BC5_UNorm: + case GraphicsFormat.RG_BC5_SNorm: + case GraphicsFormat.RGBA_BC7_UNorm: + case GraphicsFormat.RGBA_BC7_SRGB: + case GraphicsFormat.RGB_BC6H_SFloat: + case GraphicsFormat.RGB_BC6H_UFloat: + return true; + default: + return false; + } + } + + // Background DDS header reader. Throws on bad magic or unsupported format. + // Does not call any Unity API (so it is safe on a worker thread). + private static DDSPreparedHeader ParseDDSHeader(string path) + { + FileInfo fi = new FileInfo(path); + long fileLength = fi.Length; + if (fileLength < 128) + throw new IOException($"DDS file '{path}' is too small ({fileLength} bytes)"); + + using FileStream fs = File.OpenRead(path); + using BinaryReader br = new BinaryReader(fs); + + if (br.ReadUInt32() != DDSValues.uintMagic) + throw new IOException($"DDS: '{path}' is not a DDS format file"); + + DDSHeader hdr = new DDSHeader(br); + bool mipChain = (hdr.dwCaps & DDSPixelFormatCaps.MIPMAP) != 0; + bool isNormalMap = (hdr.ddspf.dwFlags & 0x80000u) != 0 || (hdr.ddspf.dwFlags & 0x80000000u) != 0; + + DDSHeaderDX10 dx10Header = default; + bool hasDx10 = (DDSFourCCBg)hdr.ddspf.dwFourCC == DDSFourCCBg.DX10; + if (hasDx10) + { + if (fileLength < 148) + throw new IOException($"DDS file '{path}' has DX10 marker but is too small for DX10 header"); + dx10Header = new DDSHeaderDX10(br); + } + + GraphicsFormat fmt = MapDDSFormat(hdr, hasDx10, dx10Header, out string error); + if (fmt == GraphicsFormat.None || error != null) + throw new IOException($"DDS: {error ?? "unknown format"}"); + + long dataOffset = hasDx10 ? 148 : 128; + return new DDSPreparedHeader + { + Width = (int)hdr.dwWidth, + Height = (int)hdr.dwHeight, + MipChain = mipChain, + IsNormalMap = isNormalMap, + Format = fmt, + DataOffset = dataOffset, + FileLength = fileLength, + }; + } + + // Background-thread-safe FourCC enum (mirrors RawAsset.DDSFourCC, which is private to RawAsset). + private enum DDSFourCCBg : uint + { + DXT1 = 0x31545844, + DXT2 = 0x32545844, + DXT3 = 0x33545844, + DXT4 = 0x34545844, + DXT5 = 0x35545844, + BC4U_ATI = 0x31495441, + BC4U = 0x55344342, + BC4S = 0x53344342, + BC5U_ATI = 0x32495441, + BC5U = 0x55354342, + BC5S = 0x53354342, + RGBG = 0x47424752, + GRGB = 0x42475247, + UYVY = 0x59565955, + YUY2 = 0x32595559, + DX10 = 0x30315844, + R16G16B16A16_UNORM = 36, + R16G16B16A16_SNORM = 110, + R16_FLOAT = 111, + R16G16_FLOAT = 112, + R16G16B16A16_FLOAT = 113, + R32_FLOAT = 114, + R32G32_FLOAT = 115, + R32G32B32A32_FLOAT = 116, + CxV8U8 = 117, + } + + // Returns GraphicsFormat.None and sets error on failure. + private static GraphicsFormat MapDDSFormat(DDSHeader hdr, bool hasDx10, DDSHeaderDX10 dx10, out string error) + { + error = null; + DDSFourCCBg fourCC = (DDSFourCCBg)hdr.ddspf.dwFourCC; + switch (fourCC) + { + case DDSFourCCBg.DXT1: return GraphicsFormatUtility.GetGraphicsFormat(TextureFormat.DXT1, true); + case DDSFourCCBg.DXT5: return GraphicsFormatUtility.GetGraphicsFormat(TextureFormat.DXT5, true); + case DDSFourCCBg.BC4U_ATI: + case DDSFourCCBg.BC4U: return GraphicsFormat.R_BC4_UNorm; + case DDSFourCCBg.BC4S: return GraphicsFormat.R_BC4_SNorm; + case DDSFourCCBg.BC5U_ATI: + case DDSFourCCBg.BC5U: return GraphicsFormat.RG_BC5_UNorm; + case DDSFourCCBg.BC5S: return GraphicsFormat.RG_BC5_SNorm; + case DDSFourCCBg.R16G16B16A16_UNORM: return GraphicsFormat.R16G16B16A16_UNorm; + case DDSFourCCBg.R16G16B16A16_SNORM: return GraphicsFormat.R16G16B16A16_SNorm; + case DDSFourCCBg.R16_FLOAT: return GraphicsFormat.R16_SFloat; + case DDSFourCCBg.R16G16_FLOAT: return GraphicsFormat.R16G16_SFloat; + case DDSFourCCBg.R16G16B16A16_FLOAT: return GraphicsFormat.R16G16B16A16_SFloat; + case DDSFourCCBg.R32_FLOAT: return GraphicsFormat.R32_SFloat; + case DDSFourCCBg.R32G32_FLOAT: return GraphicsFormat.R32G32_SFloat; + case DDSFourCCBg.R32G32B32A32_FLOAT: return GraphicsFormat.R32G32B32A32_SFloat; + case DDSFourCCBg.DX10: + if (!hasDx10) { - if (SystemInfo.operatingSystemFamily == OperatingSystemFamily.MacOSX && - (graphicsFormat == GraphicsFormat.RGBA_BC7_UNorm - || graphicsFormat == GraphicsFormat.RGBA_BC7_SRGB - || graphicsFormat == GraphicsFormat.RGB_BC6H_SFloat - || graphicsFormat == GraphicsFormat.RGB_BC6H_UFloat)) - { - SetError($"DDS: The '{graphicsFormat}' format is not supported on MacOS"); - } - else - { - SetError($"DDS: The '{graphicsFormat}' format is not supported by your GPU or OS"); - } + error = "DX10 marker without DX10 header"; + return GraphicsFormat.None; } - else + switch (dx10.dxgiFormat) { - texture2D = new Texture2D((int)dDSHeader.dwWidth, (int)dDSHeader.dwHeight, graphicsFormat, mipChain ? TextureCreationFlags.MipChain : TextureCreationFlags.None); - if (texture2D.IsNullOrDestroyed()) + case DXGI_FORMAT.DXGI_FORMAT_BC1_UNORM: return GraphicsFormat.RGBA_DXT1_UNorm; + case DXGI_FORMAT.DXGI_FORMAT_BC1_UNORM_SRGB: return GraphicsFormat.RGBA_DXT1_SRGB; + case DXGI_FORMAT.DXGI_FORMAT_BC3_UNORM: return GraphicsFormat.RGBA_DXT5_UNorm; + case DXGI_FORMAT.DXGI_FORMAT_BC3_UNORM_SRGB: return GraphicsFormat.RGBA_DXT5_SRGB; + case DXGI_FORMAT.DXGI_FORMAT_BC4_SNORM: return GraphicsFormat.R_BC4_SNorm; + case DXGI_FORMAT.DXGI_FORMAT_BC4_UNORM: return GraphicsFormat.R_BC4_UNorm; + case DXGI_FORMAT.DXGI_FORMAT_BC5_SNORM: return GraphicsFormat.RG_BC5_SNorm; + case DXGI_FORMAT.DXGI_FORMAT_BC5_UNORM: return GraphicsFormat.RG_BC5_UNorm; + case DXGI_FORMAT.DXGI_FORMAT_BC7_UNORM: return GraphicsFormat.RGBA_BC7_UNorm; + case DXGI_FORMAT.DXGI_FORMAT_BC7_UNORM_SRGB: return GraphicsFormat.RGBA_BC7_SRGB; + case DXGI_FORMAT.DXGI_FORMAT_BC6H_SF16: return GraphicsFormat.RGB_BC6H_SFloat; + case DXGI_FORMAT.DXGI_FORMAT_BC6H_UF16: return GraphicsFormat.RGB_BC6H_UFloat; + case DXGI_FORMAT.DXGI_FORMAT_R16G16B16A16_UNORM: return GraphicsFormat.R16G16B16A16_UNorm; + case DXGI_FORMAT.DXGI_FORMAT_R16G16B16A16_SNORM: return GraphicsFormat.R16G16B16A16_SNorm; + case DXGI_FORMAT.DXGI_FORMAT_R16_FLOAT: return GraphicsFormat.R16_SFloat; + case DXGI_FORMAT.DXGI_FORMAT_R16G16_FLOAT: return GraphicsFormat.R16G16_SFloat; + case DXGI_FORMAT.DXGI_FORMAT_R16G16B16A16_FLOAT: return GraphicsFormat.R16G16B16A16_SFloat; + case DXGI_FORMAT.DXGI_FORMAT_R32_FLOAT: return GraphicsFormat.R32_SFloat; + case DXGI_FORMAT.DXGI_FORMAT_R32G32_FLOAT: return GraphicsFormat.R32G32_SFloat; + case DXGI_FORMAT.DXGI_FORMAT_R32G32B32A32_FLOAT: return GraphicsFormat.R32G32B32A32_SFloat; + default: + error = $"DXT10 format '{dx10.dxgiFormat}' is not supported"; + return GraphicsFormat.None; + } + case DDSFourCCBg.DXT2: + case DDSFourCCBg.DXT3: + case DDSFourCCBg.DXT4: + case DDSFourCCBg.RGBG: + case DDSFourCCBg.GRGB: + case DDSFourCCBg.UYVY: + case DDSFourCCBg.YUY2: + case DDSFourCCBg.CxV8U8: + error = $"format '{fourCC}' is not supported, use DXT1 for RGB textures or DXT5 for RGBA textures"; + return GraphicsFormat.None; + default: + error = $"unknown dwFourCC format '0x{(uint)fourCC:X}'"; + return GraphicsFormat.None; + } + } + + // Channel-swizzle for normal maps (extracted from BitmapToCompressedNormalMapFast). + // src must hold pixel data in srcFormat; dst is written as RGBA32 (level 0 only — + // if dst has a mip chain, higher mip levels are left untouched and are populated + // by the caller's Apply(updateMipmaps: true)). + private static unsafe void SwizzleNormalMap(NativeArray src, NativeArray dst, TextureFormat srcFormat) + { + byte* s = (byte*)NativeArrayUnsafeUtility.GetUnsafeReadOnlyPtr(src); + byte* d = (byte*)NativeArrayUnsafeUtility.GetUnsafePtr(dst); + int srcLen = src.Length; + + switch (srcFormat) + { + case TextureFormat.RGBA32: + // (r, g, b, a) -> (g, g, g, r) + for (int i = 0; i < srcLen; i += 4) + { + byte r = s[i]; + byte g = s[i + 1]; + d[i] = g; d[i + 1] = g; d[i + 2] = g; d[i + 3] = r; + } + break; + case TextureFormat.ARGB32: + // (a, r, g, b) -> (g, g, g, r) + for (int i = 0; i < srcLen; i += 4) + { + byte r = s[i + 1]; + byte g = s[i + 2]; + d[i] = g; d[i + 1] = g; d[i + 2] = g; d[i + 3] = r; + } + break; + case TextureFormat.RGB24: + // (r, g, b) -> (g, g, g, r); 3-byte in, 4-byte out + { + int j = 0; + for (int i = 0; i < srcLen; i += 3) { - SetError($"DDS: Failed to load texture, unknown error"); + byte r = s[i]; + byte g = s[i + 1]; + d[j] = g; d[j + 1] = g; d[j + 2] = g; d[j + 3] = r; + j += 4; } - else - { - int position = (int)binaryReader.BaseStream.Position; - GCHandle pinnedHandle = GCHandle.Alloc(buffer, GCHandleType.Pinned); - try - { - IntPtr ptr = Marshal.UnsafeAddrOfPinnedArrayElement(buffer, position); - texture2D.LoadRawTextureData(ptr, dataLength - position); - } - finally - { - pinnedHandle.Free(); - } + } + break; + default: + throw new InvalidOperationException($"SwizzleNormalMap: unsupported source format {srcFormat}"); + } + } - texture2D.Apply(updateMipmaps: false, makeNoLongerReadable: true); - } + // Returns the most informative exception from a faulted Task without using the + // null-coalescing operator (which the type checker rejects when mixing + // AggregateException with concrete subtypes of Exception). + private static Exception UnwrapFaultedTask(Task task, string fallbackMessage) + { + AggregateException ae = task.Exception; + if (ae != null && ae.InnerException != null) + return ae.InnerException; + if (ae != null) + return ae; + return new IOException(fallbackMessage); + } + + // Iterator methods can't contain unsafe blocks in C# 8, so the AsyncReadManager + // pointer setup goes through this static helper. + private static unsafe ReadHandle BeginAsyncRead(string path, NativeArray dst, long offset, long size) + { + ReadCommand cmd = new ReadCommand + { + Buffer = NativeArrayUnsafeUtility.GetUnsafePtr(dst), + Offset = offset, + Size = size, + }; + return AsyncReadManager.Read(path, &cmd, 1); + } + + // Wraps an inner format-specific coroutine with exception capture and semaphore release. + // C# does not allow yield inside a try/catch, so we manually drive MoveNext() and + // do the catch around just the MoveNext call. + private static IEnumerator LoadTextureWrapperCoroutine(TextureLoadRequest req, IEnumerator inner, SemaphoreSlim semaphore) + { + try + { + while (true) + { + bool moved; + bool failed = false; + object current = null; + try + { + moved = inner.MoveNext(); + if (moved) + current = inner.Current; + } + catch (Exception e) + { + req.Exception = e; + req.ErrorMessage = $"{e.GetType().Name}: {e.Message}"; + req.Status = TextureLoadRequest.State.Failed; + moved = false; + failed = true; } + if (failed) + yield break; + if (!moved) + break; + yield return current; } - return new TextureInfo(file, texture2D, isNormalMap, false, true); + if (req.Status == TextureLoadRequest.State.Pending) + { + if (req.Result != null) + req.Status = TextureLoadRequest.State.Ready; + else + { + if (req.ErrorMessage == null) + req.ErrorMessage = "Loader produced no result"; + req.Status = TextureLoadRequest.State.Failed; + } + } + } + finally + { + semaphore.Release(); } + } - private TextureInfo LoadJPG() + private static IEnumerator LoadDDSCoroutine(TextureLoadRequest req) + { + string path = req.File.fullPath; + Task hdrTask = Task.Run(() => { - bool isNormal = file.name.EndsWith("NRM"); + using (s_pmParseDDSHeader.Auto()) + return ParseDDSHeader(path); + }); + while (!hdrTask.IsCompleted) + yield return null; + if (hdrTask.IsFaulted) + throw UnwrapFaultedTask(hdrTask, "DDS header parse failed"); + DDSPreparedHeader hdr = hdrTask.Result; + req.FileLength = hdr.FileLength; - if (isNormal) - { - Texture2D tex = new Texture2D(1, 1, TextureFormat.RGB24, false); - if (!ImageConversion.LoadImage(tex, buffer, false)) - return null; + if (!SupportedFormatCache.IsSupported(hdr.Format)) + { + req.ErrorMessage = $"DDS: format '{hdr.Format}' is not supported by your GPU"; + req.Status = TextureLoadRequest.State.Failed; + yield break; + } - Texture2D nrmTex = BitmapToCompressedNormalMapFast(tex); - return new TextureInfo(file, nrmTex, true, false, true); - } - else - { - Texture2D tex = new Texture2D(1, 1, TextureFormat.DXT1, false); - if (!ImageConversion.LoadImage(tex, buffer, false)) - return null; + // Unity rejects mipChain on NPOT textures for block-compressed formats. + bool mipChain = hdr.MipChain; + if (mipChain && IsBlockCompressed(hdr.Format) + && !(Numerics.IsPowerOfTwo(hdr.Width) && Numerics.IsPowerOfTwo(hdr.Height))) + { + mipChain = false; + } - return new TextureInfo(file, tex, false, true, true); - } + Texture2D tex = new Texture2D(hdr.Width, hdr.Height, hdr.Format, + mipChain ? TextureCreationFlags.MipChain : TextureCreationFlags.None); + if (tex.IsNullOrDestroyed()) + { + req.ErrorMessage = "DDS: Texture2D allocation failed"; + req.Status = TextureLoadRequest.State.Failed; + yield break; } - private TextureInfo LoadMBM() + // Wait one frame so Unity's initial GPU resource creation completes + // before AsyncReadManager writes into the staging buffer. + yield return null; + + NativeArray dst = tex.GetRawTextureData(); + long expectedSize = dst.Length; + if (hdr.FileLength - hdr.DataOffset < expectedSize) { - memoryStream = new MemoryStream(buffer, 0, dataLength); - binaryReader = new BinaryReader(memoryStream); - Texture2D texture2D = MBMReader.ReadTexture2D(buffer, binaryReader, true, true, out bool isNormalMap); - return new TextureInfo(file, texture2D, isNormalMap, false, true); + UnityEngine.Object.Destroy(tex); + req.ErrorMessage = $"DDS: file is too small for declared format (need {expectedSize} bytes after offset {hdr.DataOffset}, have {hdr.FileLength - hdr.DataOffset})"; + req.Status = TextureLoadRequest.State.Failed; + yield break; } - private static string mipMapsPNGTexturePath = Path.DirectorySeparatorChar + "Flags" + Path.DirectorySeparatorChar; + ReadHandle handle = BeginAsyncRead(path, dst, hdr.DataOffset, expectedSize); + + while (handle.Status == ReadStatus.InProgress) + yield return null; + + ReadStatus status = handle.Status; + handle.Dispose(); - private TextureInfo LoadPNG() + if (status != ReadStatus.Complete) { - if (!GetPNGSize(buffer, out uint width, out uint height)) - { - SetError("Invalid PNG file"); - return null; - } + UnityEngine.Object.Destroy(tex); + req.ErrorMessage = $"DDS: AsyncReadManager.Read failed (status={status})"; + req.Status = TextureLoadRequest.State.Failed; + yield break; + } + + tex.Apply(updateMipmaps: false, makeNoLongerReadable: true); + req.Result = new TextureInfo(req.File, tex, hdr.IsNormalMap, false, true); + req.Status = TextureLoadRequest.State.Ready; + } - bool isNormalMap = file.name.EndsWith("NRM"); - bool nonReadable = file.fullPath.Contains("@thumbs"); // KSPCF optimization : don't keep cargo icons in memory - bool hasMipMaps = file.fullPath.Contains(mipMapsPNGTexturePath); // only generate mipmaps for flags (stock behavior) - bool canCompress = hasMipMaps ? Numerics.IsPowerOfTwo(width) && Numerics.IsPowerOfTwo(height) : width % 4 == 0 && height % 4 == 0; + private static readonly string flagsPathSep = Path.DirectorySeparatorChar + "Flags" + Path.DirectorySeparatorChar; - // don't initially compress normal textures, as we need to swizzle the raw data first - TextureFormat textureFormat; - if (isNormalMap) - { - textureFormat = TextureFormat.ARGB32; - } - else if (!canCompress) + private static IEnumerator LoadUWRCoroutine(TextureLoadRequest req) + { + string filePath = req.File.fullPath; + req.FileLength = new FileInfo(filePath).Length; + string url = "file:///" + filePath.Replace('\\', '/'); + + UnityWebRequest uwr = UnityWebRequestTexture.GetTexture(url, nonReadable: false); + try + { + yield return uwr.SendWebRequest(); + + if (uwr.isNetworkError || uwr.isHttpError) { - textureFormat = TextureFormat.ARGB32; - SetWarning("Texture isn't eligible for DXT compression, width and height must be multiples of 4"); + req.ErrorMessage = $"UWR: {uwr.error}"; + req.Status = TextureLoadRequest.State.Failed; + yield break; } - else + + Texture2D src = DownloadHandlerTexture.GetContent(uwr); + if (src.IsNullOrDestroyed()) { - textureFormat = TextureFormat.DXT5; + req.ErrorMessage = "UWR: GetContent returned null"; + req.Status = TextureLoadRequest.State.Failed; + yield break; } - Texture2D texture = new Texture2D((int)width, (int)height, textureFormat, hasMipMaps); + bool isNormalMap = req.File.name.EndsWith("NRM"); + bool isFlag = filePath.Contains(flagsPathSep); + bool canCompress = src.width % 4 == 0 && src.height % 4 == 0; - if ((isNormalMap || canCompress) && textureCacheEnabled) + if (isNormalMap || isFlag) { - if (!ImageConversion.LoadImage(texture, buffer, false)) - return null; - + // Allocate a destination texture with mipchain (when POT) so that + // Apply(true) can generate mipmaps. UWR-loaded textures have no mipchain. + bool isPot = Numerics.IsPowerOfTwo(src.width) && Numerics.IsPowerOfTwo(src.height); + bool wantMipChain = isPot; + TextureFormat dstFormat = isNormalMap ? TextureFormat.RGBA32 : src.format; + Texture2D dst = new Texture2D(src.width, src.height, dstFormat, wantMipChain); if (isNormalMap) - texture = BitmapToCompressedNormalMapFast(texture, false); + dst.wrapMode = TextureWrapMode.Repeat; + + NativeArray srcData = src.GetRawTextureData(); + NativeArray dstData = dst.GetRawTextureData(); + TextureFormat srcFormat = src.format; - if (texture.graphicsFormat == GraphicsFormat.RGBA_DXT5_UNorm) + Task task; + if (isNormalMap) + { + task = Task.Run(() => + { + using (s_pmSwizzleNormalMap.Auto()) + SwizzleNormalMap(srcData, dstData, srcFormat); + }); + } + else { - SaveCachedTexture(file, texture, isNormalMap); + // Flag: copy level-0 raw data byte-for-byte (formats already match). + int level0Bytes = srcData.Length; + task = Task.Run(() => + { + using (s_pmCopyLevel0.Auto()) + NativeArray.Copy(srcData, 0, dstData, 0, level0Bytes); + }); + } - if (isNormalMap || nonReadable) - texture.Apply(true, true); + while (!task.IsCompleted) + yield return null; + if (task.IsFaulted) + { + UnityEngine.Object.Destroy(src); + UnityEngine.Object.Destroy(dst); + throw UnwrapFaultedTask(task, "texture pre-process task faulted"); } - } - else - { - if (!ImageConversion.LoadImage(texture, buffer, nonReadable)) - return null; - if (isNormalMap) - texture = BitmapToCompressedNormalMapFast(texture); + UnityEngine.Object.Destroy(src); + src = dst; + + if (wantMipChain) + src.Apply(updateMipmaps: true); } + if (canCompress) + src.Compress(highQuality: !isNormalMap); + else if (!isNormalMap) + Debug.LogWarning($"Texture '{req.File.url}' isn't eligible for DXT compression, width and height must be multiples of 4"); + + src.Apply(updateMipmaps: false, makeNoLongerReadable: true); - return new TextureInfo(file, texture, isNormalMap, !nonReadable, true); + bool isCompressed = + src.graphicsFormat == GraphicsFormat.RGBA_DXT5_UNorm + || src.graphicsFormat == GraphicsFormat.RGBA_DXT5_SRGB + || src.graphicsFormat == GraphicsFormat.RGBA_DXT1_UNorm + || src.graphicsFormat == GraphicsFormat.RGBA_DXT1_SRGB; + req.Result = new TextureInfo(req.File, src, isNormalMap, isReadable: false, isCompressed: isCompressed); + req.Status = TextureLoadRequest.State.Ready; + } + finally + { + uwr.Dispose(); } + } + + private static IEnumerator LoadTRUECOLORCoroutine(TextureLoadRequest req) + { + string path = req.File.fullPath; + Task sizeTask = Task.Run(() => + { + using (s_pmFileSize.Auto()) + return new FileInfo(path).Length; + }); + while (!sizeTask.IsCompleted) + yield return null; + if (sizeTask.IsFaulted) + throw UnwrapFaultedTask(sizeTask, "file size read failed"); - private TextureInfo LoadPNGCached() + long len = sizeTask.Result; + req.FileLength = len; + if (len <= 0 || len > int.MaxValue) { - if (cachedTextureInfo.TryCreateTexture(buffer, out Texture2D texture)) - return new TextureInfo(file, texture, cachedTextureInfo.normal, cachedTextureInfo.readable, true); + req.ErrorMessage = $"TRUECOLOR: invalid file length {len}"; + req.Status = TextureLoadRequest.State.Failed; + yield break; + } - buffer = System.IO.File.ReadAllBytes(file.fullPath); - return LoadPNG(); + NativeArray data = new NativeArray((int)len, Allocator.Persistent, NativeArrayOptions.UninitializedMemory); + ReadHandle handle = BeginAsyncRead(path, data, 0, len); + while (handle.Status == ReadStatus.InProgress) + yield return null; + ReadStatus rs = handle.Status; + handle.Dispose(); + + if (rs != ReadStatus.Complete) + { + data.Dispose(); + req.ErrorMessage = $"TRUECOLOR: AsyncReadManager.Read failed (status={rs})"; + req.Status = TextureLoadRequest.State.Failed; + yield break; } - private TextureInfo LoadTGA() + byte[] managed = data.ToArray(); + data.Dispose(); + + Texture2D tex = new Texture2D(2, 2, TextureFormat.ARGB32, false); + if (!ImageConversion.LoadImage(tex, managed, markNonReadable: false)) { - if (dataLength < 18) + UnityEngine.Object.Destroy(tex); + req.ErrorMessage = "TRUECOLOR: ImageConversion.LoadImage failed"; + req.Status = TextureLoadRequest.State.Failed; + yield break; + } + + bool isNormalMap = req.File.name.EndsWith("NRM"); + if (isNormalMap) + { + bool isPot = Numerics.IsPowerOfTwo(tex.width) && Numerics.IsPowerOfTwo(tex.height); + Texture2D dst = new Texture2D(tex.width, tex.height, TextureFormat.RGBA32, mipChain: isPot); + dst.wrapMode = TextureWrapMode.Repeat; + + NativeArray srcData = tex.GetRawTextureData(); + NativeArray dstData = dst.GetRawTextureData(); + TextureFormat srcFormat = tex.format; + Task swizzleTask = Task.Run(() => + { + using (s_pmSwizzleNormalMap.Auto()) + SwizzleNormalMap(srcData, dstData, srcFormat); + }); + while (!swizzleTask.IsCompleted) + yield return null; + if (swizzleTask.IsFaulted) { - SetError("TGA invalid length of only " + dataLength + "bytes"); - return null; + UnityEngine.Object.Destroy(tex); + UnityEngine.Object.Destroy(dst); + throw UnwrapFaultedTask(swizzleTask, "swizzle task faulted"); } + UnityEngine.Object.Destroy(tex); + tex = dst; - TGAImage tgaImage = new TGAImage(); - TGAImage.header = new TGAHeader(buffer); - TGAImage.colorData = tgaImage.ReadImage(TGAImage.header, buffer); - if (TGAImage.colorData == null) - return null; + if (isPot) + { + tex.Apply(updateMipmaps: true); + tex.Compress(highQuality: false); + } + tex.Apply(updateMipmaps: false, makeNoLongerReadable: true); + + req.Result = new TextureInfo(req.File, tex, true, isReadable: false, isCompressed: isPot); + } + else + { + tex.Apply(updateMipmaps: false, makeNoLongerReadable: false); + req.Result = new TextureInfo(req.File, tex, false, isReadable: true, isCompressed: false); + } + req.Status = TextureLoadRequest.State.Ready; + } - Texture2D texture = tgaImage.CreateTexture(mipmap: true, linear: false, compress: true, compressHighQuality: false, allowRead: true); - if (texture.IsNullOrDestroyed()) - return null; + private static IEnumerator LoadMBMCoroutine(TextureLoadRequest req) + { + string path = req.File.fullPath; + Task readTask = Task.Run(() => + { + using (s_pmReadAllBytes.Auto()) + return File.ReadAllBytes(path); + }); + while (!readTask.IsCompleted) + yield return null; + if (readTask.IsFaulted) + throw UnwrapFaultedTask(readTask, "MBM file read failed"); - bool isNormalMap = file.name.EndsWith("NRM"); - if (isNormalMap) - texture = BitmapToCompressedNormalMapFast(texture); + byte[] buffer = readTask.Result; + req.FileLength = buffer.Length; - return new TextureInfo(file, texture, isNormalMap, !isNormalMap, true); + Texture2D texture; + bool isNormalMap; + using (MemoryStream ms = new MemoryStream(buffer, 0, buffer.Length)) + using (BinaryReader br = new BinaryReader(ms)) + { + texture = MBMReader.ReadTexture2D(buffer, br, true, true, out isNormalMap); } - - private TextureInfo LoadTRUECOLOR() + if (texture.IsNullOrDestroyed()) { - bool isNormalMap = file.name.EndsWith("NRM"); + req.ErrorMessage = "MBM: ReadTexture2D failed"; + req.Status = TextureLoadRequest.State.Failed; + yield break; + } + + req.Result = new TextureInfo(req.File, texture, isNormalMap, isReadable: false, isCompressed: true); + req.Status = TextureLoadRequest.State.Ready; + } - Texture2D texture = new Texture2D(1, 1, TextureFormat.ARGB32, false); - if (!ImageConversion.LoadImage(texture, buffer, false)) - return null; + private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) + { + string path = req.File.fullPath; + Task readTask = Task.Run(() => + { + using (s_pmReadAllBytes.Auto()) + return File.ReadAllBytes(path); + }); + while (!readTask.IsCompleted) + yield return null; + if (readTask.IsFaulted) + throw UnwrapFaultedTask(readTask, "TGA file read failed"); - if (isNormalMap) - texture = BitmapToCompressedNormalMapFast(texture); + byte[] buffer = readTask.Result; + req.FileLength = buffer.Length; + if (buffer.Length < 18) + { + req.ErrorMessage = $"TGA invalid length of only {buffer.Length} bytes"; + req.Status = TextureLoadRequest.State.Failed; + yield break; + } - return new TextureInfo(file, texture, isNormalMap, !isNormalMap, false); + TGAImage tgaImage = new TGAImage(); + TGAImage.header = new TGAHeader(buffer); + TGAImage.colorData = tgaImage.ReadImage(TGAImage.header, buffer); + if (TGAImage.colorData == null) + { + req.ErrorMessage = "TGA: ReadImage failed"; + req.Status = TextureLoadRequest.State.Failed; + yield break; } - private GameObject LoadMU() + Texture2D texture = tgaImage.CreateTexture(mipmap: true, linear: false, compress: true, compressHighQuality: false, allowRead: true); + if (texture.IsNullOrDestroyed()) { - return MuParser.Parse(file.parent.url, buffer, dataLength); + req.ErrorMessage = "TGA: CreateTexture failed"; + req.Status = TextureLoadRequest.State.Failed; + yield break; } - private GameObject LoadDAE() + bool isNormalMap = req.File.name.EndsWith("NRM"); + if (isNormalMap) { - // given that this is a quite obsolete thing and that it's mess to reimplement, just call the stock - // stuff and re-load the file + bool isPot = Numerics.IsPowerOfTwo(texture.width) && Numerics.IsPowerOfTwo(texture.height); + Texture2D dst = new Texture2D(texture.width, texture.height, TextureFormat.RGBA32, mipChain: isPot); + dst.wrapMode = TextureWrapMode.Repeat; - GameObject gameObject = new DatabaseLoaderModel_DAE.DAE().Load(file, new FileInfo(file.fullPath)); - if (gameObject.IsNotNullOrDestroyed()) + NativeArray srcData = texture.GetRawTextureData(); + NativeArray dstData = dst.GetRawTextureData(); + TextureFormat srcFormat = texture.format; + Task swizzleTask = Task.Run(() => { - MeshFilter[] componentsInChildren = gameObject.GetComponentsInChildren(); - foreach (MeshFilter meshFilter in componentsInChildren) - { - if (meshFilter.gameObject.name == "node_collider") - { - meshFilter.gameObject.AddComponent().sharedMesh = meshFilter.mesh; - MeshRenderer component = meshFilter.gameObject.GetComponent(); - UnityEngine.Object.Destroy(meshFilter); - UnityEngine.Object.Destroy(component); - } - } + using (s_pmSwizzleNormalMap.Auto()) + SwizzleNormalMap(srcData, dstData, srcFormat); + }); + while (!swizzleTask.IsCompleted) + yield return null; + if (swizzleTask.IsFaulted) + { + UnityEngine.Object.Destroy(texture); + UnityEngine.Object.Destroy(dst); + throw UnwrapFaultedTask(swizzleTask, "swizzle task faulted"); } + UnityEngine.Object.Destroy(texture); + texture = dst; - return gameObject; + if (isPot) + { + texture.Apply(updateMipmaps: true); + texture.Compress(highQuality: false); + } + texture.Apply(updateMipmaps: false, makeNoLongerReadable: true); + } + + req.Result = new TextureInfo(req.File, texture, isNormalMap, isReadable: !isNormalMap, isCompressed: true); + req.Status = TextureLoadRequest.State.Ready; + } + + private static IEnumerator LoadPNGCachedCoroutine(TextureLoadRequest req) + { + CachedTextureInfo info = req.CachedInfo; + string path = info.FilePath; + Task readTask = Task.Run(() => + { + using (s_pmReadAllBytes.Auto()) + return File.ReadAllBytes(path); + }); + while (!readTask.IsCompleted) + yield return null; + if (readTask.IsFaulted) + { + Debug.LogWarning($"[KSPCF] Cached PNG read failed for '{req.File.url}', falling back to fresh load"); + FallBackFromCache(req, info); + IEnumerator fallback = LoadUWRCoroutine(req); + while (fallback.MoveNext()) + yield return fallback.Current; + yield break; + } + + byte[] buffer = readTask.Result; + req.FileLength = buffer.Length; + + if (info.TryCreateTexture(buffer, out Texture2D texture)) + { + req.Result = new TextureInfo(req.File, texture, info.normal, isReadable: false, isCompressed: true); + req.Status = TextureLoadRequest.State.Ready; + yield break; } - private static Texture2D BitmapToCompressedNormalMapFast(Texture2D original, bool makeNoLongerReadable = true) + Debug.LogWarning($"[KSPCF] Cached PNG TryCreateTexture failed for '{req.File.url}', falling back to fresh load"); + FallBackFromCache(req, info); + IEnumerator fallback2 = LoadUWRCoroutine(req); + while (fallback2.MoveNext()) + yield return fallback2.Current; + } + + private static void FallBackFromCache(TextureLoadRequest req, CachedTextureInfo info) + { + if (loader.textureCacheData.Remove(info.name)) { - // ~6 times faster than the stock BitmapToUnityNormalMap() method - // Note that this would be a lot more efficient if we didn't have to create a new texture. - // Unfortunately, Unity doesn't provide any way to add mimaps to a texture that didn't - // have them initially (but I guess this is a deeper GPU related limitation)... + loader.textureDataIds.Remove(info.id); + try { File.Delete(info.FilePath); } catch { } + loader.cacheUpdated = true; + } + req.AssetType = RawAsset.AssetType.TexturePNG; + req.CameFromCache = false; + req.CachedInfo = null; + } - TextureFormat originalFormat = original.format; - Texture2D normalMap = new Texture2D(original.width, original.height, TextureFormat.RGBA32, true); - normalMap.wrapMode = TextureWrapMode.Repeat; + private static IEnumerator TextureDriverCoroutine(List requests, HashSet loadedUrls) + { + GameDatabase gdb = GameDatabase.Instance; + SemaphoreSlim semaphore = new SemaphoreSlim(MaxConcurrentTextures, MaxConcurrentTextures); + int spawnIdx = 0; + int completeIdx = 0; + int total = requests.Count; + double nextFrameTime = ElapsedTime + minFrameTimeD; - if (originalFormat == TextureFormat.RGBA32 - || originalFormat == TextureFormat.ARGB32 - || originalFormat == TextureFormat.RGB24) + while (completeIdx < total) + { + while (spawnIdx < total && semaphore.Wait(0)) { - NativeArray originalData = original.GetRawTextureData(); - NativeArray normalMapData = normalMap.GetRawTextureData(); - int size = originalData.Length; - byte r, g; - switch (originalFormat) + TextureLoadRequest req = requests[spawnIdx++]; + IEnumerator inner; + switch (req.AssetType) { - case TextureFormat.RGBA32: - // from (r, g, b, a) - // to (g, g, g, r); - for (int i = 0; i < size; i += 4) - { - r = originalData[i]; - g = originalData[i + 1]; - normalMapData[i] = g; - normalMapData[i + 1] = g; - normalMapData[i + 2] = g; - normalMapData[i + 3] = r; - } + case RawAsset.AssetType.TextureDDS: + inner = LoadDDSCoroutine(req); break; - case TextureFormat.ARGB32: - // from (a, r, g, b) - // to (g, g, g, r); - for (int i = 0; i < size; i += 4) - { - r = originalData[i + 1]; - g = originalData[i + 2]; - normalMapData[i] = g; - normalMapData[i + 1] = g; - normalMapData[i + 2] = g; - normalMapData[i + 3] = r; - } + case RawAsset.AssetType.TexturePNG: + case RawAsset.AssetType.TextureJPG: + inner = LoadUWRCoroutine(req); break; - case TextureFormat.RGB24: - // from (r, g, b) - // to (g, g, g, r); - int j = 0; - for (int i = 0; i < size; i += 3) - { - r = originalData[i]; - g = originalData[i + 1]; - normalMapData[j] = g; - normalMapData[j + 1] = g; - normalMapData[j + 2] = g; - normalMapData[j + 3] = r; - j += 4; - } + case RawAsset.AssetType.TextureTRUECOLOR: + inner = LoadTRUECOLORCoroutine(req); + break; + case RawAsset.AssetType.TextureMBM: + inner = LoadMBMCoroutine(req); + break; + case RawAsset.AssetType.TextureTGA: + inner = LoadTGACoroutine(req); + break; + case RawAsset.AssetType.TexturePNGCached: + inner = LoadPNGCachedCoroutine(req); + break; + default: + inner = null; break; } - } - else - { - Color32[] pixels = original.GetPixels32(); - for (int i = 0; i < pixels.Length; i++) + + if (inner == null) + { + req.ErrorMessage = $"Unknown asset type {req.AssetType}"; + req.Status = TextureLoadRequest.State.Failed; + semaphore.Release(); + } + else { - Color32 pixel = pixels[i]; - pixel.a = pixel.r; - pixel.r = pixel.g; - pixel.b = pixel.g; - pixels[i] = pixel; + Debug.Log($"Load Texture: {req.File.url}"); + gdb.StartCoroutine(LoadTextureWrapperCoroutine(req, inner, semaphore)); } - normalMap.SetPixels32(pixels); } - // Unity can't convert NPOT textures to DXT5 with mipmaps - if (Numerics.IsPowerOfTwo(normalMap.width) && Numerics.IsPowerOfTwo(normalMap.height)) - { - normalMap.Apply(true); // needed to generate mipmaps, must be done before compression - normalMap.Compress(false); - normalMap.Apply(true, makeNoLongerReadable); - } - else + TextureLoadRequest head = requests[completeIdx]; + if (head.Status == TextureLoadRequest.State.Pending) { - normalMap.Apply(true, makeNoLongerReadable); + if (ElapsedTime > nextFrameTime) + { + nextFrameTime = ElapsedTime + minFrameTimeD; + gdb.progressFraction = (float)(loadedAssetCount + completeIdx) / totalAssetCount; + gdb.progressTitle = $"Loading texture asset {completeIdx}/{total}"; + } + yield return null; + continue; } - Destroy(original); - return normalMap; + InsertReadyRequest(head, loadedUrls); + completeIdx++; + loadedAssetCount++; } } + private static void InsertReadyRequest(TextureLoadRequest req, HashSet loadedUrls) + { + if (req.Status == TextureLoadRequest.State.Failed) + { + Debug.LogWarning($"LOAD FAILED: {req.File.url}: {req.ErrorMessage}"); + if (req.Result != null && req.Result.texture.IsNotNullOrDestroyed()) + UnityEngine.Object.Destroy(req.Result.texture); + return; + } + + if (!loadedUrls.Add(req.File.url)) + { + Debug.LogWarning($"Duplicate texture asset '{req.File.url}' with extension '{req.File.fileExtension}' won't be loaded"); + if (req.Result != null && req.Result.texture.IsNotNullOrDestroyed()) + UnityEngine.Object.Destroy(req.Result.texture); + return; + } + + req.Result.name = req.File.url; + req.Result.texture.name = req.File.url; + GameDatabase.Instance.databaseTexture.Add(req.Result); + texturesByUrl[req.File.url] = req.Result; + KSPCFFastLoaderReport.texturesBytesLoaded += req.FileLength; + KSPCFFastLoaderReport.texturesLoaded++; + } + #endregion #region PartLoader reimplementation @@ -2315,12 +2706,12 @@ IEnumerable instructions #region PNG texture cache - private static void SetupTextureCacheThread(List textures) + private static void SetupTextureCacheThread(List requests) { loader.SetupTextureCache(); - foreach (RawAsset rawAsset in textures) - rawAsset.CheckTextureCache(); + foreach (TextureLoadRequest req in requests) + req.CheckTextureCache(); } private void SetupTextureCache() @@ -2444,7 +2835,9 @@ public CachedTextureInfo(UrlFile urlFile, Texture2D texture, bool isNormalMap, l height = texture.height; mipCount = texture.mipmapCount; normal = isNormalMap; - readable = !isNormalMap && !name.Contains("@thumbs"); + // Always non-readable: matches the unified DDS/PNG/JPG readability policy. + // The 'readable' field in older cache entries on disk is ignored on read. + readable = false; loaded = true; } @@ -2460,7 +2853,7 @@ public bool TryCreateTexture(byte[] buffer, out Texture2D texture) { texture = new Texture2D(width, height, GraphicsFormat.RGBA_DXT5_UNorm, mipCount, mipCount == 1 ? TextureCreationFlags.None : TextureCreationFlags.MipChain); texture.LoadRawTextureData(buffer); - texture.Apply(false, !readable); + texture.Apply(false, true); loaded = true; return true; } @@ -2508,7 +2901,7 @@ private static bool GetFileStats(string path, out long size, out long time) return true; } - private static void SaveCachedTexture(UrlDir.UrlFile urlFile, Texture2D texture, bool isNormalMap) + private static void SaveCachedTextureFromBytes(UrlDir.UrlFile urlFile, Texture2D texture, bool isNormalMap, byte[] rawData) { if (!GetFileStats(urlFile.fullPath, out long size, out long creationTime)) { @@ -2517,7 +2910,7 @@ private static void SaveCachedTexture(UrlDir.UrlFile urlFile, Texture2D texture, } CachedTextureInfo cachedTextureInfo = new CachedTextureInfo(urlFile, texture, isNormalMap, size, creationTime); - cachedTextureInfo.SaveRawTextureData(texture); + File.WriteAllBytes(Path.Combine(loader.textureCachePath, cachedTextureInfo.id.ToString()), rawData); loader.textureCacheData.Add(cachedTextureInfo.name, cachedTextureInfo); loader.textureDataIds.Add(cachedTextureInfo.id); loader.cacheUpdated = true; From 961e5faf20423bd942c44f81935afce49c9e8b26 Mon Sep 17 00:00:00 2001 From: Sean Lynch Date: Wed, 29 Apr 2026 01:23:34 -0700 Subject: [PATCH 02/15] Avoid initializing texture pixels when creating them --- KSPCommunityFixes/KSPCommunityFixes.csproj | 2 + KSPCommunityFixes/Performance/FastLoader.cs | 77 +++++++++++++++++++-- 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/KSPCommunityFixes/KSPCommunityFixes.csproj b/KSPCommunityFixes/KSPCommunityFixes.csproj index c9e4754..cf4b892 100644 --- a/KSPCommunityFixes/KSPCommunityFixes.csproj +++ b/KSPCommunityFixes/KSPCommunityFixes.csproj @@ -104,6 +104,8 @@ + + diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 893257a..3807d41 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -18,6 +18,7 @@ using System.Reflection; using System.Reflection.Emit; using System.Runtime.InteropServices; +using System.Runtime.Serialization; using System.Threading; using KSPCommunityFixes.Library; using TMPro; @@ -1602,6 +1603,66 @@ private static unsafe ReadHandle BeginAsyncRead(string path, NativeArray d return AsyncReadManager.Read(path, &cmd, 1); } + // Mirrors UnityEngine.Experimental.Rendering.TextureCreationFlags but with the + // additional flag values that exist in Unity's native code but aren't exposed in + // the public managed enum. DontInitializePixels skips the zero-fill that the + // normal Texture2D constructor performs — pointless work when we're about to + // overwrite the bytes via LoadRawTextureData / AsyncReadManager / LoadImage. + // Borrowed from KSPTextureLoader (../AsyncTextureLoad/src/KSPTextureLoader/TextureUtils.cs). + [Flags] + private enum InternalTextureCreationFlags + { + None = 0, + MipChain = 1 << 0, + DontInitializePixels = 1 << 2, + DontDestroyTexture = 1 << 3, + DontCreateSharedTextureData = 1 << 4, + APIShareable = 1 << 5, + Crunch = 1 << 6, + } + + // Allocates a Texture2D without zeroing its pixel buffer. Equivalent to the + // standard Texture2D constructor except for the DontInitializePixels flag, + // which the public managed API doesn't expose for the TextureFormat overload. + private static Texture2D CreateUninitializedTexture2D( + int width, + int height, + TextureFormat format = TextureFormat.RGBA32, + bool mipChain = false, + bool linear = false, + InternalTextureCreationFlags flags = InternalTextureCreationFlags.None) + { + if (GraphicsFormatUtility.IsCrunchFormat(format)) + flags |= InternalTextureCreationFlags.Crunch; + int mipCount = !mipChain ? 1 : -1; + return CreateUninitializedTexture2D( + width, height, mipCount, + GraphicsFormatUtility.GetGraphicsFormat(format, isSRGB: !linear), + flags); + } + + private static Texture2D CreateUninitializedTexture2D( + int width, + int height, + int mipCount, + GraphicsFormat format, + InternalTextureCreationFlags flags = InternalTextureCreationFlags.None) + { + Texture2D tex = (Texture2D)FormatterServices.GetUninitializedObject(typeof(Texture2D)); + if (!tex.ValidateFormat(GraphicsFormatUtility.GetTextureFormat(format))) + return tex; + + flags |= InternalTextureCreationFlags.DontInitializePixels; + if (mipCount != 1) + flags |= InternalTextureCreationFlags.MipChain; + + Texture2D.Internal_Create( + tex, width, height, mipCount, format, + (TextureCreationFlags)flags, IntPtr.Zero); + + return tex; + } + // Wraps an inner format-specific coroutine with exception capture and semaphore release. // C# does not allow yield inside a try/catch, so we manually drive MoveNext() and // do the catch around just the MoveNext call. @@ -1683,8 +1744,10 @@ private static IEnumerator LoadDDSCoroutine(TextureLoadRequest req) mipChain = false; } - Texture2D tex = new Texture2D(hdr.Width, hdr.Height, hdr.Format, - mipChain ? TextureCreationFlags.MipChain : TextureCreationFlags.None); + Texture2D tex = CreateUninitializedTexture2D( + hdr.Width, hdr.Height, + mipChain ? -1 : 1, + hdr.Format); if (tex.IsNullOrDestroyed()) { req.ErrorMessage = "DDS: Texture2D allocation failed"; @@ -1766,7 +1829,7 @@ private static IEnumerator LoadUWRCoroutine(TextureLoadRequest req) bool isPot = Numerics.IsPowerOfTwo(src.width) && Numerics.IsPowerOfTwo(src.height); bool wantMipChain = isPot; TextureFormat dstFormat = isNormalMap ? TextureFormat.RGBA32 : src.format; - Texture2D dst = new Texture2D(src.width, src.height, dstFormat, wantMipChain); + Texture2D dst = CreateUninitializedTexture2D(src.width, src.height, dstFormat, wantMipChain); if (isNormalMap) dst.wrapMode = TextureWrapMode.Repeat; @@ -1871,7 +1934,7 @@ private static IEnumerator LoadTRUECOLORCoroutine(TextureLoadRequest req) byte[] managed = data.ToArray(); data.Dispose(); - Texture2D tex = new Texture2D(2, 2, TextureFormat.ARGB32, false); + Texture2D tex = CreateUninitializedTexture2D(2, 2, TextureFormat.ARGB32, mipChain: false); if (!ImageConversion.LoadImage(tex, managed, markNonReadable: false)) { UnityEngine.Object.Destroy(tex); @@ -1884,7 +1947,7 @@ private static IEnumerator LoadTRUECOLORCoroutine(TextureLoadRequest req) if (isNormalMap) { bool isPot = Numerics.IsPowerOfTwo(tex.width) && Numerics.IsPowerOfTwo(tex.height); - Texture2D dst = new Texture2D(tex.width, tex.height, TextureFormat.RGBA32, mipChain: isPot); + Texture2D dst = CreateUninitializedTexture2D(tex.width, tex.height, TextureFormat.RGBA32, mipChain: isPot); dst.wrapMode = TextureWrapMode.Repeat; NativeArray srcData = tex.GetRawTextureData(); @@ -2001,7 +2064,7 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) if (isNormalMap) { bool isPot = Numerics.IsPowerOfTwo(texture.width) && Numerics.IsPowerOfTwo(texture.height); - Texture2D dst = new Texture2D(texture.width, texture.height, TextureFormat.RGBA32, mipChain: isPot); + Texture2D dst = CreateUninitializedTexture2D(texture.width, texture.height, TextureFormat.RGBA32, mipChain: isPot); dst.wrapMode = TextureWrapMode.Repeat; NativeArray srcData = texture.GetRawTextureData(); @@ -2851,7 +2914,7 @@ public bool TryCreateTexture(byte[] buffer, out Texture2D texture) { try { - texture = new Texture2D(width, height, GraphicsFormat.RGBA_DXT5_UNorm, mipCount, mipCount == 1 ? TextureCreationFlags.None : TextureCreationFlags.MipChain); + texture = CreateUninitializedTexture2D(width, height, mipCount, GraphicsFormat.RGBA_DXT5_UNorm); texture.LoadRawTextureData(buffer); texture.Apply(false, true); loaded = true; From a22bc9978a7923b3e5bb713001f0f8e1cd3850b2 Mon Sep 17 00:00:00 2001 From: Sean Lynch Date: Wed, 29 Apr 2026 02:28:31 -0700 Subject: [PATCH 03/15] Add some yields to avoid UnshareTextureData overhead --- KSPCommunityFixes/Performance/FastLoader.cs | 222 +++++++++++--------- 1 file changed, 124 insertions(+), 98 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 3807d41..8bc5e9d 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -35,6 +35,7 @@ using Debug = UnityEngine.Debug; using UnityEngine.Profiling; using System.Threading.Tasks; +using KSP.UI; namespace KSPCommunityFixes.Performance { @@ -157,9 +158,14 @@ internal class KSPCFFastLoader : MonoBehaviour // max concurrent per-texture coroutines spawned by TextureDriverCoroutine. // Each in-flight request holds at most one of {ReadHandle, UnityWebRequest, background Task}, - // so a single semaphore covers all resource bounds. + // so this caps total in-flight resource usage. private const int MaxConcurrentTextures = 512; + // max new texture-load coroutines spawned in any single frame, regardless of how + // many completions free up queue slots that frame. Bounds per-frame allocation / + // dispatch overhead independently of MaxConcurrentTextures. + private const int MaxTextureSpawnsPerFrame = 128; + private static Harmony persistentHarmony; private static string PersistentHarmonyID => typeof(KSPCFFastLoader).FullName; @@ -1270,6 +1276,7 @@ private GameObject LoadDAE() private static readonly ProfilerMarker s_pmCopyLevel0 = new ProfilerMarker("KSPCF.Tex.CopyLevel0"); private static readonly ProfilerMarker s_pmFileSize = new ProfilerMarker("KSPCF.Tex.FileSize"); private static readonly ProfilerMarker s_pmReadAllBytes = new ProfilerMarker("KSPCF.Tex.ReadAllBytes"); + private static readonly ProfilerMarker s_pmCompress = new ProfilerMarker("KSPCF.Tex.Compress"); // Result/error carrier for each texture file. Replaces RawAsset for textures. private sealed class TextureLoadRequest @@ -1663,54 +1670,48 @@ private static Texture2D CreateUninitializedTexture2D( return tex; } - // Wraps an inner format-specific coroutine with exception capture and semaphore release. + // Wraps an inner format-specific coroutine with exception capture. // C# does not allow yield inside a try/catch, so we manually drive MoveNext() and - // do the catch around just the MoveNext call. - private static IEnumerator LoadTextureWrapperCoroutine(TextureLoadRequest req, IEnumerator inner, SemaphoreSlim semaphore) + // do the catch around just the MoveNext call. The driver detects completion via + // req.Status, so no other signaling is required here. + private static IEnumerator LoadTextureWrapperCoroutine(TextureLoadRequest req, IEnumerator inner) { - try + while (true) { - while (true) + bool moved; + bool failed = false; + object current = null; + try { - bool moved; - bool failed = false; - object current = null; - try - { - moved = inner.MoveNext(); - if (moved) - current = inner.Current; - } - catch (Exception e) - { - req.Exception = e; - req.ErrorMessage = $"{e.GetType().Name}: {e.Message}"; - req.Status = TextureLoadRequest.State.Failed; - moved = false; - failed = true; - } - if (failed) - yield break; - if (!moved) - break; - yield return current; + moved = inner.MoveNext(); + if (moved) + current = inner.Current; } - - if (req.Status == TextureLoadRequest.State.Pending) + catch (Exception e) { - if (req.Result != null) - req.Status = TextureLoadRequest.State.Ready; - else - { - if (req.ErrorMessage == null) - req.ErrorMessage = "Loader produced no result"; - req.Status = TextureLoadRequest.State.Failed; - } + req.Exception = e; + req.ErrorMessage = $"{e.GetType().Name}: {e.Message}"; + req.Status = TextureLoadRequest.State.Failed; + moved = false; + failed = true; } + if (failed) + yield break; + if (!moved) + break; + yield return current; } - finally + + if (req.Status == TextureLoadRequest.State.Pending) { - semaphore.Release(); + if (req.Result != null) + req.Status = TextureLoadRequest.State.Ready; + else + { + if (req.ErrorMessage == null) + req.ErrorMessage = "Loader produced no result"; + req.Status = TextureLoadRequest.State.Failed; + } } } @@ -1833,6 +1834,9 @@ private static IEnumerator LoadUWRCoroutine(TextureLoadRequest req) if (isNormalMap) dst.wrapMode = TextureWrapMode.Repeat; + // Wait a frame to avoid UnshareTextureData overhead within Unity + yield return null; + NativeArray srcData = src.GetRawTextureData(); NativeArray dstData = dst.GetRawTextureData(); TextureFormat srcFormat = src.format; @@ -1874,7 +1878,10 @@ private static IEnumerator LoadUWRCoroutine(TextureLoadRequest req) } if (canCompress) - src.Compress(highQuality: !isNormalMap); + { + using (s_pmCompress.Auto()) + src.Compress(highQuality: !isNormalMap); + } else if (!isNormalMap) Debug.LogWarning($"Texture '{req.File.url}' isn't eligible for DXT compression, width and height must be multiples of 4"); @@ -1950,6 +1957,9 @@ private static IEnumerator LoadTRUECOLORCoroutine(TextureLoadRequest req) Texture2D dst = CreateUninitializedTexture2D(tex.width, tex.height, TextureFormat.RGBA32, mipChain: isPot); dst.wrapMode = TextureWrapMode.Repeat; + // Wait a frame so we avoid a call to UnshareTextureData within unity + yield return null; + NativeArray srcData = tex.GetRawTextureData(); NativeArray dstData = dst.GetRawTextureData(); TextureFormat srcFormat = tex.format; @@ -1972,7 +1982,8 @@ private static IEnumerator LoadTRUECOLORCoroutine(TextureLoadRequest req) if (isPot) { tex.Apply(updateMipmaps: true); - tex.Compress(highQuality: false); + using (s_pmCompress.Auto()) + tex.Compress(highQuality: false); } tex.Apply(updateMipmaps: false, makeNoLongerReadable: true); @@ -2067,6 +2078,9 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) Texture2D dst = CreateUninitializedTexture2D(texture.width, texture.height, TextureFormat.RGBA32, mipChain: isPot); dst.wrapMode = TextureWrapMode.Repeat; + // Avoid UnshareTextureData overhead within Unity by waiting a frame. + yield return null; + NativeArray srcData = texture.GetRawTextureData(); NativeArray dstData = dst.GetRawTextureData(); TextureFormat srcFormat = texture.format; @@ -2089,7 +2103,8 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) if (isPot) { texture.Apply(updateMipmaps: true); - texture.Compress(highQuality: false); + using (s_pmCompress.Auto()) + texture.Compress(highQuality: false); } texture.Apply(updateMipmaps: false, makeNoLongerReadable: true); } @@ -2152,74 +2167,85 @@ private static void FallBackFromCache(TextureLoadRequest req, CachedTextureInfo private static IEnumerator TextureDriverCoroutine(List requests, HashSet loadedUrls) { GameDatabase gdb = GameDatabase.Instance; - SemaphoreSlim semaphore = new SemaphoreSlim(MaxConcurrentTextures, MaxConcurrentTextures); + Queue active = new Queue(MaxConcurrentTextures); int spawnIdx = 0; - int completeIdx = 0; int total = requests.Count; double nextFrameTime = ElapsedTime + minFrameTimeD; - while (completeIdx < total) + while (spawnIdx < total || active.Count > 0) { - while (spawnIdx < total && semaphore.Wait(0)) + // Drain completed requests at the front of the queue. + while (active.Count > 0 && active.Peek().Status != TextureLoadRequest.State.Pending) { - TextureLoadRequest req = requests[spawnIdx++]; - IEnumerator inner; - switch (req.AssetType) - { - case RawAsset.AssetType.TextureDDS: - inner = LoadDDSCoroutine(req); - break; - case RawAsset.AssetType.TexturePNG: - case RawAsset.AssetType.TextureJPG: - inner = LoadUWRCoroutine(req); - break; - case RawAsset.AssetType.TextureTRUECOLOR: - inner = LoadTRUECOLORCoroutine(req); - break; - case RawAsset.AssetType.TextureMBM: - inner = LoadMBMCoroutine(req); - break; - case RawAsset.AssetType.TextureTGA: - inner = LoadTGACoroutine(req); - break; - case RawAsset.AssetType.TexturePNGCached: - inner = LoadPNGCachedCoroutine(req); - break; - default: - inner = null; - break; - } + InsertReadyRequest(active.Peek(), loadedUrls); + active.Dequeue(); + loadedAssetCount++; + } - if (inner == null) - { - req.ErrorMessage = $"Unknown asset type {req.AssetType}"; - req.Status = TextureLoadRequest.State.Failed; - semaphore.Release(); - } - else - { - Debug.Log($"Load Texture: {req.File.url}"); - gdb.StartCoroutine(LoadTextureWrapperCoroutine(req, inner, semaphore)); - } + // Spawn new coroutines, bounded by the in-flight cap and the per-frame cap. + int spawnsThisFrame = 0; + while (spawnIdx < total + && active.Count < MaxConcurrentTextures + && spawnsThisFrame < MaxTextureSpawnsPerFrame) + { + SpawnTextureCoroutine(requests[spawnIdx++], active, gdb); + spawnsThisFrame++; } - TextureLoadRequest head = requests[completeIdx]; - if (head.Status == TextureLoadRequest.State.Pending) + if (active.Count == 0 && spawnIdx >= total) + break; + + if (ElapsedTime > nextFrameTime) { - if (ElapsedTime > nextFrameTime) - { - nextFrameTime = ElapsedTime + minFrameTimeD; - gdb.progressFraction = (float)(loadedAssetCount + completeIdx) / totalAssetCount; - gdb.progressTitle = $"Loading texture asset {completeIdx}/{total}"; - } - yield return null; - continue; + nextFrameTime = ElapsedTime + minFrameTimeD; + int completed = spawnIdx - active.Count; + gdb.progressFraction = (float)(loadedAssetCount + completed) / totalAssetCount; + gdb.progressTitle = $"Loading texture asset {completed}/{total}"; } + yield return null; + } + } - InsertReadyRequest(head, loadedUrls); - completeIdx++; - loadedAssetCount++; + private static void SpawnTextureCoroutine(TextureLoadRequest req, Queue active, GameDatabase gdb) + { + IEnumerator inner; + switch (req.AssetType) + { + case RawAsset.AssetType.TextureDDS: + inner = LoadDDSCoroutine(req); + break; + case RawAsset.AssetType.TexturePNG: + case RawAsset.AssetType.TextureJPG: + inner = LoadUWRCoroutine(req); + break; + case RawAsset.AssetType.TextureTRUECOLOR: + inner = LoadTRUECOLORCoroutine(req); + break; + case RawAsset.AssetType.TextureMBM: + inner = LoadMBMCoroutine(req); + break; + case RawAsset.AssetType.TextureTGA: + inner = LoadTGACoroutine(req); + break; + case RawAsset.AssetType.TexturePNGCached: + inner = LoadPNGCachedCoroutine(req); + break; + default: + inner = null; + break; + } + + if (inner == null) + { + req.ErrorMessage = $"Unknown asset type {req.AssetType}"; + req.Status = TextureLoadRequest.State.Failed; + } + else + { + Debug.Log($"Load Texture: {req.File.url}"); + gdb.StartCoroutine(LoadTextureWrapperCoroutine(req, inner)); } + active.Enqueue(req); } private static void InsertReadyRequest(TextureLoadRequest req, HashSet loadedUrls) From 9f436f0610fcb45630fe01ae5b8221219eda5fb9 Mon Sep 17 00:00:00 2001 From: Sean Lynch Date: Wed, 29 Apr 2026 19:34:28 -0700 Subject: [PATCH 04/15] Add some more tracing spans --- KSPCommunityFixes/Performance/FastLoader.cs | 45 ++++++++++++++++----- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 8bc5e9d..e7b2d20 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -159,7 +159,7 @@ internal class KSPCFFastLoader : MonoBehaviour // max concurrent per-texture coroutines spawned by TextureDriverCoroutine. // Each in-flight request holds at most one of {ReadHandle, UnityWebRequest, background Task}, // so this caps total in-flight resource usage. - private const int MaxConcurrentTextures = 512; + private const int MaxConcurrentTextures = 16384; // max new texture-load coroutines spawned in any single frame, regardless of how // many completions free up queue slots that frame. Bounds per-frame allocation / @@ -1277,6 +1277,11 @@ private GameObject LoadDAE() private static readonly ProfilerMarker s_pmFileSize = new ProfilerMarker("KSPCF.Tex.FileSize"); private static readonly ProfilerMarker s_pmReadAllBytes = new ProfilerMarker("KSPCF.Tex.ReadAllBytes"); private static readonly ProfilerMarker s_pmCompress = new ProfilerMarker("KSPCF.Tex.Compress"); + private static readonly ProfilerMarker s_pmGetRawDataDDS = new ProfilerMarker("KSPCF.Tex.LoadDDS.GetRawTextureData"); + private static readonly ProfilerMarker s_pmGetRawDataUWR = new ProfilerMarker("KSPCF.Tex.LoadUWR.GetRawTextureData"); + private static readonly ProfilerMarker s_pmGetRawDataTGA = new ProfilerMarker("KSPCF.Tex.LoadTGA.GetRawTextureData"); + private static readonly ProfilerMarker s_pmGetRawDataPNGCached = new ProfilerMarker("KSPCF.Tex.LoadPNGCached.GetRawTextureData"); + private static readonly ProfilerMarker s_pmGetRawDataSaveCache = new ProfilerMarker("KSPCF.Tex.SaveCache.GetRawTextureData"); // Result/error carrier for each texture file. Replaces RawAsset for textures. private sealed class TextureLoadRequest @@ -1760,7 +1765,9 @@ private static IEnumerator LoadDDSCoroutine(TextureLoadRequest req) // before AsyncReadManager writes into the staging buffer. yield return null; - NativeArray dst = tex.GetRawTextureData(); + NativeArray dst; + using (s_pmGetRawDataDDS.Auto()) + dst = tex.GetRawTextureData(); long expectedSize = dst.Length; if (hdr.FileLength - hdr.DataOffset < expectedSize) { @@ -1837,8 +1844,13 @@ private static IEnumerator LoadUWRCoroutine(TextureLoadRequest req) // Wait a frame to avoid UnshareTextureData overhead within Unity yield return null; - NativeArray srcData = src.GetRawTextureData(); - NativeArray dstData = dst.GetRawTextureData(); + NativeArray srcData; + NativeArray dstData; + using (s_pmGetRawDataUWR.Auto()) + { + srcData = src.GetRawTextureData(); + dstData = dst.GetRawTextureData(); + } TextureFormat srcFormat = src.format; Task task; @@ -1942,7 +1954,7 @@ private static IEnumerator LoadTRUECOLORCoroutine(TextureLoadRequest req) data.Dispose(); Texture2D tex = CreateUninitializedTexture2D(2, 2, TextureFormat.ARGB32, mipChain: false); - if (!ImageConversion.LoadImage(tex, managed, markNonReadable: false)) + if (!tex.LoadImage(managed, markNonReadable: false)) { UnityEngine.Object.Destroy(tex); req.ErrorMessage = "TRUECOLOR: ImageConversion.LoadImage failed"; @@ -1960,8 +1972,13 @@ private static IEnumerator LoadTRUECOLORCoroutine(TextureLoadRequest req) // Wait a frame so we avoid a call to UnshareTextureData within unity yield return null; - NativeArray srcData = tex.GetRawTextureData(); - NativeArray dstData = dst.GetRawTextureData(); + NativeArray srcData; + NativeArray dstData; + using (s_pmGetRawDataTGA.Auto()) + { + srcData = tex.GetRawTextureData(); + dstData = dst.GetRawTextureData(); + } TextureFormat srcFormat = tex.format; Task swizzleTask = Task.Run(() => { @@ -2081,8 +2098,14 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) // Avoid UnshareTextureData overhead within Unity by waiting a frame. yield return null; - NativeArray srcData = texture.GetRawTextureData(); - NativeArray dstData = dst.GetRawTextureData(); + NativeArray srcData; + NativeArray dstData; + using (s_pmGetRawDataPNGCached.Auto()) + { + srcData = texture.GetRawTextureData(); + dstData = dst.GetRawTextureData(); + } + TextureFormat srcFormat = texture.format; Task swizzleTask = Task.Run(() => { @@ -2932,7 +2955,9 @@ public CachedTextureInfo(UrlFile urlFile, Texture2D texture, bool isNormalMap, l public void SaveRawTextureData(Texture2D texture) { - byte[] rawData = texture.GetRawTextureData(); + byte[] rawData; + using (s_pmGetRawDataSaveCache.Auto()) + rawData = texture.GetRawTextureData(); File.WriteAllBytes(Path.Combine(loader.textureCachePath, id.ToString()), rawData); } From 5e7e396d5d092ec4f2b64e18ed82507f2f7973c5 Mon Sep 17 00:00:00 2001 From: Sean Lynch Date: Wed, 29 Apr 2026 20:09:18 -0700 Subject: [PATCH 05/15] Clean up some messy code --- KSPCommunityFixes/Performance/FastLoader.cs | 43 ++++++++++----------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index e7b2d20..7eb4a9b 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -36,6 +36,7 @@ using UnityEngine.Profiling; using System.Threading.Tasks; using KSP.UI; +using System.Security.Cryptography; namespace KSPCommunityFixes.Performance { @@ -1683,40 +1684,36 @@ private static IEnumerator LoadTextureWrapperCoroutine(TextureLoadRequest req, I { while (true) { - bool moved; - bool failed = false; - object current = null; + object current; try { - moved = inner.MoveNext(); - if (moved) - current = inner.Current; + if (!inner.MoveNext()) + break; + + current = inner.Current; } catch (Exception e) { req.Exception = e; req.ErrorMessage = $"{e.GetType().Name}: {e.Message}"; req.Status = TextureLoadRequest.State.Failed; - moved = false; - failed = true; - } - if (failed) yield break; - if (!moved) - break; + } + yield return current; } - if (req.Status == TextureLoadRequest.State.Pending) + if (req.Status != TextureLoadRequest.State.Pending) + yield break; + + if (req.Result != null) { - if (req.Result != null) - req.Status = TextureLoadRequest.State.Ready; - else - { - if (req.ErrorMessage == null) - req.ErrorMessage = "Loader produced no result"; - req.Status = TextureLoadRequest.State.Failed; - } + req.Status = TextureLoadRequest.State.Ready; + } + else + { + req.ErrorMessage ??= "Loader produced no result"; + req.Status = TextureLoadRequest.State.Failed; } } @@ -1725,8 +1722,8 @@ private static IEnumerator LoadDDSCoroutine(TextureLoadRequest req) string path = req.File.fullPath; Task hdrTask = Task.Run(() => { - using (s_pmParseDDSHeader.Auto()) - return ParseDDSHeader(path); + using var scope = s_pmParseDDSHeader.Auto(); + return ParseDDSHeader(path); }); while (!hdrTask.IsCompleted) yield return null; From bf639efa8739ee7a3aa6c6de0c05dd27537b6952 Mon Sep 17 00:00:00 2001 From: Sean Lynch Date: Wed, 29 Apr 2026 20:36:02 -0700 Subject: [PATCH 06/15] Simplify a comment --- KSPCommunityFixes/Performance/FastLoader.cs | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 7eb4a9b..905bf79 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -157,14 +157,8 @@ internal class KSPCFFastLoader : MonoBehaviour // min amount of files to try to keep in memory, regardless of maxBufferSize private const int minFileRead = 10; - // max concurrent per-texture coroutines spawned by TextureDriverCoroutine. - // Each in-flight request holds at most one of {ReadHandle, UnityWebRequest, background Task}, - // so this caps total in-flight resource usage. - private const int MaxConcurrentTextures = 16384; - - // max new texture-load coroutines spawned in any single frame, regardless of how - // many completions free up queue slots that frame. Bounds per-frame allocation / - // dispatch overhead independently of MaxConcurrentTextures. + // Max number of new texture load coroutines that will be spawned each frame. + // This should roughly limit the max frame time spent on loading textures. private const int MaxTextureSpawnsPerFrame = 128; private static Harmony persistentHarmony; @@ -2187,7 +2181,7 @@ private static void FallBackFromCache(TextureLoadRequest req, CachedTextureInfo private static IEnumerator TextureDriverCoroutine(List requests, HashSet loadedUrls) { GameDatabase gdb = GameDatabase.Instance; - Queue active = new Queue(MaxConcurrentTextures); + Queue active = new Queue(); int spawnIdx = 0; int total = requests.Count; double nextFrameTime = ElapsedTime + minFrameTimeD; @@ -2204,9 +2198,7 @@ private static IEnumerator TextureDriverCoroutine(List reque // Spawn new coroutines, bounded by the in-flight cap and the per-frame cap. int spawnsThisFrame = 0; - while (spawnIdx < total - && active.Count < MaxConcurrentTextures - && spawnsThisFrame < MaxTextureSpawnsPerFrame) + while (spawnIdx < total && spawnsThisFrame < MaxTextureSpawnsPerFrame) { SpawnTextureCoroutine(requests[spawnIdx++], active, gdb); spawnsThisFrame++; From 6f35b32d3f51de584944964872e8b7e09cdb8c68 Mon Sep 17 00:00:00 2001 From: Sean Lynch Date: Wed, 29 Apr 2026 23:48:04 -0700 Subject: [PATCH 07/15] Allow NPOT dds textures to have mipmaps --- KSPCommunityFixes/Performance/FastLoader.cs | 36 ++------------------- 1 file changed, 2 insertions(+), 34 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 905bf79..63c92eb 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -1366,31 +1366,7 @@ public static void Build() public static bool IsSupported(GraphicsFormat fmt) => supported != null && supported.Contains(fmt); } - // Block-compressed format check used to ignore mipChain on NPOT textures - // (Unity rejects DXT5 + mipmaps on non-power-of-two sources). - private static bool IsBlockCompressed(GraphicsFormat fmt) - { - switch (fmt) - { - case GraphicsFormat.RGBA_DXT1_UNorm: - case GraphicsFormat.RGBA_DXT1_SRGB: - case GraphicsFormat.RGBA_DXT5_UNorm: - case GraphicsFormat.RGBA_DXT5_SRGB: - case GraphicsFormat.R_BC4_UNorm: - case GraphicsFormat.R_BC4_SNorm: - case GraphicsFormat.RG_BC5_UNorm: - case GraphicsFormat.RG_BC5_SNorm: - case GraphicsFormat.RGBA_BC7_UNorm: - case GraphicsFormat.RGBA_BC7_SRGB: - case GraphicsFormat.RGB_BC6H_SFloat: - case GraphicsFormat.RGB_BC6H_UFloat: - return true; - default: - return false; - } - } - - // Background DDS header reader. Throws on bad magic or unsupported format. +// Background DDS header reader. Throws on bad magic or unsupported format. // Does not call any Unity API (so it is safe on a worker thread). private static DDSPreparedHeader ParseDDSHeader(string path) { @@ -1733,17 +1709,9 @@ private static IEnumerator LoadDDSCoroutine(TextureLoadRequest req) yield break; } - // Unity rejects mipChain on NPOT textures for block-compressed formats. - bool mipChain = hdr.MipChain; - if (mipChain && IsBlockCompressed(hdr.Format) - && !(Numerics.IsPowerOfTwo(hdr.Width) && Numerics.IsPowerOfTwo(hdr.Height))) - { - mipChain = false; - } - Texture2D tex = CreateUninitializedTexture2D( hdr.Width, hdr.Height, - mipChain ? -1 : 1, + hdr.MipChain ? -1 : 1, hdr.Format); if (tex.IsNullOrDestroyed()) { From bfe11aa976a41852e87df191bd4451652d351095 Mon Sep 17 00:00:00 2001 From: Sean Lynch Date: Thu, 30 Apr 2026 00:02:10 -0700 Subject: [PATCH 08/15] Cleanup cached PNG path --- KSPCommunityFixes/GlobalSuppressions.cs | 1 - KSPCommunityFixes/Internal/PatchSettings.cs | 18 -- KSPCommunityFixes/Performance/FastLoader.cs | 338 +------------------- 3 files changed, 9 insertions(+), 348 deletions(-) diff --git a/KSPCommunityFixes/GlobalSuppressions.cs b/KSPCommunityFixes/GlobalSuppressions.cs index 2c49b00..431d157 100644 --- a/KSPCommunityFixes/GlobalSuppressions.cs +++ b/KSPCommunityFixes/GlobalSuppressions.cs @@ -9,6 +9,5 @@ [assembly: SuppressMessage("Style", "IDE0017:Simplify object initialization")] [assembly: SuppressMessage("Design", "CA1031:Do not catch general exception types")] [assembly: SuppressMessage("Style", "IDE0016:Use 'throw' expression", Justification = "", Scope = "member", Target = "~M:KSPCommunityFixes.BasePatch.PatchInfo.#ctor(KSPCommunityFixes.BasePatch.PatchMethodType,System.Reflection.MethodBase,KSPCommunityFixes.BasePatch,System.String,System.Int32)")] -[assembly: SuppressMessage("Design", "CA1034:Nested types should not be visible", Justification = "", Scope = "type", Target = "~T:KSPCommunityFixes.TextureLoaderOptimizations.CachedTextureInfo")] [assembly: SuppressMessage("Style", "IDE0028:Simplify collection initialization", Justification = "", Scope = "member", Target = "~M:KSPCommunityFixes.LocalizationUtils.GenerateLocTemplate(System.String)")] diff --git a/KSPCommunityFixes/Internal/PatchSettings.cs b/KSPCommunityFixes/Internal/PatchSettings.cs index 700076c..554625f 100644 --- a/KSPCommunityFixes/Internal/PatchSettings.cs +++ b/KSPCommunityFixes/Internal/PatchSettings.cs @@ -38,9 +38,6 @@ protected override void ApplyPatches() if (disableMHPatch != null) entryCount++; - if (KSPCFFastLoader.IsPatchEnabled) - entryCount++; - // NoIVA is always enabled entryCount++; } @@ -107,21 +104,6 @@ static void GameplaySettingsScreen_DrawMiniSettings_Postfix(ref DialogGUIBase[] count++; } - if (KSPCFFastLoader.IsPatchEnabled) - { - DialogGUIToggle toggle = new DialogGUIToggle(KSPCFFastLoader.TextureCacheEnabled, - () => (KSPCFFastLoader.TextureCacheEnabled) - ? Localizer.Format("#autoLOC_900889") //"Enabled" - : Localizer.Format("#autoLOC_900890"), //"Disabled" - KSPCFFastLoader.OnToggleCacheFromSettings, 150f); - toggle.tooltipText = KSPCFFastLoader.LOC_SettingsTooltip; - - modifiedResult[count] = new DialogGUIHorizontalLayout(TextAnchor.MiddleLeft, - new DialogGUILabel(() => KSPCFFastLoader.LOC_SettingsTitle, 150f), - toggle, new DialogGUIFlexibleSpace()); - count++; - } - DialogGUISlider noIVAslider = new DialogGUISlider(NoIVA.PatchStateToFloat, 0f, 2f, true, 100f, 20f, NoIVA.SwitchPatchState); noIVAslider.tooltipText = NoIVA.LOC_SettingsTooltip; DialogGUILabel valueLabel = new DialogGUILabel(NoIVA.PatchStateTitle); diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 63c92eb..90d3a0f 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -130,12 +130,6 @@ void OnDestroy() [KSPAddon(KSPAddon.Startup.Instantly, true)] internal class KSPCFFastLoader : MonoBehaviour { - public static string LOC_SettingsTitle = "Texture caching optimization"; - public static string LOC_SettingsTooltip = - "Cache PNG textures on disk instead of converting them on every KSP launch." + - "\nSpeedup loading time but increase disk space usage." + - "\nChanges will take effect after relaunching KSP"; - public static string LOC_PopupL1 = "KSPCommunityFixes can cache converted PNG textures on disk to speed up loading time."; public static string LOC_F_PopupL2 = @@ -173,22 +167,14 @@ internal class KSPCFFastLoader : MonoBehaviour public static KSPCFFastLoader loader; public static bool IsPatchEnabled { get; private set; } - public static bool TextureCacheEnabled => textureCacheEnabled; + // Vestigial: kept so the popup can persist its choice across launches once it is repurposed. private static bool textureCacheEnabled; private static string ModPath => Path.GetDirectoryName(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)); private static string ConfigPath => Path.Combine(ModPath, "PluginData", "PNGTextureCache.cfg"); private bool userOptInChoiceDone; - private const string textureCacheVersion = "V2"; private string configPath; - private string textureCachePath; - private string textureCacheDataPath; - private string textureProgressMarkerPath; - - private Dictionary textureCacheData; - private HashSet textureDataIds; - private bool cacheUpdated = false; internal static Dictionary modelsByUrl; internal static Dictionary modelsByDirectoryUrl; @@ -257,7 +243,6 @@ private void Awake() GameEvents.OnGameDatabaseLoaded.Add(OnGameDatabaseLoaded); configPath = ConfigPath; - textureCachePath = Path.Combine(ModPath, "PluginData", "TextureCache"); if (File.Exists(configPath)) { @@ -269,17 +254,6 @@ private void Awake() if (!config.TryGetValue(nameof(textureCacheEnabled), ref textureCacheEnabled)) userOptInChoiceDone = false; } - -#if DEBUG && !DEBUG_TEXTURE_CACHE - userOptInChoiceDone = true; - textureCacheEnabled = false; -#endif - } - - void Start() - { - if (IsPatchEnabled && !userOptInChoiceDone) - StartCoroutine(WaitForUserOptIn()); } /// @@ -468,10 +442,6 @@ static IEnumerator FastAssetLoader(List configFileTypes) KSPCFFastLoaderReport.wConfigTranslate.Stop(); yield return null; - gdb.progressTitle = "Waiting for PNGTextureCache opt-in..."; - while (!loader.userOptInChoiceDone) - yield return null; - // Start load asset bundles in the background while we load other assets. PreloadAssetBundleObjects(gdb); @@ -577,9 +547,6 @@ static IEnumerator FastAssetLoader(List configFileTypes) } } - // PNG/DDS cache temporarily disabled — opt-in popup and settings are still - // wired up, but no cache I/O is performed during loading. - gdb.progressTitle = "Loading sound assets..."; KSPCFFastLoaderReport.wAudioLoading.Restart(); yield return null; @@ -1022,7 +989,6 @@ public enum AssetType TextureJPG, TextureMBM, TexturePNG, - TexturePNGCached, TextureTGA, TextureTRUECOLOR, ModelMU, @@ -1035,7 +1001,6 @@ public enum AssetType "JPG texture", "MBM texture", "PNG texture", - "Cached PNG Texture", "TGA texture", "TRUECOLOR texture", "MU model", @@ -1274,9 +1239,8 @@ private GameObject LoadDAE() private static readonly ProfilerMarker s_pmCompress = new ProfilerMarker("KSPCF.Tex.Compress"); private static readonly ProfilerMarker s_pmGetRawDataDDS = new ProfilerMarker("KSPCF.Tex.LoadDDS.GetRawTextureData"); private static readonly ProfilerMarker s_pmGetRawDataUWR = new ProfilerMarker("KSPCF.Tex.LoadUWR.GetRawTextureData"); + private static readonly ProfilerMarker s_pmGetRawDataTRUECOLOR = new ProfilerMarker("KSPCF.Tex.LoadTRUECOLOR.GetRawTextureData"); private static readonly ProfilerMarker s_pmGetRawDataTGA = new ProfilerMarker("KSPCF.Tex.LoadTGA.GetRawTextureData"); - private static readonly ProfilerMarker s_pmGetRawDataPNGCached = new ProfilerMarker("KSPCF.Tex.LoadPNGCached.GetRawTextureData"); - private static readonly ProfilerMarker s_pmGetRawDataSaveCache = new ProfilerMarker("KSPCF.Tex.SaveCache.GetRawTextureData"); // Result/error carrier for each texture file. Replaces RawAsset for textures. private sealed class TextureLoadRequest @@ -1286,8 +1250,6 @@ public enum State : byte { Pending, Ready, Failed } public UrlFile File; public RawAsset.AssetType AssetType; public long FileLength; - public CachedTextureInfo CachedInfo; - public bool CameFromCache; public volatile State Status; public TextureInfo Result; public string ErrorMessage; @@ -1299,17 +1261,6 @@ public TextureLoadRequest(UrlFile file, RawAsset.AssetType assetType) AssetType = assetType; Status = State.Pending; } - - // Called from the texture cache reader thread before the driver runs. - public void CheckTextureCache() - { - CachedTextureInfo info = GetCachedTextureInfo(File); - if (info == null) - return; - AssetType = RawAsset.AssetType.TexturePNGCached; - CachedInfo = info; - CameFromCache = true; - } } // Result of background DDS header parsing. @@ -1933,7 +1884,7 @@ private static IEnumerator LoadTRUECOLORCoroutine(TextureLoadRequest req) NativeArray srcData; NativeArray dstData; - using (s_pmGetRawDataTGA.Auto()) + using (s_pmGetRawDataTRUECOLOR.Auto()) { srcData = tex.GetRawTextureData(); dstData = dst.GetRawTextureData(); @@ -2059,7 +2010,7 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) NativeArray srcData; NativeArray dstData; - using (s_pmGetRawDataPNGCached.Auto()) + using (s_pmGetRawDataTGA.Auto()) { srcData = texture.GetRawTextureData(); dstData = dst.GetRawTextureData(); @@ -2095,57 +2046,6 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) req.Status = TextureLoadRequest.State.Ready; } - private static IEnumerator LoadPNGCachedCoroutine(TextureLoadRequest req) - { - CachedTextureInfo info = req.CachedInfo; - string path = info.FilePath; - Task readTask = Task.Run(() => - { - using (s_pmReadAllBytes.Auto()) - return File.ReadAllBytes(path); - }); - while (!readTask.IsCompleted) - yield return null; - if (readTask.IsFaulted) - { - Debug.LogWarning($"[KSPCF] Cached PNG read failed for '{req.File.url}', falling back to fresh load"); - FallBackFromCache(req, info); - IEnumerator fallback = LoadUWRCoroutine(req); - while (fallback.MoveNext()) - yield return fallback.Current; - yield break; - } - - byte[] buffer = readTask.Result; - req.FileLength = buffer.Length; - - if (info.TryCreateTexture(buffer, out Texture2D texture)) - { - req.Result = new TextureInfo(req.File, texture, info.normal, isReadable: false, isCompressed: true); - req.Status = TextureLoadRequest.State.Ready; - yield break; - } - - Debug.LogWarning($"[KSPCF] Cached PNG TryCreateTexture failed for '{req.File.url}', falling back to fresh load"); - FallBackFromCache(req, info); - IEnumerator fallback2 = LoadUWRCoroutine(req); - while (fallback2.MoveNext()) - yield return fallback2.Current; - } - - private static void FallBackFromCache(TextureLoadRequest req, CachedTextureInfo info) - { - if (loader.textureCacheData.Remove(info.name)) - { - loader.textureDataIds.Remove(info.id); - try { File.Delete(info.FilePath); } catch { } - loader.cacheUpdated = true; - } - req.AssetType = RawAsset.AssetType.TexturePNG; - req.CameFromCache = false; - req.CachedInfo = null; - } - private static IEnumerator TextureDriverCoroutine(List requests, HashSet loadedUrls) { GameDatabase gdb = GameDatabase.Instance; @@ -2207,9 +2107,6 @@ private static void SpawnTextureCoroutine(TextureLoadRequest req, Queue instructions } #endregion - #region PNG texture cache - - private static void SetupTextureCacheThread(List requests) - { - loader.SetupTextureCache(); - - foreach (TextureLoadRequest req in requests) - req.CheckTextureCache(); - } - - private void SetupTextureCache() - { - textureCacheDataPath = Path.Combine(textureCachePath, "textureData.json"); - textureProgressMarkerPath = Path.Combine(textureCachePath, "progressMarker"); - - textureCacheData = new Dictionary(2000); - textureDataIds = new HashSet(2000); - - if (Directory.Exists(textureCachePath)) - { - if (File.Exists(textureProgressMarkerPath)) - { - // If progress marker is still here, the game somehow crashed during loading on - // the previous run, so we delete the whole cache to avoid orphan cached texture - // files from lying around - Directory.Delete(textureCachePath, true); - Directory.CreateDirectory(textureCachePath); - } - else if (File.Exists(textureCacheDataPath)) - { - string[] textureCacheDataContent = File.ReadAllLines(textureCacheDataPath); - - if (textureCacheDataContent.Length > 0 && textureCacheDataContent[0].StartsWith(textureCacheVersion)) - { - for (int i = 1; i < textureCacheDataContent.Length; i++) - { - string json = textureCacheDataContent[i]; - CachedTextureInfo cachedTextureInfo = JsonUtility.FromJson(json); - textureCacheData.Add(cachedTextureInfo.name, cachedTextureInfo); - textureDataIds.Add(cachedTextureInfo.id); - } - } - else - { - Directory.Delete(textureCachePath, true); - Directory.CreateDirectory(textureCachePath); - } - } - } - else - { - Directory.CreateDirectory(textureCachePath); - } - - File.WriteAllText(textureProgressMarkerPath, string.Empty); - } - - private void WriteTextureCache() - { - if (!userOptInChoiceDone || !textureCacheEnabled) - { - if (Directory.Exists(textureCachePath)) - Directory.Delete(textureCachePath, true); - } - else - { - foreach (CachedTextureInfo cachedTextureInfo in textureCacheData.Values) - { - if (!cachedTextureInfo.loaded) - { - cacheUpdated = true; - File.Delete(cachedTextureInfo.FilePath); - } - } + #region User opt-in popup (vestigial) - if (cacheUpdated) - { - File.Delete(textureCacheDataPath); - - List textureCacheDataContent = new List(textureCacheData.Count + 1); - textureCacheDataContent.Add(textureCacheVersion); - - foreach (CachedTextureInfo cachedTextureInfo in textureCacheData.Values) - if (cachedTextureInfo.loaded) - textureCacheDataContent.Add(JsonUtility.ToJson(cachedTextureInfo)); - - File.WriteAllLines(textureCacheDataPath, textureCacheDataContent); - } - - File.Delete(textureProgressMarkerPath); - } - } - - [Serializable] - private class CachedTextureInfo - { - private static readonly System.Random random = new System.Random(); - - public string name; - public uint id; - public long time; - public long size; - public int width; - public int height; - public int mipCount; - public bool readable; - public bool normal; - [NonSerialized] public bool loaded = false; - - public string FilePath => Path.Combine(loader.textureCachePath, id.ToString()); - - public CachedTextureInfo() { } - - public CachedTextureInfo(UrlFile urlFile, Texture2D texture, bool isNormalMap, long size, long time) - { - name = urlFile.url; - do - { - unchecked - { - id = (uint)random.Next(); - } - } - while (loader.textureDataIds.Contains(id)); - - this.size = size; - this.time = time; - width = texture.width; - height = texture.height; - mipCount = texture.mipmapCount; - normal = isNormalMap; - // Always non-readable: matches the unified DDS/PNG/JPG readability policy. - // The 'readable' field in older cache entries on disk is ignored on read. - readable = false; - loaded = true; - } - - public void SaveRawTextureData(Texture2D texture) - { - byte[] rawData; - using (s_pmGetRawDataSaveCache.Auto()) - rawData = texture.GetRawTextureData(); - File.WriteAllBytes(Path.Combine(loader.textureCachePath, id.ToString()), rawData); - } - - public bool TryCreateTexture(byte[] buffer, out Texture2D texture) - { - try - { - texture = CreateUninitializedTexture2D(width, height, mipCount, GraphicsFormat.RGBA_DXT5_UNorm); - texture.LoadRawTextureData(buffer); - texture.Apply(false, true); - loaded = true; - return true; - } - catch (Exception e) - { - Debug.LogWarning($"[KSPCF] Failed to load cached PNG texture '{name}'\n{e}"); - texture = null; - return false; - } - } - } - - private static CachedTextureInfo GetCachedTextureInfo(UrlDir.UrlFile file) - { - if (!loader.textureCacheData.TryGetValue(file.url, out CachedTextureInfo cachedTextureInfo)) - return null; - - if (!GetFileStats(file.fullPath, out long size, out long time) || size != cachedTextureInfo.size || time != cachedTextureInfo.time) - { - loader.textureCacheData.Remove(file.url); - loader.textureDataIds.Remove(cachedTextureInfo.id); - File.Delete(cachedTextureInfo.FilePath); - loader.cacheUpdated = true; - return null; - } - - return cachedTextureInfo; - } - - private static bool GetFileStats(string path, out long size, out long time) - { - MonoIO.GetFileStat(path, out MonoIOStat stat, out MonoIOError error); - if (error == MonoIOError.ERROR_FILE_NOT_FOUND || error == MonoIOError.ERROR_PATH_NOT_FOUND || error == MonoIOError.ERROR_NOT_READY) - { - size = 0; - time = 0; - return false; - } - - size = stat.Length; - time = Math.Max(stat.CreationTime, stat.LastWriteTime); - if (size <= 0 || time <= 0) - return false; - - return true; - } - - private static void SaveCachedTextureFromBytes(UrlDir.UrlFile urlFile, Texture2D texture, bool isNormalMap, byte[] rawData) - { - if (!GetFileStats(urlFile.fullPath, out long size, out long creationTime)) - { - Debug.LogWarning($"[KSPCF] PNG texture '{urlFile.url}' couldn't be cached : IO error"); - return; - } - - CachedTextureInfo cachedTextureInfo = new CachedTextureInfo(urlFile, texture, isNormalMap, size, creationTime); - File.WriteAllBytes(Path.Combine(loader.textureCachePath, cachedTextureInfo.id.ToString()), rawData); - loader.textureCacheData.Add(cachedTextureInfo.name, cachedTextureInfo); - loader.textureDataIds.Add(cachedTextureInfo.id); - loader.cacheUpdated = true; - Debug.Log($"[KSPCF] PNG texture '{urlFile.url}' was converted to DXT5 and has been cached for future reloads"); - } + // The popup, opt-in flow, and PNG-cache-size estimator below are intentionally + // kept around — the cache they were originally tied to has been removed, but the + // popup is going to be repurposed for an upcoming feature. Nothing currently + // triggers WaitForUserOptIn; it must be invoked explicitly by the new feature. private static IEnumerator WaitForUserOptIn() { @@ -3068,15 +2757,6 @@ private static void SetOptIn(bool optIn, ref bool? choosed) config.Save(ConfigPath); } - internal static void OnToggleCacheFromSettings(bool cacheEnabled) - { - textureCacheEnabled = cacheEnabled; - ConfigNode config = new ConfigNode(); - config.AddValue(nameof(userOptInChoiceDone), true); - config.AddValue(nameof(textureCacheEnabled), cacheEnabled); - config.Save(ConfigPath); - } - private static readonly string flagsPath = Path.DirectorySeparatorChar + "Flags" + Path.DirectorySeparatorChar; private static bool GetPngCacheSize(string path, out int cacheSize, out bool isNormal) From 8ab1d5d5d917e86b9ba51da207f4c67c1ce44b07 Mon Sep 17 00:00:00 2001 From: Sean Lynch Date: Thu, 30 Apr 2026 01:00:43 -0700 Subject: [PATCH 09/15] Avoid allocating secondary textures for swizzles --- KSPCommunityFixes/Performance/FastLoader.cs | 220 +++++++++++--------- 1 file changed, 119 insertions(+), 101 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 90d3a0f..3e7f367 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -1233,7 +1233,6 @@ private GameObject LoadDAE() // appears under that thread in the Unity profiler. private static readonly ProfilerMarker s_pmParseDDSHeader = new ProfilerMarker("KSPCF.Tex.ParseDDSHeader"); private static readonly ProfilerMarker s_pmSwizzleNormalMap = new ProfilerMarker("KSPCF.Tex.SwizzleNormalMap"); - private static readonly ProfilerMarker s_pmCopyLevel0 = new ProfilerMarker("KSPCF.Tex.CopyLevel0"); private static readonly ProfilerMarker s_pmFileSize = new ProfilerMarker("KSPCF.Tex.FileSize"); private static readonly ProfilerMarker s_pmReadAllBytes = new ProfilerMarker("KSPCF.Tex.ReadAllBytes"); private static readonly ProfilerMarker s_pmCompress = new ProfilerMarker("KSPCF.Tex.Compress"); @@ -1317,7 +1316,7 @@ public static void Build() public static bool IsSupported(GraphicsFormat fmt) => supported != null && supported.Contains(fmt); } -// Background DDS header reader. Throws on bad magic or unsupported format. + // Background DDS header reader. Throws on bad magic or unsupported format. // Does not call any Unity API (so it is safe on a worker thread). private static DDSPreparedHeader ParseDDSHeader(string path) { @@ -1463,10 +1462,33 @@ private static GraphicsFormat MapDDSFormat(DDSHeader hdr, bool hasDx10, DDSHeade } } - // Channel-swizzle for normal maps (extracted from BitmapToCompressedNormalMapFast). - // src must hold pixel data in srcFormat; dst is written as RGBA32 (level 0 only — - // if dst has a mip chain, higher mip levels are left untouched and are populated - // by the caller's Apply(updateMipmaps: true)). + // In-place channel-swizzle for RGBA32 normal maps. Operates on the texture's + // entire raw byte buffer, which means every populated mip level when the texture + // was created with a mip chain. + // + // Channel swizzle is per-pixel and the box-filter mip generator is linear, so + // swizzling pre-built mips matches what stock KSP produced by swizzling level-0 + // and regenerating from there (BitmapToCompressedNormalMapFast). + private static unsafe void SwizzleNormalMap(NativeArray data) + { + byte* p = (byte*)NativeArrayUnsafeUtility.GetUnsafePtr(data); + int len = data.Length; + // (r, g, b, a) -> (g, g, g, r) + for (int i = 0; i < len; i += 4) + { + byte r = p[i]; + byte g = p[i + 1]; + p[i] = g; + p[i + 1] = g; + p[i + 2] = g; + p[i + 3] = r; + } + } + + // Legacy src->dst swizzle, kept for the rare TGA RGB24 path (where source and + // destination have different pixel sizes so an in-place transform is impossible). + // Writes level 0 only; the caller is expected to call Apply(updateMipmaps: true) + // afterwards if a mip chain is wanted. private static unsafe void SwizzleNormalMap(NativeArray src, NativeArray dst, TextureFormat srcFormat) { byte* s = (byte*)NativeArrayUnsafeUtility.GetUnsafeReadOnlyPtr(src); @@ -1708,8 +1730,6 @@ private static IEnumerator LoadDDSCoroutine(TextureLoadRequest req) req.Status = TextureLoadRequest.State.Ready; } - private static readonly string flagsPathSep = Path.DirectorySeparatorChar + "Flags" + Path.DirectorySeparatorChar; - private static IEnumerator LoadUWRCoroutine(TextureLoadRequest req) { string filePath = req.File.fullPath; @@ -1737,66 +1757,33 @@ private static IEnumerator LoadUWRCoroutine(TextureLoadRequest req) } bool isNormalMap = req.File.name.EndsWith("NRM"); - bool isFlag = filePath.Contains(flagsPathSep); bool canCompress = src.width % 4 == 0 && src.height % 4 == 0; - if (isNormalMap || isFlag) + // UWR returns a Texture2D with a mipchain already populated, so for normal + // maps we swizzle every level of its CPU buffer in place — no dst alloc, + // no copy, no Apply(true). + if (isNormalMap) { - // Allocate a destination texture with mipchain (when POT) so that - // Apply(true) can generate mipmaps. UWR-loaded textures have no mipchain. - bool isPot = Numerics.IsPowerOfTwo(src.width) && Numerics.IsPowerOfTwo(src.height); - bool wantMipChain = isPot; - TextureFormat dstFormat = isNormalMap ? TextureFormat.RGBA32 : src.format; - Texture2D dst = CreateUninitializedTexture2D(src.width, src.height, dstFormat, wantMipChain); - if (isNormalMap) - dst.wrapMode = TextureWrapMode.Repeat; + src.wrapMode = TextureWrapMode.Repeat; // Wait a frame to avoid UnshareTextureData overhead within Unity yield return null; - NativeArray srcData; - NativeArray dstData; + NativeArray allLevels; using (s_pmGetRawDataUWR.Auto()) + allLevels = src.GetRawTextureData(); + Task swizzleTask = Task.Run(() => { - srcData = src.GetRawTextureData(); - dstData = dst.GetRawTextureData(); - } - TextureFormat srcFormat = src.format; - - Task task; - if (isNormalMap) - { - task = Task.Run(() => - { - using (s_pmSwizzleNormalMap.Auto()) - SwizzleNormalMap(srcData, dstData, srcFormat); - }); - } - else - { - // Flag: copy level-0 raw data byte-for-byte (formats already match). - int level0Bytes = srcData.Length; - task = Task.Run(() => - { - using (s_pmCopyLevel0.Auto()) - NativeArray.Copy(srcData, 0, dstData, 0, level0Bytes); - }); - } - - while (!task.IsCompleted) + using (s_pmSwizzleNormalMap.Auto()) + SwizzleNormalMap(allLevels); + }); + while (!swizzleTask.IsCompleted) yield return null; - if (task.IsFaulted) + if (swizzleTask.IsFaulted) { UnityEngine.Object.Destroy(src); - UnityEngine.Object.Destroy(dst); - throw UnwrapFaultedTask(task, "texture pre-process task faulted"); + throw UnwrapFaultedTask(swizzleTask, "swizzle task faulted"); } - - UnityEngine.Object.Destroy(src); - src = dst; - - if (wantMipChain) - src.Apply(updateMipmaps: true); } if (canCompress) @@ -1863,7 +1850,11 @@ private static IEnumerator LoadTRUECOLORCoroutine(TextureLoadRequest req) byte[] managed = data.ToArray(); data.Dispose(); - Texture2D tex = CreateUninitializedTexture2D(2, 2, TextureFormat.ARGB32, mipChain: false); + // Create as RGBA32 with mipchain when this is a normal map: LoadImage will + // populate every mip level for us, so we can swizzle the whole thing in place. + // Non-normals keep the existing single-mip readable behavior. + bool isNormalMap = req.File.name.EndsWith("NRM"); + Texture2D tex = CreateUninitializedTexture2D(2, 2, TextureFormat.RGBA32, mipChain: isNormalMap); if (!tex.LoadImage(managed, markNonReadable: false)) { UnityEngine.Object.Destroy(tex); @@ -1872,43 +1863,32 @@ private static IEnumerator LoadTRUECOLORCoroutine(TextureLoadRequest req) yield break; } - bool isNormalMap = req.File.name.EndsWith("NRM"); if (isNormalMap) { bool isPot = Numerics.IsPowerOfTwo(tex.width) && Numerics.IsPowerOfTwo(tex.height); - Texture2D dst = CreateUninitializedTexture2D(tex.width, tex.height, TextureFormat.RGBA32, mipChain: isPot); - dst.wrapMode = TextureWrapMode.Repeat; + tex.wrapMode = TextureWrapMode.Repeat; // Wait a frame so we avoid a call to UnshareTextureData within unity yield return null; - NativeArray srcData; - NativeArray dstData; + NativeArray allLevels; using (s_pmGetRawDataTRUECOLOR.Auto()) - { - srcData = tex.GetRawTextureData(); - dstData = dst.GetRawTextureData(); - } - TextureFormat srcFormat = tex.format; + allLevels = tex.GetRawTextureData(); Task swizzleTask = Task.Run(() => { using (s_pmSwizzleNormalMap.Auto()) - SwizzleNormalMap(srcData, dstData, srcFormat); + SwizzleNormalMap(allLevels); }); while (!swizzleTask.IsCompleted) yield return null; if (swizzleTask.IsFaulted) { UnityEngine.Object.Destroy(tex); - UnityEngine.Object.Destroy(dst); throw UnwrapFaultedTask(swizzleTask, "swizzle task faulted"); } - UnityEngine.Object.Destroy(tex); - tex = dst; if (isPot) { - tex.Apply(updateMipmaps: true); using (s_pmCompress.Auto()) tex.Compress(highQuality: false); } @@ -2002,44 +1982,82 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) if (isNormalMap) { bool isPot = Numerics.IsPowerOfTwo(texture.width) && Numerics.IsPowerOfTwo(texture.height); - Texture2D dst = CreateUninitializedTexture2D(texture.width, texture.height, TextureFormat.RGBA32, mipChain: isPot); - dst.wrapMode = TextureWrapMode.Repeat; - - // Avoid UnshareTextureData overhead within Unity by waiting a frame. - yield return null; - NativeArray srcData; - NativeArray dstData; - using (s_pmGetRawDataTGA.Auto()) + if (texture.format == TextureFormat.RGBA32) { - srcData = texture.GetRawTextureData(); - dstData = dst.GetRawTextureData(); - } + // tgaImage.CreateTexture(mipmap: true, ...) already calls Apply(true) + // and the texture is readable, so the CPU buffer holds every populated + // mip level. Swizzle the whole thing in place. + texture.wrapMode = TextureWrapMode.Repeat; - TextureFormat srcFormat = texture.format; - Task swizzleTask = Task.Run(() => - { - using (s_pmSwizzleNormalMap.Auto()) - SwizzleNormalMap(srcData, dstData, srcFormat); - }); - while (!swizzleTask.IsCompleted) + // Avoid UnshareTextureData overhead within Unity by waiting a frame. yield return null; - if (swizzleTask.IsFaulted) + + NativeArray allLevels; + using (s_pmGetRawDataTGA.Auto()) + allLevels = texture.GetRawTextureData(); + Task swizzleTask = Task.Run(() => + { + using (s_pmSwizzleNormalMap.Auto()) + SwizzleNormalMap(allLevels); + }); + while (!swizzleTask.IsCompleted) + yield return null; + if (swizzleTask.IsFaulted) + { + UnityEngine.Object.Destroy(texture); + throw UnwrapFaultedTask(swizzleTask, "swizzle task faulted"); + } + + if (isPot) + { + using (s_pmCompress.Auto()) + texture.Compress(highQuality: false); + } + texture.Apply(updateMipmaps: false, makeNoLongerReadable: true); + } + else { + // RGB24 (24bpp TGA): pixel size differs from RGBA32, so we can't + // swizzle in place. Fall back to the legacy src->dst expansion path. + Texture2D dst = CreateUninitializedTexture2D(texture.width, texture.height, TextureFormat.RGBA32, mipChain: isPot); + dst.wrapMode = TextureWrapMode.Repeat; + + yield return null; + + NativeArray srcData; + NativeArray dstData; + using (s_pmGetRawDataTGA.Auto()) + { + srcData = texture.GetRawTextureData(); + dstData = dst.GetRawTextureData(); + } + + TextureFormat srcFormat = texture.format; + Task swizzleTask = Task.Run(() => + { + using (s_pmSwizzleNormalMap.Auto()) + SwizzleNormalMap(srcData, dstData, srcFormat); + }); + while (!swizzleTask.IsCompleted) + yield return null; + if (swizzleTask.IsFaulted) + { + UnityEngine.Object.Destroy(texture); + UnityEngine.Object.Destroy(dst); + throw UnwrapFaultedTask(swizzleTask, "swizzle task faulted"); + } UnityEngine.Object.Destroy(texture); - UnityEngine.Object.Destroy(dst); - throw UnwrapFaultedTask(swizzleTask, "swizzle task faulted"); - } - UnityEngine.Object.Destroy(texture); - texture = dst; + texture = dst; - if (isPot) - { - texture.Apply(updateMipmaps: true); - using (s_pmCompress.Auto()) - texture.Compress(highQuality: false); + if (isPot) + { + texture.Apply(updateMipmaps: true); + using (s_pmCompress.Auto()) + texture.Compress(highQuality: false); + } + texture.Apply(updateMipmaps: false, makeNoLongerReadable: true); } - texture.Apply(updateMipmaps: false, makeNoLongerReadable: true); } req.Result = new TextureInfo(req.File, texture, isNormalMap, isReadable: !isNormalMap, isCompressed: true); From 126ab2b8f78cbe8b9cca2f1414e7782dbf1e9cec Mon Sep 17 00:00:00 2001 From: Sean Lynch Date: Thu, 30 Apr 2026 01:02:40 -0700 Subject: [PATCH 10/15] Actually compress TGA images --- KSPCommunityFixes/Performance/FastLoader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 3e7f367..a9bdd61 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -1970,7 +1970,7 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) yield break; } - Texture2D texture = tgaImage.CreateTexture(mipmap: true, linear: false, compress: true, compressHighQuality: false, allowRead: true); + Texture2D texture = tgaImage.CreateTexture(mipmap: true, linear: false, compress: true, compressHighQuality: true, allowRead: true); if (texture.IsNullOrDestroyed()) { req.ErrorMessage = "TGA: CreateTexture failed"; From c24c9c0ed46395c1e4e6ecd573496128494a7a89 Mon Sep 17 00:00:00 2001 From: Sean Lynch Date: Thu, 30 Apr 2026 01:30:35 -0700 Subject: [PATCH 11/15] Fix a double-count in loaded textures --- KSPCommunityFixes/Performance/FastLoader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index a9bdd61..5e48bae 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -2097,7 +2097,7 @@ private static IEnumerator TextureDriverCoroutine(List reque { nextFrameTime = ElapsedTime + minFrameTimeD; int completed = spawnIdx - active.Count; - gdb.progressFraction = (float)(loadedAssetCount + completed) / totalAssetCount; + gdb.progressFraction = (float)loadedAssetCount / totalAssetCount; gdb.progressTitle = $"Loading texture asset {completed}/{total}"; } yield return null; From a6f11a2eb810ed4cada459b5e95c3a91ce28fecb Mon Sep 17 00:00:00 2001 From: Sean Lynch Date: Thu, 30 Apr 2026 01:48:18 -0700 Subject: [PATCH 12/15] Fix an OOB write in the TGA loader --- KSPCommunityFixes/Performance/FastLoader.cs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 5e48bae..2b7e507 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -1487,8 +1487,8 @@ private static unsafe void SwizzleNormalMap(NativeArray data) // Legacy src->dst swizzle, kept for the rare TGA RGB24 path (where source and // destination have different pixel sizes so an in-place transform is impossible). - // Writes level 0 only; the caller is expected to call Apply(updateMipmaps: true) - // afterwards if a mip chain is wanted. + // Walks src.Length end-to-end; the caller must size dst with a mip chain that + // matches src's so the constant 3:4 (or 4:4) byte-count ratio fills dst exactly. private static unsafe void SwizzleNormalMap(NativeArray src, NativeArray dst, TextureFormat srcFormat) { byte* s = (byte*)NativeArrayUnsafeUtility.GetUnsafeReadOnlyPtr(src); @@ -2019,8 +2019,11 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) else { // RGB24 (24bpp TGA): pixel size differs from RGBA32, so we can't - // swizzle in place. Fall back to the legacy src->dst expansion path. - Texture2D dst = CreateUninitializedTexture2D(texture.width, texture.height, TextureFormat.RGBA32, mipChain: isPot); + // swizzle in place. Fall back to the legacy src->dst expansion + // path. dst is allocated with a full mip chain so its byte layout + // matches the mipmapped src (CreateTexture(mipmap: true) populates + // every level), letting the swizzle fill dst end-to-end. + Texture2D dst = CreateUninitializedTexture2D(texture.width, texture.height, TextureFormat.RGBA32, mipChain: true); dst.wrapMode = TextureWrapMode.Repeat; yield return null; @@ -2052,7 +2055,6 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) if (isPot) { - texture.Apply(updateMipmaps: true); using (s_pmCompress.Auto()) texture.Compress(highQuality: false); } From 03b3e60d966f1e165a1dfa8f0c78c6b0184bf41d Mon Sep 17 00:00:00 2001 From: Phantomical Date: Sat, 2 May 2026 14:45:58 -0700 Subject: [PATCH 13/15] Delay Load Texture log until the texture load has actually completed --- KSPCommunityFixes/Performance/FastLoader.cs | 177 ++++++++++++++------ 1 file changed, 126 insertions(+), 51 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 2b7e507..667b9c5 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -1316,8 +1316,6 @@ public static void Build() public static bool IsSupported(GraphicsFormat fmt) => supported != null && supported.Contains(fmt); } - // Background DDS header reader. Throws on bad magic or unsupported format. - // Does not call any Unity API (so it is safe on a worker thread). private static DDSPreparedHeader ParseDDSHeader(string path) { FileInfo fi = new FileInfo(path); @@ -1336,7 +1334,7 @@ private static DDSPreparedHeader ParseDDSHeader(string path) bool isNormalMap = (hdr.ddspf.dwFlags & 0x80000u) != 0 || (hdr.ddspf.dwFlags & 0x80000000u) != 0; DDSHeaderDX10 dx10Header = default; - bool hasDx10 = (DDSFourCCBg)hdr.ddspf.dwFourCC == DDSFourCCBg.DX10; + bool hasDx10 = (DDSFourCC)hdr.ddspf.dwFourCC == DDSFourCC.DX10; if (hasDx10) { if (fileLength < 148) @@ -1361,8 +1359,7 @@ private static DDSPreparedHeader ParseDDSHeader(string path) }; } - // Background-thread-safe FourCC enum (mirrors RawAsset.DDSFourCC, which is private to RawAsset). - private enum DDSFourCCBg : uint + private enum DDSFourCC : uint { DXT1 = 0x31545844, DXT2 = 0x32545844, @@ -1395,26 +1392,26 @@ private enum DDSFourCCBg : uint private static GraphicsFormat MapDDSFormat(DDSHeader hdr, bool hasDx10, DDSHeaderDX10 dx10, out string error) { error = null; - DDSFourCCBg fourCC = (DDSFourCCBg)hdr.ddspf.dwFourCC; + DDSFourCC fourCC = (DDSFourCC)hdr.ddspf.dwFourCC; switch (fourCC) { - case DDSFourCCBg.DXT1: return GraphicsFormatUtility.GetGraphicsFormat(TextureFormat.DXT1, true); - case DDSFourCCBg.DXT5: return GraphicsFormatUtility.GetGraphicsFormat(TextureFormat.DXT5, true); - case DDSFourCCBg.BC4U_ATI: - case DDSFourCCBg.BC4U: return GraphicsFormat.R_BC4_UNorm; - case DDSFourCCBg.BC4S: return GraphicsFormat.R_BC4_SNorm; - case DDSFourCCBg.BC5U_ATI: - case DDSFourCCBg.BC5U: return GraphicsFormat.RG_BC5_UNorm; - case DDSFourCCBg.BC5S: return GraphicsFormat.RG_BC5_SNorm; - case DDSFourCCBg.R16G16B16A16_UNORM: return GraphicsFormat.R16G16B16A16_UNorm; - case DDSFourCCBg.R16G16B16A16_SNORM: return GraphicsFormat.R16G16B16A16_SNorm; - case DDSFourCCBg.R16_FLOAT: return GraphicsFormat.R16_SFloat; - case DDSFourCCBg.R16G16_FLOAT: return GraphicsFormat.R16G16_SFloat; - case DDSFourCCBg.R16G16B16A16_FLOAT: return GraphicsFormat.R16G16B16A16_SFloat; - case DDSFourCCBg.R32_FLOAT: return GraphicsFormat.R32_SFloat; - case DDSFourCCBg.R32G32_FLOAT: return GraphicsFormat.R32G32_SFloat; - case DDSFourCCBg.R32G32B32A32_FLOAT: return GraphicsFormat.R32G32B32A32_SFloat; - case DDSFourCCBg.DX10: + case DDSFourCC.DXT1: return GraphicsFormatUtility.GetGraphicsFormat(TextureFormat.DXT1, true); + case DDSFourCC.DXT5: return GraphicsFormatUtility.GetGraphicsFormat(TextureFormat.DXT5, true); + case DDSFourCC.BC4U_ATI: + case DDSFourCC.BC4U: return GraphicsFormat.R_BC4_UNorm; + case DDSFourCC.BC4S: return GraphicsFormat.R_BC4_SNorm; + case DDSFourCC.BC5U_ATI: + case DDSFourCC.BC5U: return GraphicsFormat.RG_BC5_UNorm; + case DDSFourCC.BC5S: return GraphicsFormat.RG_BC5_SNorm; + case DDSFourCC.R16G16B16A16_UNORM: return GraphicsFormat.R16G16B16A16_UNorm; + case DDSFourCC.R16G16B16A16_SNORM: return GraphicsFormat.R16G16B16A16_SNorm; + case DDSFourCC.R16_FLOAT: return GraphicsFormat.R16_SFloat; + case DDSFourCC.R16G16_FLOAT: return GraphicsFormat.R16G16_SFloat; + case DDSFourCC.R16G16B16A16_FLOAT: return GraphicsFormat.R16G16B16A16_SFloat; + case DDSFourCC.R32_FLOAT: return GraphicsFormat.R32_SFloat; + case DDSFourCC.R32G32_FLOAT: return GraphicsFormat.R32G32_SFloat; + case DDSFourCC.R32G32B32A32_FLOAT: return GraphicsFormat.R32G32B32A32_SFloat; + case DDSFourCC.DX10: if (!hasDx10) { error = "DX10 marker without DX10 header"; @@ -1446,14 +1443,14 @@ private static GraphicsFormat MapDDSFormat(DDSHeader hdr, bool hasDx10, DDSHeade error = $"DXT10 format '{dx10.dxgiFormat}' is not supported"; return GraphicsFormat.None; } - case DDSFourCCBg.DXT2: - case DDSFourCCBg.DXT3: - case DDSFourCCBg.DXT4: - case DDSFourCCBg.RGBG: - case DDSFourCCBg.GRGB: - case DDSFourCCBg.UYVY: - case DDSFourCCBg.YUY2: - case DDSFourCCBg.CxV8U8: + case DDSFourCC.DXT2: + case DDSFourCC.DXT3: + case DDSFourCC.DXT4: + case DDSFourCC.RGBG: + case DDSFourCC.GRGB: + case DDSFourCC.UYVY: + case DDSFourCC.YUY2: + case DDSFourCC.CxV8U8: error = $"format '{fourCC}' is not supported, use DXT1 for RGB textures or DXT5 for RGBA textures"; return GraphicsFormat.None; default: @@ -1471,6 +1468,8 @@ private static GraphicsFormat MapDDSFormat(DDSHeader hdr, bool hasDx10, DDSHeade // and regenerating from there (BitmapToCompressedNormalMapFast). private static unsafe void SwizzleNormalMap(NativeArray data) { + using var scope = s_pmSwizzleNormalMap.Auto(); + byte* p = (byte*)NativeArrayUnsafeUtility.GetUnsafePtr(data); int len = data.Length; // (r, g, b, a) -> (g, g, g, r) @@ -1491,6 +1490,8 @@ private static unsafe void SwizzleNormalMap(NativeArray data) // matches src's so the constant 3:4 (or 4:4) byte-count ratio fills dst exactly. private static unsafe void SwizzleNormalMap(NativeArray src, NativeArray dst, TextureFormat srcFormat) { + using var scope = s_pmSwizzleNormalMap.Auto(); + byte* s = (byte*)NativeArrayUnsafeUtility.GetUnsafeReadOnlyPtr(src); byte* d = (byte*)NativeArrayUnsafeUtility.GetUnsafePtr(dst); int srcLen = src.Length; @@ -1533,9 +1534,7 @@ private static unsafe void SwizzleNormalMap(NativeArray src, NativeArray reque { GameDatabase gdb = GameDatabase.Instance; Queue active = new Queue(); - int spawnIdx = 0; int total = requests.Count; - double nextFrameTime = ElapsedTime + minFrameTimeD; + var iter = requests.GetEnumerator(); + int completed = 0; - while (spawnIdx < total || active.Count > 0) + while (true) { - // Drain completed requests at the front of the queue. - while (active.Count > 0 && active.Peek().Status != TextureLoadRequest.State.Pending) + while (active.TryPeek(out var pending)) { - InsertReadyRequest(active.Peek(), loadedUrls); + if (pending.Status == TextureLoadRequest.State.Pending) + break; + active.Dequeue(); + InsertReadyRequest(pending, loadedUrls); loadedAssetCount++; + completed++; } - // Spawn new coroutines, bounded by the in-flight cap and the per-frame cap. - int spawnsThisFrame = 0; - while (spawnIdx < total && spawnsThisFrame < MaxTextureSpawnsPerFrame) + for (int i = 0; i < MaxTextureSpawnsPerFrame; ++i) { - SpawnTextureCoroutine(requests[spawnIdx++], active, gdb); - spawnsThisFrame++; + if (!iter.MoveNext()) + goto WINDDOWN; + var request = iter.Current; + + gdb.StartCoroutine(LoadTextureCoroutine(request)); + active.Enqueue(request); } - if (active.Count == 0 && spawnIdx >= total) - break; + gdb.progressFraction = (float)loadedAssetCount / totalAssetCount; + gdb.progressTitle = $"Loading texture asset {completed}/{total}"; + yield return null; + } - if (ElapsedTime > nextFrameTime) + WINDDOWN: + while (active.TryDequeue(out var pending)) + { + while (pending.Status == TextureLoadRequest.State.Pending) { - nextFrameTime = ElapsedTime + minFrameTimeD; - int completed = spawnIdx - active.Count; gdb.progressFraction = (float)loadedAssetCount / totalAssetCount; gdb.progressTitle = $"Loading texture asset {completed}/{total}"; + yield return null; } - yield return null; + + InsertReadyRequest(pending, loadedUrls); + loadedAssetCount++; + completed++; + } + } + + private static IEnumerator LoadTextureCoroutine(TextureLoadRequest req) + { + IEnumerator inner; + switch (req.AssetType) + { + case RawAsset.AssetType.TextureDDS: + inner = LoadDDSCoroutine(req); + break; + case RawAsset.AssetType.TexturePNG: + case RawAsset.AssetType.TextureJPG: + inner = LoadUWRCoroutine(req); + break; + case RawAsset.AssetType.TextureTRUECOLOR: + inner = LoadTRUECOLORCoroutine(req); + break; + case RawAsset.AssetType.TextureMBM: + inner = LoadMBMCoroutine(req); + break; + case RawAsset.AssetType.TextureTGA: + inner = LoadTGACoroutine(req); + break; + default: + req.ErrorMessage = $"Unknown asset type {req.AssetType}"; + req.Status = TextureLoadRequest.State.Failed; + yield break; + } + + while (true) + { + object current; + try + { + if (!inner.MoveNext()) + break; + + current = inner.Current; + } + catch (Exception e) + { + req.Exception = e; + req.ErrorMessage = $"{e.GetType().Name}: {e.Message}"; + req.Status = TextureLoadRequest.State.Failed; + yield break; + } + + yield return current; + } + + if (req.Status != TextureLoadRequest.State.Pending) + yield break; + + if (req.Result != null) + { + req.Status = TextureLoadRequest.State.Ready; + } + else + { + req.ErrorMessage ??= "Loader produced no result"; + req.Status = TextureLoadRequest.State.Failed; } } @@ -2147,6 +2220,8 @@ private static void SpawnTextureCoroutine(TextureLoadRequest req, Queue loadedUrls) { + Debug.Log($"Load Texture: {req.File.url}"); + if (req.Status == TextureLoadRequest.State.Failed) { Debug.LogWarning($"LOAD FAILED: {req.File.url}: {req.ErrorMessage}"); From ddec969b85f5a21bc45221f3637663a88ffc16de Mon Sep 17 00:00:00 2001 From: Phantomical Date: Sat, 2 May 2026 15:36:04 -0700 Subject: [PATCH 14/15] Properly delay until render thread is finished --- KSPCommunityFixes/Performance/FastLoader.cs | 78 +++++++++++++++++---- 1 file changed, 66 insertions(+), 12 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 667b9c5..aa0eb09 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -37,6 +37,7 @@ using System.Threading.Tasks; using KSP.UI; using System.Security.Cryptography; +using UnityEngine.Rendering; namespace KSPCommunityFixes.Performance { @@ -153,7 +154,7 @@ internal class KSPCFFastLoader : MonoBehaviour // Max number of new texture load coroutines that will be spawned each frame. // This should roughly limit the max frame time spent on loading textures. - private const int MaxTextureSpawnsPerFrame = 128; + private const int MaxTextureSpawnsPerFrame = 512; private static Harmony persistentHarmony; private static string PersistentHarmonyID => typeof(KSPCFFastLoader).FullName; @@ -1692,9 +1693,9 @@ private static IEnumerator LoadDDSCoroutine(TextureLoadRequest req) yield break; } - // Wait one frame so Unity's initial GPU resource creation completes - // before AsyncReadManager writes into the staging buffer. - yield return null; + // Wait until the texture is finished uploading so unity doesn't + // copy its internal buffer when we call GetRawTextureData + yield return WaitForGraphicsThread(); NativeArray dst; using (s_pmGetRawDataDDS.Auto()) @@ -1755,6 +1756,10 @@ private static IEnumerator LoadUWRCoroutine(TextureLoadRequest req) yield break; } + // Wait until the texture is finished uploading so unity doesn't + // copy its internal buffer when we operate on it. + yield return WaitForGraphicsThread(); + bool isNormalMap = req.File.name.EndsWith("NRM"); bool canCompress = src.width % 4 == 0 && src.height % 4 == 0; @@ -1765,9 +1770,6 @@ private static IEnumerator LoadUWRCoroutine(TextureLoadRequest req) { src.wrapMode = TextureWrapMode.Repeat; - // Wait a frame to avoid UnshareTextureData overhead within Unity - yield return null; - NativeArray allLevels; using (s_pmGetRawDataUWR.Auto()) allLevels = src.GetRawTextureData(); @@ -1867,8 +1869,9 @@ private static IEnumerator LoadTRUECOLORCoroutine(TextureLoadRequest req) bool isPot = Numerics.IsPowerOfTwo(tex.width) && Numerics.IsPowerOfTwo(tex.height); tex.wrapMode = TextureWrapMode.Repeat; - // Wait a frame so we avoid a call to UnshareTextureData within unity - yield return null; + // Wait until the texture is finished uploading so unity doesn't + // copy its internal buffer when we call GetRawTextureData + yield return WaitForGraphicsThread(); NativeArray allLevels; using (s_pmGetRawDataTRUECOLOR.Auto()) @@ -1989,8 +1992,9 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) // mip level. Swizzle the whole thing in place. texture.wrapMode = TextureWrapMode.Repeat; - // Avoid UnshareTextureData overhead within Unity by waiting a frame. - yield return null; + // Wait until the texture is finished uploading so unity doesn't + // copy its internal buffer when we call GetRawTextureData + yield return WaitForGraphicsThread(); NativeArray allLevels; using (s_pmGetRawDataTGA.Auto()) @@ -2101,7 +2105,7 @@ private static IEnumerator TextureDriverCoroutine(List reque yield return null; } - WINDDOWN: + WINDDOWN: while (active.TryDequeue(out var pending)) { while (pending.Status == TextureLoadRequest.State.Pending) @@ -3036,6 +3040,56 @@ public void Show() } } + // A helper that yields until it has been processed on the render thread. + // Use this to delay until the render thread is no longer using a texture + // (or any other resource). + private unsafe class WaitForGraphicsThreadInst : CustomYieldInstruction + { + static CommandBuffer DispatchCB; + static readonly IntPtr NotifyPtr = (IntPtr)Marshal.GetFunctionPointerForDelegate((Action)Notify); + static readonly int GchandleOffset = UnsafeUtility.GetFieldOffset( + typeof(WaitForGraphicsThreadInst).GetField(nameof(gchandle), BindingFlags.Instance | BindingFlags.NonPublic)); + static readonly int ReadyOffset = UnsafeUtility.GetFieldOffset( + typeof(WaitForGraphicsThreadInst).GetField(nameof(ready), BindingFlags.Instance | BindingFlags.NonPublic)); + + ulong gchandle = 0; + bool ready = false; + + public override bool keepWaiting => !ready; + + public WaitForGraphicsThreadInst() + { + DispatchCB ??= new CommandBuffer() + { + name = "KSPCF.WaitForGraphicsThreadCB" + }; + + void* addr = UnsafeUtility.PinGCObjectAndGetAddress(this, out gchandle); + try + { + DispatchCB.Clear(); + DispatchCB.IssuePluginEventAndData(NotifyPtr, 0, (IntPtr)addr); + Graphics.ExecuteCommandBuffer(DispatchCB); + } + catch + { + UnsafeUtility.ReleaseGCObject(gchandle); + throw; + } + } + + static void Notify(int _, IntPtr data) + { + ulong gchandle = *(ulong*)((byte*)data + GchandleOffset); + bool* ready = (bool*)((byte*)data + ReadyOffset); + + *ready = true; + UnsafeUtility.ReleaseGCObject(gchandle); + } + } + + private static WaitForGraphicsThreadInst WaitForGraphicsThread() => + new WaitForGraphicsThreadInst(); #endregion From 40de810e052763ec7ae79a615ac0a748524bace7 Mon Sep 17 00:00:00 2001 From: Phantomical Date: Sat, 2 May 2026 22:18:27 -0700 Subject: [PATCH 15/15] Move completed texture loading after dispatches --- KSPCommunityFixes/Performance/FastLoader.cs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index aa0eb09..db07aab 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -2079,6 +2079,16 @@ private static IEnumerator TextureDriverCoroutine(List reque while (true) { + for (int i = 0; i < MaxTextureSpawnsPerFrame; ++i) + { + if (!iter.MoveNext()) + goto WINDDOWN; + var request = iter.Current; + + gdb.StartCoroutine(LoadTextureCoroutine(request)); + active.Enqueue(request); + } + while (active.TryPeek(out var pending)) { if (pending.Status == TextureLoadRequest.State.Pending) @@ -2090,16 +2100,6 @@ private static IEnumerator TextureDriverCoroutine(List reque completed++; } - for (int i = 0; i < MaxTextureSpawnsPerFrame; ++i) - { - if (!iter.MoveNext()) - goto WINDDOWN; - var request = iter.Current; - - gdb.StartCoroutine(LoadTextureCoroutine(request)); - active.Enqueue(request); - } - gdb.progressFraction = (float)loadedAssetCount / totalAssetCount; gdb.progressTitle = $"Loading texture asset {completed}/{total}"; yield return null;