diff --git a/docs/changelog.rst b/docs/changelog.rst index 43f779b2d4..e605971980 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -34,6 +34,7 @@ Changelog * Runtime: added configurable physics `solver_iterations` and `sleep_threshold` via the `physics` property of :doc:`boot.config `. * Data Compiler: missing resource references no longer stop data compilation and print a warning instead. * Data Compiler: the data index is now validated at startup to catch missing data blobs and other edge cases. +* Data Compiler: the shader compiler has been optimized with caching, more parallelism, and by skipping unnecessary work, resulting in much faster material compilation times. * Lua: added ``GameSave`` singleton to save/load savegames. * Lua: added a comprehensive set of ``PhysicsWorld.joint_*`` functions for creating and manipulating various joint types. * Lua: ``print()`` now shows tables in a more readable format. diff --git a/src/resource/data_compiler.cpp b/src/resource/data_compiler.cpp index 66aa5fd8b4..bbaa32c1ae 100644 --- a/src/resource/data_compiler.cpp +++ b/src/resource/data_compiler.cpp @@ -34,6 +34,7 @@ #include "resource/data_compiler.h" #include "resource/mesh.h" #include "resource/resource_id.inl" +#include "resource/shader_resource.h" #include "resource/types.h" #include #include @@ -1481,7 +1482,9 @@ bool DataCompiler::compile_internal(const char *data_dir, const char *platform_n bool DataCompiler::compile(const char *data_dir, const char *platform_name) { profiler_globals::clear(); + shader_compiler::clear_metadata_cache(); bool success = compile_internal(data_dir, platform_name); + shader_compiler::clear_metadata_cache(); profiler_globals::flush(); return success; } diff --git a/src/resource/material_resource.cpp b/src/resource/material_resource.cpp index a7449177b8..a796f67778 100644 --- a/src/resource/material_resource.cpp +++ b/src/resource/material_resource.cpp @@ -360,13 +360,14 @@ namespace material_resource_internal Buffer shader_code(default_allocator()); FileBuffer shader_fb(shader_code); StringView shader_name; + StringId32 shader_id; DynamicString shader_library(ta); DynamicString shader(ta); RETURN_IF_ERROR(sjson::parse_string(shader, obj["shader"])); shader_name_defines(shader_name, defines, shader.c_str()); - s32 err = shader_compiler::compile_variant(shader_fb, &data.uniforms_meta, shader_library, shader_name, defines, opts, true, &samplers_meta); + s32 err = shader_compiler::compile_variant(shader_fb, &data.uniforms_meta, shader_library, shader_id, shader_name, defines, opts, true, &samplers_meta); ENSURE_OR_RETURN(MATERIAL_RESOURCE, err == 0, opts); opts.add_requirement("shader", shader_library.c_str()); @@ -401,7 +402,7 @@ namespace material_resource_internal MaterialResource mr; mr.version = RESOURCE_HEADER(RESOURCE_VERSION_MATERIAL); - mr.shader = shader.to_string_id(); + mr.shader = shader_id; mr.num_textures = array::size(data.textures); mr.texture_data_offset = sizeof(mr); mr.num_uniforms = array::size(data.uniforms); diff --git a/src/resource/shader_resource.cpp b/src/resource/shader_resource.cpp index 0ddac1050b..dbd3ba5ffd 100644 --- a/src/resource/shader_resource.cpp +++ b/src/resource/shader_resource.cpp @@ -724,6 +724,26 @@ namespace shader_resource_internal return pr.spawn(array::begin(argv), CROWN_PROCESS_STDOUT_PIPE | CROWN_PROCESS_STDERR_MERGE); } + static void wait_shaderc_process(CompileOptions &opts, Process &pr) + { + if (!pr.spawned()) + return; + + // Drain output before wait() when abandoning overlapping shaderc work. + TempAllocator4096 ta; + StringStream output(ta); + opts.read_output(output, pr); + pr.wait(); + } + + static void wait_shaderc_processes(CompileOptions &opts, Process *pr_vert, Process *pr_frag, u32 count) + { + for (u32 i = 0; i < count; ++i) { + wait_shaderc_process(opts, pr_vert[i]); + wait_shaderc_process(opts, pr_frag[i]); + } + } + struct RenderState { ALLOCATOR_AWARE; @@ -1233,6 +1253,127 @@ namespace shader_resource_internal return 0; } + struct MetadataCacheEntry + { + ALLOCATOR_AWARE; + + DynamicString shader_library; + Vector dependencies; + Vector uniform_meta; + Vector sampler_meta; + + explicit MetadataCacheEntry(Allocator &a) + : shader_library(a) + , dependencies(a) + , uniform_meta(a) + , sampler_meta(a) + { + } + }; + + typedef HashMap MetadataCache; + typedef HashMap ShaderLibraryCache; + + static MetadataCache *s_metadata_cache = NULL; + static ShaderLibraryCache *s_shader_library_cache = NULL; + + static MetadataCache &metadata_cache() + { + if (s_metadata_cache == NULL) + s_metadata_cache = CE_NEW(default_allocator(), MetadataCache)(default_allocator()); + + return *s_metadata_cache; + } + + static ShaderLibraryCache &shader_library_cache() + { + if (s_shader_library_cache == NULL) + s_shader_library_cache = CE_NEW(default_allocator(), ShaderLibraryCache)(default_allocator()); + + return *s_shader_library_cache; + } + + static void metadata_cache_key(DynamicString &key + , Platform::Enum platform + , const DynamicString &shader_library + , const DynamicString &shader + , const Vector &defines + ) + { + StringStream ss(default_allocator()); + ss << u32(platform) << "|"; + ss << shader_library.c_str() << "|"; + ss << shader.c_str(); + + for (u32 i = 0; i < vector::size(defines); ++i) + ss << "+" << defines[i].c_str(); + + key = string_stream::c_str(ss); + } + + static bool load_metadata_cache(const DynamicString &cache_key + , CompileOptions &opts + , DynamicString &shader_library + , Vector *uniform_meta + , Vector *sampler_meta + ) + { + if (!hash_map::has(metadata_cache(), cache_key)) + return false; + + const MetadataCacheEntry cached(default_allocator()); + const MetadataCacheEntry &entry = hash_map::get(metadata_cache(), cache_key, cached); + + // Cache hits skip opts.read(), so replay source dependencies for invalidation. + for (u32 i = 0; i < vector::size(entry.dependencies); ++i) + opts.fake_read(entry.dependencies[i].c_str()); + + shader_library = entry.shader_library; + + if (uniform_meta != NULL) + *uniform_meta = entry.uniform_meta; + + if (sampler_meta != NULL) + *sampler_meta = entry.sampler_meta; + + return true; + } + + static void store_metadata_cache(const DynamicString &cache_key + , const DynamicString &shader_library + , const HashSet &dependencies + , const Vector *uniform_meta + , const Vector *sampler_meta + ) + { + if (hash_map::has(metadata_cache(), cache_key)) + return; + + MetadataCacheEntry entry(default_allocator()); + entry.shader_library = shader_library; + + auto cur = hash_set::begin(dependencies); + auto end = hash_set::end(dependencies); + for (; cur != end; ++cur) { + HASH_SET_SKIP_HOLE(dependencies, cur); + vector::push_back(entry.dependencies, *cur); + } + + if (uniform_meta != NULL) + entry.uniform_meta = *uniform_meta; + + if (sampler_meta != NULL) + entry.sampler_meta = *sampler_meta; + + hash_map::set(metadata_cache(), cache_key, entry); + } + + static void cache_shader_library(const DynamicString &shader, const DynamicString &shader_library) + { + if (!hash_map::has(shader_library_cache(), shader)) + hash_map::set(shader_library_cache(), shader, shader_library); + } + struct BgfxShader { ALLOCATOR_AWARE; @@ -1297,6 +1438,7 @@ namespace shader_resource_internal HashMap _shaders; Vector _static_compile; + DynamicString _shader_library; DynamicString _vs_path; DynamicString _fs_path; DynamicString _varying_path; @@ -1321,6 +1463,7 @@ namespace shader_resource_internal , _bgfx_shaders(default_allocator()) , _shaders(default_allocator()) , _static_compile(default_allocator()) + , _shader_library(default_allocator()) , _vs_path(default_allocator()) , _fs_path(default_allocator()) , _varying_path(default_allocator()) @@ -1346,6 +1489,7 @@ namespace shader_resource_internal hash_map::clear(_bgfx_shaders); hash_map::clear(_shaders); vector::clear(_static_compile); + _shader_library = ""; } s32 parse(const char *path, bool is_include) @@ -1354,6 +1498,14 @@ namespace shader_resource_internal DynamicString path_str(ta); path_str = path; + if (!is_include) { + const char *type = resource_type(path); + if (type != NULL) + _shader_library.set(path, u32(type - path - 1)); + else + _shader_library = path; + } + if (hash_set::has(_parsed_includes, path_str)) return 0; @@ -2063,11 +2215,22 @@ namespace shader_resource_internal return 0; } + static void shader_variant(DynamicString &variant, const char *shader, const Vector &defines) + { + variant = shader; + + for (u32 jj = 0; jj < vector::size(defines); ++jj) { + variant += "+"; + variant += defines[jj]; + } + } + s32 parse_static_compile(const char *json) { TempAllocator4096 ta; JsonArray static_compile(ta); RETURN_IF_ERROR(sjson::parse_array(static_compile, json)); + HashSet static_compile_variants(default_allocator()); for (u32 ii = 0; ii < array::size(static_compile); ++ii) { JsonObject obj(ta); @@ -2083,6 +2246,13 @@ namespace shader_resource_internal RETURN_IF_ERROR(sjson::parse_string(def, defines[jj])); vector::push_back(sc._defines, def); } + std::sort(vector::begin(sc._defines), vector::end(sc._defines)); + + DynamicString variant(ta); + shader_variant(variant, sc._shader.c_str(), sc._defines); + if (hash_set::has(static_compile_variants, variant)) + continue; + hash_set::insert(static_compile_variants, variant); vector::push_back(_static_compile, sc); } @@ -2101,14 +2271,14 @@ namespace shader_resource_internal _opts.delete_file(_fs_bin_path.c_str()); } - static void shader_variant(DynamicString &variant, const char *shader, const Vector &defines) + void delete_temp_files(DynamicString * const *vs_bin_paths, DynamicString * const *fs_bin_paths, u32 count) { - variant = shader; - - for (u32 jj = 0; jj < vector::size(defines); ++jj) { - variant += "+"; - variant += defines[jj]; + for (u32 i = 1; i < count; ++i) { + _opts.delete_file(vs_bin_paths[i]->c_str()); + _opts.delete_file(fs_bin_paths[i]->c_str()); } + + delete_temp_files(); } static StringId32 shader_variant_id(const char *shader, const Vector &defines) @@ -2158,7 +2328,7 @@ namespace shader_resource_internal bw.write(state.encode()); // Render state bw.write(stencil_front); // Stencil bw.write(stencil_back); // - return compile_bgfx_shader(fb, meta, sampler_meta, bgfx_shader.c_str(), defines, metadata_only); // Sampler states and shader code + return compile_bgfx_shader(fb, meta, sampler_meta, shader, bgfx_shader.c_str(), defines, metadata_only); // Sampler states and shader code } s32 compile() @@ -2177,6 +2347,8 @@ namespace shader_resource_internal const DynamicString &shader = sc._shader; const Vector &defines = sc._defines; + cache_shader_library(shader, _shader_library); + RETURN_IF_FALSE(SHADER_RESOURCE, hash_map::has(_shaders, shader) , _opts , "Unknown shader: '%s'" @@ -2321,6 +2493,7 @@ namespace shader_resource_internal s32 compile_bgfx_shader(FileBuffer &fb , Vector *meta , Vector *sampler_meta + , const DynamicString &shader_name , const char *bgfx_shader , const Vector &defines , bool metadata_only @@ -2332,6 +2505,13 @@ namespace shader_resource_internal key = bgfx_shader; const BgfxShader shader_default(default_allocator()); const BgfxShader &shader = hash_map::get(_bgfx_shaders, key, shader_default); + const bool has_sampler_metadata = hash_map::size(shader._samplers) > 0; + // Full static compiles can seed metadata for later material-only compiles. + const bool cache_static_metadata = !metadata_only + && meta == NULL + && has_sampler_metadata + && !_shader_library.empty() + ; StringStream code(default_allocator()); s32 err = bgfx_shader_collect_code(code, shader); @@ -2351,23 +2531,115 @@ namespace shader_resource_internal fs_code << shader._fs_input_output.c_str(); fs_code << string_stream::c_str(code); fs_code << shader._fs_code.c_str(); + const bool need_preprocess = meta != NULL || has_sampler_metadata; - StringStream vs_source(default_allocator()); - StringStream fs_source(default_allocator()); - err = inject_sampler_stage_comments(vs_source, string_stream::c_str(vs_code), _opts); - ENSURE_OR_RETURN(SHADER_RESOURCE, err == 0, _opts); - err = inject_sampler_stage_comments(fs_source, string_stream::c_str(fs_code), _opts); - ENSURE_OR_RETURN(SHADER_RESOURCE, err == 0, _opts); + if (need_preprocess) { + StringStream vs_source(default_allocator()); + StringStream fs_source(default_allocator()); + err = inject_sampler_stage_comments(vs_source, string_stream::c_str(vs_code), _opts); + ENSURE_OR_RETURN(SHADER_RESOURCE, err == 0, _opts); + err = inject_sampler_stage_comments(fs_source, string_stream::c_str(fs_code), _opts); + ENSURE_OR_RETURN(SHADER_RESOURCE, err == 0, _opts); - _opts.write_temporary(_vs_path.c_str(), vs_source); - _opts.write_temporary(_fs_path.c_str(), fs_source); + _opts.write_temporary(_vs_path.c_str(), vs_source); + _opts.write_temporary(_fs_path.c_str(), fs_source); + } else { + _opts.write_temporary(_vs_path.c_str(), vs_code); + _opts.write_temporary(_fs_path.c_str(), fs_code); + } _opts.write_temporary(_varying_path.c_str(), varying_code); + const ShadercTargetList targets = shaderc_targets(_opts._platform); + Process binary_pr_vert[ShaderBackend::COUNT]; + Process binary_pr_frag[ShaderBackend::COUNT]; + // Each target needs a unique output path because all binary compiles are spawned before + // their outputs are read back. + DynamicString vs_bin_path_0(default_allocator()); + DynamicString vs_bin_path_1(default_allocator()); + DynamicString vs_bin_path_2(default_allocator()); + DynamicString vs_bin_path_3(default_allocator()); + DynamicString fs_bin_path_0(default_allocator()); + DynamicString fs_bin_path_1(default_allocator()); + DynamicString fs_bin_path_2(default_allocator()); + DynamicString fs_bin_path_3(default_allocator()); + DynamicString *vs_bin_paths[ShaderBackend::COUNT] = + { + &vs_bin_path_0, + &vs_bin_path_1, + &vs_bin_path_2, + &vs_bin_path_3 + }; + DynamicString *fs_bin_paths[ShaderBackend::COUNT] = + { + &fs_bin_path_0, + &fs_bin_path_1, + &fs_bin_path_2, + &fs_bin_path_3 + }; + + for (u32 ti = 0; ti < targets.count; ++ti) { + const ShadercTarget &target = targets.targets[ti]; + *vs_bin_paths[ti] = _vs_bin_path.c_str(); + *fs_bin_paths[ti] = _fs_bin_path.c_str(); + + if (ti > 0) { + *vs_bin_paths[ti] += "."; + *vs_bin_paths[ti] += target.profile; + *fs_bin_paths[ti] += "."; + *fs_bin_paths[ti] += target.profile; + } + } + + if (!metadata_only) { + // Start binary shaderc work up front so it overlaps with the preprocess pass below. + for (u32 ti = 0; ti < targets.count; ++ti) { + const ShadercTarget &target = targets.targets[ti]; + + s32 sc = run_shaderc(binary_pr_vert[ti] + , _opts + , target + , _vs_path.c_str() + , vs_bin_paths[ti]->c_str() + , _varying_path.c_str() + , "vertex" + , defines + ); + if (sc != 0) { + wait_shaderc_processes(_opts, binary_pr_vert, binary_pr_frag, targets.count); + delete_temp_files(vs_bin_paths, fs_bin_paths, targets.count); + RETURN_IF_FALSE(SHADER_RESOURCE, sc == 0 + , _opts + , "Failed to spawn shaderc" + ); + } + + sc = run_shaderc(binary_pr_frag[ti] + , _opts + , target + , _fs_path.c_str() + , fs_bin_paths[ti]->c_str() + , _varying_path.c_str() + , "fragment" + , defines + ); + if (sc != 0) { + wait_shaderc_processes(_opts, binary_pr_vert, binary_pr_frag, targets.count); + delete_temp_files(vs_bin_paths, fs_bin_paths, targets.count); + RETURN_IF_FALSE(SHADER_RESOURCE, sc == 0 + , _opts + , "Failed to spawn shaderc" + ); + } + } + } + const ShadercTarget &metadata_target = default_shaderc_target[_opts._platform]; HashMap sampler_stages(default_allocator()); + Vector cached_uniform_meta(default_allocator()); + const bool collect_uniform_metadata = meta != NULL || cache_static_metadata; // Run preprocess pass on shaders. - { + if (need_preprocess) { s32 sc; Process pr_vert; Process pr_frag; @@ -2383,7 +2655,8 @@ namespace shader_resource_internal , ShadercFlags::PREPROCESS | ShadercFlags::KEEPCOMMENTS ); if (sc != 0) { - delete_temp_files(); + wait_shaderc_processes(_opts, binary_pr_vert, binary_pr_frag, targets.count); + delete_temp_files(vs_bin_paths, fs_bin_paths, targets.count); RETURN_IF_FALSE(SHADER_RESOURCE, sc == 0 , _opts , "Failed to spawn shaderc" @@ -2401,7 +2674,8 @@ namespace shader_resource_internal , ShadercFlags::PREPROCESS | ShadercFlags::KEEPCOMMENTS ); if (sc != 0) { - delete_temp_files(); + wait_shaderc_processes(_opts, binary_pr_vert, binary_pr_frag, targets.count); + delete_temp_files(vs_bin_paths, fs_bin_paths, targets.count); RETURN_IF_FALSE(SHADER_RESOURCE, sc == 0 , _opts , "Failed to spawn shaderc" @@ -2417,8 +2691,9 @@ namespace shader_resource_internal _opts.read_output(output_vert, pr_vert); ec = pr_vert.wait(); if (ec != 0) { - pr_frag.wait(); - delete_temp_files(); + wait_shaderc_process(_opts, pr_frag); + wait_shaderc_processes(_opts, binary_pr_vert, binary_pr_frag, targets.count); + delete_temp_files(vs_bin_paths, fs_bin_paths, targets.count); RETURN_IF_FALSE(SHADER_RESOURCE, false , _opts , "Failed to preprocess vertex shader `%s`:\n%s" @@ -2430,7 +2705,8 @@ namespace shader_resource_internal _opts.read_output(output_frag, pr_frag); ec = pr_frag.wait(); if (ec != 0) { - delete_temp_files(); + wait_shaderc_processes(_opts, binary_pr_vert, binary_pr_frag, targets.count); + delete_temp_files(vs_bin_paths, fs_bin_paths, targets.count); RETURN_IF_FALSE(SHADER_RESOURCE, false , _opts , "Failed to preprocess fragment shader `%s`:\n%s" @@ -2446,15 +2722,33 @@ namespace shader_resource_internal array::push_back(fs_pp_data, '\0'); err = parse_sampler_stage_markers(sampler_stages, vs_pp_data, _opts); - ENSURE_OR_RETURN(SHADER_RESOURCE, err == 0, _opts); + if (err != 0) { + wait_shaderc_processes(_opts, binary_pr_vert, binary_pr_frag, targets.count); + delete_temp_files(vs_bin_paths, fs_bin_paths, targets.count); + return err; + } err = parse_sampler_stage_markers(sampler_stages, fs_pp_data, _opts); - ENSURE_OR_RETURN(SHADER_RESOURCE, err == 0, _opts); + if (err != 0) { + wait_shaderc_processes(_opts, binary_pr_vert, binary_pr_frag, targets.count); + delete_temp_files(vs_bin_paths, fs_bin_paths, targets.count); + return err; + } - if (meta != NULL) { - err = parse_metadata(*meta, vs_pp_data, _opts); - ENSURE_OR_RETURN(SHADER_RESOURCE, err == 0, _opts); - err = parse_metadata(*meta, fs_pp_data, _opts); - ENSURE_OR_RETURN(SHADER_RESOURCE, err == 0, _opts); + if (collect_uniform_metadata) { + // Static compiles do not request material metadata, so collect it locally for caching. + Vector &metadata = meta != NULL ? *meta : cached_uniform_meta; + err = parse_metadata(metadata, vs_pp_data, _opts); + if (err != 0) { + wait_shaderc_processes(_opts, binary_pr_vert, binary_pr_frag, targets.count); + delete_temp_files(vs_bin_paths, fs_bin_paths, targets.count); + return err; + } + err = parse_metadata(metadata, fs_pp_data, _opts); + if (err != 0) { + wait_shaderc_processes(_opts, binary_pr_vert, binary_pr_frag, targets.count); + delete_temp_files(vs_bin_paths, fs_bin_paths, targets.count); + return err; + } } } @@ -2482,14 +2776,19 @@ namespace shader_resource_internal vector::push_back(samplers, sampler); } - RETURN_IF_FALSE(SHADER_RESOURCE, vector::size(samplers) <= ShaderResource::MAX_SAMPLERS - , _opts - , "Too many active samplers in shader '%s': %u" - , bgfx_shader - , vector::size(samplers) - ); + if (vector::size(samplers) > ShaderResource::MAX_SAMPLERS) { + wait_shaderc_processes(_opts, binary_pr_vert, binary_pr_frag, targets.count); + delete_temp_files(vs_bin_paths, fs_bin_paths, targets.count); + RETURN_IF_FALSE(SHADER_RESOURCE, false + , _opts + , "Too many active samplers in shader '%s': %u" + , bgfx_shader + , vector::size(samplers) + ); + } if (metadata_only) { + // Metadata-only compiles do not spawn the binary jobs or per-target output paths. delete_temp_files(); return 0; } @@ -2501,48 +2800,11 @@ namespace shader_resource_internal bw.write(samplers[si].stage); } - // Compile shader binaries. - const ShadercTargetList targets = shaderc_targets(_opts._platform); + // The binary jobs were spawned above; wait and serialize them in target order. bw.write(targets.count); for (u32 ti = 0; ti < targets.count; ++ti) { const ShadercTarget &target = targets.targets[ti]; - Process pr_vert; - Process pr_frag; - - s32 sc = run_shaderc(pr_vert - , _opts - , target - , _vs_path.c_str() - , _vs_bin_path.c_str() - , _varying_path.c_str() - , "vertex" - , defines - ); - if (sc != 0) { - delete_temp_files(); - RETURN_IF_FALSE(SHADER_RESOURCE, sc == 0 - , _opts - , "Failed to spawn shaderc" - ); - } - - sc = run_shaderc(pr_frag - , _opts - , target - , _fs_path.c_str() - , _fs_bin_path.c_str() - , _varying_path.c_str() - , "fragment" - , defines - ); - if (sc != 0) { - delete_temp_files(); - RETURN_IF_FALSE(SHADER_RESOURCE, sc == 0 - , _opts - , "Failed to spawn shaderc" - ); - } // Check exit code. s32 ec; @@ -2550,11 +2812,11 @@ namespace shader_resource_internal StringStream output_vert(ta); StringStream output_frag(ta); - _opts.read_output(output_vert, pr_vert); - ec = pr_vert.wait(); + _opts.read_output(output_vert, binary_pr_vert[ti]); + ec = binary_pr_vert[ti].wait(); if (ec != 0) { - pr_frag.wait(); - delete_temp_files(); + wait_shaderc_processes(_opts, binary_pr_vert, binary_pr_frag, targets.count); + delete_temp_files(vs_bin_paths, fs_bin_paths, targets.count); RETURN_IF_FALSE(SHADER_RESOURCE, false , _opts , "Failed to compile vertex shader `%s` for %s/%s:\n%s" @@ -2565,10 +2827,11 @@ namespace shader_resource_internal ); } - _opts.read_output(output_frag, pr_frag); - ec = pr_frag.wait(); + _opts.read_output(output_frag, binary_pr_frag[ti]); + ec = binary_pr_frag[ti].wait(); if (ec != 0) { - delete_temp_files(); + wait_shaderc_processes(_opts, binary_pr_vert, binary_pr_frag, targets.count); + delete_temp_files(vs_bin_paths, fs_bin_paths, targets.count); RETURN_IF_FALSE(SHADER_RESOURCE, false , _opts , "Failed to compile fragment shader `%s` for %s/%s:\n%s" @@ -2579,8 +2842,8 @@ namespace shader_resource_internal ); } - Buffer vs_data = _opts.read_temporary(_vs_bin_path.c_str()); - Buffer fs_data = _opts.read_temporary(_fs_bin_path.c_str()); + Buffer vs_data = _opts.read_temporary(vs_bin_paths[ti]->c_str()); + Buffer fs_data = _opts.read_temporary(fs_bin_paths[ti]->c_str()); bw.write(u32(target.backend)); bw.write(array::size(vs_data)); @@ -2589,7 +2852,14 @@ namespace shader_resource_internal bw.write(fs_data); } - delete_temp_files(); + if (cache_static_metadata) { + // Reuse metadata parsed during static compiles when compiling matching materials. + DynamicString cache_key(default_allocator()); + metadata_cache_key(cache_key, _opts._platform, _shader_library, shader_name, defines); + store_metadata_cache(cache_key, _shader_library, _parsed_includes, &cached_uniform_meta, &samplers); + } + + delete_temp_files(vs_bin_paths, fs_bin_paths, targets.count); return 0; } @@ -2721,9 +2991,23 @@ namespace shader_compiler { using namespace shader_resource_internal; + void clear_metadata_cache() + { + if (s_metadata_cache != NULL) { + CE_DELETE(default_allocator(), s_metadata_cache); + s_metadata_cache = NULL; + } + + if (s_shader_library_cache != NULL) { + CE_DELETE(default_allocator(), s_shader_library_cache); + s_shader_library_cache = NULL; + } + } + s32 compile_variant(FileBuffer &fb , Vector *uniform_meta , DynamicString &shader_library + , StringId32 &shader_id , StringView &shader , Vector &defines , CompileOptions &opts @@ -2735,6 +3019,7 @@ namespace shader_compiler Vector defines_dyn(default_allocator()); DynamicString shader_library_path(default_allocator()); DynamicString shader_name(default_allocator()); + DynamicString cache_key(default_allocator()); for (u32 i = 0; i < vector::size(defines); ++i) { TempAllocator64 ta; @@ -2742,6 +3027,26 @@ namespace shader_compiler tmp = defines[i]; vector::push_back(defines_dyn, tmp); } + std::sort(vector::begin(defines_dyn), vector::end(defines_dyn)); + + shader_name = shader; + shader_id = ShaderCompiler::shader_variant_id(shader_name.c_str(), defines_dyn); + + if (metadata_only) { + metadata_cache_key(cache_key, opts._platform, shader_library, shader_name, defines_dyn); + + if (load_metadata_cache(cache_key, opts, shader_library, uniform_meta, sampler_meta)) + return 0; + + if (shader_library.empty() && s_shader_library_cache != NULL && hash_map::has(*s_shader_library_cache, shader_name)) { + const DynamicString empty(default_allocator()); + shader_library = hash_map::get(*s_shader_library_cache, shader_name, empty); + metadata_cache_key(cache_key, opts._platform, shader_library, shader_name, defines_dyn); + if (load_metadata_cache(cache_key, opts, shader_library, uniform_meta, sampler_meta)) + return 0; + shader_library = ""; + } + } // Find a shader library that contains the specified shader if none provided. This is slow // and ugly and it only exists for backwards compatibility with older material formats. @@ -2761,6 +3066,9 @@ namespace shader_compiler if (sc.has_shader(shader)) { const char *sp = shader_library_path.c_str(); shader_library.set(sp, (u32)(resource_type(sp) - sp - 1)); + cache_shader_library(shader_name, shader_library); + if (metadata_only) + metadata_cache_key(cache_key, opts._platform, shader_library, shader_name, defines_dyn); break; } } @@ -2772,8 +3080,19 @@ namespace shader_compiler sc.parse(shader_library_path.c_str(), false); } - shader_name = shader; - return sc.compile_variant(fb, uniform_meta, sampler_meta, shader_name, defines_dyn, metadata_only); + s32 err = sc.compile_variant(fb, uniform_meta, sampler_meta, shader_name, defines_dyn, metadata_only); + ENSURE_OR_RETURN(SHADER_RESOURCE, err == 0, opts); + + if (metadata_only) { + store_metadata_cache(cache_key + , shader_library + , sc._parsed_includes + , uniform_meta + , sampler_meta + ); + } + + return 0; } } // namespace shader_compiler diff --git a/src/resource/shader_resource.h b/src/resource/shader_resource.h index d76858f582..85f0528797 100644 --- a/src/resource/shader_resource.h +++ b/src/resource/shader_resource.h @@ -110,12 +110,16 @@ struct ShaderData #if CROWN_CAN_COMPILE namespace shader_compiler { + /// Clears cached shader metadata used by material compilation. + void clear_metadata_cache(); + /// Compiles a @a shader variant and writes it to @a fb. The shader must be defined inside @a /// shader_library. If @a shader_library is empty it tries to find a suitable one in the source /// directories and returns it via @a shader_library itself. s32 compile_variant(FileBuffer &fb , Vector *uniform_meta , DynamicString &shader_library + , StringId32 &shader_id , StringView &shader , Vector &defines , CompileOptions &opts