Resource: optimize shader and material compile times

docs/changelog.rst

@@ -34,6 +34,7 @@ Changelog
 * Runtime: added configurable physics `solver_iterations` and `sleep_threshold` via the `physics` property of :doc:`boot.config <reference/boot_config>`.
 * Data Compiler: missing resource references no longer stop data compilation and print a warning instead.
 * Data Compiler: the data index is now validated at startup to catch missing data blobs and other edge cases.
+* Data Compiler: the shader compiler has been optimized with caching, more parallelism, and by skipping unnecessary work, resulting in much faster material compilation times.
 * Lua: added ``GameSave`` singleton to save/load savegames.
 * Lua: added a comprehensive set of ``PhysicsWorld.joint_*`` functions for creating and manipulating various joint types.
 * Lua: ``print()`` now shows tables in a more readable format.

src/resource/data_compiler.cpp

@@ -34,6 +34,7 @@
 #include "resource/data_compiler.h"
 #include "resource/mesh.h"
 #include "resource/resource_id.inl"
+#include "resource/shader_resource.h"
 #include "resource/types.h"
 #include <algorithm>
 #include <inttypes.h>
@@ -1481,7 +1482,9 @@ bool DataCompiler::compile_internal(const char *data_dir, const char *platform_n
 bool DataCompiler::compile(const char *data_dir, const char *platform_name)
 {
 	profiler_globals::clear();
+	shader_compiler::clear_metadata_cache();
 	bool success = compile_internal(data_dir, platform_name);
+	shader_compiler::clear_metadata_cache();
 	profiler_globals::flush();
 	return success;
 }

src/resource/material_resource.cpp

@@ -360,13 +360,14 @@ namespace material_resource_internal
 		Buffer shader_code(default_allocator());
 		FileBuffer shader_fb(shader_code);
 		StringView shader_name;
+		StringId32 shader_id;
 		DynamicString shader_library(ta);
 		DynamicString shader(ta);
 
 		RETURN_IF_ERROR(sjson::parse_string(shader, obj["shader"]));
 		shader_name_defines(shader_name, defines, shader.c_str());
 
-		s32 err = shader_compiler::compile_variant(shader_fb, &data.uniforms_meta, shader_library, shader_name, defines, opts, true, &samplers_meta);
+		s32 err = shader_compiler::compile_variant(shader_fb, &data.uniforms_meta, shader_library, shader_id, shader_name, defines, opts, true, &samplers_meta);
 		ENSURE_OR_RETURN(MATERIAL_RESOURCE, err == 0, opts);
 		opts.add_requirement("shader", shader_library.c_str());
 
@@ -401,7 +402,7 @@ namespace material_resource_internal
 
 		MaterialResource mr;
 		mr.version             = RESOURCE_HEADER(RESOURCE_VERSION_MATERIAL);
-		mr.shader              = shader.to_string_id();
+		mr.shader              = shader_id;
 		mr.num_textures        = array::size(data.textures);
 		mr.texture_data_offset = sizeof(mr);
 		mr.num_uniforms        = array::size(data.uniforms);