From 01fff5443162adde8edb66b1ed7da42810ffe1e4 Mon Sep 17 00:00:00 2001 From: Ali Cheraghi Date: Fri, 4 Oct 2024 11:09:40 +0330 Subject: [PATCH 01/11] stash Signed-off-by: Ali Cheraghi --- src/nbl/system/ISystem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nbl/system/ISystem.cpp b/src/nbl/system/ISystem.cpp index 39f722369..64406c647 100644 --- a/src/nbl/system/ISystem.cpp +++ b/src/nbl/system/ISystem.cpp @@ -122,7 +122,7 @@ bool ISystem::deleteDirectory(const system::path& p) bool nbl::system::ISystem::deleteFile(const system::path& p) { - if (std::filesystem::exists(p) && !std::filesystem::is_directory(p))) + if (std::filesystem::exists(p) && !std::filesystem::is_directory(p)) return std::filesystem::remove(p); else return false; From c1c490809756bb0a63aa4f0ff466055e68ee5531 Mon Sep 17 00:00:00 2001 From: Ali Cheraghi Date: Fri, 4 Oct 2024 16:23:30 +0330 Subject: [PATCH 02/11] ShaderCompiler: Polishing --- include/nbl/asset/utils/IShaderCompiler.h | 31 +++++++++---------- src/nbl/asset/utils/IShaderCompiler.cpp | 12 +++++-- .../utils/shaderCompiler_serialization.h | 2 -- src/nbl/asset/utils/waveContext.h | 2 +- 4 files changed, 25 insertions(+), 22 deletions(-) diff --git a/include/nbl/asset/utils/IShaderCompiler.h b/include/nbl/asset/utils/IShaderCompiler.h index 1ed613f27..0ac247a48 100644 --- a/include/nbl/asset/utils/IShaderCompiler.h +++ b/include/nbl/asset/utils/IShaderCompiler.h @@ -33,7 +33,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted { system::path absolutePath = {}; std::string contents = {}; - std::array hash = {}; // TODO: we're not yet using IFile::getPrecomputedHash(), so for builtins we can maybe use that in the future + std::array hash = {}; // TODO: we're not yet using IFile::getPrecomputedHash(), so for builtins we can maybe use that in the future // Could be used in the future for early rejection of cache hit //nbl::system::IFileBase::time_point_t lastWriteTime = {}; @@ -185,7 +185,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted // Used to check compatibility of Caches before reading constexpr static inline std::string_view VERSION = "1.0.0"; - using hash_t = std::array; + using hash_t = std::array; static auto const SHADER_BUFFER_SIZE_BYTES = sizeof(uint64_t) / sizeof(uint8_t); // It's obviously 8 struct SEntry @@ -196,11 +196,9 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted { public: // Perf note: hashing while preprocessor lexing is likely to be slower than just hashing the whole array like this - inline SPreprocessingDependency(const system::path& _requestingSourceDir, const std::string_view& _identifier, const std::string_view& _contents, bool _standardInclude, std::array _hash) : - requestingSourceDir(_requestingSourceDir), identifier(_identifier), contents(_contents), standardInclude(_standardInclude), hash(_hash) - { - assert(!_contents.empty()); - } + inline SPreprocessingDependency(const system::path& _requestingSourceDir, const std::string_view& _identifier, bool _standardInclude, std::array _hash) : + requestingSourceDir(_requestingSourceDir), identifier(_identifier), standardInclude(_standardInclude), hash(_hash) + {} inline SPreprocessingDependency(SPreprocessingDependency&) = default; inline SPreprocessingDependency& operator=(SPreprocessingDependency&) = delete; @@ -218,11 +216,8 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted // path or identifier system::path requestingSourceDir = ""; std::string identifier = ""; - // file contents - // TODO: change to `core::vector` a compressed blob of LZMA, and store all contents together in the `SEntry` - std::string contents = ""; // hash of the contents - used to check against a found_t - std::array hash = {}; + std::array hash = {}; // If true, then `getIncludeStandard` was used to find, otherwise `getIncludeRelative` bool standardInclude = false; }; @@ -351,9 +346,13 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted // Now add the mainFileContents and produce both lookup and early equality rejection hashes hashable.insert(hashable.end(), mainFileContents.begin(), mainFileContents.end()); - hash = nbl::core::XXHash_256(hashable.data(), hashable.size()); - lookupHash = hash[0]; - for (auto i = 1u; i < 4; i++) { + + core::blake3_hasher hasher; + hasher.update(hashable.data(), hashable.size()); + hash = { *static_cast(hasher).data }; + // ALI:TODO + lookupHash = std::bit_cast({hash[0], hash[1], hash[2], hash[3], hash[4], hash[5], hash[6], hash[7]}); + for (auto i = 8u; i < 32; i++) { core::hash_combine(lookupHash, hash[i]); } } @@ -374,7 +373,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted // TODO: make some of these private std::string mainFileContents; SCompilerArgs compilerArgs; - std::array hash; + std::array hash; size_t lookupHash; dependency_container_t dependencies; core::smart_refctd_ptr cpuShader; @@ -429,7 +428,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted }; - using EntrySet = core::unordered_multiset; + using EntrySet = core::unordered_set; EntrySet m_container; NBL_API2 EntrySet::const_iterator find_impl(const SEntry& mainFile, const CIncludeFinder* finder) const; diff --git a/src/nbl/asset/utils/IShaderCompiler.cpp b/src/nbl/asset/utils/IShaderCompiler.cpp index ba23ba628..d4efafd80 100644 --- a/src/nbl/asset/utils/IShaderCompiler.cpp +++ b/src/nbl/asset/utils/IShaderCompiler.cpp @@ -116,7 +116,10 @@ auto IShaderCompiler::CIncludeFinder::getIncludeStandard(const system::path& req retVal = std::move(contents); else retVal = m_defaultFileSystemLoader->getInclude(requestingSourceDir.string(), includeName); - retVal.hash = nbl::core::XXHash_256((uint8_t*)(retVal.contents.data()), retVal.contents.size() * (sizeof(char) / sizeof(uint8_t))); + + core::blake3_hasher hasher; + hasher.update((uint8_t*)(retVal.contents.data()), retVal.contents.size() * (sizeof(char) / sizeof(uint8_t))); + retVal.hash = { *static_cast(hasher).data }; return retVal; } @@ -129,7 +132,10 @@ auto IShaderCompiler::CIncludeFinder::getIncludeRelative(const system::path& req if (auto contents = m_defaultFileSystemLoader->getInclude(requestingSourceDir.string(), includeName)) retVal = std::move(contents); else retVal = std::move(trySearchPaths(includeName)); - retVal.hash = nbl::core::XXHash_256((uint8_t*)(retVal.contents.data()), retVal.contents.size() * (sizeof(char) / sizeof(uint8_t))); + + core::blake3_hasher hasher; + hasher.update((uint8_t*)(retVal.contents.data()), retVal.contents.size() * (sizeof(char) / sizeof(uint8_t))); + retVal.hash = { *static_cast(hasher).data }; return retVal; } @@ -238,7 +244,7 @@ IShaderCompiler::CCache::EntrySet::const_iterator IShaderCompiler::CCache::find_ else header = finder->getIncludeRelative(dependency.requestingSourceDir, dependency.identifier); - if (header.hash != dependency.hash || header.contents != dependency.contents) + if (header.hash != dependency.hash) { allDependenciesMatch = false; break; diff --git a/src/nbl/asset/utils/shaderCompiler_serialization.h b/src/nbl/asset/utils/shaderCompiler_serialization.h index cd964d568..11ca678b4 100644 --- a/src/nbl/asset/utils/shaderCompiler_serialization.h +++ b/src/nbl/asset/utils/shaderCompiler_serialization.h @@ -116,7 +116,6 @@ inline void to_json(json& j, const SEntry::SPreprocessingDependency& dependency) j = json{ { "requestingSourceDir", dependency.requestingSourceDir }, { "identifier", dependency.identifier }, - { "contents", dependency.contents }, { "hash", dependency.hash }, { "standardInclude", dependency.standardInclude }, }; @@ -126,7 +125,6 @@ inline void from_json(const json& j, SEntry::SPreprocessingDependency& dependenc { j.at("requestingSourceDir").get_to(dependency.requestingSourceDir); j.at("identifier").get_to(dependency.identifier); - j.at("contents").get_to(dependency.contents); j.at("hash").get_to(dependency.hash); j.at("standardInclude").get_to(dependency.standardInclude); } diff --git a/src/nbl/asset/utils/waveContext.h b/src/nbl/asset/utils/waveContext.h index 696a2b9fb..a37439157 100644 --- a/src/nbl/asset/utils/waveContext.h +++ b/src/nbl/asset/utils/waveContext.h @@ -533,7 +533,7 @@ template<> inline bool boost::wave::impl::pp_iterator_functor Date: Fri, 4 Oct 2024 16:58:12 +0330 Subject: [PATCH 03/11] ShaderCompiler: Remove comment --- include/nbl/asset/utils/IShaderCompiler.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/nbl/asset/utils/IShaderCompiler.h b/include/nbl/asset/utils/IShaderCompiler.h index 0ac247a48..38e9d38dd 100644 --- a/include/nbl/asset/utils/IShaderCompiler.h +++ b/include/nbl/asset/utils/IShaderCompiler.h @@ -350,7 +350,6 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted core::blake3_hasher hasher; hasher.update(hashable.data(), hashable.size()); hash = { *static_cast(hasher).data }; - // ALI:TODO lookupHash = std::bit_cast({hash[0], hash[1], hash[2], hash[3], hash[4], hash[5], hash[6], hash[7]}); for (auto i = 8u; i < 32; i++) { core::hash_combine(lookupHash, hash[i]); From d821878620bb2d45ab8b65aec8b937c641578b89 Mon Sep 17 00:00:00 2001 From: Ali Cheraghi Date: Sat, 5 Oct 2024 11:27:12 +0330 Subject: [PATCH 04/11] use blake3_hash_t and std-specialization Signed-off-by: Ali Cheraghi --- include/nbl/asset/utils/IShaderCompiler.h | 16 ++++++---------- src/nbl/asset/utils/IShaderCompiler.cpp | 4 ++-- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/include/nbl/asset/utils/IShaderCompiler.h b/include/nbl/asset/utils/IShaderCompiler.h index 38e9d38dd..05b42db34 100644 --- a/include/nbl/asset/utils/IShaderCompiler.h +++ b/include/nbl/asset/utils/IShaderCompiler.h @@ -33,7 +33,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted { system::path absolutePath = {}; std::string contents = {}; - std::array hash = {}; // TODO: we're not yet using IFile::getPrecomputedHash(), so for builtins we can maybe use that in the future + core::blake3_hash_t hash = {}; // TODO: we're not yet using IFile::getPrecomputedHash(), so for builtins we can maybe use that in the future // Could be used in the future for early rejection of cache hit //nbl::system::IFileBase::time_point_t lastWriteTime = {}; @@ -185,7 +185,6 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted // Used to check compatibility of Caches before reading constexpr static inline std::string_view VERSION = "1.0.0"; - using hash_t = std::array; static auto const SHADER_BUFFER_SIZE_BYTES = sizeof(uint64_t) / sizeof(uint8_t); // It's obviously 8 struct SEntry @@ -196,7 +195,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted { public: // Perf note: hashing while preprocessor lexing is likely to be slower than just hashing the whole array like this - inline SPreprocessingDependency(const system::path& _requestingSourceDir, const std::string_view& _identifier, bool _standardInclude, std::array _hash) : + inline SPreprocessingDependency(const system::path& _requestingSourceDir, const std::string_view& _identifier, bool _standardInclude, core::blake3_hash_t _hash) : requestingSourceDir(_requestingSourceDir), identifier(_identifier), standardInclude(_standardInclude), hash(_hash) {} @@ -217,7 +216,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted system::path requestingSourceDir = ""; std::string identifier = ""; // hash of the contents - used to check against a found_t - std::array hash = {}; + core::blake3_hash_t hash = {}; // If true, then `getIncludeStandard` was used to find, otherwise `getIncludeRelative` bool standardInclude = false; }; @@ -349,11 +348,8 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted core::blake3_hasher hasher; hasher.update(hashable.data(), hashable.size()); - hash = { *static_cast(hasher).data }; - lookupHash = std::bit_cast({hash[0], hash[1], hash[2], hash[3], hash[4], hash[5], hash[6], hash[7]}); - for (auto i = 8u; i < 32; i++) { - core::hash_combine(lookupHash, hash[i]); - } + hash = static_cast(hasher); + lookupHash = std::hash{}(hash); } // Needed to get the vector deserialization automatically @@ -372,7 +368,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted // TODO: make some of these private std::string mainFileContents; SCompilerArgs compilerArgs; - std::array hash; + core::blake3_hash_t hash; size_t lookupHash; dependency_container_t dependencies; core::smart_refctd_ptr cpuShader; diff --git a/src/nbl/asset/utils/IShaderCompiler.cpp b/src/nbl/asset/utils/IShaderCompiler.cpp index d4efafd80..843207acc 100644 --- a/src/nbl/asset/utils/IShaderCompiler.cpp +++ b/src/nbl/asset/utils/IShaderCompiler.cpp @@ -119,7 +119,7 @@ auto IShaderCompiler::CIncludeFinder::getIncludeStandard(const system::path& req core::blake3_hasher hasher; hasher.update((uint8_t*)(retVal.contents.data()), retVal.contents.size() * (sizeof(char) / sizeof(uint8_t))); - retVal.hash = { *static_cast(hasher).data }; + retVal.hash = static_cast(hasher); return retVal; } @@ -135,7 +135,7 @@ auto IShaderCompiler::CIncludeFinder::getIncludeRelative(const system::path& req core::blake3_hasher hasher; hasher.update((uint8_t*)(retVal.contents.data()), retVal.contents.size() * (sizeof(char) / sizeof(uint8_t))); - retVal.hash = { *static_cast(hasher).data }; + retVal.hash = static_cast(hasher); return retVal; } From 8c70f06b971f4e80ccf5f8359d13f80ac6f743b8 Mon Sep 17 00:00:00 2001 From: Ali Cheraghi Date: Sat, 5 Oct 2024 11:43:18 +0330 Subject: [PATCH 05/11] fix serialization Signed-off-by: Ali Cheraghi --- include/nbl/asset/utils/IShaderCompiler.h | 2 +- src/nbl/asset/utils/shaderCompiler_serialization.h | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/nbl/asset/utils/IShaderCompiler.h b/include/nbl/asset/utils/IShaderCompiler.h index 05b42db34..696478d56 100644 --- a/include/nbl/asset/utils/IShaderCompiler.h +++ b/include/nbl/asset/utils/IShaderCompiler.h @@ -183,7 +183,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted public: // Used to check compatibility of Caches before reading - constexpr static inline std::string_view VERSION = "1.0.0"; + constexpr static inline std::string_view VERSION = "1.1.0"; static auto const SHADER_BUFFER_SIZE_BYTES = sizeof(uint64_t) / sizeof(uint8_t); // It's obviously 8 diff --git a/src/nbl/asset/utils/shaderCompiler_serialization.h b/src/nbl/asset/utils/shaderCompiler_serialization.h index 11ca678b4..ab2358843 100644 --- a/src/nbl/asset/utils/shaderCompiler_serialization.h +++ b/src/nbl/asset/utils/shaderCompiler_serialization.h @@ -116,7 +116,7 @@ inline void to_json(json& j, const SEntry::SPreprocessingDependency& dependency) j = json{ { "requestingSourceDir", dependency.requestingSourceDir }, { "identifier", dependency.identifier }, - { "hash", dependency.hash }, + { "hash", dependency.hash.data }, { "standardInclude", dependency.standardInclude }, }; } @@ -125,7 +125,7 @@ inline void from_json(const json& j, SEntry::SPreprocessingDependency& dependenc { j.at("requestingSourceDir").get_to(dependency.requestingSourceDir); j.at("identifier").get_to(dependency.identifier); - j.at("hash").get_to(dependency.hash); + j.at("hash").get_to(dependency.hash.data); j.at("standardInclude").get_to(dependency.standardInclude); } @@ -177,7 +177,7 @@ inline void to_json(json& j, const SEntry& entry) j = json{ { "mainFileContents", entry.mainFileContents }, { "compilerArgs", entry.compilerArgs }, - { "hash", entry.hash }, + { "hash", entry.hash.data }, { "lookupHash", entry.lookupHash }, { "dependencies", entry.dependencies }, }; @@ -187,7 +187,7 @@ inline void from_json(const json& j, SEntry& entry) { j.at("mainFileContents").get_to(entry.mainFileContents); j.at("compilerArgs").get_to(entry.compilerArgs); - j.at("hash").get_to(entry.hash); + j.at("hash").get_to(entry.hash.data); j.at("lookupHash").get_to(entry.lookupHash); j.at("dependencies").get_to(entry.dependencies); entry.cpuShader = nullptr; From 99f93869fc2d871bc60d74121e7a6b5e076f5a11 Mon Sep 17 00:00:00 2001 From: Ali Cheraghi Date: Sat, 5 Oct 2024 20:44:31 +0330 Subject: [PATCH 06/11] not working yet Signed-off-by: Ali Cheraghi --- include/nbl/asset/utils/IShaderCompiler.h | 52 +++------- src/nbl/asset/utils/CHLSLCompiler.cpp | 5 + src/nbl/asset/utils/IShaderCompiler.cpp | 97 +++++++++++++++++-- .../utils/shaderCompiler_serialization.h | 4 +- 4 files changed, 110 insertions(+), 48 deletions(-) diff --git a/include/nbl/asset/utils/IShaderCompiler.h b/include/nbl/asset/utils/IShaderCompiler.h index 696478d56..0c0dfef3e 100644 --- a/include/nbl/asset/utils/IShaderCompiler.h +++ b/include/nbl/asset/utils/IShaderCompiler.h @@ -239,6 +239,8 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted return true; } + std::string sourceIdentifier; + private: friend class SCompilerArgs; friend class SEntry; @@ -262,7 +264,6 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted // Sort them so equality and hashing are well defined std::sort(extraDefines.begin(), extraDefines.end(), [](const SMacroDefinition& lhs, const SMacroDefinition& rhs) {return lhs.identifier < rhs.identifier; }); }; - std::string sourceIdentifier; std::vector extraDefines; }; // TODO: SPreprocessorArgs could just be folded into `SCompilerArgs` to have less classes and operators @@ -282,6 +283,9 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted return retVal; } + IShader::E_SHADER_STAGE stage; + SPreprocessorArgs preprocessorArgs; + private: friend class SEntry; friend void to_json(nlohmann::json&, const SCompilerArgs&); @@ -306,11 +310,9 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted } } - IShader::E_SHADER_STAGE stage; E_SPIRV_VERSION targetSpirvVersion; std::vector optimizerPasses; core::bitflag debugInfoFlags; - SPreprocessorArgs preprocessorArgs; }; // The ordering is important here, the dependencies MUST be added to the array IN THE ORDER THE PREPROCESSOR INCLUDED THEM! @@ -358,20 +360,23 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted // Making the copy constructor deep-copy everything but the shader inline SEntry(const SEntry& other) : mainFileContents(other.mainFileContents), compilerArgs(other.compilerArgs), hash(other.hash), lookupHash(other.lookupHash), - dependencies(other.dependencies), cpuShader(other.cpuShader) {} + dependencies(other.dependencies), spirv(other.spirv) {} inline SEntry& operator=(SEntry& other) = delete; inline SEntry(SEntry&& other) = default; // Used for late initialization while looking up a cache, so as not to always initialize an entry even if caching was not requested inline SEntry& operator=(SEntry&& other) = default; + core::smart_refctd_ptr decodeShader() const; + // TODO: make some of these private std::string mainFileContents; SCompilerArgs compilerArgs; core::blake3_hash_t hash; size_t lookupHash; dependency_container_t dependencies; - core::smart_refctd_ptr cpuShader; + core::smart_refctd_ptr spirv; + size_t uncompressedSize; }; inline void insert(SEntry&& entry) @@ -429,42 +434,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted NBL_API2 EntrySet::const_iterator find_impl(const SEntry& mainFile, const CIncludeFinder* finder) const; }; - inline core::smart_refctd_ptr compileToSPIRV(const std::string_view code, const SCompilerOptions& options) const - { - CCache::SEntry entry; - std::vector dependencies; - if (options.readCache || options.writeCache) - entry = std::move(CCache::SEntry(code, options)); - - if (options.readCache) - { - auto found = options.readCache->find_impl(entry, options.preprocessorOptions.includeFinder); - if (found != options.readCache->m_container.end()) - { - if (options.writeCache) - { - CCache::SEntry writeEntry = *found; - options.writeCache->insert(std::move(writeEntry)); - } - return found->cpuShader; - } - } - - auto retVal = compileToSPIRV_impl(code, options, options.writeCache ? &dependencies : nullptr); - // compute the SPIR-V shader content hash - { - auto backingBuffer = retVal->getContent(); - const_cast(backingBuffer)->setContentHash(backingBuffer->computeContentHash()); - } - - if (options.writeCache) - { - entry.dependencies = std::move(dependencies); - entry.cpuShader = retVal; - options.writeCache->insert(std::move(entry)); - } - return retVal; - } + core::smart_refctd_ptr compileToSPIRV(const std::string_view code, const SCompilerOptions& options) const; inline core::smart_refctd_ptr compileToSPIRV(const char* code, const SCompilerOptions& options) const { diff --git a/src/nbl/asset/utils/CHLSLCompiler.cpp b/src/nbl/asset/utils/CHLSLCompiler.cpp index ca0af41ba..b6d46768e 100644 --- a/src/nbl/asset/utils/CHLSLCompiler.cpp +++ b/src/nbl/asset/utils/CHLSLCompiler.cpp @@ -18,6 +18,8 @@ #include #include +#include "lzma/C/LzmaEnc.h" + using namespace nbl; using namespace nbl::asset; using Microsoft::WRL::ComPtr; @@ -373,6 +375,9 @@ std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADE return preprocessShader(std::move(code), stage, preprocessOptions, extra_dxc_compile_flags); } +static void* SzAlloc(ISzAllocPtr p, size_t size) { p = p; return _NBL_ALIGNED_MALLOC(size, _NBL_SIMD_ALIGNMENT); } +static void SzFree(ISzAllocPtr p, void* address) { p = p; _NBL_ALIGNED_FREE(address); } + core::smart_refctd_ptr CHLSLCompiler::compileToSPIRV_impl(const std::string_view code, const IShaderCompiler::SCompilerOptions& options, std::vector* dependencies) const { auto hlslOptions = option_cast(options); diff --git a/src/nbl/asset/utils/IShaderCompiler.cpp b/src/nbl/asset/utils/IShaderCompiler.cpp index 843207acc..8520580b7 100644 --- a/src/nbl/asset/utils/IShaderCompiler.cpp +++ b/src/nbl/asset/utils/IShaderCompiler.cpp @@ -13,6 +13,9 @@ #include #include +#include +#include + using namespace nbl; using namespace nbl::asset; @@ -22,6 +25,71 @@ IShaderCompiler::IShaderCompiler(core::smart_refctd_ptr&& syste m_defaultIncludeFinder = core::make_smart_refctd_ptr(core::smart_refctd_ptr(m_system)); } +static void* SzAlloc(ISzAllocPtr p, size_t size) { p = p; return _NBL_ALIGNED_MALLOC(size, _NBL_SIMD_ALIGNMENT); } +static void SzFree(ISzAllocPtr p, void* address) { p = p; _NBL_ALIGNED_FREE(address); } + +inline core::smart_refctd_ptr nbl::asset::IShaderCompiler::compileToSPIRV(const std::string_view code, const SCompilerOptions& options) const +{ + CCache::SEntry entry; + std::vector dependencies; + if (options.readCache || options.writeCache) + entry = std::move(CCache::SEntry(code, options)); + + if (options.readCache) + { + auto found = options.readCache->find_impl(entry, options.preprocessorOptions.includeFinder); + if (found != options.readCache->m_container.end()) + { + if (options.writeCache) + { + CCache::SEntry writeEntry = *found; + options.writeCache->insert(std::move(writeEntry)); + } + return found->decodeShader(); + } + } + + auto retVal = compileToSPIRV_impl(code, options, options.writeCache ? &dependencies : nullptr); + // compute the SPIR-V shader content hash + { + auto backingBuffer = retVal->getContent(); + const_cast(backingBuffer)->setContentHash(backingBuffer->computeContentHash()); + } + + std::cout << "writeCache: " << options.writeCache << "\n"; + if (options.writeCache) + { + auto* spirvBuffer = retVal->getContent(); + size_t propsSize = LZMA_PROPS_SIZE; + size_t destLen = spirvBuffer->getSize() + spirvBuffer->getSize() / 3 + 128; + auto compressedSpirvBuffer = core::make_smart_refctd_ptr(propsSize + destLen); + + CLzmaEncProps props; + LzmaEncProps_Init(&props); + props.dictSize = 1 << 16; // 64 KB + props.writeEndMark = 1; // 0 or 1 + + ISzAlloc alloc = { SzAlloc, SzFree }; + int res = LzmaEncode( + reinterpret_cast(compressedSpirvBuffer->getPointer()) + LZMA_PROPS_SIZE, &destLen, + reinterpret_cast(spirvBuffer->getPointer()), spirvBuffer->getSize(), + &props, reinterpret_cast(compressedSpirvBuffer->getPointer()), &propsSize, props.writeEndMark, + nullptr, &alloc, &alloc); + + assert(propsSize == LZMA_PROPS_SIZE); + assert(res == SZ_OK); + + entry.dependencies = std::move(dependencies); + entry.spirv = std::move(compressedSpirvBuffer); + entry.uncompressedSize = spirvBuffer->getSize(); + + std::cout << "original: " << spirvBuffer->getSize() << ", compressed: " << compressedSpirvBuffer->getSize() << "\n"; + + options.writeCache->insert(std::move(entry)); + } + return retVal; +} + std::string IShaderCompiler::preprocessShader( system::IFile* sourcefile, IShader::E_SHADER_STAGE stage, @@ -224,7 +292,7 @@ auto IShaderCompiler::CIncludeFinder::tryIncludeGenerators(const std::string& in core::smart_refctd_ptr IShaderCompiler::CCache::find(const SEntry& mainFile, const IShaderCompiler::CIncludeFinder* finder) const { - return find_impl(mainFile, finder)->cpuShader; + return find_impl(mainFile, finder)->decodeShader(); } IShaderCompiler::CCache::EntrySet::const_iterator IShaderCompiler::CCache::find_impl(const SEntry& mainFile, const IShaderCompiler::CIncludeFinder* finder) const @@ -273,10 +341,10 @@ core::smart_refctd_ptr IShaderCompiler::CCache::serialize() const // We keep a copy of the offsets and the sizes of each shader. This is so that later on, when we add the shaders to the buffer after json creation // (where the params array has been moved) we don't have to read the json to get the offsets again offsets[i] = shaderBufferSize; - sizes[i] = entry.cpuShader->getContent()->getSize(); + sizes[i] = entry.spirv->getSize(); // And add the params to the shader creation parameters array - shaderCreationParams.emplace_back(entry.cpuShader->getStage(), entry.cpuShader->getContentType(), entry.cpuShader->getFilepathHint(), sizes[i], shaderBufferSize); + shaderCreationParams.emplace_back(entry.compilerArgs.stage, IShader::E_CONTENT_TYPE::ECT_SPIRV, entry.compilerArgs.preprocessorArgs.sourceIdentifier.data(), sizes[i], shaderBufferSize); // Enlarge the shader buffer by the size of the current shader shaderBufferSize += sizes[i]; i++; @@ -300,7 +368,7 @@ core::smart_refctd_ptr IShaderCompiler::CCache::serialize() const // Loop over entries again, adding each one's shader to the buffer. i = 0u; for (auto& entry : m_container) { - memcpy(retVal.data() + SHADER_BUFFER_SIZE_BYTES + offsets[i], entry.cpuShader->getContent()->getPointer(), sizes[i]); + memcpy(retVal.data() + SHADER_BUFFER_SIZE_BYTES + offsets[i], entry.spirv->getPointer(), sizes[i]); i++; } @@ -343,13 +411,30 @@ core::smart_refctd_ptr IShaderCompiler::CCache::deseria // Create buffer to hold the code auto code = core::make_smart_refctd_ptr(shaderCreationParams[i].codeByteSize); // Copy the shader bytecode into the buffer + memcpy(code->getPointer(), serializedCache.data() + SHADER_BUFFER_SIZE_BYTES + shaderCreationParams[i].offset, shaderCreationParams[i].codeByteSize); code->setContentHash(code->computeContentHash()); // Create the ICPUShader - entries[i].cpuShader = core::make_smart_refctd_ptr(std::move(code), shaderCreationParams[i].stage, shaderCreationParams[i].contentType, std::move(shaderCreationParams[i].filepathHint)); + entries[i].spirv = std::move(code); retVal->insert(std::move(entries[i])); } return retVal; -} \ No newline at end of file +} + +core::smart_refctd_ptr nbl::asset::IShaderCompiler::CCache::SEntry::decodeShader() const +{ + auto uncompressedBuf = core::make_smart_refctd_ptr(uncompressedSize); + size_t dstSize = uncompressedBuf->getSize(); + size_t srcSize = spirv->getSize() - LZMA_PROPS_SIZE; + ELzmaStatus status; + ISzAlloc alloc = { SzAlloc, SzFree }; + SRes res = LzmaDecode( + reinterpret_cast(uncompressedBuf->getPointer()), &dstSize, + reinterpret_cast(spirv->getPointer()) + LZMA_PROPS_SIZE, &srcSize, + reinterpret_cast(spirv->getPointer()), LZMA_PROPS_SIZE, + LZMA_FINISH_ANY, &status, &alloc); + assert(res == SZ_OK); + return core::make_smart_refctd_ptr(std::move(uncompressedBuf), compilerArgs.stage, IShader::E_CONTENT_TYPE::ECT_SPIRV, compilerArgs.preprocessorArgs.sourceIdentifier.data()); +} diff --git a/src/nbl/asset/utils/shaderCompiler_serialization.h b/src/nbl/asset/utils/shaderCompiler_serialization.h index ab2358843..e038d3bf9 100644 --- a/src/nbl/asset/utils/shaderCompiler_serialization.h +++ b/src/nbl/asset/utils/shaderCompiler_serialization.h @@ -180,6 +180,7 @@ inline void to_json(json& j, const SEntry& entry) { "hash", entry.hash.data }, { "lookupHash", entry.lookupHash }, { "dependencies", entry.dependencies }, + { "uncompressedSize", entry.uncompressedSize }, }; } @@ -190,7 +191,8 @@ inline void from_json(const json& j, SEntry& entry) j.at("hash").get_to(entry.hash.data); j.at("lookupHash").get_to(entry.lookupHash); j.at("dependencies").get_to(entry.dependencies); - entry.cpuShader = nullptr; + j.at("uncompressedSize").get_to(entry.uncompressedSize); + entry.spirv = nullptr; } } From afa49359e09c67a1fbb72d9136d700dd0b485286 Mon Sep 17 00:00:00 2001 From: Ali Cheraghi Date: Sat, 5 Oct 2024 21:06:45 +0330 Subject: [PATCH 07/11] maybe Signed-off-by: Ali Cheraghi --- src/nbl/asset/utils/CHLSLCompiler.cpp | 5 ----- src/nbl/asset/utils/IShaderCompiler.cpp | 1 - 2 files changed, 6 deletions(-) diff --git a/src/nbl/asset/utils/CHLSLCompiler.cpp b/src/nbl/asset/utils/CHLSLCompiler.cpp index b6d46768e..ca0af41ba 100644 --- a/src/nbl/asset/utils/CHLSLCompiler.cpp +++ b/src/nbl/asset/utils/CHLSLCompiler.cpp @@ -18,8 +18,6 @@ #include #include -#include "lzma/C/LzmaEnc.h" - using namespace nbl; using namespace nbl::asset; using Microsoft::WRL::ComPtr; @@ -375,9 +373,6 @@ std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADE return preprocessShader(std::move(code), stage, preprocessOptions, extra_dxc_compile_flags); } -static void* SzAlloc(ISzAllocPtr p, size_t size) { p = p; return _NBL_ALIGNED_MALLOC(size, _NBL_SIMD_ALIGNMENT); } -static void SzFree(ISzAllocPtr p, void* address) { p = p; _NBL_ALIGNED_FREE(address); } - core::smart_refctd_ptr CHLSLCompiler::compileToSPIRV_impl(const std::string_view code, const IShaderCompiler::SCompilerOptions& options, std::vector* dependencies) const { auto hlslOptions = option_cast(options); diff --git a/src/nbl/asset/utils/IShaderCompiler.cpp b/src/nbl/asset/utils/IShaderCompiler.cpp index 8520580b7..ef685d08f 100644 --- a/src/nbl/asset/utils/IShaderCompiler.cpp +++ b/src/nbl/asset/utils/IShaderCompiler.cpp @@ -414,7 +414,6 @@ core::smart_refctd_ptr IShaderCompiler::CCache::deseria memcpy(code->getPointer(), serializedCache.data() + SHADER_BUFFER_SIZE_BYTES + shaderCreationParams[i].offset, shaderCreationParams[i].codeByteSize); code->setContentHash(code->computeContentHash()); - // Create the ICPUShader entries[i].spirv = std::move(code); retVal->insert(std::move(entries[i])); From b2927babcfb17bcabcda7cf5e95dc3a783221589 Mon Sep 17 00:00:00 2001 From: Ali Cheraghi Date: Sun, 6 Oct 2024 12:11:25 +0330 Subject: [PATCH 08/11] ready Signed-off-by: Ali Cheraghi --- include/nbl/asset/utils/IShaderCompiler.h | 2 +- src/nbl/asset/utils/IShaderCompiler.cpp | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/include/nbl/asset/utils/IShaderCompiler.h b/include/nbl/asset/utils/IShaderCompiler.h index 0c0dfef3e..991d45351 100644 --- a/include/nbl/asset/utils/IShaderCompiler.h +++ b/include/nbl/asset/utils/IShaderCompiler.h @@ -360,7 +360,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted // Making the copy constructor deep-copy everything but the shader inline SEntry(const SEntry& other) : mainFileContents(other.mainFileContents), compilerArgs(other.compilerArgs), hash(other.hash), lookupHash(other.lookupHash), - dependencies(other.dependencies), spirv(other.spirv) {} + dependencies(other.dependencies), spirv(other.spirv), uncompressedSize(other.uncompressedSize) {} inline SEntry& operator=(SEntry& other) = delete; inline SEntry(SEntry&& other) = default; diff --git a/src/nbl/asset/utils/IShaderCompiler.cpp b/src/nbl/asset/utils/IShaderCompiler.cpp index ef685d08f..d36c87f62 100644 --- a/src/nbl/asset/utils/IShaderCompiler.cpp +++ b/src/nbl/asset/utils/IShaderCompiler.cpp @@ -62,29 +62,31 @@ inline core::smart_refctd_ptr nbl::asset::IShaderCompiler::compileTo auto* spirvBuffer = retVal->getContent(); size_t propsSize = LZMA_PROPS_SIZE; size_t destLen = spirvBuffer->getSize() + spirvBuffer->getSize() / 3 + 128; - auto compressedSpirvBuffer = core::make_smart_refctd_ptr(propsSize + destLen); + std::vector compressedSpirv = {}; + compressedSpirv.resize(propsSize + destLen); CLzmaEncProps props; LzmaEncProps_Init(&props); - props.dictSize = 1 << 16; // 64 KB - props.writeEndMark = 1; // 0 or 1 + props.dictSize = 1 << 16; // 64KB + props.writeEndMark = 1; ISzAlloc alloc = { SzAlloc, SzFree }; int res = LzmaEncode( - reinterpret_cast(compressedSpirvBuffer->getPointer()) + LZMA_PROPS_SIZE, &destLen, + compressedSpirv.data() + LZMA_PROPS_SIZE, &destLen, reinterpret_cast(spirvBuffer->getPointer()), spirvBuffer->getSize(), - &props, reinterpret_cast(compressedSpirvBuffer->getPointer()), &propsSize, props.writeEndMark, + &props, compressedSpirv.data(), &propsSize, props.writeEndMark, nullptr, &alloc, &alloc); assert(propsSize == LZMA_PROPS_SIZE); assert(res == SZ_OK); + auto compressedSpirvBuffer = core::make_smart_refctd_ptr(propsSize + destLen); + memcpy(compressedSpirvBuffer->getPointer(), compressedSpirv.data(), compressedSpirvBuffer->getSize()); + entry.dependencies = std::move(dependencies); entry.spirv = std::move(compressedSpirvBuffer); entry.uncompressedSize = spirvBuffer->getSize(); - std::cout << "original: " << spirvBuffer->getSize() << ", compressed: " << compressedSpirvBuffer->getSize() << "\n"; - options.writeCache->insert(std::move(entry)); } return retVal; @@ -398,7 +400,6 @@ core::smart_refctd_ptr IShaderCompiler::CCache::deseria return nullptr; } } - // Now retrieve two vectors, one with the entries and one with the extra data to recreate the CPUShaders std::vector entries; From ff7ed067aa4b9e79958f6d0b84da24833d047c57 Mon Sep 17 00:00:00 2001 From: Ali Cheraghi Date: Mon, 7 Oct 2024 23:36:27 +0330 Subject: [PATCH 09/11] review fixes Signed-off-by: Ali Cheraghi --- include/nbl/asset/utils/IShaderCompiler.h | 29 ++++++++++++------- src/nbl/asset/utils/IShaderCompiler.cpp | 17 +++++------ .../utils/shaderCompiler_serialization.h | 13 ++++----- 3 files changed, 31 insertions(+), 28 deletions(-) diff --git a/include/nbl/asset/utils/IShaderCompiler.h b/include/nbl/asset/utils/IShaderCompiler.h index 991d45351..3899c1266 100644 --- a/include/nbl/asset/utils/IShaderCompiler.h +++ b/include/nbl/asset/utils/IShaderCompiler.h @@ -239,11 +239,10 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted return true; } - std::string sourceIdentifier; - private: friend class SCompilerArgs; friend class SEntry; + friend class CCache; friend void to_json(nlohmann::json&, const SPreprocessorArgs&); friend void from_json(const nlohmann::json&, SPreprocessorArgs&); @@ -264,9 +263,10 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted // Sort them so equality and hashing are well defined std::sort(extraDefines.begin(), extraDefines.end(), [](const SMacroDefinition& lhs, const SMacroDefinition& rhs) {return lhs.identifier < rhs.identifier; }); }; + std::string sourceIdentifier; std::vector extraDefines; }; - // TODO: SPreprocessorArgs could just be folded into `SCompilerArgs` to have less classes and operators + // TODO: SPreprocessorArgs could just be folded into `SCompilerArgs` to have less classes and decompressShader struct SCompilerArgs final { public: @@ -283,11 +283,9 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted return retVal; } - IShader::E_SHADER_STAGE stage; - SPreprocessorArgs preprocessorArgs; - private: friend class SEntry; + friend class CCache; friend void to_json(nlohmann::json&, const SCompilerArgs&); friend void from_json(const nlohmann::json&, SCompilerArgs&); @@ -310,9 +308,11 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted } } + IShader::E_SHADER_STAGE stage; E_SPIRV_VERSION targetSpirvVersion; std::vector optimizerPasses; core::bitflag debugInfoFlags; + SPreprocessorArgs preprocessorArgs; }; // The ordering is important here, the dependencies MUST be added to the array IN THE ORDER THE PREPROCESSOR INCLUDED THEM! @@ -354,20 +354,28 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted lookupHash = std::hash{}(hash); } + // Making an entry to insert into write cache + inline SEntry(const SEntry& other, dependency_container_t dependencies, core::smart_refctd_ptr spirv, + core::blake3_hash_t uncompressedContentHash, size_t uncompressedSize) + : mainFileContents(other.mainFileContents), compilerArgs(other.compilerArgs), hash(other.hash), + lookupHash(other.lookupHash), dependencies(dependencies), spirv(spirv), + uncompressedContentHash(uncompressedContentHash), uncompressedSize(uncompressedSize) {} + // Needed to get the vector deserialization automatically inline SEntry() {} // Making the copy constructor deep-copy everything but the shader - inline SEntry(const SEntry& other) - : mainFileContents(other.mainFileContents), compilerArgs(other.compilerArgs), hash(other.hash), lookupHash(other.lookupHash), - dependencies(other.dependencies), spirv(other.spirv), uncompressedSize(other.uncompressedSize) {} + inline SEntry(const SEntry& other) + : mainFileContents(other.mainFileContents), compilerArgs(other.compilerArgs), hash(other.hash), + lookupHash(other.lookupHash), dependencies(other.dependencies), spirv(other.spirv), + uncompressedContentHash(other.uncompressedContentHash), uncompressedSize(other.uncompressedSize) {} inline SEntry& operator=(SEntry& other) = delete; inline SEntry(SEntry&& other) = default; // Used for late initialization while looking up a cache, so as not to always initialize an entry even if caching was not requested inline SEntry& operator=(SEntry&& other) = default; - core::smart_refctd_ptr decodeShader() const; + core::smart_refctd_ptr decompressShader() const; // TODO: make some of these private std::string mainFileContents; @@ -376,6 +384,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted size_t lookupHash; dependency_container_t dependencies; core::smart_refctd_ptr spirv; + core::blake3_hash_t uncompressedContentHash; size_t uncompressedSize; }; diff --git a/src/nbl/asset/utils/IShaderCompiler.cpp b/src/nbl/asset/utils/IShaderCompiler.cpp index d36c87f62..20c8f20cf 100644 --- a/src/nbl/asset/utils/IShaderCompiler.cpp +++ b/src/nbl/asset/utils/IShaderCompiler.cpp @@ -45,7 +45,7 @@ inline core::smart_refctd_ptr nbl::asset::IShaderCompiler::compileTo CCache::SEntry writeEntry = *found; options.writeCache->insert(std::move(writeEntry)); } - return found->decodeShader(); + return found->decompressShader(); } } @@ -56,7 +56,6 @@ inline core::smart_refctd_ptr nbl::asset::IShaderCompiler::compileTo const_cast(backingBuffer)->setContentHash(backingBuffer->computeContentHash()); } - std::cout << "writeCache: " << options.writeCache << "\n"; if (options.writeCache) { auto* spirvBuffer = retVal->getContent(); @@ -83,11 +82,7 @@ inline core::smart_refctd_ptr nbl::asset::IShaderCompiler::compileTo auto compressedSpirvBuffer = core::make_smart_refctd_ptr(propsSize + destLen); memcpy(compressedSpirvBuffer->getPointer(), compressedSpirv.data(), compressedSpirvBuffer->getSize()); - entry.dependencies = std::move(dependencies); - entry.spirv = std::move(compressedSpirvBuffer); - entry.uncompressedSize = spirvBuffer->getSize(); - - options.writeCache->insert(std::move(entry)); + options.writeCache->insert(std::move(SEntry(entry, std::move(dependencies), std::move(compressedSpirvBuffer), spirvBuffer->getContentHash(), spirvBuffer->getSize()))); } return retVal; } @@ -294,7 +289,7 @@ auto IShaderCompiler::CIncludeFinder::tryIncludeGenerators(const std::string& in core::smart_refctd_ptr IShaderCompiler::CCache::find(const SEntry& mainFile, const IShaderCompiler::CIncludeFinder* finder) const { - return find_impl(mainFile, finder)->decodeShader(); + return find_impl(mainFile, finder)->decompressShader(); } IShaderCompiler::CCache::EntrySet::const_iterator IShaderCompiler::CCache::find_impl(const SEntry& mainFile, const IShaderCompiler::CIncludeFinder* finder) const @@ -346,7 +341,7 @@ core::smart_refctd_ptr IShaderCompiler::CCache::serialize() const sizes[i] = entry.spirv->getSize(); // And add the params to the shader creation parameters array - shaderCreationParams.emplace_back(entry.compilerArgs.stage, IShader::E_CONTENT_TYPE::ECT_SPIRV, entry.compilerArgs.preprocessorArgs.sourceIdentifier.data(), sizes[i], shaderBufferSize); + shaderCreationParams.emplace_back(entry.compilerArgs.stage, entry.compilerArgs.preprocessorArgs.sourceIdentifier.data(), sizes[i], shaderBufferSize); // Enlarge the shader buffer by the size of the current shader shaderBufferSize += sizes[i]; i++; @@ -423,9 +418,11 @@ core::smart_refctd_ptr IShaderCompiler::CCache::deseria return retVal; } -core::smart_refctd_ptr nbl::asset::IShaderCompiler::CCache::SEntry::decodeShader() const +core::smart_refctd_ptr nbl::asset::IShaderCompiler::CCache::SEntry::decompressShader() const { auto uncompressedBuf = core::make_smart_refctd_ptr(uncompressedSize); + uncompressedBuf->setContentHash(uncompressedContentHash); + size_t dstSize = uncompressedBuf->getSize(); size_t srcSize = spirv->getSize() - LZMA_PROPS_SIZE; ELzmaStatus status; diff --git a/src/nbl/asset/utils/shaderCompiler_serialization.h b/src/nbl/asset/utils/shaderCompiler_serialization.h index e038d3bf9..ca81df640 100644 --- a/src/nbl/asset/utils/shaderCompiler_serialization.h +++ b/src/nbl/asset/utils/shaderCompiler_serialization.h @@ -133,13 +133,12 @@ inline void from_json(const json& j, SEntry::SPreprocessingDependency& dependenc struct CPUShaderCreationParams { IShader::E_SHADER_STAGE stage; - IShader::E_CONTENT_TYPE contentType; //I think this one could be skipped since it's always going to be SPIR-V std::string filepathHint; uint64_t codeByteSize = 0; uint64_t offset = 0; // Offset into the serialized .bin for the Cache where code starts - CPUShaderCreationParams(IShader::E_SHADER_STAGE _stage, IShader::E_CONTENT_TYPE _contentType, std::string_view _filepathHint, uint64_t _codeByteSize, uint64_t _offset) - : stage(_stage), contentType(_contentType), filepathHint(_filepathHint), codeByteSize(_codeByteSize), offset(_offset) + CPUShaderCreationParams(IShader::E_SHADER_STAGE _stage, std::string_view _filepathHint, uint64_t _codeByteSize, uint64_t _offset) + : stage(_stage), filepathHint(_filepathHint), codeByteSize(_codeByteSize), offset(_offset) {} CPUShaderCreationParams() {}; @@ -148,10 +147,8 @@ struct CPUShaderCreationParams { inline void to_json(json& j, const CPUShaderCreationParams& creationParams) { uint32_t stage = static_cast(creationParams.stage); - uint32_t contentType = static_cast(creationParams.contentType); j = json{ { "stage", stage }, - { "contentType", contentType }, { "filepathHint", creationParams.filepathHint }, { "codeByteSize", creationParams.codeByteSize }, { "offset", creationParams.offset }, @@ -160,14 +157,12 @@ inline void to_json(json& j, const CPUShaderCreationParams& creationParams) inline void from_json(const json& j, CPUShaderCreationParams& creationParams) { - uint32_t stage, contentType; + uint32_t stage; j.at("stage").get_to(stage); - j.at("contentType").get_to(contentType); j.at("filepathHint").get_to(creationParams.filepathHint); j.at("codeByteSize").get_to(creationParams.codeByteSize); j.at("offset").get_to(creationParams.offset); creationParams.stage = static_cast(stage); - creationParams.contentType = static_cast(stage); } // Serialize SEntry, keeping some fields as extra serialization to keep them separate on disk @@ -180,6 +175,7 @@ inline void to_json(json& j, const SEntry& entry) { "hash", entry.hash.data }, { "lookupHash", entry.lookupHash }, { "dependencies", entry.dependencies }, + { "uncompressedContentHash", entry.uncompressedContentHash.data }, { "uncompressedSize", entry.uncompressedSize }, }; } @@ -191,6 +187,7 @@ inline void from_json(const json& j, SEntry& entry) j.at("hash").get_to(entry.hash.data); j.at("lookupHash").get_to(entry.lookupHash); j.at("dependencies").get_to(entry.dependencies); + j.at("uncompressedContentHash").get_to(entry.uncompressedContentHash.data); j.at("uncompressedSize").get_to(entry.uncompressedSize); entry.spirv = nullptr; } From 3445f6123f0dce03af132311f49b63bbd1072f10 Mon Sep 17 00:00:00 2001 From: Ali Cheraghi Date: Tue, 8 Oct 2024 22:27:39 +0330 Subject: [PATCH 10/11] resolve reviews Signed-off-by: Ali Cheraghi --- include/nbl/asset/utils/IShaderCompiler.h | 9 +-- src/nbl/asset/utils/IShaderCompiler.cpp | 68 +++++++++++++---------- 2 files changed, 41 insertions(+), 36 deletions(-) diff --git a/include/nbl/asset/utils/IShaderCompiler.h b/include/nbl/asset/utils/IShaderCompiler.h index 3899c1266..bac2379ed 100644 --- a/include/nbl/asset/utils/IShaderCompiler.h +++ b/include/nbl/asset/utils/IShaderCompiler.h @@ -354,13 +354,6 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted lookupHash = std::hash{}(hash); } - // Making an entry to insert into write cache - inline SEntry(const SEntry& other, dependency_container_t dependencies, core::smart_refctd_ptr spirv, - core::blake3_hash_t uncompressedContentHash, size_t uncompressedSize) - : mainFileContents(other.mainFileContents), compilerArgs(other.compilerArgs), hash(other.hash), - lookupHash(other.lookupHash), dependencies(dependencies), spirv(spirv), - uncompressedContentHash(uncompressedContentHash), uncompressedSize(uncompressedSize) {} - // Needed to get the vector deserialization automatically inline SEntry() {} @@ -375,6 +368,8 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted // Used for late initialization while looking up a cache, so as not to always initialize an entry even if caching was not requested inline SEntry& operator=(SEntry&& other) = default; + void setContent(const asset::ICPUBuffer* uncompressedSpirvBuffer, dependency_container_t&& dependencies); + core::smart_refctd_ptr decompressShader() const; // TODO: make some of these private diff --git a/src/nbl/asset/utils/IShaderCompiler.cpp b/src/nbl/asset/utils/IShaderCompiler.cpp index 20c8f20cf..2ce889189 100644 --- a/src/nbl/asset/utils/IShaderCompiler.cpp +++ b/src/nbl/asset/utils/IShaderCompiler.cpp @@ -25,9 +25,6 @@ IShaderCompiler::IShaderCompiler(core::smart_refctd_ptr&& syste m_defaultIncludeFinder = core::make_smart_refctd_ptr(core::smart_refctd_ptr(m_system)); } -static void* SzAlloc(ISzAllocPtr p, size_t size) { p = p; return _NBL_ALIGNED_MALLOC(size, _NBL_SIMD_ALIGNMENT); } -static void SzFree(ISzAllocPtr p, void* address) { p = p; _NBL_ALIGNED_FREE(address); } - inline core::smart_refctd_ptr nbl::asset::IShaderCompiler::compileToSPIRV(const std::string_view code, const SCompilerOptions& options) const { CCache::SEntry entry; @@ -58,31 +55,8 @@ inline core::smart_refctd_ptr nbl::asset::IShaderCompiler::compileTo if (options.writeCache) { - auto* spirvBuffer = retVal->getContent(); - size_t propsSize = LZMA_PROPS_SIZE; - size_t destLen = spirvBuffer->getSize() + spirvBuffer->getSize() / 3 + 128; - std::vector compressedSpirv = {}; - compressedSpirv.resize(propsSize + destLen); - - CLzmaEncProps props; - LzmaEncProps_Init(&props); - props.dictSize = 1 << 16; // 64KB - props.writeEndMark = 1; - - ISzAlloc alloc = { SzAlloc, SzFree }; - int res = LzmaEncode( - compressedSpirv.data() + LZMA_PROPS_SIZE, &destLen, - reinterpret_cast(spirvBuffer->getPointer()), spirvBuffer->getSize(), - &props, compressedSpirv.data(), &propsSize, props.writeEndMark, - nullptr, &alloc, &alloc); - - assert(propsSize == LZMA_PROPS_SIZE); - assert(res == SZ_OK); - - auto compressedSpirvBuffer = core::make_smart_refctd_ptr(propsSize + destLen); - memcpy(compressedSpirvBuffer->getPointer(), compressedSpirv.data(), compressedSpirvBuffer->getSize()); - - options.writeCache->insert(std::move(SEntry(entry, std::move(dependencies), std::move(compressedSpirvBuffer), spirvBuffer->getContentHash(), spirvBuffer->getSize()))); + entry.setContent(retVal->getContent(), std::move(dependencies)); + options.writeCache->insert(std::move(entry)); } return retVal; } @@ -289,7 +263,10 @@ auto IShaderCompiler::CIncludeFinder::tryIncludeGenerators(const std::string& in core::smart_refctd_ptr IShaderCompiler::CCache::find(const SEntry& mainFile, const IShaderCompiler::CIncludeFinder* finder) const { - return find_impl(mainFile, finder)->decompressShader(); + const auto found = find_impl(mainFile, finder); + if (found==m_container.end()) + return nullptr; + return found->decompressShader(); } IShaderCompiler::CCache::EntrySet::const_iterator IShaderCompiler::CCache::find_impl(const SEntry& mainFile, const IShaderCompiler::CIncludeFinder* finder) const @@ -418,6 +395,39 @@ core::smart_refctd_ptr IShaderCompiler::CCache::deseria return retVal; } +static void* SzAlloc(ISzAllocPtr p, size_t size) { p = p; return _NBL_ALIGNED_MALLOC(size, _NBL_SIMD_ALIGNMENT); } +static void SzFree(ISzAllocPtr p, void* address) { p = p; _NBL_ALIGNED_FREE(address); } + +void nbl::asset::IShaderCompiler::CCache::SEntry::setContent(const asset::ICPUBuffer* uncompressedSpirvBuffer, dependency_container_t&& dependencies) +{ + dependencies = std::move(dependencies); + uncompressedContentHash = uncompressedSpirvBuffer->getContentHash(); + uncompressedSize = uncompressedSpirvBuffer->getSize(); + + size_t propsSize = LZMA_PROPS_SIZE; + size_t destLen = uncompressedSpirvBuffer->getSize() + uncompressedSpirvBuffer->getSize() / 3 + 128; + std::vector compressedSpirv = {}; + compressedSpirv.resize(propsSize + destLen); + + CLzmaEncProps props; + LzmaEncProps_Init(&props); + props.dictSize = 1 << 16; // 64KB + props.writeEndMark = 1; + + ISzAlloc alloc = { SzAlloc, SzFree }; + int res = LzmaEncode( + compressedSpirv.data() + LZMA_PROPS_SIZE, &destLen, + reinterpret_cast(uncompressedSpirvBuffer->getPointer()), uncompressedSpirvBuffer->getSize(), + &props, compressedSpirv.data(), &propsSize, props.writeEndMark, + nullptr, &alloc, &alloc); + + assert(propsSize == LZMA_PROPS_SIZE); + assert(res == SZ_OK); + + spirv = core::make_smart_refctd_ptr(propsSize + destLen); + memcpy(spirv->getPointer(), compressedSpirv.data(), spirv->getSize()); +} + core::smart_refctd_ptr nbl::asset::IShaderCompiler::CCache::SEntry::decompressShader() const { auto uncompressedBuf = core::make_smart_refctd_ptr(uncompressedSize); From f3961db7ac57ded32e52c4f0581c28e5e0b50f23 Mon Sep 17 00:00:00 2001 From: Ali Cheraghi Date: Wed, 9 Oct 2024 12:33:06 +0330 Subject: [PATCH 11/11] resolve reviews Signed-off-by: Ali Cheraghi --- include/nbl/asset/utils/IShaderCompiler.h | 2 +- src/nbl/asset/utils/IShaderCompiler.cpp | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/include/nbl/asset/utils/IShaderCompiler.h b/include/nbl/asset/utils/IShaderCompiler.h index bac2379ed..ac7ed5eb1 100644 --- a/include/nbl/asset/utils/IShaderCompiler.h +++ b/include/nbl/asset/utils/IShaderCompiler.h @@ -368,7 +368,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted // Used for late initialization while looking up a cache, so as not to always initialize an entry even if caching was not requested inline SEntry& operator=(SEntry&& other) = default; - void setContent(const asset::ICPUBuffer* uncompressedSpirvBuffer, dependency_container_t&& dependencies); + bool setContent(const asset::ICPUBuffer* uncompressedSpirvBuffer, dependency_container_t&& dependencies); core::smart_refctd_ptr decompressShader() const; diff --git a/src/nbl/asset/utils/IShaderCompiler.cpp b/src/nbl/asset/utils/IShaderCompiler.cpp index 2ce889189..f8c9e545d 100644 --- a/src/nbl/asset/utils/IShaderCompiler.cpp +++ b/src/nbl/asset/utils/IShaderCompiler.cpp @@ -55,8 +55,8 @@ inline core::smart_refctd_ptr nbl::asset::IShaderCompiler::compileTo if (options.writeCache) { - entry.setContent(retVal->getContent(), std::move(dependencies)); - options.writeCache->insert(std::move(entry)); + if (entry.setContent(retVal->getContent(), std::move(dependencies))) + options.writeCache->insert(std::move(entry)); } return retVal; } @@ -398,7 +398,7 @@ core::smart_refctd_ptr IShaderCompiler::CCache::deseria static void* SzAlloc(ISzAllocPtr p, size_t size) { p = p; return _NBL_ALIGNED_MALLOC(size, _NBL_SIMD_ALIGNMENT); } static void SzFree(ISzAllocPtr p, void* address) { p = p; _NBL_ALIGNED_FREE(address); } -void nbl::asset::IShaderCompiler::CCache::SEntry::setContent(const asset::ICPUBuffer* uncompressedSpirvBuffer, dependency_container_t&& dependencies) +bool nbl::asset::IShaderCompiler::CCache::SEntry::setContent(const asset::ICPUBuffer* uncompressedSpirvBuffer, dependency_container_t&& dependencies) { dependencies = std::move(dependencies); uncompressedContentHash = uncompressedSpirvBuffer->getContentHash(); @@ -421,11 +421,12 @@ void nbl::asset::IShaderCompiler::CCache::SEntry::setContent(const asset::ICPUBu &props, compressedSpirv.data(), &propsSize, props.writeEndMark, nullptr, &alloc, &alloc); - assert(propsSize == LZMA_PROPS_SIZE); - assert(res == SZ_OK); + if (res != SZ_OK || propsSize != LZMA_PROPS_SIZE) return false; spirv = core::make_smart_refctd_ptr(propsSize + destLen); memcpy(spirv->getPointer(), compressedSpirv.data(), spirv->getSize()); + + return true; } core::smart_refctd_ptr nbl::asset::IShaderCompiler::CCache::SEntry::decompressShader() const