Index: source/graphics/PreprocessorWrapper.h =================================================================== --- source/graphics/PreprocessorWrapper.h +++ source/graphics/PreprocessorWrapper.h @@ -21,6 +21,9 @@ #include "ps/CStr.h" #include "third_party/ogre3d_preprocessor/OgreGLSLPreprocessor.h" +#include +#include + class CShaderDefines; /** @@ -29,7 +32,10 @@ class CPreprocessorWrapper { public: + using IncludeRetrieverCallback = std::function; + CPreprocessorWrapper(); + CPreprocessorWrapper(const IncludeRetrieverCallback& includeCallback); void AddDefine(const char* name, const char* value); @@ -37,12 +43,20 @@ bool TestConditional(const CStr& expr); + // Find all #include directives in the input and replace them by + // by a file content from the directive's argument. Parsing is strict + // and simple. The directive will be expanded in comments and multiline + // strings. + CStr ResolveIncludes(CStr source); + CStr Preprocess(const CStr& input); static void PyrogenesisShaderError(int iLine, const char* iError, const Ogre::CPreprocessor::Token* iToken); private: Ogre::CPreprocessor m_Preprocessor; + IncludeRetrieverCallback m_IncludeCallback; + std::map m_IncludeCache; }; #endif // INCLUDED_PREPROCESSORWRAPPER Index: source/graphics/PreprocessorWrapper.cpp =================================================================== --- source/graphics/PreprocessorWrapper.cpp +++ source/graphics/PreprocessorWrapper.cpp @@ -22,6 +22,88 @@ #include "graphics/ShaderDefines.h" #include "ps/CLogger.h" +#include + +namespace +{ + +struct MatchIncludeResult +{ + bool found; + bool error; + size_t nextLineStart; + size_t pathFirst, pathLast; + + static MatchIncludeResult MakeNotFound(const CStr& source, size_t pos) + { + while (pos < source.size() && source[pos] != '\n') + ++pos; + return MatchIncludeResult{ + false, false, pos < source.size() ? pos + 1 : source.size(), 0, 0}; + } + + static MatchIncludeResult MakeError( + const char* message, const CStr& source, const size_t lineStart, const size_t currentPos) + { + ENSURE(currentPos >= lineStart); + size_t lineEnd = currentPos; + while (lineEnd < source.size() && source[lineEnd] != '\n' && source[lineEnd] != '\r') + ++lineEnd; + const CStr line = source.substr(lineStart, lineEnd - lineStart); + while (lineEnd < source.size() && source[lineEnd] != '\n') + ++lineEnd; + const size_t nextLineStart = lineEnd < source.size() ? lineEnd + 1 : source.size(); + LOGERROR("Preprocessor error: %s: '%s'\n", message, line.c_str()); + return MatchIncludeResult{false, true, nextLineStart, 0, 0}; + } +}; + +MatchIncludeResult MatchIncludeUntilEOLorEOS(const CStr& source, const size_t lineStart) +{ + // We need to match a line like this: + // ^[ \t]*#[ \t]*include[ \t]*"[^"]+".*$ + // ^ ^^ ^ ^ ^^ ^ + // 1 23 4 5 67 8 <- steps + const CStr INCLUDE = "include"; + size_t pos = lineStart; + // Matching step #1. + while (pos < source.size() && std::isblank(source[pos])) + ++pos; + // Matching step #2. + if (pos == source.size() || source[pos] != '#') + return MatchIncludeResult::MakeNotFound(source, pos); + ++pos; + // Matching step #3. + while (pos < source.size() && std::isblank(source[pos])) + ++pos; + // Matching step #4. + if (pos + INCLUDE.size() >= source.size() || source.substr(pos, INCLUDE.size()) != INCLUDE) + return MatchIncludeResult::MakeNotFound(source, pos); + pos += INCLUDE.size(); + // Matching step #5. + while (pos < source.size() && std::isblank(source[pos])) + ++pos; + // Matching step #6. + if (pos == source.size() || source[pos] != '"') + return MatchIncludeResult::MakeError("#include should be followed by quote", source, lineStart, pos); + ++pos; + // Matching step #7. + const size_t pathFirst = pos; + while (pos < source.size() && source[pos] != '"' && source[pos] != '\n') + ++pos; + const size_t pathLast = pos; + // Matching step #8. + if (pos == source.size() || source[pos] != '"') + return MatchIncludeResult::MakeError("#include has invalid quote pair", source, lineStart, pos); + if (pathLast - pathFirst <= 1) + return MatchIncludeResult::MakeError("#include path shouldn't be empty", source, lineStart, pos); + while (pos < source.size() && source[pos] != '\n') + ++pos; + return MatchIncludeResult{true, false, pos < source.size() ? pos + 1 : source.size(), pathFirst, pathLast}; +} + +} // anonymous namespace + void CPreprocessorWrapper::PyrogenesisShaderError(int iLine, const char* iError, const Ogre::CPreprocessor::Token* iToken) { if (iToken) @@ -31,6 +113,12 @@ } CPreprocessorWrapper::CPreprocessorWrapper() + : CPreprocessorWrapper(IncludeRetrieverCallback{}) +{ +} + +CPreprocessorWrapper::CPreprocessorWrapper(const IncludeRetrieverCallback& includeCallback) + : m_IncludeCallback(includeCallback) { Ogre::CPreprocessor::ErrorHandler = CPreprocessorWrapper::PyrogenesisShaderError; } @@ -78,10 +166,63 @@ } +CStr CPreprocessorWrapper::ResolveIncludes(CStr source) +{ + const CStr lineDirective = "#line "; + for (size_t lineStart = 0, line = 1; lineStart < source.size(); ++line) + { + MatchIncludeResult match = MatchIncludeUntilEOLorEOS(source, lineStart); + if (match.error) + return {}; + else if (!match.found) + { + if (lineStart + lineDirective.size() < source.size() && + source.substr(lineStart, lineDirective.size()) == lineDirective) + { + size_t newLineNumber = 0; + size_t pos = lineStart + lineDirective.size(); + while (pos < match.nextLineStart && std::isdigit(source[pos])) + { + newLineNumber = newLineNumber * 10 + (source[pos] - '0'); + ++pos; + } + if (newLineNumber > 0) + line = newLineNumber - 1; + } + + lineStart = match.nextLineStart; + continue; + } + const CStr path = source.substr(match.pathFirst, match.pathLast - match.pathFirst); + auto it = m_IncludeCache.find(path); + if (it == m_IncludeCache.end()) + { + CStr includeContent; + if (!m_IncludeCallback(path, includeContent)) + { + LOGERROR("Preprocessor error: line %zu: Can't load #include file: '%s'", line, path.c_str()); + return {}; + } + it = m_IncludeCache.emplace(path, std::move(includeContent)).first; + } + // We need to insert #line directives to have correct line numbers in errors. + source = + source.substr(0, lineStart) + + lineDirective + "1\n" + it->second + "\n" + lineDirective + CStr::FromUInt(line + 1) + "\n" + + source.substr(match.nextLineStart); + --line; + } + return source; +} + CStr CPreprocessorWrapper::Preprocess(const CStr& input) { + PROFILE("Preprocess shader source"); + + CStr source = ResolveIncludes(input); + size_t len = 0; - char* output = m_Preprocessor.Parse(input.c_str(), input.size(), len); + char* output = m_Preprocessor.Parse(source.c_str(), source.size(), len); if (!output) { @@ -92,7 +233,7 @@ CStr ret(output, len); // Free output if it's not inside the source string - if (!(output >= input.c_str() && output < input.c_str() + input.size())) + if (!(output >= source.c_str() && output < source.c_str() + source.size())) free(output); return ret; Index: source/graphics/ShaderManager.h =================================================================== --- source/graphics/ShaderManager.h +++ source/graphics/ShaderManager.h @@ -125,6 +125,11 @@ static Status ReloadChangedFileCB(void* param, const VfsPath& path); Status ReloadChangedFile(const VfsPath& path); + + /** + * Associates the file with the program to be reloaded if the file has changed. + */ + void AddProgramFileDependency(const CShaderProgramPtr& program, const VfsPath& path); }; #endif // INCLUDED_SHADERMANAGER Index: source/graphics/ShaderManager.cpp =================================================================== --- source/graphics/ShaderManager.cpp +++ source/graphics/ShaderManager.cpp @@ -1,4 +1,4 @@ -/* Copyright (C) 2019 Wildfire Games. +/* Copyright (C) 2020 Wildfire Games. * This file is part of 0 A.D. * * 0 A.D. is free software: you can redistribute it and/or modify @@ -269,8 +269,8 @@ program->Reload(); // m_HotloadFiles[xmlFilename].insert(program); // TODO: should reload somehow when the XML changes - m_HotloadFiles[vertexFile].insert(program); - m_HotloadFiles[fragmentFile].insert(program); + for (const VfsPath& path : program->GetFileDependencies()) + AddProgramFileDependency(program, path); return true; } @@ -566,17 +566,20 @@ { // Find all shaders using this file HotloadFilesMap::iterator files = m_HotloadFiles.find(path); - if (files != m_HotloadFiles.end()) - { - // Reload all shaders using this file - for (std::set >::iterator it = files->second.begin(); it != files->second.end(); ++it) - { - if (std::shared_ptr program = it->lock()) - program->Reload(); - } - } + if (files == m_HotloadFiles.end()) + return INFO::OK; + + // Reload all shaders using this file + for (const std::weak_ptr& ptr : files->second) + if (std::shared_ptr program = ptr.lock()) + program->Reload(); // TODO: hotloading changes to shader XML files and effect XML files would be nice return INFO::OK; } + +void CShaderManager::AddProgramFileDependency(const CShaderProgramPtr& program, const VfsPath& path) +{ + m_HotloadFiles[path].insert(program); +} Index: source/graphics/ShaderProgram.h =================================================================== --- source/graphics/ShaderProgram.h +++ source/graphics/ShaderProgram.h @@ -25,6 +25,7 @@ #include "lib/res/handle.h" #include +#include struct CColor; class CMatrix3D; @@ -193,6 +194,8 @@ */ void AssertPointersBound(); + virtual std::vector GetFileDependencies() const = 0; + protected: CShaderProgram(int streamflags); Index: source/graphics/ShaderProgram.cpp =================================================================== --- source/graphics/ShaderProgram.cpp +++ source/graphics/ShaderProgram.cpp @@ -235,6 +235,11 @@ Uniform(id, v[0]); } + virtual std::vector GetFileDependencies() const override + { + return {m_VertexFile, m_FragmentFile}; + } + private: VfsPath m_VertexFile; VfsPath m_FragmentFile; @@ -271,6 +276,7 @@ m_Program = 0; m_VertexShader = pglCreateShaderObjectARB(GL_VERTEX_SHADER); m_FragmentShader = pglCreateShaderObjectARB(GL_FRAGMENT_SHADER); + m_FileDependencies = {m_VertexFile, m_FragmentFile}; } ~CShaderProgramGLSL() @@ -430,7 +436,17 @@ if (fragmentFile.Load(g_VFS, m_FragmentFile) != PSRETURN_OK) return; - CPreprocessorWrapper preprocessor; + std::vector newFileDependencies = {m_VertexFile, m_FragmentFile}; + + CPreprocessorWrapper preprocessor([&newFileDependencies](const CStr& includePath, CStr& out) -> bool { + const VfsPath includeFilePath(L"shaders/glsl/" + wstring_from_utf8(includePath)); + CVFSFile includeFile; + if (includeFile.Load(g_VFS, includeFilePath) != PSRETURN_OK) + return false; + out = includeFile.GetAsString(); + newFileDependencies.push_back(includeFilePath); + return true; + }); preprocessor.AddDefines(m_Defines); #if CONFIG2_GLES @@ -443,6 +459,8 @@ CStr vertexCode = preprocessor.Preprocess(vertexFile.GetAsString()); CStr fragmentCode = preprocessor.Preprocess(fragmentFile.GetAsString()); + m_FileDependencies = std::move(newFileDependencies); + #if CONFIG2_GLES // Ugly hack to replace desktop GLSL 1.10/1.20 with GLSL ES 1.00, // and also to set default float precision for fragment shaders @@ -636,9 +654,15 @@ } } + virtual std::vector GetFileDependencies() const + { + return m_FileDependencies; + } + private: VfsPath m_VertexFile; VfsPath m_FragmentFile; + std::vector m_FileDependencies; CShaderDefines m_Defines; std::map m_VertexAttribs; Index: source/graphics/ShaderProgramFFP.cpp =================================================================== --- source/graphics/ShaderProgramFFP.cpp +++ source/graphics/ShaderProgramFFP.cpp @@ -113,6 +113,11 @@ { } + virtual std::vector GetFileDependencies() const + { + return {}; + } + protected: std::map m_UniformIndexes; Index: source/third_party/ogre3d_preprocessor/tests/test_Preprocessor.h =================================================================== --- source/third_party/ogre3d_preprocessor/tests/test_Preprocessor.h +++ source/third_party/ogre3d_preprocessor/tests/test_Preprocessor.h @@ -21,6 +21,8 @@ #include "ps/CStr.h" #include "third_party/ogre3d_preprocessor/OgreGLSLPreprocessor.h" +#include + class TestPreprocessor : public CxxTest::TestSuite { public: @@ -35,6 +37,16 @@ CStr8 loggerOutput; }; + // Replaces consecutive spaces/tabs by single space/tab. + CStr CompressWhiteSpaces(const CStr& source) + { + CStr result; + for (char ch : source) + if (!std::isblank(ch) || (result.empty() || result.back() != ch)) + result += ch; + return result; + } + PreprocessorResult Parse(const char* in) { PreprocessorResult result; @@ -52,6 +64,18 @@ return result; } + PreprocessorResult ParseWithIncludes(const char* in, CPreprocessorWrapper::IncludeRetrieverCallback includeCallback) + { + PreprocessorResult result; + TestLogger logger; + + CPreprocessorWrapper preprocessor(includeCallback); + result.output = preprocessor.Preprocess(in); + result.loggerOutput = logger.GetOutput(); + + return result; + } + void test_basic() { PreprocessorResult result = Parse("#define TEST 2\n1+1=TEST\n"); @@ -113,4 +137,126 @@ TS_ASSERT_EQUALS(result.output.Trim(PS_TRIM_BOTH), ""); TS_ASSERT_STR_CONTAINS(result.loggerOutput, "ERROR: Preprocessor error: line 2: Division by zero"); } + + void test_include() + { + bool includeRetrieved = false; + CPreprocessorWrapper::IncludeRetrieverCallback includeCallback = [&includeRetrieved]( + const CStr& includePath, CStr& out) { + TS_ASSERT_EQUALS(includePath, "test.h"); + out = "42"; + includeRetrieved = true; + return true; + }; + + PreprocessorResult result = ParseWithIncludes( + R"(#include "test.h" // Includes something.)", includeCallback); + + TS_ASSERT(includeRetrieved); + TS_ASSERT_EQUALS(result.output.Trim(PS_TRIM_BOTH), "#line 1\n42\n#line 2"); + } + + void test_include_double() + { + CPreprocessorWrapper::IncludeRetrieverCallback includeCallback = []( + const CStr& includePath, CStr& out) { + TS_ASSERT_EQUALS(includePath, "test.h"); + out = "42"; + return true; + }; + + PreprocessorResult result = ParseWithIncludes(R"( + #include "test.h" + #include "test.h" + #include "test.h" + )", includeCallback); + + TS_ASSERT_EQUALS(result.output.Trim(PS_TRIM_BOTH), "#line 1\n42\n#line 3\n#line 1\n42\n#line 4\n#line 1\n42\n#line 5"); + } + + void test_include_double_with_guards() + { + CPreprocessorWrapper::IncludeRetrieverCallback includeCallback = []( + const CStr& includePath, CStr& out) { + TS_ASSERT_EQUALS(includePath, "test.h"); + out = R"(#ifndef INCLUDED_TEST + #define INCLUDED_TEST + 42 + #endif)"; + return true; + }; + + PreprocessorResult result = ParseWithIncludes(R"( + #include "test.h" + #include "test.h" + #include "test.h" + )", includeCallback); + + TS_ASSERT_EQUALS(result.output.Trim(PS_TRIM_BOTH), "#line 1\n\n\t\t\t\t\n\t\t\t\t42\n\t\t\t\n#line 3\n#line 1\n\n\n\n\n#line 4\n#line 1\n\n\n\n\n#line 5"); + } + + void test_include_invalid_argument() + { + int includeRetrievedCounter = 0; + CPreprocessorWrapper::IncludeRetrieverCallback includeCallback = [&includeRetrievedCounter]( + const CStr& includePath, CStr& out) { + out = "42"; + ++includeRetrievedCounter; + return true; + }; + + PreprocessorResult result = ParseWithIncludes(R"( + #include + #include test.h + )", includeCallback); + + TS_ASSERT_EQUALS(includeRetrievedCounter, 0); + } + + void test_include_invalid_file() + { + CPreprocessorWrapper::IncludeRetrieverCallback includeCallback = []( + const CStr& includePath, CStr& out) { + return false; + }; + + PreprocessorResult result = ParseWithIncludes(R"( + #include "missed_file.h" + )", includeCallback); + + TS_ASSERT_STR_CONTAINS(result.loggerOutput, "ERROR: Preprocessor error: line 2: Can't load #include file: 'missed_file.h'"); + } + + void test_include_with_defines() + { + CPreprocessorWrapper::IncludeRetrieverCallback includeCallback = []( + const CStr& includePath, CStr& out) { + out = R"( + #if defined(A) + #define X 41 + #elif defined(B) + #define X 42 + #else + #define X 43 + #endif + #ifdef Y + #undef Y + #define Y 256 + #endif + vec3 color(); + )"; + return true; + }; + + PreprocessorResult result = ParseWithIncludes(R"( + #define Y 128 + #define B 1 + #include "test.h" + X Y + )", includeCallback); + + TS_ASSERT_EQUALS( + CompressWhiteSpaces(result.output.Trim(PS_TRIM_BOTH)), + "#line 1\n\n\t\n\n\n\t\n\t\n\n\n\t\n\t\n\t\n\t\n\tvec3 color();\n\t\n#line 5\n\t42 256"); + } };