Index: ps/trunk/source/graphics/PreprocessorWrapper.h =================================================================== --- ps/trunk/source/graphics/PreprocessorWrapper.h +++ ps/trunk/source/graphics/PreprocessorWrapper.h @@ -47,7 +47,7 @@ // by a file content from the directive's argument. Parsing is strict // and simple. The directive will be expanded in comments and multiline // strings. - CStr ResolveIncludes(CStr source); + CStr ResolveIncludes(const CStr& source); CStr Preprocess(const CStr& input); Index: ps/trunk/source/graphics/PreprocessorWrapper.cpp =================================================================== --- ps/trunk/source/graphics/PreprocessorWrapper.cpp +++ ps/trunk/source/graphics/PreprocessorWrapper.cpp @@ -24,6 +24,9 @@ #include "ps/Profile.h" #include +#include +#include +#include namespace { @@ -35,7 +38,7 @@ size_t nextLineStart; size_t pathFirst, pathLast; - static MatchIncludeResult MakeNotFound(const CStr& source, size_t pos) + static MatchIncludeResult MakeNotFound(const std::string_view& source, size_t pos) { while (pos < source.size() && source[pos] != '\n') ++pos; @@ -44,22 +47,22 @@ } static MatchIncludeResult MakeError( - const char* message, const CStr& source, const size_t lineStart, const size_t currentPos) + const char* message, const std::string_view& source, const size_t lineStart, const size_t currentPos) { ENSURE(currentPos >= lineStart); size_t lineEnd = currentPos; while (lineEnd < source.size() && source[lineEnd] != '\n' && source[lineEnd] != '\r') ++lineEnd; - const CStr line = source.substr(lineStart, lineEnd - lineStart); + const std::string_view line = source.substr(lineStart, lineEnd - lineStart); while (lineEnd < source.size() && source[lineEnd] != '\n') ++lineEnd; const size_t nextLineStart = lineEnd < source.size() ? lineEnd + 1 : source.size(); - LOGERROR("Preprocessor error: %s: '%s'\n", message, line.c_str()); + LOGERROR("Preprocessor error: %s: '%s'\n", message, std::string(line).c_str()); return MatchIncludeResult{false, true, nextLineStart, 0, 0}; } }; -MatchIncludeResult MatchIncludeUntilEOLorEOS(const CStr& source, const size_t lineStart) +MatchIncludeResult MatchIncludeUntilEOLorEOS(const std::string_view& source, const size_t lineStart) { // We need to match a line like this: // ^[ \t]*#[ \t]*include[ \t]*"[^"]+".*$ @@ -103,6 +106,60 @@ return MatchIncludeResult{true, false, pos < source.size() ? pos + 1 : source.size(), pathFirst, pathLast}; } +bool ResolveIncludesImpl( + std::string_view currentPart, + std::unordered_map& includeCache, const CPreprocessorWrapper::IncludeRetrieverCallback& includeCallback, + std::vector& chunks, std::vector& processedParts) +{ + static const CStr lineDirective = "#line "; + for (size_t lineStart = 0, line = 1; lineStart < currentPart.size(); ++line) + { + MatchIncludeResult match = MatchIncludeUntilEOLorEOS(currentPart, lineStart); + if (match.error) + return {}; + else if (!match.found) + { + if (lineStart + lineDirective.size() < currentPart.size() && + currentPart.substr(lineStart, lineDirective.size()) == lineDirective) + { + size_t newLineNumber = 0; + size_t pos = lineStart + lineDirective.size(); + while (pos < match.nextLineStart && std::isdigit(currentPart[pos])) + { + newLineNumber = newLineNumber * 10 + (currentPart[pos] - '0'); + ++pos; + } + if (newLineNumber > 0) + line = newLineNumber - 1; + } + + lineStart = match.nextLineStart; + continue; + } + const std::string path(currentPart.substr(match.pathFirst, match.pathLast - match.pathFirst)); + auto it = includeCache.find(path); + if (it == includeCache.end()) + { + CStr includeContent; + if (!includeCallback(path, includeContent)) + { + LOGERROR("Preprocessor error: line %zu: Can't load #include file: '%s'", line, path.c_str()); + return false; + } + it = includeCache.emplace(path, std::move(includeContent)).first; + } + // We need to insert #line directives to have correct line numbers in errors. + chunks.emplace_back(lineDirective + "1\n" + it->second + "\n" + lineDirective + CStr::FromUInt(line + 1) + "\n"); + processedParts.emplace_back(currentPart.substr(0, lineStart)); + ResolveIncludesImpl(chunks.back(), includeCache, includeCallback, chunks, processedParts); + currentPart = currentPart.substr(match.nextLineStart); + lineStart = 0; + } + if (!currentPart.empty()) + processedParts.emplace_back(currentPart); + return true; +} + } // anonymous namespace void CPreprocessorWrapper::PyrogenesisShaderError(int iLine, const char* iError, const Ogre::CPreprocessor::Token* iToken) @@ -167,53 +224,24 @@ } -CStr CPreprocessorWrapper::ResolveIncludes(CStr source) +CStr CPreprocessorWrapper::ResolveIncludes(const CStr& source) { - const CStr lineDirective = "#line "; - for (size_t lineStart = 0, line = 1; lineStart < source.size(); ++line) - { - MatchIncludeResult match = MatchIncludeUntilEOLorEOS(source, lineStart); - if (match.error) - return {}; - else if (!match.found) - { - if (lineStart + lineDirective.size() < source.size() && - source.substr(lineStart, lineDirective.size()) == lineDirective) - { - size_t newLineNumber = 0; - size_t pos = lineStart + lineDirective.size(); - while (pos < match.nextLineStart && std::isdigit(source[pos])) - { - newLineNumber = newLineNumber * 10 + (source[pos] - '0'); - ++pos; - } - if (newLineNumber > 0) - line = newLineNumber - 1; - } - - lineStart = match.nextLineStart; - continue; - } - const CStr path = source.substr(match.pathFirst, match.pathLast - match.pathFirst); - auto it = m_IncludeCache.find(path); - if (it == m_IncludeCache.end()) - { - CStr includeContent; - if (!m_IncludeCallback(path, includeContent)) - { - LOGERROR("Preprocessor error: line %zu: Can't load #include file: '%s'", line, path.c_str()); - return {}; - } - it = m_IncludeCache.emplace(path, std::move(includeContent)).first; - } - // We need to insert #line directives to have correct line numbers in errors. - source = - source.substr(0, lineStart) + - lineDirective + "1\n" + it->second + "\n" + lineDirective + CStr::FromUInt(line + 1) + "\n" + - source.substr(match.nextLineStart); - --line; - } - return source; + // Stores intermediate blocks of text to avoid additional copying. Should + // be constructed before views and destroyed after (currently guaranteed + // by stack). + std::vector chunks; + // After resolving the following vector should contain a complete list + // to concatenate. + std::vector processedParts; + ResolveIncludesImpl(source, m_IncludeCache, m_IncludeCallback, chunks, processedParts); + std::size_t totalSize = 0; + for (const std::string_view& part : processedParts) + totalSize += part.size(); + std::string processedSource; + processedSource.reserve(totalSize); + for (const std::string_view& part : processedParts) + processedSource.append(part); + return processedSource; } CStr CPreprocessorWrapper::Preprocess(const CStr& input) Index: ps/trunk/source/third_party/ogre3d_preprocessor/tests/test_Preprocessor.h =================================================================== --- ps/trunk/source/third_party/ogre3d_preprocessor/tests/test_Preprocessor.h +++ ps/trunk/source/third_party/ogre3d_preprocessor/tests/test_Preprocessor.h @@ -273,7 +273,7 @@ if (depth < 4) { std::stringstream nextIncludes; - for (int idx = 0; idx < 8; ++idx) + for (int idx = 0; idx < 16; ++idx) nextIncludes << "#include \"" << depth + 1 << ".h\"\n"; out = nextIncludes.str(); } @@ -292,6 +292,6 @@ )", includeCallback); const double finish = timer_Time(); printf("Total: %lfs\n", finish - start); - TS_ASSERT_EQUALS(result.output.Trim(PS_TRIM_BOTH).size(), 132824u); + TS_ASSERT_EQUALS(result.output.Trim(PS_TRIM_BOTH).size(), 2075304u); } };