Index: ps/trunk/source/graphics/TextureConverter.cpp =================================================================== --- ps/trunk/source/graphics/TextureConverter.cpp (revision 25765) +++ ps/trunk/source/graphics/TextureConverter.cpp (revision 25766) @@ -1,599 +1,603 @@ /* Copyright (C) 2021 Wildfire Games. * This file is part of 0 A.D. * * 0 A.D. is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 2 of the License, or * (at your option) any later version. * * 0 A.D. is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with 0 A.D. If not, see . */ #include "precompiled.h" #include "TextureConverter.h" #include "lib/regex.h" #include "lib/timer.h" #include "lib/allocators/shared_ptr.h" #include "lib/tex/tex.h" #include "maths/MD5.h" #include "ps/CLogger.h" #include "ps/CStr.h" #include "ps/Profiler2.h" #include "ps/Threading.h" #include "ps/XML/Xeromyces.h" #if CONFIG2_NVTT #include "nvtt/nvtt.h" // We assume NVTT is recent enough to support the alpha flag in the DXT1a format. If users try to use an // old version of NVTT, the game will crash when trying to decode dds files generated by NVTT. // // The support was added upstream in https://github.com/castano/nvidia-texture-tools/commit/782a127071895f538c1ae49925a6e15687e3c966 // so, in theory, 2.0.7 and newer should be enough, but all 2.0.x releases define NVTT_VERSION as 200, so // we can't distinguish them. NVTT_VERSION is 201 in all development versions of the 2.0.x era, so we also // have to exclude that value. #if !defined NVTT_VERSION || NVTT_VERSION == 200 || NVTT_VERSION == 201 #error Please use NVTT 2.1.0 or newer. \ If your system does not provide it, you should use the bundled version by NOT passing --with-system-nvtt to premake. #endif /** * Output handler to collect NVTT's output into a simplistic buffer. * WARNING: Used in the worker thread - must be thread-safe. */ struct BufferOutputHandler : public nvtt::OutputHandler { std::vector buffer; virtual void beginImage(int UNUSED(size), int UNUSED(width), int UNUSED(height), int UNUSED(depth), int UNUSED(face), int UNUSED(miplevel)) { } virtual bool writeData(const void* data, int size) { size_t off = buffer.size(); buffer.resize(off + size); memcpy(&buffer[off], data, size); return true; } virtual void endImage() { } }; /** * Request for worker thread to process. */ struct CTextureConverter::ConversionRequest { VfsPath dest; CTexturePtr texture; nvtt::InputOptions inputOptions; nvtt::CompressionOptions compressionOptions; nvtt::OutputOptions outputOptions; }; /** * Result from worker thread. */ struct CTextureConverter::ConversionResult { VfsPath dest; CTexturePtr texture; BufferOutputHandler output; bool ret; // true if the conversion succeeded }; #endif // CONFIG2_NVTT void CTextureConverter::Settings::Hash(MD5& hash) { hash.Update((const u8*)&format, sizeof(format)); hash.Update((const u8*)&mipmap, sizeof(mipmap)); hash.Update((const u8*)&normal, sizeof(normal)); hash.Update((const u8*)&alpha, sizeof(alpha)); hash.Update((const u8*)&filter, sizeof(filter)); hash.Update((const u8*)&kaiserWidth, sizeof(kaiserWidth)); hash.Update((const u8*)&kaiserAlpha, sizeof(kaiserAlpha)); hash.Update((const u8*)&kaiserStretch, sizeof(kaiserStretch)); } CTextureConverter::SettingsFile* CTextureConverter::LoadSettings(const VfsPath& path) const { CXeromyces XeroFile; if (XeroFile.Load(m_VFS, path, "texture") != PSRETURN_OK) return NULL; // Define all the elements used in the XML file #define EL(x) int el_##x = XeroFile.GetElementID(#x) #define AT(x) int at_##x = XeroFile.GetAttributeID(#x) EL(textures); EL(file); AT(pattern); AT(format); AT(mipmap); AT(normal); AT(alpha); AT(filter); AT(kaiserwidth); AT(kaiseralpha); AT(kaiserstretch); #undef AT #undef EL XMBElement root = XeroFile.GetRoot(); if (root.GetNodeName() != el_textures) { LOGERROR("Invalid texture settings file \"%s\" (unrecognised root element)", path.string8()); return NULL; } std::unique_ptr settings = std::make_unique(); XERO_ITER_EL(root, child) { if (child.GetNodeName() == el_file) { Match p; XERO_ITER_ATTR(child, attr) { if (attr.Name == at_pattern) { p.pattern = attr.Value.FromUTF8(); } else if (attr.Name == at_format) { CStr v(attr.Value); if (v == "dxt1") p.settings.format = FMT_DXT1; else if (v == "dxt3") p.settings.format = FMT_DXT3; else if (v == "dxt5") p.settings.format = FMT_DXT5; else if (v == "rgba") p.settings.format = FMT_RGBA; else if (v == "alpha") p.settings.format = FMT_ALPHA; else LOGERROR("Invalid attribute value ", v.c_str()); } else if (attr.Name == at_mipmap) { CStr v(attr.Value); if (v == "true") p.settings.mipmap = MIP_TRUE; else if (v == "false") p.settings.mipmap = MIP_FALSE; else LOGERROR("Invalid attribute value ", v.c_str()); } else if (attr.Name == at_normal) { CStr v(attr.Value); if (v == "true") p.settings.normal = NORMAL_TRUE; else if (v == "false") p.settings.normal = NORMAL_FALSE; else LOGERROR("Invalid attribute value ", v.c_str()); } else if (attr.Name == at_alpha) { CStr v(attr.Value); if (v == "none") p.settings.alpha = ALPHA_NONE; else if (v == "player") p.settings.alpha = ALPHA_PLAYER; else if (v == "transparency") p.settings.alpha = ALPHA_TRANSPARENCY; else LOGERROR("Invalid attribute value ", v.c_str()); } else if (attr.Name == at_filter) { CStr v(attr.Value); if (v == "box") p.settings.filter = FILTER_BOX; else if (v == "triangle") p.settings.filter = FILTER_TRIANGLE; else if (v == "kaiser") p.settings.filter = FILTER_KAISER; else LOGERROR("Invalid attribute value ", v.c_str()); } else if (attr.Name == at_kaiserwidth) { p.settings.kaiserWidth = CStr(attr.Value).ToFloat(); } else if (attr.Name == at_kaiseralpha) { p.settings.kaiserAlpha = CStr(attr.Value).ToFloat(); } else if (attr.Name == at_kaiserstretch) { p.settings.kaiserStretch = CStr(attr.Value).ToFloat(); } else { LOGERROR("Invalid attribute name ", XeroFile.GetAttributeString(attr.Name)); } } settings->patterns.push_back(p); } } return settings.release(); } CTextureConverter::Settings CTextureConverter::ComputeSettings(const std::wstring& filename, const std::vector& settingsFiles) const { // Set sensible defaults Settings settings; settings.format = FMT_DXT1; settings.mipmap = MIP_TRUE; settings.normal = NORMAL_FALSE; settings.alpha = ALPHA_NONE; settings.filter = FILTER_BOX; settings.kaiserWidth = 3.f; settings.kaiserAlpha = 4.f; settings.kaiserStretch = 1.f; for (size_t i = 0; i < settingsFiles.size(); ++i) { for (size_t j = 0; j < settingsFiles[i]->patterns.size(); ++j) { Match p = settingsFiles[i]->patterns[j]; // Check that the pattern matches the texture file if (!match_wildcard(filename.c_str(), p.pattern.c_str())) continue; if (p.settings.format != FMT_UNSPECIFIED) settings.format = p.settings.format; if (p.settings.mipmap != MIP_UNSPECIFIED) settings.mipmap = p.settings.mipmap; if (p.settings.normal != NORMAL_UNSPECIFIED) settings.normal = p.settings.normal; if (p.settings.alpha != ALPHA_UNSPECIFIED) settings.alpha = p.settings.alpha; if (p.settings.filter != FILTER_UNSPECIFIED) settings.filter = p.settings.filter; if (p.settings.kaiserWidth != -1.f) settings.kaiserWidth = p.settings.kaiserWidth; if (p.settings.kaiserAlpha != -1.f) settings.kaiserAlpha = p.settings.kaiserAlpha; if (p.settings.kaiserStretch != -1.f) settings.kaiserStretch = p.settings.kaiserStretch; } } return settings; } CTextureConverter::CTextureConverter(PIVFS vfs, bool highQuality) : m_VFS(vfs), m_HighQuality(highQuality), m_Shutdown(false) { +#if CONFIG2_NVTT // Verify that we are running with at least the version we were compiled with, // to avoid bugs caused by ABI changes -#if CONFIG2_NVTT ENSURE(nvtt::version() >= NVTT_VERSION); -#endif m_WorkerThread = std::thread(Threading::HandleExceptions::Wrapper, this); // Maybe we should share some centralised pool of worker threads? // For now we'll just stick with a single thread for this specific use. +#endif // CONFIG2_NVTT } CTextureConverter::~CTextureConverter() { +#if CONFIG2_NVTT // Tell the thread to shut down { std::lock_guard lock(m_WorkerMutex); m_Shutdown = true; } while (true) { // Wake the thread up so that it shutdowns. // If we send the message once, there is a chance it will be missed, // so keep sending until shtudown becomes false again, indicating that the thread has shut down. std::lock_guard lock(m_WorkerMutex); m_WorkerCV.notify_all(); if (!m_Shutdown) break; } // Wait for it to shut down cleanly m_WorkerThread.join(); +#endif // CONFIG2_NVTT } bool CTextureConverter::ConvertTexture(const CTexturePtr& texture, const VfsPath& src, const VfsPath& dest, const Settings& settings) { std::shared_ptr file; size_t fileSize; if (m_VFS->LoadFile(src, file, fileSize) < 0) { LOGERROR("Failed to load texture \"%s\"", src.string8()); return false; } Tex tex; if (tex.decode(file, fileSize) < 0) { LOGERROR("Failed to decode texture \"%s\"", src.string8()); return false; } // Check whether there's any alpha channel bool hasAlpha = ((tex.m_Flags & TEX_ALPHA) != 0); if (settings.format == FMT_ALPHA) { // Convert to uncompressed 8-bit with no mipmaps if (tex.transform_to((tex.m_Flags | TEX_GREY) & ~(TEX_DXT | TEX_MIPMAPS | TEX_ALPHA)) < 0) { LOGERROR("Failed to transform texture \"%s\"", src.string8()); return false; } } else { // Convert to uncompressed BGRA with no mipmaps if (tex.transform_to((tex.m_Flags | TEX_BGR | TEX_ALPHA) & ~(TEX_DXT | TEX_MIPMAPS)) < 0) { LOGERROR("Failed to transform texture \"%s\"", src.string8()); return false; } } // Check if the texture has all alpha=255, so we can automatically // switch from DXT3/DXT5 to DXT1 with no loss if (hasAlpha) { hasAlpha = false; u8* data = tex.get_data(); for (size_t i = 0; i < tex.m_Width * tex.m_Height; ++i) { if (data[i*4+3] != 0xFF) { hasAlpha = true; break; } } } #if CONFIG2_NVTT std::shared_ptr request = std::make_shared(); request->dest = dest; request->texture = texture; // Apply the chosen settings: request->inputOptions.setMipmapGeneration(settings.mipmap == MIP_TRUE); if (settings.alpha == ALPHA_TRANSPARENCY) request->inputOptions.setAlphaMode(nvtt::AlphaMode_Transparency); else request->inputOptions.setAlphaMode(nvtt::AlphaMode_None); if (settings.format == FMT_RGBA) { request->compressionOptions.setFormat(nvtt::Format_RGBA); // Change the default component order (see tex_dds.cpp decode_pf) request->compressionOptions.setPixelFormat(32, 0xFF, 0xFF00, 0xFF0000, 0xFF000000u); } else if (settings.format == FMT_ALPHA) { request->compressionOptions.setFormat(nvtt::Format_RGBA); request->compressionOptions.setPixelFormat(8, 0x00, 0x00, 0x00, 0xFF); } else if (!hasAlpha) { // if no alpha channel then there's no point using DXT3 or DXT5 request->compressionOptions.setFormat(nvtt::Format_DXT1); } else if (settings.format == FMT_DXT1) { request->compressionOptions.setFormat(nvtt::Format_DXT1a); } else if (settings.format == FMT_DXT3) { request->compressionOptions.setFormat(nvtt::Format_DXT3); } else if (settings.format == FMT_DXT5) { request->compressionOptions.setFormat(nvtt::Format_DXT5); } if (settings.filter == FILTER_BOX) request->inputOptions.setMipmapFilter(nvtt::MipmapFilter_Box); else if (settings.filter == FILTER_TRIANGLE) request->inputOptions.setMipmapFilter(nvtt::MipmapFilter_Triangle); else if (settings.filter == FILTER_KAISER) request->inputOptions.setMipmapFilter(nvtt::MipmapFilter_Kaiser); if (settings.normal == NORMAL_TRUE) request->inputOptions.setNormalMap(true); request->inputOptions.setKaiserParameters(settings.kaiserWidth, settings.kaiserAlpha, settings.kaiserStretch); request->inputOptions.setWrapMode(nvtt::WrapMode_Mirror); // TODO: should this be configurable? request->compressionOptions.setQuality(m_HighQuality ? nvtt::Quality_Production : nvtt::Quality_Fastest); // TODO: normal maps, gamma, etc // Load the texture data request->inputOptions.setTextureLayout(nvtt::TextureType_2D, tex.m_Width, tex.m_Height); if (tex.m_Bpp == 32) { request->inputOptions.setMipmapData(tex.get_data(), tex.m_Width, tex.m_Height); } else // bpp == 8 { // NVTT requires 32-bit input data, so convert const u8* input = tex.get_data(); u8* rgba = new u8[tex.m_Width * tex.m_Height * 4]; u8* p = rgba; for (size_t i = 0; i < tex.m_Width * tex.m_Height; i++) { p[0] = p[1] = p[2] = p[3] = *input++; p += 4; } request->inputOptions.setMipmapData(rgba, tex.m_Width, tex.m_Height); delete[] rgba; } { std::lock_guard lock(m_WorkerMutex); m_RequestQueue.push_back(request); } // Wake up the worker thread m_WorkerCV.notify_all(); return true; -#else +#else // CONFIG2_NVTT LOGERROR("Failed to convert texture \"%s\" (NVTT not available)", src.string8()); return false; -#endif +#endif // !CONFIG2_NVTT } bool CTextureConverter::Poll(CTexturePtr& texture, VfsPath& dest, bool& ok) { #if CONFIG2_NVTT std::shared_ptr result; // Grab the first result (if any) { std::lock_guard lock(m_WorkerMutex); if (!m_ResultQueue.empty()) { result = m_ResultQueue.front(); m_ResultQueue.pop_front(); } } if (!result) { // no work to do return false; } if (!result->ret) { // conversion had failed ok = false; return true; } // Move output into a correctly-aligned buffer size_t size = result->output.buffer.size(); std::shared_ptr file; AllocateAligned(file, size, maxSectorSize); memcpy(file.get(), &result->output.buffer[0], size); if (m_VFS->CreateFile(result->dest, file, size) < 0) { // error writing file ok = false; return true; } // Succeeded in converting texture texture = result->texture; dest = result->dest; ok = true; return true; -#else // #if CONFIG2_NVTT +#else // CONFIG2_NVTT return false; -#endif +#endif // !CONFIG2_NVTT } bool CTextureConverter::IsBusy() { +#if CONFIG2_NVTT std::lock_guard lock(m_WorkerMutex); return !m_RequestQueue.empty(); +#else // CONFIG2_NVTT + return false; +#endif // !CONFIG2_NVTT } void CTextureConverter::RunThread(CTextureConverter* textureConverter) { +#if CONFIG2_NVTT debug_SetThreadName("TextureConverter"); g_Profiler2.RegisterCurrentThread("texconv"); - -#if CONFIG2_NVTT - // Wait until the main thread wakes us up while (true) { // We may have several textures in the incoming queue, process them all before going back to sleep. if (!textureConverter->IsBusy()) { std::unique_lock wait_lock(textureConverter->m_WorkerMutex); // Use the no-condition variant because spurious wake-ups don't matter that much here. textureConverter->m_WorkerCV.wait(wait_lock); } g_Profiler2.RecordSyncMarker(); PROFILE2_EVENT("wakeup"); std::shared_ptr request; { std::lock_guard wait_lock(textureConverter->m_WorkerMutex); if (textureConverter->m_Shutdown) break; // If we weren't woken up for shutdown, we must have been woken up for // a new request, so grab it from the queue request = textureConverter->m_RequestQueue.front(); textureConverter->m_RequestQueue.pop_front(); } // Set up the result object std::shared_ptr result = std::make_shared(); result->dest = request->dest; result->texture = request->texture; request->outputOptions.setOutputHandler(&result->output); // TIMER(L"TextureConverter compress"); { PROFILE2("compress"); // Perform the compression nvtt::Compressor compressor; result->ret = compressor.process(request->inputOptions, request->compressionOptions, request->outputOptions); } // Push the result onto the queue std::lock_guard wait_lock(textureConverter->m_WorkerMutex); textureConverter->m_ResultQueue.push_back(result); } std::lock_guard wait_lock(textureConverter->m_WorkerMutex); textureConverter->m_Shutdown = false; -#endif +#endif // CONFIG2_NVTT } Index: ps/trunk/source/graphics/TextureConverter.h =================================================================== --- ps/trunk/source/graphics/TextureConverter.h (revision 25765) +++ ps/trunk/source/graphics/TextureConverter.h (revision 25766) @@ -1,222 +1,224 @@ /* Copyright (C) 2021 Wildfire Games. * This file is part of 0 A.D. * * 0 A.D. is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 2 of the License, or * (at your option) any later version. * * 0 A.D. is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with 0 A.D. If not, see . */ #ifndef INCLUDED_TEXTURECONVERTER #define INCLUDED_TEXTURECONVERTER #include "lib/file/vfs/vfs.h" #include "TextureManager.h" #include #include #include #include class MD5; /** * Texture conversion helper class. * Provides an asynchronous API to convert input image files into compressed DDS, * given various conversion settings. * (The (potentially very slow) compression is a performed in a background thread, * so the game can remain responsive). * Also provides an API to load conversion settings from XML files. * * XML files are of the form: * @code * * * * * @endcode * * 'pattern' is a wildcard expression, matching on filenames. * All other attributes are optional. Later elements override attributes from * earlier elements. * * 'format' is 'dxt1', 'dxt3', 'dxt5' or 'rgba'. * * 'mipmap' is 'true' or 'false'. * * 'normal' is 'true' or 'false'. * * 'alpha' is 'transparency' or 'player' (it determines whether the color value of * 0-alpha pixels is significant or not). * * 'filter' is 'box', 'triangle' or 'kaiser'. * * 'kaiserwidth', 'kaiseralpha', 'kaiserstretch' are floats * (see http://code.google.com/p/nvidia-texture-tools/wiki/ApiDocumentation#Mipmap_Generation) */ class CTextureConverter { public: enum EFormat { FMT_UNSPECIFIED, FMT_DXT1, FMT_DXT3, FMT_DXT5, FMT_RGBA, FMT_ALPHA }; enum EMipmap { MIP_UNSPECIFIED, MIP_TRUE, MIP_FALSE }; enum ENormalMap { NORMAL_UNSPECIFIED, NORMAL_TRUE, NORMAL_FALSE }; enum EAlpha { ALPHA_UNSPECIFIED, ALPHA_NONE, ALPHA_PLAYER, ALPHA_TRANSPARENCY }; enum EFilter { FILTER_UNSPECIFIED, FILTER_BOX, FILTER_TRIANGLE, FILTER_KAISER }; /** * Texture conversion settings. */ struct Settings { Settings() : format(FMT_UNSPECIFIED), mipmap(MIP_UNSPECIFIED), normal(NORMAL_UNSPECIFIED), alpha(ALPHA_UNSPECIFIED), filter(FILTER_UNSPECIFIED), kaiserWidth(-1.f), kaiserAlpha(-1.f), kaiserStretch(-1.f) { } /** * Append this object's state to the given hash. */ void Hash(MD5& hash); EFormat format; EMipmap mipmap; ENormalMap normal; EAlpha alpha; EFilter filter; float kaiserWidth; float kaiserAlpha; float kaiserStretch; }; /** * Representation of \ line from settings XML file. */ struct Match { std::wstring pattern; Settings settings; }; /** * Representation of settings XML file. */ struct SettingsFile { std::vector patterns; }; /** * Construct texture converter, for use with files in the given vfs. */ CTextureConverter(PIVFS vfs, bool highQuality); /** * Destroy texture converter and wait to shut down worker thread. * This might take a long time (maybe seconds) if the worker is busy * processing a texture. */ ~CTextureConverter(); /** * Load a texture conversion settings XML file. * Returns NULL on failure. */ SettingsFile* LoadSettings(const VfsPath& path) const; /** * Match a sequence of settings files against a given texture filename, * and return the resulting settings. * Later entries in settingsFiles override earlier entries. */ Settings ComputeSettings(const std::wstring& filename, const std::vector& settingsFiles) const; /** * Begin converting a texture, using the given settings. * This will load src and return false on failure. * Otherwise it will return true and start an asynchronous conversion request, * whose result will be returned from Poll() (with the texture and dest passed * into this function). */ bool ConvertTexture(const CTexturePtr& texture, const VfsPath& src, const VfsPath& dest, const Settings& settings); /** * Returns the result of a successful ConvertTexture call. * If no result is available yet, returns false. * Otherwise, if the conversion succeeded, it sets ok to true and sets * texture and dest to the corresponding values passed into ConvertTexture(), * then returns true. * If the conversion failed, it sets ok to false and doesn't touch the other arguments, * then returns true. */ bool Poll(CTexturePtr& texture, VfsPath& dest, bool& ok); /** * Returns whether there is currently a queued request from ConvertTexture(). * (Note this may return false while the worker thread is still converting the last texture.) */ bool IsBusy(); private: static void RunThread(CTextureConverter* data); PIVFS m_VFS; bool m_HighQuality; +#if CONFIG2_NVTT std::thread m_WorkerThread; std::mutex m_WorkerMutex; std::condition_variable m_WorkerCV; +#endif // CONFIG2_NVTT struct ConversionRequest; struct ConversionResult; std::deque> m_RequestQueue; // protected by m_WorkerMutex std::deque> m_ResultQueue; // protected by m_WorkerMutex bool m_Shutdown; // protected by m_WorkerMutex }; #endif // INCLUDED_TEXTURECONVERTER