Index: ps/trunk/source/graphics/TextureConverter.cpp
===================================================================
--- ps/trunk/source/graphics/TextureConverter.cpp (revision 22771)
+++ ps/trunk/source/graphics/TextureConverter.cpp (revision 22772)
@@ -1,596 +1,602 @@
/* Copyright (C) 2019 Wildfire Games.
* This file is part of 0 A.D.
*
* 0 A.D. is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 2 of the License, or
* (at your option) any later version.
*
* 0 A.D. is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with 0 A.D. If not, see .
*/
#include "precompiled.h"
#include "TextureConverter.h"
#include "lib/regex.h"
#include "lib/timer.h"
#include "lib/allocators/shared_ptr.h"
#include "lib/tex/tex.h"
#include "maths/MD5.h"
#include "ps/CLogger.h"
#include "ps/CStr.h"
#include "ps/Profiler2.h"
#include "ps/XML/Xeromyces.h"
#if CONFIG2_NVTT
#include "nvtt/nvtt.h"
/**
* Output handler to collect NVTT's output into a simplistic buffer.
* WARNING: Used in the worker thread - must be thread-safe.
*/
struct BufferOutputHandler : public nvtt::OutputHandler
{
std::vector buffer;
virtual void beginImage(int UNUSED(size), int UNUSED(width), int UNUSED(height), int UNUSED(depth), int UNUSED(face), int UNUSED(miplevel))
{
}
virtual bool writeData(const void* data, int size)
{
size_t off = buffer.size();
buffer.resize(off + size);
memcpy(&buffer[off], data, size);
return true;
}
};
/**
* Request for worker thread to process.
*/
struct CTextureConverter::ConversionRequest
{
VfsPath dest;
CTexturePtr texture;
nvtt::InputOptions inputOptions;
nvtt::CompressionOptions compressionOptions;
nvtt::OutputOptions outputOptions;
bool isDXT1a; // see comment in RunThread
bool is8bpp;
};
/**
* Result from worker thread.
*/
struct CTextureConverter::ConversionResult
{
VfsPath dest;
CTexturePtr texture;
BufferOutputHandler output;
bool ret; // true if the conversion succeeded
};
#endif // CONFIG2_NVTT
void CTextureConverter::Settings::Hash(MD5& hash)
{
hash.Update((const u8*)&format, sizeof(format));
hash.Update((const u8*)&mipmap, sizeof(mipmap));
hash.Update((const u8*)&normal, sizeof(normal));
hash.Update((const u8*)&alpha, sizeof(alpha));
hash.Update((const u8*)&filter, sizeof(filter));
hash.Update((const u8*)&kaiserWidth, sizeof(kaiserWidth));
hash.Update((const u8*)&kaiserAlpha, sizeof(kaiserAlpha));
hash.Update((const u8*)&kaiserStretch, sizeof(kaiserStretch));
}
CTextureConverter::SettingsFile* CTextureConverter::LoadSettings(const VfsPath& path) const
{
CXeromyces XeroFile;
if (XeroFile.Load(m_VFS, path, "texture") != PSRETURN_OK)
return NULL;
// Define all the elements used in the XML file
#define EL(x) int el_##x = XeroFile.GetElementID(#x)
#define AT(x) int at_##x = XeroFile.GetAttributeID(#x)
EL(textures);
EL(file);
AT(pattern);
AT(format);
AT(mipmap);
AT(normal);
AT(alpha);
AT(filter);
AT(kaiserwidth);
AT(kaiseralpha);
AT(kaiserstretch);
#undef AT
#undef EL
XMBElement root = XeroFile.GetRoot();
if (root.GetNodeName() != el_textures)
{
LOGERROR("Invalid texture settings file \"%s\" (unrecognised root element)", path.string8());
return NULL;
}
std::unique_ptr settings(new SettingsFile());
XERO_ITER_EL(root, child)
{
if (child.GetNodeName() == el_file)
{
Match p;
XERO_ITER_ATTR(child, attr)
{
if (attr.Name == at_pattern)
{
p.pattern = attr.Value.FromUTF8();
}
else if (attr.Name == at_format)
{
CStr v(attr.Value);
if (v == "dxt1")
p.settings.format = FMT_DXT1;
else if (v == "dxt3")
p.settings.format = FMT_DXT3;
else if (v == "dxt5")
p.settings.format = FMT_DXT5;
else if (v == "rgba")
p.settings.format = FMT_RGBA;
else if (v == "alpha")
p.settings.format = FMT_ALPHA;
else
LOGERROR("Invalid attribute value ", v.c_str());
}
else if (attr.Name == at_mipmap)
{
CStr v(attr.Value);
if (v == "true")
p.settings.mipmap = MIP_TRUE;
else if (v == "false")
p.settings.mipmap = MIP_FALSE;
else
LOGERROR("Invalid attribute value ", v.c_str());
}
else if (attr.Name == at_normal)
{
CStr v(attr.Value);
if (v == "true")
p.settings.normal = NORMAL_TRUE;
else if (v == "false")
p.settings.normal = NORMAL_FALSE;
else
LOGERROR("Invalid attribute value ", v.c_str());
}
else if (attr.Name == at_alpha)
{
CStr v(attr.Value);
if (v == "none")
p.settings.alpha = ALPHA_NONE;
else if (v == "player")
p.settings.alpha = ALPHA_PLAYER;
else if (v == "transparency")
p.settings.alpha = ALPHA_TRANSPARENCY;
else
LOGERROR("Invalid attribute value ", v.c_str());
}
else if (attr.Name == at_filter)
{
CStr v(attr.Value);
if (v == "box")
p.settings.filter = FILTER_BOX;
else if (v == "triangle")
p.settings.filter = FILTER_TRIANGLE;
else if (v == "kaiser")
p.settings.filter = FILTER_KAISER;
else
LOGERROR("Invalid attribute value ", v.c_str());
}
else if (attr.Name == at_kaiserwidth)
{
p.settings.kaiserWidth = CStr(attr.Value).ToFloat();
}
else if (attr.Name == at_kaiseralpha)
{
p.settings.kaiserAlpha = CStr(attr.Value).ToFloat();
}
else if (attr.Name == at_kaiserstretch)
{
p.settings.kaiserStretch = CStr(attr.Value).ToFloat();
}
else
{
LOGERROR("Invalid attribute name ", XeroFile.GetAttributeString(attr.Name).c_str());
}
}
settings->patterns.push_back(p);
}
}
return settings.release();
}
CTextureConverter::Settings CTextureConverter::ComputeSettings(const std::wstring& filename, const std::vector& settingsFiles) const
{
// Set sensible defaults
Settings settings;
settings.format = FMT_DXT1;
settings.mipmap = MIP_TRUE;
settings.normal = NORMAL_FALSE;
settings.alpha = ALPHA_NONE;
settings.filter = FILTER_BOX;
settings.kaiserWidth = 3.f;
settings.kaiserAlpha = 4.f;
settings.kaiserStretch = 1.f;
for (size_t i = 0; i < settingsFiles.size(); ++i)
{
for (size_t j = 0; j < settingsFiles[i]->patterns.size(); ++j)
{
Match p = settingsFiles[i]->patterns[j];
// Check that the pattern matches the texture file
if (!match_wildcard(filename.c_str(), p.pattern.c_str()))
continue;
if (p.settings.format != FMT_UNSPECIFIED)
settings.format = p.settings.format;
if (p.settings.mipmap != MIP_UNSPECIFIED)
settings.mipmap = p.settings.mipmap;
if (p.settings.normal != NORMAL_UNSPECIFIED)
settings.normal = p.settings.normal;
if (p.settings.alpha != ALPHA_UNSPECIFIED)
settings.alpha = p.settings.alpha;
if (p.settings.filter != FILTER_UNSPECIFIED)
settings.filter = p.settings.filter;
if (p.settings.kaiserWidth != -1.f)
settings.kaiserWidth = p.settings.kaiserWidth;
if (p.settings.kaiserAlpha != -1.f)
settings.kaiserAlpha = p.settings.kaiserAlpha;
if (p.settings.kaiserStretch != -1.f)
settings.kaiserStretch = p.settings.kaiserStretch;
}
}
return settings;
}
CTextureConverter::CTextureConverter(PIVFS vfs, bool highQuality) :
m_VFS(vfs), m_HighQuality(highQuality), m_Shutdown(false)
{
// Verify that we are running with at least the version we were compiled with,
// to avoid bugs caused by ABI changes
#if CONFIG2_NVTT
ENSURE(nvtt::version() >= NVTT_VERSION);
#endif
- // Set up the worker thread:
-
- // Use SDL semaphores since OS X doesn't implement sem_init
- m_WorkerSem = SDL_CreateSemaphore(0);
- ENSURE(m_WorkerSem);
-
m_WorkerThread = std::thread(RunThread, this);
// Maybe we should share some centralised pool of worker threads?
// For now we'll just stick with a single thread for this specific use.
}
CTextureConverter::~CTextureConverter()
{
// Tell the thread to shut down
{
std::lock_guard lock(m_WorkerMutex);
m_Shutdown = true;
}
- // Wake it up so it sees the notification
- SDL_SemPost(m_WorkerSem);
+ while (true)
+ {
+ // Wake the thread up so that it shutdowns.
+ // If we send the message once, there is a chance it will be missed,
+ // so keep sending until shtudown becomes false again, indicating that the thread has shut down.
+ std::lock_guard lock(m_WorkerMutex);
+ m_WorkerCV.notify_all();
+ if (!m_Shutdown)
+ break;
+ }
// Wait for it to shut down cleanly
m_WorkerThread.join();
-
- // Clean up resources
- SDL_DestroySemaphore(m_WorkerSem);
}
bool CTextureConverter::ConvertTexture(const CTexturePtr& texture, const VfsPath& src, const VfsPath& dest, const Settings& settings)
{
shared_ptr file;
size_t fileSize;
if (m_VFS->LoadFile(src, file, fileSize) < 0)
{
LOGERROR("Failed to load texture \"%s\"", src.string8());
return false;
}
Tex tex;
if (tex.decode(file, fileSize) < 0)
{
LOGERROR("Failed to decode texture \"%s\"", src.string8());
return false;
}
// Check whether there's any alpha channel
bool hasAlpha = ((tex.m_Flags & TEX_ALPHA) != 0);
if (settings.format == FMT_ALPHA)
{
// Convert to uncompressed 8-bit with no mipmaps
if (tex.transform_to((tex.m_Flags | TEX_GREY) & ~(TEX_DXT | TEX_MIPMAPS | TEX_ALPHA)) < 0)
{
LOGERROR("Failed to transform texture \"%s\"", src.string8());
return false;
}
}
else
{
// Convert to uncompressed BGRA with no mipmaps
if (tex.transform_to((tex.m_Flags | TEX_BGR | TEX_ALPHA) & ~(TEX_DXT | TEX_MIPMAPS)) < 0)
{
LOGERROR("Failed to transform texture \"%s\"", src.string8());
return false;
}
}
// Check if the texture has all alpha=255, so we can automatically
// switch from DXT3/DXT5 to DXT1 with no loss
if (hasAlpha)
{
hasAlpha = false;
u8* data = tex.get_data();
for (size_t i = 0; i < tex.m_Width * tex.m_Height; ++i)
{
if (data[i*4+3] != 0xFF)
{
hasAlpha = true;
break;
}
}
}
#if CONFIG2_NVTT
shared_ptr request(new ConversionRequest);
request->dest = dest;
request->texture = texture;
// Apply the chosen settings:
request->inputOptions.setMipmapGeneration(settings.mipmap == MIP_TRUE);
if (settings.alpha == ALPHA_TRANSPARENCY)
request->inputOptions.setAlphaMode(nvtt::AlphaMode_Transparency);
else
request->inputOptions.setAlphaMode(nvtt::AlphaMode_None);
request->isDXT1a = false;
request->is8bpp = false;
if (settings.format == FMT_RGBA)
{
request->compressionOptions.setFormat(nvtt::Format_RGBA);
// Change the default component order (see tex_dds.cpp decode_pf)
request->compressionOptions.setPixelFormat(32, 0xFF, 0xFF00, 0xFF0000, 0xFF000000u);
}
else if (settings.format == FMT_ALPHA)
{
request->compressionOptions.setFormat(nvtt::Format_RGBA);
request->compressionOptions.setPixelFormat(8, 0x00, 0x00, 0x00, 0xFF);
request->is8bpp = true;
}
else if (!hasAlpha)
{
// if no alpha channel then there's no point using DXT3 or DXT5
request->compressionOptions.setFormat(nvtt::Format_DXT1);
}
else if (settings.format == FMT_DXT1)
{
request->compressionOptions.setFormat(nvtt::Format_DXT1a);
request->isDXT1a = true;
}
else if (settings.format == FMT_DXT3)
{
request->compressionOptions.setFormat(nvtt::Format_DXT3);
}
else if (settings.format == FMT_DXT5)
{
request->compressionOptions.setFormat(nvtt::Format_DXT5);
}
if (settings.filter == FILTER_BOX)
request->inputOptions.setMipmapFilter(nvtt::MipmapFilter_Box);
else if (settings.filter == FILTER_TRIANGLE)
request->inputOptions.setMipmapFilter(nvtt::MipmapFilter_Triangle);
else if (settings.filter == FILTER_KAISER)
request->inputOptions.setMipmapFilter(nvtt::MipmapFilter_Kaiser);
if (settings.normal == NORMAL_TRUE)
request->inputOptions.setNormalMap(true);
request->inputOptions.setKaiserParameters(settings.kaiserWidth, settings.kaiserAlpha, settings.kaiserStretch);
request->inputOptions.setWrapMode(nvtt::WrapMode_Mirror); // TODO: should this be configurable?
request->compressionOptions.setQuality(m_HighQuality ? nvtt::Quality_Production : nvtt::Quality_Fastest);
// TODO: normal maps, gamma, etc
// Load the texture data
request->inputOptions.setTextureLayout(nvtt::TextureType_2D, tex.m_Width, tex.m_Height);
if (tex.m_Bpp == 32)
{
request->inputOptions.setMipmapData(tex.get_data(), tex.m_Width, tex.m_Height);
}
else // bpp == 8
{
// NVTT requires 32-bit input data, so convert
const u8* input = tex.get_data();
u8* rgba = new u8[tex.m_Width * tex.m_Height * 4];
u8* p = rgba;
for (size_t i = 0; i < tex.m_Width * tex.m_Height; i++)
{
p[0] = p[1] = p[2] = p[3] = *input++;
p += 4;
}
request->inputOptions.setMipmapData(rgba, tex.m_Width, tex.m_Height);
delete[] rgba;
}
{
std::lock_guard lock(m_WorkerMutex);
m_RequestQueue.push_back(request);
}
// Wake up the worker thread
- SDL_SemPost(m_WorkerSem);
+ m_WorkerCV.notify_all();
return true;
#else
LOGERROR("Failed to convert texture \"%s\" (NVTT not available)", src.string8());
return false;
#endif
}
bool CTextureConverter::Poll(CTexturePtr& texture, VfsPath& dest, bool& ok)
{
#if CONFIG2_NVTT
shared_ptr result;
// Grab the first result (if any)
{
std::lock_guard lock(m_WorkerMutex);
if (!m_ResultQueue.empty())
{
result = m_ResultQueue.front();
m_ResultQueue.pop_front();
}
}
if (!result)
{
// no work to do
return false;
}
if (!result->ret)
{
// conversion had failed
ok = false;
return true;
}
// Move output into a correctly-aligned buffer
size_t size = result->output.buffer.size();
shared_ptr file;
AllocateAligned(file, size, maxSectorSize);
memcpy(file.get(), &result->output.buffer[0], size);
if (m_VFS->CreateFile(result->dest, file, size) < 0)
{
// error writing file
ok = false;
return true;
}
// Succeeded in converting texture
texture = result->texture;
dest = result->dest;
ok = true;
return true;
#else // #if CONFIG2_NVTT
return false;
#endif
}
bool CTextureConverter::IsBusy()
{
std::lock_guard lock(m_WorkerMutex);
- bool busy = !m_RequestQueue.empty();
-
- return busy;
+ return !m_RequestQueue.empty();
}
void CTextureConverter::RunThread(CTextureConverter* textureConverter)
{
debug_SetThreadName("TextureConverter");
g_Profiler2.RegisterCurrentThread("texconv");
#if CONFIG2_NVTT
// Wait until the main thread wakes us up
- while (SDL_SemWait(textureConverter->m_WorkerSem) == 0)
+ while (true)
{
+ // We may have several textures in the incoming queue, process them all before going back to sleep.
+ if (!textureConverter->IsBusy()) {
+ std::unique_lock wait_lock(textureConverter->m_WorkerMutex);
+ // Use the no-condition variant because spurious wake-ups don't matter that much here.
+ textureConverter->m_WorkerCV.wait(wait_lock);
+ }
+
g_Profiler2.RecordSyncMarker();
PROFILE2_EVENT("wakeup");
shared_ptr request;
{
- std::lock_guard lock(textureConverter->m_WorkerMutex);
+ std::lock_guard wait_lock(textureConverter->m_WorkerMutex);
if (textureConverter->m_Shutdown)
break;
// If we weren't woken up for shutdown, we must have been woken up for
// a new request, so grab it from the queue
request = textureConverter->m_RequestQueue.front();
textureConverter->m_RequestQueue.pop_front();
}
// Set up the result object
shared_ptr result(new ConversionResult());
result->dest = request->dest;
result->texture = request->texture;
request->outputOptions.setOutputHandler(&result->output);
// TIMER(L"TextureConverter compress");
{
PROFILE2("compress");
// Perform the compression
nvtt::Compressor compressor;
result->ret = compressor.process(request->inputOptions, request->compressionOptions, request->outputOptions);
}
// Ugly hack: NVTT 2.0 doesn't set DDPF_ALPHAPIXELS for DXT1a, so we can't
// distinguish it from DXT1. (It's fixed in trunk by
// http://code.google.com/p/nvidia-texture-tools/source/detail?r=924&path=/trunk).
// Rather than using a trunk NVTT (unstable, makes packaging harder)
// or patching our copy (makes packaging harder), we'll just manually
// set the flag here.
if (request->isDXT1a && result->ret && result->output.buffer.size() > 80)
result->output.buffer[80] |= 1; // DDPF_ALPHAPIXELS in DDS_PIXELFORMAT.dwFlags
// Ugly hack: NVTT always sets DDPF_RGB, even if we're trying to output 8-bit
// alpha-only DDS with no RGB components. Unset that flag.
if (request->is8bpp)
result->output.buffer[80] &= ~0x40; // DDPF_RGB in DDS_PIXELFORMAT.dwFlags
// Push the result onto the queue
- std::lock_guard lock(textureConverter->m_WorkerMutex);
+ std::lock_guard wait_lock(textureConverter->m_WorkerMutex);
textureConverter->m_ResultQueue.push_back(result);
}
+ std::lock_guard wait_lock(textureConverter->m_WorkerMutex);
+ textureConverter->m_Shutdown = false;
#endif
}
Index: ps/trunk/source/graphics/TextureConverter.h
===================================================================
--- ps/trunk/source/graphics/TextureConverter.h (revision 22771)
+++ ps/trunk/source/graphics/TextureConverter.h (revision 22772)
@@ -1,221 +1,221 @@
/* Copyright (C) 2019 Wildfire Games.
* This file is part of 0 A.D.
*
* 0 A.D. is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 2 of the License, or
* (at your option) any later version.
*
* 0 A.D. is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with 0 A.D. If not, see .
*/
#ifndef INCLUDED_TEXTURECONVERTER
#define INCLUDED_TEXTURECONVERTER
#include "lib/file/vfs/vfs.h"
-#include "lib/external_libraries/libsdl.h"
#include "TextureManager.h"
+#include
#include
#include
class MD5;
/**
* Texture conversion helper class.
* Provides an asynchronous API to convert input image files into compressed DDS,
* given various conversion settings.
* (The (potentially very slow) compression is a performed in a background thread,
* so the game can remain responsive).
* Also provides an API to load conversion settings from XML files.
*
* XML files are of the form:
* @code
*
*
*
*
* @endcode
*
* 'pattern' is a wildcard expression, matching on filenames.
* All other attributes are optional. Later elements override attributes from
* earlier elements.
*
* 'format' is 'dxt1', 'dxt3', 'dxt5' or 'rgba'.
*
* 'mipmap' is 'true' or 'false'.
*
* 'normal' is 'true' or 'false'.
*
* 'alpha' is 'transparency' or 'player' (it determines whether the color value of
* 0-alpha pixels is significant or not).
*
* 'filter' is 'box', 'triangle' or 'kaiser'.
*
* 'kaiserwidth', 'kaiseralpha', 'kaiserstretch' are floats
* (see http://code.google.com/p/nvidia-texture-tools/wiki/ApiDocumentation#Mipmap_Generation)
*/
class CTextureConverter
{
public:
enum EFormat
{
FMT_UNSPECIFIED,
FMT_DXT1,
FMT_DXT3,
FMT_DXT5,
FMT_RGBA,
FMT_ALPHA
};
enum EMipmap
{
MIP_UNSPECIFIED,
MIP_TRUE,
MIP_FALSE
};
enum ENormalMap
{
NORMAL_UNSPECIFIED,
NORMAL_TRUE,
NORMAL_FALSE
};
enum EAlpha
{
ALPHA_UNSPECIFIED,
ALPHA_NONE,
ALPHA_PLAYER,
ALPHA_TRANSPARENCY
};
enum EFilter
{
FILTER_UNSPECIFIED,
FILTER_BOX,
FILTER_TRIANGLE,
FILTER_KAISER
};
/**
* Texture conversion settings.
*/
struct Settings
{
Settings() :
format(FMT_UNSPECIFIED), mipmap(MIP_UNSPECIFIED), normal(NORMAL_UNSPECIFIED),
alpha(ALPHA_UNSPECIFIED), filter(FILTER_UNSPECIFIED),
kaiserWidth(-1.f), kaiserAlpha(-1.f), kaiserStretch(-1.f)
{
}
/**
* Append this object's state to the given hash.
*/
void Hash(MD5& hash);
EFormat format;
EMipmap mipmap;
ENormalMap normal;
EAlpha alpha;
EFilter filter;
float kaiserWidth;
float kaiserAlpha;
float kaiserStretch;
};
/**
* Representation of \ line from settings XML file.
*/
struct Match
{
std::wstring pattern;
Settings settings;
};
/**
* Representation of settings XML file.
*/
struct SettingsFile
{
std::vector patterns;
};
/**
* Construct texture converter, for use with files in the given vfs.
*/
CTextureConverter(PIVFS vfs, bool highQuality);
/**
* Destroy texture converter and wait to shut down worker thread.
* This might take a long time (maybe seconds) if the worker is busy
* processing a texture.
*/
~CTextureConverter();
/**
* Load a texture conversion settings XML file.
* Returns NULL on failure.
*/
SettingsFile* LoadSettings(const VfsPath& path) const;
/**
* Match a sequence of settings files against a given texture filename,
* and return the resulting settings.
* Later entries in settingsFiles override earlier entries.
*/
Settings ComputeSettings(const std::wstring& filename, const std::vector& settingsFiles) const;
/**
* Begin converting a texture, using the given settings.
* This will load src and return false on failure.
* Otherwise it will return true and start an asynchronous conversion request,
* whose result will be returned from Poll() (with the texture and dest passed
* into this function).
*/
bool ConvertTexture(const CTexturePtr& texture, const VfsPath& src, const VfsPath& dest, const Settings& settings);
/**
* Returns the result of a successful ConvertTexture call.
* If no result is available yet, returns false.
* Otherwise, if the conversion succeeded, it sets ok to true and sets
* texture and dest to the corresponding values passed into ConvertTexture(),
* then returns true.
* If the conversion failed, it sets ok to false and doesn't touch the other arguments,
* then returns true.
*/
bool Poll(CTexturePtr& texture, VfsPath& dest, bool& ok);
/**
* Returns whether there is currently a queued request from ConvertTexture().
* (Note this may return false while the worker thread is still converting the last texture.)
*/
bool IsBusy();
private:
static void RunThread(CTextureConverter* data);
PIVFS m_VFS;
bool m_HighQuality;
std::thread m_WorkerThread;
std::mutex m_WorkerMutex;
- SDL_sem* m_WorkerSem;
+ std::condition_variable m_WorkerCV;
struct ConversionRequest;
struct ConversionResult;
std::deque > m_RequestQueue; // protected by m_WorkerMutex
std::deque > m_ResultQueue; // protected by m_WorkerMutex
bool m_Shutdown; // protected by m_WorkerMutex
};
#endif // INCLUDED_TEXTURECONVERTER
Index: ps/trunk/source/graphics/TextureManager.cpp
===================================================================
--- ps/trunk/source/graphics/TextureManager.cpp (revision 22771)
+++ ps/trunk/source/graphics/TextureManager.cpp (revision 22772)
@@ -1,685 +1,684 @@
/* Copyright (C) 2018 Wildfire Games.
* This file is part of 0 A.D.
*
* 0 A.D. is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 2 of the License, or
* (at your option) any later version.
*
* 0 A.D. is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with 0 A.D. If not, see .
*/
#include "precompiled.h"
#include "TextureManager.h"
#include
#include
#include
#include
#include "graphics/TextureConverter.h"
#include "lib/allocators/shared_ptr.h"
#include "lib/res/h_mgr.h"
#include "lib/file/vfs/vfs_tree.h"
#include "lib/res/graphics/ogl_tex.h"
#include "lib/timer.h"
#include "maths/MD5.h"
#include "ps/CacheLoader.h"
#include "ps/CLogger.h"
#include "ps/Filesystem.h"
#include "ps/Profile.h"
struct TPhash
: std::unary_function,
std::unary_function
{
std::size_t operator()(CTextureProperties const& a) const
{
std::size_t seed = 0;
boost::hash_combine(seed, a.m_Path);
boost::hash_combine(seed, a.m_Filter);
boost::hash_combine(seed, a.m_WrapS);
boost::hash_combine(seed, a.m_WrapT);
boost::hash_combine(seed, a.m_Aniso);
boost::hash_combine(seed, a.m_Format);
return seed;
}
std::size_t operator()(CTexturePtr const& a) const
{
return (*this)(a->m_Properties);
}
};
struct TPequal_to
: std::binary_function,
std::binary_function
{
bool operator()(CTextureProperties const& a, CTextureProperties const& b) const
{
return a.m_Path == b.m_Path && a.m_Filter == b.m_Filter
&& a.m_WrapS == b.m_WrapS && a.m_WrapT == b.m_WrapT
&& a.m_Aniso == b.m_Aniso && a.m_Format == b.m_Format;
}
bool operator()(CTexturePtr const& a, CTexturePtr const& b) const
{
return (*this)(a->m_Properties, b->m_Properties);
}
};
std::size_t hash_value(const CTexturePtr& v)
{
TPhash h;
return h(v);
}
std::size_t hash_value(const CTextureProperties& v)
{
TPhash h;
return h(v);
}
class CTextureManagerImpl
{
friend class CTexture;
public:
CTextureManagerImpl(PIVFS vfs, bool highQuality, bool disableGL) :
m_VFS(vfs), m_CacheLoader(vfs, L".dds"), m_DisableGL(disableGL), m_TextureConverter(vfs, highQuality),
m_DefaultHandle(0), m_ErrorHandle(0)
{
// Initialise some textures that will always be available,
// without needing to load any files
// Default placeholder texture (grey)
if (!m_DisableGL)
{
// Construct 1x1 24-bit texture
shared_ptr data(new u8[3], ArrayDeleter());
data.get()[0] = 64;
data.get()[1] = 64;
data.get()[2] = 64;
Tex t;
(void)t.wrap(1, 1, 24, 0, data, 0);
m_DefaultHandle = ogl_tex_wrap(&t, m_VFS, L"(default texture)");
(void)ogl_tex_set_filter(m_DefaultHandle, GL_LINEAR);
if (!m_DisableGL)
(void)ogl_tex_upload(m_DefaultHandle);
}
// Error texture (magenta)
if (!m_DisableGL)
{
// Construct 1x1 24-bit texture
shared_ptr data(new u8[3], ArrayDeleter());
data.get()[0] = 255;
data.get()[1] = 0;
data.get()[2] = 255;
Tex t;
(void)t.wrap(1, 1, 24, 0, data, 0);
m_ErrorHandle = ogl_tex_wrap(&t, m_VFS, L"(error texture)");
(void)ogl_tex_set_filter(m_ErrorHandle, GL_LINEAR);
if (!m_DisableGL)
(void)ogl_tex_upload(m_ErrorHandle);
// Construct a CTexture to return to callers who want an error texture
CTextureProperties props(L"(error texture)");
m_ErrorTexture = CTexturePtr(new CTexture(m_ErrorHandle, props, this));
m_ErrorTexture->m_State = CTexture::LOADED;
m_ErrorTexture->m_Self = m_ErrorTexture;
}
// Allow hotloading of textures
RegisterFileReloadFunc(ReloadChangedFileCB, this);
}
~CTextureManagerImpl()
{
UnregisterFileReloadFunc(ReloadChangedFileCB, this);
(void)ogl_tex_free(m_DefaultHandle);
(void)ogl_tex_free(m_ErrorHandle);
}
CTexturePtr GetErrorTexture()
{
return m_ErrorTexture;
}
/**
* See CTextureManager::CreateTexture
*/
CTexturePtr CreateTexture(const CTextureProperties& props)
{
// Construct a new default texture with the given properties to use as the search key
CTexturePtr texture(new CTexture(m_DefaultHandle, props, this));
// Try to find an existing texture with the given properties
TextureCache::iterator it = m_TextureCache.find(texture);
if (it != m_TextureCache.end())
return *it;
// Can't find an existing texture - finish setting up this new texture
texture->m_Self = texture;
m_TextureCache.insert(texture);
m_HotloadFiles[props.m_Path].insert(texture);
return texture;
}
/**
* Load the given file into the texture object and upload it to OpenGL.
* Assumes the file already exists.
*/
void LoadTexture(const CTexturePtr& texture, const VfsPath& path)
{
if (m_DisableGL)
return;
PROFILE2("load texture");
PROFILE2_ATTR("name: %ls", path.string().c_str());
Handle h = ogl_tex_load(m_VFS, path, RES_UNIQUE);
if (h <= 0)
{
LOGERROR("Texture failed to load; \"%s\"", texture->m_Properties.m_Path.string8());
// Replace with error texture to make it obvious
texture->SetHandle(m_ErrorHandle);
return;
}
// Get some flags for later use
size_t flags = 0;
(void)ogl_tex_get_format(h, &flags, NULL);
// Initialise base color from the texture
(void)ogl_tex_get_average_color(h, &texture->m_BaseColor);
// Set GL upload properties
(void)ogl_tex_set_wrap(h, texture->m_Properties.m_WrapS, texture->m_Properties.m_WrapT);
(void)ogl_tex_set_anisotropy(h, texture->m_Properties.m_Aniso);
// Prevent ogl_tex automatically generating mipmaps (which is slow and unwanted),
// by avoiding mipmapped filters unless the source texture already has mipmaps
GLint filter = texture->m_Properties.m_Filter;
if (!(flags & TEX_MIPMAPS))
{
switch (filter)
{
case GL_NEAREST_MIPMAP_NEAREST:
case GL_NEAREST_MIPMAP_LINEAR:
filter = GL_NEAREST;
break;
case GL_LINEAR_MIPMAP_NEAREST:
case GL_LINEAR_MIPMAP_LINEAR:
filter = GL_LINEAR;
break;
}
}
(void)ogl_tex_set_filter(h, filter);
// Upload to GL
if (!m_DisableGL && ogl_tex_upload(h, texture->m_Properties.m_Format) < 0)
{
LOGERROR("Texture failed to upload: \"%s\"", texture->m_Properties.m_Path.string8());
ogl_tex_free(h);
// Replace with error texture to make it obvious
texture->SetHandle(m_ErrorHandle);
return;
}
// Let the texture object take ownership of this handle
texture->SetHandle(h, true);
}
/**
* Set up some parameters for the loose cache filename code.
*/
void PrepareCacheKey(const CTexturePtr& texture, MD5& hash, u32& version)
{
// Hash the settings, so we won't use an old loose cache file if the
// settings have changed
CTextureConverter::Settings settings = GetConverterSettings(texture);
settings.Hash(hash);
// Arbitrary version number - change this if we update the code and
// need to invalidate old users' caches
version = 1;
}
/**
* Attempts to load a cached version of a texture.
* If the texture is loaded (or there was an error), returns true.
* Otherwise, returns false to indicate the caller should generate the cached version.
*/
bool TryLoadingCached(const CTexturePtr& texture)
{
MD5 hash;
u32 version;
PrepareCacheKey(texture, hash, version);
VfsPath loadPath;
Status ret = m_CacheLoader.TryLoadingCached(texture->m_Properties.m_Path, hash, version, loadPath);
if (ret == INFO::OK)
{
// Found a cached texture - load it
LoadTexture(texture, loadPath);
return true;
}
else if (ret == INFO::SKIPPED)
{
// No cached version was found - we'll need to create it
return false;
}
else
{
ENSURE(ret < 0);
// No source file or archive cache was found, so we can't load the
// real texture at all - return the error texture instead
LOGERROR("CCacheLoader failed to find archived or source file for: \"%s\"", texture->m_Properties.m_Path.string8());
texture->SetHandle(m_ErrorHandle);
return true;
}
}
/**
* Initiates an asynchronous conversion process, from the texture's
* source file to the corresponding loose cache file.
*/
void ConvertTexture(const CTexturePtr& texture)
{
VfsPath sourcePath = texture->m_Properties.m_Path;
PROFILE2("convert texture");
PROFILE2_ATTR("name: %ls", sourcePath.string().c_str());
MD5 hash;
u32 version;
PrepareCacheKey(texture, hash, version);
VfsPath looseCachePath = m_CacheLoader.LooseCachePath(sourcePath, hash, version);
// LOGWARNING("Converting texture \"%s\"", srcPath.c_str());
CTextureConverter::Settings settings = GetConverterSettings(texture);
m_TextureConverter.ConvertTexture(texture, sourcePath, looseCachePath, settings);
}
bool TextureExists(const VfsPath& path) const
{
return m_VFS->GetFileInfo(m_CacheLoader.ArchiveCachePath(path), 0) == INFO::OK ||
m_VFS->GetFileInfo(path, 0) == INFO::OK;
}
bool GenerateCachedTexture(const VfsPath& sourcePath, VfsPath& archiveCachePath)
{
archiveCachePath = m_CacheLoader.ArchiveCachePath(sourcePath);
CTextureProperties textureProps(sourcePath);
CTexturePtr texture = CreateTexture(textureProps);
CTextureConverter::Settings settings = GetConverterSettings(texture);
if (!m_TextureConverter.ConvertTexture(texture, sourcePath, VfsPath("cache") / archiveCachePath, settings))
return false;
while (true)
{
CTexturePtr textureOut;
VfsPath dest;
bool ok;
if (m_TextureConverter.Poll(textureOut, dest, ok))
return ok;
- // Spin-loop is dumb but it works okay for now
- SDL_Delay(0);
+ std::this_thread::sleep_for(std::chrono::microseconds(1));
}
}
bool MakeProgress()
{
// Process any completed conversion tasks
{
CTexturePtr texture;
VfsPath dest;
bool ok;
if (m_TextureConverter.Poll(texture, dest, ok))
{
if (ok)
{
LoadTexture(texture, dest);
}
else
{
LOGERROR("Texture failed to convert: \"%s\"", texture->m_Properties.m_Path.string8());
texture->SetHandle(m_ErrorHandle);
}
texture->m_State = CTexture::LOADED;
return true;
}
}
// We'll only push new conversion requests if it's not already busy
bool converterBusy = m_TextureConverter.IsBusy();
if (!converterBusy)
{
// Look for all high-priority textures needing conversion.
// (Iterating over all textures isn't optimally efficient, but it
// doesn't seem to be a problem yet and it's simpler than maintaining
// multiple queues.)
for (TextureCache::iterator it = m_TextureCache.begin(); it != m_TextureCache.end(); ++it)
{
if ((*it)->m_State == CTexture::HIGH_NEEDS_CONVERTING)
{
// Start converting this texture
(*it)->m_State = CTexture::HIGH_IS_CONVERTING;
ConvertTexture(*it);
return true;
}
}
}
// Try loading prefetched textures from their cache
for (TextureCache::iterator it = m_TextureCache.begin(); it != m_TextureCache.end(); ++it)
{
if ((*it)->m_State == CTexture::PREFETCH_NEEDS_LOADING)
{
if (TryLoadingCached(*it))
{
(*it)->m_State = CTexture::LOADED;
}
else
{
(*it)->m_State = CTexture::PREFETCH_NEEDS_CONVERTING;
}
return true;
}
}
// If we've got nothing better to do, then start converting prefetched textures.
if (!converterBusy)
{
for (TextureCache::iterator it = m_TextureCache.begin(); it != m_TextureCache.end(); ++it)
{
if ((*it)->m_State == CTexture::PREFETCH_NEEDS_CONVERTING)
{
(*it)->m_State = CTexture::PREFETCH_IS_CONVERTING;
ConvertTexture(*it);
return true;
}
}
}
return false;
}
/**
* Compute the conversion settings that apply to a given texture, by combining
* the textures.xml files from its directory and all parent directories
* (up to the VFS root).
*/
CTextureConverter::Settings GetConverterSettings(const CTexturePtr& texture)
{
fs::wpath srcPath = texture->m_Properties.m_Path.string();
std::vector files;
VfsPath p;
for (fs::wpath::iterator it = srcPath.begin(); it != srcPath.end(); ++it)
{
VfsPath settingsPath = p / "textures.xml";
m_HotloadFiles[settingsPath].insert(texture);
CTextureConverter::SettingsFile* f = GetSettingsFile(settingsPath);
if (f)
files.push_back(f);
p = p / GetWstringFromWpath(*it);
}
return m_TextureConverter.ComputeSettings(GetWstringFromWpath(srcPath.leaf()), files);
}
/**
* Return the (cached) settings file with the given filename,
* or NULL if it doesn't exist.
*/
CTextureConverter::SettingsFile* GetSettingsFile(const VfsPath& path)
{
SettingsFilesMap::iterator it = m_SettingsFiles.find(path);
if (it != m_SettingsFiles.end())
return it->second.get();
if (m_VFS->GetFileInfo(path, NULL) >= 0)
{
shared_ptr settings(m_TextureConverter.LoadSettings(path));
m_SettingsFiles.insert(std::make_pair(path, settings));
return settings.get();
}
else
{
m_SettingsFiles.insert(std::make_pair(path, shared_ptr()));
return NULL;
}
}
static Status ReloadChangedFileCB(void* param, const VfsPath& path)
{
return static_cast(param)->ReloadChangedFile(path);
}
Status ReloadChangedFile(const VfsPath& path)
{
// Uncache settings file, if this is one
m_SettingsFiles.erase(path);
// Find all textures using this file
HotloadFilesMap::iterator files = m_HotloadFiles.find(path);
if (files != m_HotloadFiles.end())
{
// Flag all textures using this file as needing reloading
for (std::set >::iterator it = files->second.begin(); it != files->second.end(); ++it)
{
if (std::shared_ptr texture = it->lock())
{
texture->m_State = CTexture::UNLOADED;
texture->SetHandle(m_DefaultHandle);
}
}
}
return INFO::OK;
}
size_t GetBytesUploaded() const
{
size_t size = 0;
for (TextureCache::const_iterator it = m_TextureCache.begin(); it != m_TextureCache.end(); ++it)
size += (*it)->GetUploadedSize();
return size;
}
private:
PIVFS m_VFS;
CCacheLoader m_CacheLoader;
bool m_DisableGL;
CTextureConverter m_TextureConverter;
Handle m_DefaultHandle;
Handle m_ErrorHandle;
CTexturePtr m_ErrorTexture;
// Cache of all loaded textures
typedef boost::unordered_set TextureCache;
TextureCache m_TextureCache;
// TODO: we ought to expire unused textures from the cache eventually
// Store the set of textures that need to be reloaded when the given file
// (a source file or settings.xml) is modified
typedef boost::unordered_map, std::owner_less>>> HotloadFilesMap;
HotloadFilesMap m_HotloadFiles;
// Cache for the conversion settings files
typedef boost::unordered_map > SettingsFilesMap;
SettingsFilesMap m_SettingsFiles;
};
CTexture::CTexture(Handle handle, const CTextureProperties& props, CTextureManagerImpl* textureManager) :
m_Handle(handle), m_BaseColor(0), m_State(UNLOADED), m_Properties(props), m_TextureManager(textureManager)
{
// Add a reference to the handle (it might be shared by multiple CTextures
// so we can't take ownership of it)
if (m_Handle)
h_add_ref(m_Handle);
}
CTexture::~CTexture()
{
if (m_Handle)
ogl_tex_free(m_Handle);
}
void CTexture::Bind(size_t unit)
{
ogl_tex_bind(GetHandle(), unit);
}
Handle CTexture::GetHandle()
{
// TODO: TryLoad might call ogl_tex_upload which enables GL_TEXTURE_2D
// on texture unit 0, regardless of 'unit', which callers might
// not be expecting. Ideally that wouldn't happen.
TryLoad();
return m_Handle;
}
bool CTexture::TryLoad()
{
// If we haven't started loading, then try loading, and if that fails then request conversion.
// If we have already tried prefetch loading, and it failed, bump the conversion request to HIGH priority.
if (m_State == UNLOADED || m_State == PREFETCH_NEEDS_LOADING || m_State == PREFETCH_NEEDS_CONVERTING)
{
if (std::shared_ptr self = m_Self.lock())
{
if (m_State != PREFETCH_NEEDS_CONVERTING && m_TextureManager->TryLoadingCached(self))
m_State = LOADED;
else
m_State = HIGH_NEEDS_CONVERTING;
}
}
return (m_State == LOADED);
}
void CTexture::Prefetch()
{
if (m_State == UNLOADED)
{
if (std::shared_ptr self = m_Self.lock())
{
m_State = PREFETCH_NEEDS_LOADING;
}
}
}
bool CTexture::IsLoaded()
{
return (m_State == LOADED);
}
void CTexture::SetHandle(Handle handle, bool takeOwnership)
{
if (handle == m_Handle)
return;
if (!takeOwnership)
h_add_ref(handle);
ogl_tex_free(m_Handle);
m_Handle = handle;
}
size_t CTexture::GetWidth() const
{
size_t w = 0;
(void)ogl_tex_get_size(m_Handle, &w, 0, 0);
return w;
}
size_t CTexture::GetHeight() const
{
size_t h = 0;
(void)ogl_tex_get_size(m_Handle, 0, &h, 0);
return h;
}
bool CTexture::HasAlpha() const
{
size_t flags = 0;
(void)ogl_tex_get_format(m_Handle, &flags, 0);
return (flags & TEX_ALPHA) != 0;
}
u32 CTexture::GetBaseColor() const
{
return m_BaseColor;
}
size_t CTexture::GetUploadedSize() const
{
size_t size = 0;
(void)ogl_tex_get_uploaded_size(m_Handle, &size);
return size;
}
// CTextureManager: forward all calls to impl:
CTextureManager::CTextureManager(PIVFS vfs, bool highQuality, bool disableGL) :
m(new CTextureManagerImpl(vfs, highQuality, disableGL))
{
}
CTextureManager::~CTextureManager()
{
delete m;
}
CTexturePtr CTextureManager::CreateTexture(const CTextureProperties& props)
{
return m->CreateTexture(props);
}
bool CTextureManager::TextureExists(const VfsPath& path) const
{
return m->TextureExists(path);
}
CTexturePtr CTextureManager::GetErrorTexture()
{
return m->GetErrorTexture();
}
bool CTextureManager::MakeProgress()
{
return m->MakeProgress();
}
bool CTextureManager::GenerateCachedTexture(const VfsPath& path, VfsPath& outputPath)
{
return m->GenerateCachedTexture(path, outputPath);
}
size_t CTextureManager::GetBytesUploaded() const
{
return m->GetBytesUploaded();
}
Index: ps/trunk/source/pch/engine/precompiled.h
===================================================================
--- ps/trunk/source/pch/engine/precompiled.h (revision 22771)
+++ ps/trunk/source/pch/engine/precompiled.h (revision 22772)
@@ -1,30 +1,29 @@
/* Copyright (C) 2019 Wildfire Games.
* This file is part of 0 A.D.
*
* 0 A.D. is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 2 of the License, or
* (at your option) any later version.
*
* 0 A.D. is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with 0 A.D. If not, see .
*/
#include "lib/precompiled.h" // common precompiled header
// "engine"-specific PCH:
#include "ps/Pyrogenesis.h" // old error system
#if HAVE_PCH
#include "SDL.h"
-#include "SDL_thread.h"
#include "SDL_endian.h"
#endif // HAVE_PCH
Index: ps/trunk/source/ps/UserReport.cpp
===================================================================
--- ps/trunk/source/ps/UserReport.cpp (revision 22771)
+++ ps/trunk/source/ps/UserReport.cpp (revision 22772)
@@ -1,649 +1,636 @@
/* Copyright (C) 2019 Wildfire Games.
* This file is part of 0 A.D.
*
* 0 A.D. is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 2 of the License, or
* (at your option) any later version.
*
* 0 A.D. is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with 0 A.D. If not, see .
*/
#include "precompiled.h"
#include "UserReport.h"
#include "lib/timer.h"
#include "lib/utf8.h"
#include "lib/external_libraries/curl.h"
-#include "lib/external_libraries/libsdl.h"
#include "lib/external_libraries/zlib.h"
#include "lib/file/archive/stream.h"
#include "lib/os_path.h"
#include "lib/sysdep/sysdep.h"
#include "ps/ConfigDB.h"
#include "ps/Filesystem.h"
#include "ps/Profiler2.h"
+#include
#include
#include
#include
#include
#define DEBUG_UPLOADS 0
/*
* The basic idea is that the game submits reports to us, which we send over
* HTTP to a server for storage and analysis.
*
* We can't use libcurl's asynchronous 'multi' API, because DNS resolution can
* be synchronous and slow (which would make the game pause).
* So we use the 'easy' API in a background thread.
* The main thread submits reports, toggles whether uploading is enabled,
* and polls for the current status (typically to display in the GUI);
* the worker thread does all of the uploading.
*
* It'd be nice to extend this in the future to handle things like crash reports.
* The game should store the crashlogs (suitably anonymised) in a directory, and
* we should detect those files and upload them when we're restarted and online.
*/
/**
* Version number stored in config file when the user agrees to the reporting.
* Reporting will be disabled if the config value is missing or is less than
* this value. If we start reporting a lot more data, we should increase this
* value and get the user to re-confirm.
*/
static const int REPORTER_VERSION = 1;
/**
* Time interval (seconds) at which the worker thread will check its reconnection
* timers. (This should be relatively high so the thread doesn't waste much time
* continually waking up.)
*/
static const double TIMER_CHECK_INTERVAL = 10.0;
/**
* Seconds we should wait before reconnecting to the server after a failure.
*/
static const double RECONNECT_INVERVAL = 60.0;
CUserReporter g_UserReporter;
struct CUserReport
{
time_t m_Time;
std::string m_Type;
int m_Version;
std::string m_Data;
};
class CUserReporterWorker
{
public:
CUserReporterWorker(const std::string& userID, const std::string& url) :
m_URL(url), m_UserID(userID), m_Enabled(false), m_Shutdown(false), m_Status("disabled"),
m_PauseUntilTime(timer_Time()), m_LastUpdateTime(timer_Time())
{
// Set up libcurl:
m_Curl = curl_easy_init();
ENSURE(m_Curl);
#if DEBUG_UPLOADS
curl_easy_setopt(m_Curl, CURLOPT_VERBOSE, 1L);
#endif
// Capture error messages
curl_easy_setopt(m_Curl, CURLOPT_ERRORBUFFER, m_ErrorBuffer);
// Disable signal handlers (required for multithreaded applications)
curl_easy_setopt(m_Curl, CURLOPT_NOSIGNAL, 1L);
// To minimise security risks, don't support redirects
curl_easy_setopt(m_Curl, CURLOPT_FOLLOWLOCATION, 0L);
// Prevent this thread from blocking the engine shutdown for 5 minutes in case the server is unavailable
curl_easy_setopt(m_Curl, CURLOPT_CONNECTTIMEOUT, 10L);
// Set IO callbacks
curl_easy_setopt(m_Curl, CURLOPT_WRITEFUNCTION, ReceiveCallback);
curl_easy_setopt(m_Curl, CURLOPT_WRITEDATA, this);
curl_easy_setopt(m_Curl, CURLOPT_READFUNCTION, SendCallback);
curl_easy_setopt(m_Curl, CURLOPT_READDATA, this);
// Set URL to POST to
curl_easy_setopt(m_Curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(m_Curl, CURLOPT_POST, 1L);
// Set up HTTP headers
m_Headers = NULL;
// Set the UA string
std::string ua = "User-Agent: 0ad ";
ua += curl_version();
ua += " (http://play0ad.com/)";
m_Headers = curl_slist_append(m_Headers, ua.c_str());
// Override the default application/x-www-form-urlencoded type since we're not using that type
m_Headers = curl_slist_append(m_Headers, "Content-Type: application/octet-stream");
// Disable the Accept header because it's a waste of a dozen bytes
m_Headers = curl_slist_append(m_Headers, "Accept: ");
curl_easy_setopt(m_Curl, CURLOPT_HTTPHEADER, m_Headers);
-
- // Set up the worker thread:
-
- // Use SDL semaphores since OS X doesn't implement sem_init
- m_WorkerSem = SDL_CreateSemaphore(0);
- ENSURE(m_WorkerSem);
-
m_WorkerThread = std::thread(RunThread, this);
}
~CUserReporterWorker()
{
- // Clean up resources
-
- SDL_DestroySemaphore(m_WorkerSem);
-
curl_slist_free_all(m_Headers);
curl_easy_cleanup(m_Curl);
}
/**
* Called by main thread, when the online reporting is enabled/disabled.
*/
void SetEnabled(bool enabled)
{
std::lock_guard lock(m_WorkerMutex);
if (enabled != m_Enabled)
{
m_Enabled = enabled;
// Wake up the worker thread
- SDL_SemPost(m_WorkerSem);
+ m_WorkerCV.notify_all();
}
}
/**
* Called by main thread to request shutdown.
* Returns true if we've shut down successfully.
* Returns false if shutdown is taking too long (we might be blocked on a
* sync network operation) - you mustn't destroy this object, just leak it
* and terminate.
*/
bool Shutdown()
{
{
std::lock_guard lock(m_WorkerMutex);
m_Shutdown = true;
}
// Wake up the worker thread
- SDL_SemPost(m_WorkerSem);
+ m_WorkerCV.notify_all();
// Wait for it to shut down cleanly
// TODO: should have a timeout in case of network hangs
m_WorkerThread.join();
return true;
}
/**
* Called by main thread to determine the current status of the uploader.
*/
std::string GetStatus()
{
std::lock_guard lock(m_WorkerMutex);
return m_Status;
}
/**
* Called by main thread to add a new report to the queue.
*/
void Submit(const shared_ptr& report)
{
{
std::lock_guard lock(m_WorkerMutex);
m_ReportQueue.push_back(report);
}
// Wake up the worker thread
- SDL_SemPost(m_WorkerSem);
+ m_WorkerCV.notify_all();
}
/**
* Called by the main thread every frame, so we can check
* retransmission timers.
*/
void Update()
{
double now = timer_Time();
if (now > m_LastUpdateTime + TIMER_CHECK_INTERVAL)
{
// Wake up the worker thread
- SDL_SemPost(m_WorkerSem);
+ m_WorkerCV.notify_all();
m_LastUpdateTime = now;
}
}
private:
static void RunThread(CUserReporterWorker* data)
{
debug_SetThreadName("CUserReportWorker");
g_Profiler2.RegisterCurrentThread("userreport");
data->Run();
}
void Run()
{
// Set libcurl's proxy configuration
// (This has to be done in the thread because it's potentially very slow)
SetStatus("proxy");
std::wstring proxy;
{
PROFILE2("get proxy config");
if (sys_get_proxy_config(wstring_from_utf8(m_URL), proxy) == INFO::OK)
curl_easy_setopt(m_Curl, CURLOPT_PROXY, utf8_from_wstring(proxy).c_str());
}
SetStatus("waiting");
/*
- * We use a semaphore to let the thread be woken up when it has
+ * We use a condition_variable to let the thread be woken up when it has
* work to do. Various actions from the main thread can wake it:
* * SetEnabled()
* * Shutdown()
* * Submit()
* * Retransmission timeouts, once every several seconds
*
* If multiple actions have triggered wakeups, we might respond to
* all of those actions after the first wakeup, which is okay (we'll do
* nothing during the subsequent wakeups). We should never hang due to
* processing fewer actions than wakeups.
*
- * Retransmission timeouts are triggered via the main thread - we can't simply
- * use SDL_SemWaitTimeout because on Linux it's implemented as an inefficient
- * busy-wait loop, and we can't use a manual busy-wait with a long delay time
- * because we'd lose responsiveness. So the main thread pings the worker
- * occasionally so it can check its timer.
+ * Retransmission timeouts are triggered via the main thread.
*/
// Wait until the main thread wakes us up
while (true)
{
- g_Profiler2.RecordRegionEnter("semaphore wait");
+ g_Profiler2.RecordRegionEnter("condition_variable wait");
- ENSURE(SDL_SemWait(m_WorkerSem) == 0);
+ std::unique_lock lock(m_WorkerMutex);
+ m_WorkerCV.wait(lock, [this] { return m_Enabled || m_Shutdown; });
+ lock.unlock();
g_Profiler2.RecordRegionLeave();
// Handle shutdown requests as soon as possible
if (GetShutdown())
return;
// If we're not enabled, ignore this wakeup
if (!GetEnabled())
continue;
// If we're still pausing due to a failed connection,
// go back to sleep again
if (timer_Time() < m_PauseUntilTime)
continue;
// We're enabled, so process as many reports as possible
while (ProcessReport())
{
// Handle shutdowns while we were sending the report
if (GetShutdown())
return;
}
}
}
bool GetEnabled()
{
std::lock_guard lock(m_WorkerMutex);
return m_Enabled;
}
bool GetShutdown()
{
std::lock_guard lock(m_WorkerMutex);
return m_Shutdown;
}
void SetStatus(const std::string& status)
{
std::lock_guard lock(m_WorkerMutex);
m_Status = status;
#if DEBUG_UPLOADS
debug_printf(">>> CUserReporterWorker status: %s\n", status.c_str());
#endif
}
bool ProcessReport()
{
PROFILE2("process report");
shared_ptr report;
{
std::lock_guard lock(m_WorkerMutex);
if (m_ReportQueue.empty())
return false;
report = m_ReportQueue.front();
m_ReportQueue.pop_front();
}
ConstructRequestData(*report);
m_RequestDataOffset = 0;
m_ResponseData.clear();
m_ErrorBuffer[0] = '\0';
curl_easy_setopt(m_Curl, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)m_RequestData.size());
SetStatus("connecting");
#if DEBUG_UPLOADS
TIMER(L"CUserReporterWorker request");
#endif
CURLcode err = curl_easy_perform(m_Curl);
#if DEBUG_UPLOADS
printf(">>>\n%s\n<<<\n", m_ResponseData.c_str());
#endif
if (err == CURLE_OK)
{
long code = -1;
curl_easy_getinfo(m_Curl, CURLINFO_RESPONSE_CODE, &code);
SetStatus("completed:" + CStr::FromInt(code));
// Check for success code
if (code == 200)
return true;
// If the server returns the 410 Gone status, interpret that as meaning
// it no longer supports uploads (at least from this version of the game),
// so shut down and stop talking to it (to avoid wasting bandwidth)
if (code == 410)
{
std::lock_guard lock(m_WorkerMutex);
m_Shutdown = true;
return false;
}
}
else
{
std::string errorString(m_ErrorBuffer);
if (errorString.empty())
errorString = curl_easy_strerror(err);
SetStatus("failed:" + CStr::FromInt(err) + ":" + errorString);
}
// We got an unhandled return code or a connection failure;
// push this report back onto the queue and try again after
// a long interval
{
std::lock_guard lock(m_WorkerMutex);
m_ReportQueue.push_front(report);
}
m_PauseUntilTime = timer_Time() + RECONNECT_INVERVAL;
return false;
}
void ConstructRequestData(const CUserReport& report)
{
// Construct the POST request data in the application/x-www-form-urlencoded format
std::string r;
r += "user_id=";
AppendEscaped(r, m_UserID);
r += "&time=" + CStr::FromInt64(report.m_Time);
r += "&type=";
AppendEscaped(r, report.m_Type);
r += "&version=" + CStr::FromInt(report.m_Version);
r += "&data=";
AppendEscaped(r, report.m_Data);
// Compress the content with zlib to save bandwidth.
// (Note that we send a request with unlabelled compressed data instead
// of using Content-Encoding, because Content-Encoding is a mess and causes
// problems with servers and breaks Content-Length and this is much easier.)
std::string compressed;
compressed.resize(compressBound(r.size()));
uLongf destLen = compressed.size();
int ok = compress((Bytef*)compressed.c_str(), &destLen, (const Bytef*)r.c_str(), r.size());
ENSURE(ok == Z_OK);
compressed.resize(destLen);
m_RequestData.swap(compressed);
}
void AppendEscaped(std::string& buffer, const std::string& str)
{
char* escaped = curl_easy_escape(m_Curl, str.c_str(), str.size());
buffer += escaped;
curl_free(escaped);
}
static size_t ReceiveCallback(void* buffer, size_t size, size_t nmemb, void* userp)
{
CUserReporterWorker* self = static_cast(userp);
if (self->GetShutdown())
return 0; // signals an error
self->m_ResponseData += std::string((char*)buffer, (char*)buffer+size*nmemb);
return size*nmemb;
}
static size_t SendCallback(char* bufptr, size_t size, size_t nmemb, void* userp)
{
CUserReporterWorker* self = static_cast(userp);
if (self->GetShutdown())
return CURL_READFUNC_ABORT; // signals an error
// We can return as much data as available, up to the buffer size
size_t amount = std::min(self->m_RequestData.size() - self->m_RequestDataOffset, size*nmemb);
// ...But restrict to sending a small amount at once, so that we remain
// responsive to shutdown requests even if the network is pretty slow
amount = std::min((size_t)1024, amount);
if(amount != 0) // (avoids invalid operator[] call where index=size)
{
memcpy(bufptr, &self->m_RequestData[self->m_RequestDataOffset], amount);
self->m_RequestDataOffset += amount;
}
self->SetStatus("sending:" + CStr::FromDouble((double)self->m_RequestDataOffset / self->m_RequestData.size()));
return amount;
}
private:
// Thread-related members:
std::thread m_WorkerThread;
std::mutex m_WorkerMutex;
- SDL_sem* m_WorkerSem;
+ std::condition_variable m_WorkerCV;
// Shared by main thread and worker thread:
// These variables are all protected by m_WorkerMutex
std::deque > m_ReportQueue;
bool m_Enabled;
bool m_Shutdown;
std::string m_Status;
// Initialised in constructor by main thread; otherwise used only by worker thread:
std::string m_URL;
std::string m_UserID;
CURL* m_Curl;
curl_slist* m_Headers;
double m_PauseUntilTime;
// Only used by worker thread:
std::string m_ResponseData;
std::string m_RequestData;
size_t m_RequestDataOffset;
char m_ErrorBuffer[CURL_ERROR_SIZE];
// Only used by main thread:
double m_LastUpdateTime;
};
CUserReporter::CUserReporter() :
m_Worker(NULL)
{
}
CUserReporter::~CUserReporter()
{
ENSURE(!m_Worker); // Deinitialize should have been called before shutdown
}
std::string CUserReporter::LoadUserID()
{
std::string userID;
// Read the user ID from user.cfg (if there is one)
CFG_GET_VAL("userreport.id", userID);
// If we don't have a validly-formatted user ID, generate a new one
if (userID.length() != 16)
{
u8 bytes[8] = {0};
sys_generate_random_bytes(bytes, ARRAY_SIZE(bytes));
// ignore failures - there's not much we can do about it
userID = "";
for (size_t i = 0; i < ARRAY_SIZE(bytes); ++i)
{
char hex[3];
sprintf_s(hex, ARRAY_SIZE(hex), "%02x", (unsigned int)bytes[i]);
userID += hex;
}
g_ConfigDB.SetValueString(CFG_USER, "userreport.id", userID);
g_ConfigDB.WriteValueToFile(CFG_USER, "userreport.id", userID);
}
return userID;
}
bool CUserReporter::IsReportingEnabled()
{
int version = -1;
CFG_GET_VAL("userreport.enabledversion", version);
return (version >= REPORTER_VERSION);
}
void CUserReporter::SetReportingEnabled(bool enabled)
{
CStr val = CStr::FromInt(enabled ? REPORTER_VERSION : 0);
g_ConfigDB.SetValueString(CFG_USER, "userreport.enabledversion", val);
g_ConfigDB.WriteValueToFile(CFG_USER, "userreport.enabledversion", val);
if (m_Worker)
m_Worker->SetEnabled(enabled);
}
std::string CUserReporter::GetStatus()
{
if (!m_Worker)
return "disabled";
return m_Worker->GetStatus();
}
void CUserReporter::Initialize()
{
ENSURE(!m_Worker); // must only be called once
std::string userID = LoadUserID();
std::string url;
CFG_GET_VAL("userreport.url_upload", url);
m_Worker = new CUserReporterWorker(userID, url);
m_Worker->SetEnabled(IsReportingEnabled());
}
void CUserReporter::Deinitialize()
{
if (!m_Worker)
return;
if (m_Worker->Shutdown())
{
// Worker was shut down cleanly
SAFE_DELETE(m_Worker);
}
else
{
// Worker failed to shut down in a reasonable time
// Leak the resources (since that's better than hanging or crashing)
m_Worker = NULL;
}
}
void CUserReporter::Update()
{
if (m_Worker)
m_Worker->Update();
}
void CUserReporter::SubmitReport(const std::string& type, int version, const std::string& data, const std::string& dataHumanReadable)
{
// Write to logfile, enabling users to assess privacy concerns before the data is submitted
if (!dataHumanReadable.empty())
{
OsPath path = psLogDir() / OsPath("userreport_" + type + ".txt");
std::ofstream stream(OsString(path), std::ofstream::trunc);
if (stream)
{
debug_printf("UserReport written to %s\n", path.string8().c_str());
stream << dataHumanReadable << std::endl;
stream.close();
}
else
debug_printf("Failed to write UserReport to %s\n", path.string8().c_str());
}
// If not initialised, discard the report
if (!m_Worker)
return;
// Actual submit
shared_ptr report(new CUserReport);
report->m_Time = time(NULL);
report->m_Type = type;
report->m_Version = version;
report->m_Data = data;
m_Worker->Submit(report);
}