Differential D2915 Diff 13035 source/ps/XML/Xeromyces.cpp

Changeset View

Standalone View

source/ps/XML/Xeromyces.cpp

Show All 11 Lines
* GNU General Public License for more details.		* GNU General Public License for more details.
*		*
* You should have received a copy of the GNU General Public License		* You should have received a copy of the GNU General Public License
* along with 0 A.D. If not, see <http://www.gnu.org/licenses/>.		* along with 0 A.D. If not, see <http://www.gnu.org/licenses/>.
*/		*/

#include "precompiled.h"		#include "precompiled.h"

		#include "Xeromyces.h"

#include <vector>		#include <vector>
#include <set>		#include <set>
#include <map>		#include <map>
#include <mutex>		#include <mutex>
#include <stack>		#include <stack>
#include <algorithm>		#include <algorithm>

#include "maths/MD5.h"		#include "maths/MD5.h"
#include "ps/CacheLoader.h"		#include "ps/CacheLoader.h"
#include "ps/CLogger.h"		#include "ps/CLogger.h"
		#include "ps/DataTreeXML.h"
#include "ps/Filesystem.h"		#include "ps/Filesystem.h"
#include "RelaxNG.h"		#include "RelaxNG.h"
#include "Xeromyces.h"

#include <libxml/parser.h>		#include <libxml/parser.h>
		#include <unordered_map>

static std::mutex g_ValidatorCacheLock;		static std::mutex g_ValidatorCacheLock;
static std::map<const std::string, RelaxNGValidator> g_ValidatorCache;		static std::map<const std::string, RelaxNGValidator> g_ValidatorCache;
static bool g_XeromycesStarted = false;		static bool g_XeromycesStarted = false;

		static std::unordered_map<std::string, bool> m_ValidXMBFiles;

static void errorHandler(void* UNUSED(userData), xmlErrorPtr error)		static void errorHandler(void* UNUSED(userData), xmlErrorPtr error)
{		{
// Strip a trailing newline		// Strip a trailing newline
std::string message = error->message;		std::string message = error->message;
if (message.length() > 0 && message[message.length()-1] == '\n')		if (message.length() > 0 && message[message.length()-1] == '\n')
message.erase(message.length()-1);		message.erase(message.length()-1);

LOGERROR("CXeromyces: Parse %s: %s:%d: %s",		LOGERROR("CXeromyces: Parse %s: %s:%d: %s",
error->level == XML_ERR_WARNING ? "warning" : "error",		error->level == XML_ERR_WARNING ? "warning" : "error",
error->file, error->line, message);		error->file, error->line, message);
// TODO: The (non-fatal) warnings and errors don't get stored in the XMB,		// TODO: The (non-fatal) warnings and errors don't get stored in the XMB,
// so the caching is less transparent than it should be		// so the caching is less transparent than it should be
}		}

void CXeromyces::Startup()		void CXeromyces::Startup()
{		{
ENSURE(!g_XeromycesStarted);		ENSURE(!g_XeromycesStarted);
xmlInitParser();		xmlInitParser();
xmlSetStructuredErrorFunc(NULL, &errorHandler);		xmlSetStructuredErrorFunc(NULL, &errorHandler);
std::lock_guard<std::mutex> lock(g_ValidatorCacheLock);		std::lock_guard<std::mutex> lock(g_ValidatorCacheLock);
g_ValidatorCache.insert(std::make_pair(std::string(), RelaxNGValidator()));		g_ValidatorCache.insert(std::make_pair(std::string(), RelaxNGValidator()));
		m_ValidXMBFiles.clear();
g_XeromycesStarted = true;		g_XeromycesStarted = true;
}		}

void CXeromyces::Terminate()		void CXeromyces::Terminate()
{		{
ENSURE(g_XeromycesStarted);		ENSURE(g_XeromycesStarted);
g_XeromycesStarted = false;		g_XeromycesStarted = false;
ClearSchemaCache();		ClearSchemaCache();
std::lock_guard<std::mutex> lock(g_ValidatorCacheLock);		std::lock_guard<std::mutex> lock(g_ValidatorCacheLock);
g_ValidatorCache.clear();		g_ValidatorCache.clear();
		m_ValidXMBFiles.clear();
xmlSetStructuredErrorFunc(NULL, NULL);		xmlSetStructuredErrorFunc(NULL, NULL);
xmlCleanupParser();		xmlCleanupParser();
}		}

bool CXeromyces::AddValidator(const PIVFS& vfs, const std::string& name, const VfsPath& grammarPath)		bool CXeromyces::AddValidator(const PIVFS& vfs, const std::string& name, const VfsPath& grammarPath)
{		{
ENSURE(g_XeromycesStarted);		ENSURE(g_XeromycesStarted);

▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines	PSRETURN CXeromyces::Load(const PIVFS& vfs, const VfsPath& filename, const std::string& validatorName /* = "" */)
}		}
VfsPath xmbPath;		VfsPath xmbPath;
Status ret = cacheLoader.TryLoadingCached(filename, validatorGrammarHash, XMBVersion, xmbPath);		Status ret = cacheLoader.TryLoadingCached(filename, validatorGrammarHash, XMBVersion, xmbPath);

if (ret == INFO::OK)		if (ret == INFO::OK)
{		{
// Found a cached XMB - load it		// Found a cached XMB - load it
if (ReadXMBFile(vfs, xmbPath))		if (ReadXMBFile(vfs, xmbPath))
		{
		// Check that dependencies aren't invalid themselves.
		bool valid = true;
		bool unknowns = false;
		// Iterate through dependencies - first check for any explicitly invalid xmb files.
		for (const std::string& dep : m_Dependencies)
		{
		std::unordered_map<std::string, bool>::const_iterator it = m_ValidXMBFiles.find(dep);
		if (it != m_ValidXMBFiles.end() && !it->second)
		{
		// A dependency isn't valid - break
		valid = false;
		break;
		} else if (it != m_ValidXMBFiles.end())
		unknowns = true;
		}
		// Then check for any unknowns - load them and verify.
		// This is done in a 2nd pass as opening files is slower and might not be needed if we're lucky.
		if (unknowns)
		{
		for (const std::string& dep : m_Dependencies)
		{
		std::unordered_map<std::string, bool>::const_iterator it = m_ValidXMBFiles.find(dep);
		if (it != m_ValidXMBFiles.end())
		continue;
		CXeromyces temp;
		temp.Load(vfs, dep);
		it = m_ValidXMBFiles.find(dep);
		ENSURE(it != m_ValidXMBFiles.end());
		valid &= it->second;
		}
		}

		if (valid)
		{
		// TODO c++17: try_emplace
		if (m_ValidXMBFiles.find(filename.string8()) == m_ValidXMBFiles.end())
		m_ValidXMBFiles.emplace(filename.string8(), true);
return PSRETURN_OK;		return PSRETURN_OK;
		}
		}
// If this fails then we'll continue and (re)create the loose cache -		// If this fails then we'll continue and (re)create the loose cache -
// this failure legitimately happens due to partially-written XMB files.		// this failure legitimately happens due to partially-written XMB files.
}		}
else if (ret == INFO::SKIPPED)		else if (ret == INFO::SKIPPED)
{		{
// No cached version was found - we'll need to create it		// No cached version was found - we'll need to create it
}		}
else		else
{		{
ENSURE(ret < 0);		ENSURE(ret < 0);

		m_ValidXMBFiles.emplace(filename.string8(), false);
// No source file or archive cache was found, so we can't load the		// No source file or archive cache was found, so we can't load the
// XML file at all		// XML file at all
LOGERROR("CCacheLoader failed to find archived or source file for: \"%s\"", filename.string8());		LOGERROR("CCacheLoader failed to find archived or source file for: \"%s\"", filename.string8());
return PSRETURN_Xeromyces_XMLOpenFailed;		return PSRETURN_Xeromyces_XMLOpenFailed;
}		}

		m_ValidXMBFiles.emplace(filename.string8(), false);

// XMB isn't up to date with the XML, so rebuild it		// XMB isn't up to date with the XML, so rebuild it
return ConvertFile(vfs, filename, xmbPath, validatorName);		return ConvertFile(vfs, filename, xmbPath, validatorName);
}		}

bool CXeromyces::GenerateCachedXMB(const PIVFS& vfs, const VfsPath& sourcePath, VfsPath& archiveCachePath, const std::string& validatorName /* = "" */)		bool CXeromyces::GenerateCachedXMB(const PIVFS& vfs, const VfsPath& sourcePath, VfsPath& archiveCachePath, const std::string& validatorName /* = "" */)
{		{
CCacheLoader cacheLoader(vfs, L".xmb");		CCacheLoader cacheLoader(vfs, L".xmb");

archiveCachePath = cacheLoader.ArchiveCachePath(sourcePath);		archiveCachePath = cacheLoader.ArchiveCachePath(sourcePath);

return (ConvertFile(vfs, sourcePath, VfsPath("cache") / archiveCachePath, validatorName) == PSRETURN_OK);		return (ConvertFile(vfs, sourcePath, VfsPath("cache") / archiveCachePath, validatorName) == PSRETURN_OK);
}		}

PSRETURN CXeromyces::ConvertFile(const PIVFS& vfs, const VfsPath& filename, const VfsPath& xmbPath, const std::string& validatorName)		PSRETURN CXeromyces::ConvertFile(const PIVFS& vfs, const VfsPath& filename, const VfsPath& xmbPath, const std::string& validatorName)
{		{
CVFSFile input;		WriteBuffer writeBuffer;
if (input.Load(vfs, filename))
{
LOGERROR("CXeromyces: Failed to open XML file %s", filename.string8());
return PSRETURN_Xeromyces_XMLOpenFailed;
}

xmlDocPtr doc = xmlReadMemory((const char*)input.GetBuffer(), input.GetBufferSize(), CStrW(filename.string()).ToUTF8().c_str(), NULL,		{
XML_PARSE_NONET\|XML_PARSE_NOCDATA);		DataTreeXML xmlDoc;
if (!doc)		xmlDoc.Load(vfs, filename);
		if (!xmlDoc)
{		{
LOGERROR("CXeromyces: Failed to parse XML file %s", filename.string8());		LOGERROR("CXeromyces: Failed to parse XML file %s", filename.string8());
return PSRETURN_Xeromyces_XMLParseError;		return PSRETURN_Xeromyces_XMLParseError;
}		}

{		{
std::lock_guard<std::mutex> lock(g_ValidatorCacheLock);		std::lock_guard<std::mutex> lock(g_ValidatorCacheLock);
RelaxNGValidator& validator = GetValidator(validatorName);		RelaxNGValidator& validator = GetValidator(validatorName);
if (validator.CanValidate() && !validator.ValidateEncoded(doc))		if (validator.CanValidate() && !validator.ValidateEncoded(xmlDoc))
{		{
LOGERROR("CXeromyces: failed to validate XML file %s", filename.string8());		LOGERROR("CXeromyces: failed to validate XML file %s", filename.string8());
return PSRETURN_Xeromyces_XMLValidationFailed;		return PSRETURN_Xeromyces_XMLValidationFailed;
}		}
}		}

WriteBuffer writeBuffer;		CreateXMB(xmlDoc, writeBuffer);
CreateXMB(doc, writeBuffer);		}

xmlFreeDoc(doc);

// Save the file to disk, so it can be loaded quickly next time.		// Save the file to disk, so it can be loaded quickly next time.
// Don't save if invalid, because we want the syntax error every program start.		// Don't save if invalid, because we want the syntax error every program start.
vfs->CreateFile(xmbPath, writeBuffer.Data(), writeBuffer.Size());		vfs->CreateFile(xmbPath, writeBuffer.Data(), writeBuffer.Size());

m_XMBBuffer = writeBuffer.Data(); // add a reference		m_XMBBuffer = writeBuffer.Data(); // add a reference

// Set up the XMBFile		// Set up the XMBFile
Show All 39 Lines	if (!doc)
if (validator.CanValidate() && !validator.ValidateEncoded(doc))		if (validator.CanValidate() && !validator.ValidateEncoded(doc))
{		{
LOGERROR("CXeromyces: failed to validate XML string");		LOGERROR("CXeromyces: failed to validate XML string");
return PSRETURN_Xeromyces_XMLValidationFailed;		return PSRETURN_Xeromyces_XMLValidationFailed;
}		}
}		}

WriteBuffer writeBuffer;		WriteBuffer writeBuffer;
CreateXMB(doc, writeBuffer);		DataTreeXML tree;
		tree.SetDoc(doc);
xmlFreeDoc(doc);		CreateXMB(tree, writeBuffer);

m_XMBBuffer = writeBuffer.Data(); // add a reference		m_XMBBuffer = writeBuffer.Data(); // add a reference

// Set up the XMBFile		// Set up the XMBFile
const bool ok = Initialise((const char*)m_XMBBuffer.get());		const bool ok = Initialise((const char*)m_XMBBuffer.get());
ENSURE(ok);		ENSURE(ok);

return PSRETURN_OK;		return PSRETURN_OK;
▲ Show 20 Lines • Show All 114 Lines • ▼ Show 20 Lines	for (xmlNodePtr child = node->children; child; child = child->next)
if (child->type == XML_ELEMENT_NODE)		if (child->type == XML_ELEMENT_NODE)
OutputElement(child, writeBuffer, elementIDs, attributeIDs);		OutputElement(child, writeBuffer, elementIDs, attributeIDs);

// Go back and fill in the length		// Go back and fill in the length
u32 length = (u32)(writeBuffer.Size() - posLength);		u32 length = (u32)(writeBuffer.Size() - posLength);
writeBuffer.Overwrite(&length, 4, posLength);		writeBuffer.Overwrite(&length, 4, posLength);
}		}

PSRETURN CXeromyces::CreateXMB(const xmlDocPtr doc, WriteBuffer& writeBuffer)		PSRETURN CXeromyces::CreateXMB(const DataTreeXML& doc, WriteBuffer& writeBuffer)
{		{
// Header		// Header
writeBuffer.Append(UnfinishedHeaderMagicStr, 4);		writeBuffer.Append(UnfinishedHeaderMagicStr, 4);
// Version		// Version
writeBuffer.Append(&XMBVersion, 4);		writeBuffer.Append(&XMBVersion, 4);

		const std::set<VfsPath>& deps = doc.GetDependencies();
		u32 ndep = deps.size();
		writeBuffer.Append(&ndep, 4);

		// Output dependencies
		for (const VfsPath& dep : deps)
		{
		std::string st = dep.string8();
		u32 textLen = (u32)st.length()+1;
		writeBuffer.Append(&textLen, 4);
		writeBuffer.Append((void*)st.c_str(), textLen);
		}

u32 i;		u32 i;

// Find the unique element/attribute names		// Find the unique element/attribute names
std::set<std::string> elementNames;		std::set<std::string> elementNames;
std::set<std::string> attributeNames;		std::set<std::string> attributeNames;
FindNames(xmlDocGetRootElement(doc), elementNames, attributeNames);		FindNames(xmlDocGetRootElement(doc), elementNames, attributeNames);

std::map<std::string, u32> elementIDs;		std::map<std::string, u32> elementIDs;
Show All 33 Lines