Changeset View
Changeset View
Standalone View
Standalone View
source/ps/XML/Xeromyces.cpp
Show All 11 Lines | |||||
* GNU General Public License for more details. | * GNU General Public License for more details. | ||||
* | * | ||||
* You should have received a copy of the GNU General Public License | * You should have received a copy of the GNU General Public License | ||||
* along with 0 A.D. If not, see <http://www.gnu.org/licenses/>. | * along with 0 A.D. If not, see <http://www.gnu.org/licenses/>. | ||||
*/ | */ | ||||
#include "precompiled.h" | #include "precompiled.h" | ||||
#include "Xeromyces.h" | |||||
#include <vector> | #include <vector> | ||||
#include <set> | #include <set> | ||||
#include <map> | #include <map> | ||||
#include <mutex> | #include <mutex> | ||||
#include <stack> | #include <stack> | ||||
#include <algorithm> | #include <algorithm> | ||||
#include "maths/MD5.h" | #include "maths/MD5.h" | ||||
#include "ps/CacheLoader.h" | #include "ps/CacheLoader.h" | ||||
#include "ps/CLogger.h" | #include "ps/CLogger.h" | ||||
#include "ps/DataTreeXML.h" | |||||
#include "ps/Filesystem.h" | #include "ps/Filesystem.h" | ||||
#include "RelaxNG.h" | #include "RelaxNG.h" | ||||
#include "Xeromyces.h" | |||||
#include <libxml/parser.h> | #include <libxml/parser.h> | ||||
#include <unordered_map> | |||||
static std::mutex g_ValidatorCacheLock; | static std::mutex g_ValidatorCacheLock; | ||||
static std::map<const std::string, RelaxNGValidator> g_ValidatorCache; | static std::map<const std::string, RelaxNGValidator> g_ValidatorCache; | ||||
static bool g_XeromycesStarted = false; | static bool g_XeromycesStarted = false; | ||||
static std::unordered_map<std::string, bool> m_ValidXMBFiles; | |||||
static void errorHandler(void* UNUSED(userData), xmlErrorPtr error) | static void errorHandler(void* UNUSED(userData), xmlErrorPtr error) | ||||
{ | { | ||||
// Strip a trailing newline | // Strip a trailing newline | ||||
std::string message = error->message; | std::string message = error->message; | ||||
if (message.length() > 0 && message[message.length()-1] == '\n') | if (message.length() > 0 && message[message.length()-1] == '\n') | ||||
message.erase(message.length()-1); | message.erase(message.length()-1); | ||||
LOGERROR("CXeromyces: Parse %s: %s:%d: %s", | LOGERROR("CXeromyces: Parse %s: %s:%d: %s", | ||||
error->level == XML_ERR_WARNING ? "warning" : "error", | error->level == XML_ERR_WARNING ? "warning" : "error", | ||||
error->file, error->line, message); | error->file, error->line, message); | ||||
// TODO: The (non-fatal) warnings and errors don't get stored in the XMB, | // TODO: The (non-fatal) warnings and errors don't get stored in the XMB, | ||||
// so the caching is less transparent than it should be | // so the caching is less transparent than it should be | ||||
} | } | ||||
void CXeromyces::Startup() | void CXeromyces::Startup() | ||||
{ | { | ||||
ENSURE(!g_XeromycesStarted); | ENSURE(!g_XeromycesStarted); | ||||
xmlInitParser(); | xmlInitParser(); | ||||
xmlSetStructuredErrorFunc(NULL, &errorHandler); | xmlSetStructuredErrorFunc(NULL, &errorHandler); | ||||
std::lock_guard<std::mutex> lock(g_ValidatorCacheLock); | std::lock_guard<std::mutex> lock(g_ValidatorCacheLock); | ||||
g_ValidatorCache.insert(std::make_pair(std::string(), RelaxNGValidator())); | g_ValidatorCache.insert(std::make_pair(std::string(), RelaxNGValidator())); | ||||
m_ValidXMBFiles.clear(); | |||||
g_XeromycesStarted = true; | g_XeromycesStarted = true; | ||||
} | } | ||||
void CXeromyces::Terminate() | void CXeromyces::Terminate() | ||||
{ | { | ||||
ENSURE(g_XeromycesStarted); | ENSURE(g_XeromycesStarted); | ||||
g_XeromycesStarted = false; | g_XeromycesStarted = false; | ||||
ClearSchemaCache(); | ClearSchemaCache(); | ||||
std::lock_guard<std::mutex> lock(g_ValidatorCacheLock); | std::lock_guard<std::mutex> lock(g_ValidatorCacheLock); | ||||
g_ValidatorCache.clear(); | g_ValidatorCache.clear(); | ||||
m_ValidXMBFiles.clear(); | |||||
xmlSetStructuredErrorFunc(NULL, NULL); | xmlSetStructuredErrorFunc(NULL, NULL); | ||||
xmlCleanupParser(); | xmlCleanupParser(); | ||||
} | } | ||||
bool CXeromyces::AddValidator(const PIVFS& vfs, const std::string& name, const VfsPath& grammarPath) | bool CXeromyces::AddValidator(const PIVFS& vfs, const std::string& name, const VfsPath& grammarPath) | ||||
{ | { | ||||
ENSURE(g_XeromycesStarted); | ENSURE(g_XeromycesStarted); | ||||
Show All 24 Lines | |||||
*/ | */ | ||||
RelaxNGValidator& CXeromyces::GetValidator(const std::string& name) | RelaxNGValidator& CXeromyces::GetValidator(const std::string& name) | ||||
{ | { | ||||
if (g_ValidatorCache.find(name) == g_ValidatorCache.end()) | if (g_ValidatorCache.find(name) == g_ValidatorCache.end()) | ||||
return g_ValidatorCache.find("")->second; | return g_ValidatorCache.find("")->second; | ||||
return g_ValidatorCache.find(name)->second; | return g_ValidatorCache.find(name)->second; | ||||
} | } | ||||
PSRETURN CXeromyces::Load(const PIVFS& vfs, const VfsPath& filename, const std::string& validatorName /* = "" */) | PSRETURN CXeromyces::Load(const PIVFS& vfs, const VfsPath& filename, const std::string& validatorName /* = "" */, const VfsPath& includePath /* = "" */) | ||||
{ | { | ||||
ENSURE(g_XeromycesStarted); | ENSURE(g_XeromycesStarted); | ||||
CCacheLoader cacheLoader(vfs, L".xmb"); | CCacheLoader cacheLoader(vfs, L".xmb"); | ||||
MD5 validatorGrammarHash; | MD5 validatorGrammarHash; | ||||
{ | { | ||||
std::lock_guard<std::mutex> lock(g_ValidatorCacheLock); | std::lock_guard<std::mutex> lock(g_ValidatorCacheLock); | ||||
validatorGrammarHash = GetValidator(validatorName).GetGrammarHash(); | validatorGrammarHash = GetValidator(validatorName).GetGrammarHash(); | ||||
} | } | ||||
VfsPath xmbPath; | VfsPath xmbPath; | ||||
Status ret = cacheLoader.TryLoadingCached(filename, validatorGrammarHash, XMBVersion, xmbPath); | Status ret = cacheLoader.TryLoadingCached(filename, validatorGrammarHash, XMBVersion, xmbPath); | ||||
if (ret == INFO::OK) | if (ret == INFO::OK) | ||||
{ | { | ||||
// Found a cached XMB - load it | // Found a cached XMB - load it | ||||
if (ReadXMBFile(vfs, xmbPath)) | if (ReadXMBFile(vfs, xmbPath)) | ||||
{ | |||||
// Check that dependencies aren't invalid themselves. | |||||
bool valid = true; | |||||
bool unknowns = false; | |||||
// Iterate through dependencies - first check for any explicitly invalid xmb files. | |||||
for (const std::string& dep : m_Dependencies) | |||||
{ | |||||
std::unordered_map<std::string, bool>::const_iterator it = m_ValidXMBFiles.find(dep); | |||||
if (it != m_ValidXMBFiles.end() && !it->second) | |||||
{ | |||||
// A dependency isn't valid - break | |||||
valid = false; | |||||
break; | |||||
} else if (it != m_ValidXMBFiles.end()) | |||||
unknowns = true; | |||||
} | |||||
// Then check for any unknowns - load them and verify. | |||||
// This is done in a 2nd pass as opening files is slower and might not be needed if we're lucky. | |||||
if (unknowns) | |||||
{ | |||||
for (const std::string& dep : m_Dependencies) | |||||
{ | |||||
std::unordered_map<std::string, bool>::const_iterator it = m_ValidXMBFiles.find(dep); | |||||
if (it != m_ValidXMBFiles.end()) | |||||
continue; | |||||
CXeromyces temp; | |||||
temp.Load(vfs, dep); | |||||
it = m_ValidXMBFiles.find(dep); | |||||
ENSURE(it != m_ValidXMBFiles.end()); | |||||
valid &= it->second; | |||||
} | |||||
} | |||||
if (valid) | |||||
{ | |||||
// TODO c++17: try_emplace | |||||
if (m_ValidXMBFiles.find(filename.string8()) == m_ValidXMBFiles.end()) | |||||
m_ValidXMBFiles.emplace(filename.string8(), true); | |||||
return PSRETURN_OK; | return PSRETURN_OK; | ||||
} | |||||
} | |||||
// If this fails then we'll continue and (re)create the loose cache - | // If this fails then we'll continue and (re)create the loose cache - | ||||
// this failure legitimately happens due to partially-written XMB files. | // this failure legitimately happens due to partially-written XMB files. | ||||
} | } | ||||
else if (ret == INFO::SKIPPED) | else if (ret == INFO::SKIPPED) | ||||
{ | { | ||||
// No cached version was found - we'll need to create it | // No cached version was found - we'll need to create it | ||||
} | } | ||||
else | else | ||||
{ | { | ||||
ENSURE(ret < 0); | ENSURE(ret < 0); | ||||
m_ValidXMBFiles.emplace(filename.string8(), false); | |||||
// No source file or archive cache was found, so we can't load the | // No source file or archive cache was found, so we can't load the | ||||
// XML file at all | // XML file at all | ||||
LOGERROR("CCacheLoader failed to find archived or source file for: \"%s\"", filename.string8()); | LOGERROR("CCacheLoader failed to find archived or source file for: \"%s\"", filename.string8()); | ||||
return PSRETURN_Xeromyces_XMLOpenFailed; | return PSRETURN_Xeromyces_XMLOpenFailed; | ||||
} | } | ||||
m_ValidXMBFiles.emplace(filename.string8(), false); | |||||
// XMB isn't up to date with the XML, so rebuild it | // XMB isn't up to date with the XML, so rebuild it | ||||
return ConvertFile(vfs, filename, xmbPath, validatorName); | return ConvertFile(vfs, filename, xmbPath, validatorName, includePath); | ||||
} | } | ||||
bool CXeromyces::GenerateCachedXMB(const PIVFS& vfs, const VfsPath& sourcePath, VfsPath& archiveCachePath, const std::string& validatorName /* = "" */) | bool CXeromyces::GenerateCachedXMB(const PIVFS& vfs, const VfsPath& sourcePath, VfsPath& archiveCachePath, const std::string& validatorName /* = "" */) | ||||
{ | { | ||||
CCacheLoader cacheLoader(vfs, L".xmb"); | CCacheLoader cacheLoader(vfs, L".xmb"); | ||||
archiveCachePath = cacheLoader.ArchiveCachePath(sourcePath); | archiveCachePath = cacheLoader.ArchiveCachePath(sourcePath); | ||||
return (ConvertFile(vfs, sourcePath, VfsPath("cache") / archiveCachePath, validatorName) == PSRETURN_OK); | return (ConvertFile(vfs, sourcePath, VfsPath("cache") / archiveCachePath, validatorName, "") == PSRETURN_OK); | ||||
} | } | ||||
PSRETURN CXeromyces::ConvertFile(const PIVFS& vfs, const VfsPath& filename, const VfsPath& xmbPath, const std::string& validatorName) | PSRETURN CXeromyces::ConvertFile(const PIVFS& vfs, const VfsPath& filename, const VfsPath& xmbPath, const std::string& validatorName, const VfsPath& includePath) | ||||
{ | { | ||||
CVFSFile input; | WriteBuffer writeBuffer; | ||||
if (input.Load(vfs, filename)) | |||||
{ | |||||
LOGERROR("CXeromyces: Failed to open XML file %s", filename.string8()); | |||||
return PSRETURN_Xeromyces_XMLOpenFailed; | |||||
} | |||||
xmlDocPtr doc = xmlReadMemory((const char*)input.GetBuffer(), input.GetBufferSize(), CStrW(filename.string()).ToUTF8().c_str(), NULL, | { | ||||
XML_PARSE_NONET|XML_PARSE_NOCDATA); | DataTreeXML xmlDoc; | ||||
if (!doc) | if (includePath.string8().size() != 0) | ||||
xmlDoc.AddIncludePath(includePath); | |||||
xmlDoc.Load(vfs, filename); | |||||
if (!xmlDoc) | |||||
{ | { | ||||
LOGERROR("CXeromyces: Failed to parse XML file %s", filename.string8()); | LOGERROR("CXeromyces: Failed to parse XML file %s", filename.string8()); | ||||
return PSRETURN_Xeromyces_XMLParseError; | return PSRETURN_Xeromyces_XMLParseError; | ||||
} | } | ||||
{ | { | ||||
std::lock_guard<std::mutex> lock(g_ValidatorCacheLock); | std::lock_guard<std::mutex> lock(g_ValidatorCacheLock); | ||||
RelaxNGValidator& validator = GetValidator(validatorName); | RelaxNGValidator& validator = GetValidator(validatorName); | ||||
if (validator.CanValidate() && !validator.ValidateEncoded(doc)) | if (validator.CanValidate() && !validator.ValidateEncoded(xmlDoc)) | ||||
{ | { | ||||
LOGERROR("CXeromyces: failed to validate XML file %s", filename.string8()); | LOGERROR("CXeromyces: failed to validate XML file %s", filename.string8()); | ||||
return PSRETURN_Xeromyces_XMLValidationFailed; | return PSRETURN_Xeromyces_XMLValidationFailed; | ||||
} | } | ||||
} | } | ||||
WriteBuffer writeBuffer; | CreateXMB(xmlDoc, writeBuffer); | ||||
CreateXMB(doc, writeBuffer); | } | ||||
xmlFreeDoc(doc); | |||||
// Save the file to disk, so it can be loaded quickly next time. | // Save the file to disk, so it can be loaded quickly next time. | ||||
// Don't save if invalid, because we want the syntax error every program start. | // Don't save if invalid, because we want the syntax error every program start. | ||||
vfs->CreateFile(xmbPath, writeBuffer.Data(), writeBuffer.Size()); | vfs->CreateFile(xmbPath, writeBuffer.Data(), writeBuffer.Size()); | ||||
m_XMBBuffer = writeBuffer.Data(); // add a reference | m_XMBBuffer = writeBuffer.Data(); // add a reference | ||||
// Set up the XMBFile | // Set up the XMBFile | ||||
Show All 39 Lines | if (!doc) | ||||
if (validator.CanValidate() && !validator.ValidateEncoded(doc)) | if (validator.CanValidate() && !validator.ValidateEncoded(doc)) | ||||
{ | { | ||||
LOGERROR("CXeromyces: failed to validate XML string"); | LOGERROR("CXeromyces: failed to validate XML string"); | ||||
return PSRETURN_Xeromyces_XMLValidationFailed; | return PSRETURN_Xeromyces_XMLValidationFailed; | ||||
} | } | ||||
} | } | ||||
WriteBuffer writeBuffer; | WriteBuffer writeBuffer; | ||||
CreateXMB(doc, writeBuffer); | DataTreeXML tree; | ||||
tree.SetDoc(doc); | |||||
xmlFreeDoc(doc); | CreateXMB(tree, writeBuffer); | ||||
m_XMBBuffer = writeBuffer.Data(); // add a reference | m_XMBBuffer = writeBuffer.Data(); // add a reference | ||||
// Set up the XMBFile | // Set up the XMBFile | ||||
const bool ok = Initialise((const char*)m_XMBBuffer.get()); | const bool ok = Initialise((const char*)m_XMBBuffer.get()); | ||||
ENSURE(ok); | ENSURE(ok); | ||||
return PSRETURN_OK; | return PSRETURN_OK; | ||||
▲ Show 20 Lines • Show All 114 Lines • ▼ Show 20 Lines | for (xmlNodePtr child = node->children; child; child = child->next) | ||||
if (child->type == XML_ELEMENT_NODE) | if (child->type == XML_ELEMENT_NODE) | ||||
OutputElement(child, writeBuffer, elementIDs, attributeIDs); | OutputElement(child, writeBuffer, elementIDs, attributeIDs); | ||||
// Go back and fill in the length | // Go back and fill in the length | ||||
u32 length = (u32)(writeBuffer.Size() - posLength); | u32 length = (u32)(writeBuffer.Size() - posLength); | ||||
writeBuffer.Overwrite(&length, 4, posLength); | writeBuffer.Overwrite(&length, 4, posLength); | ||||
} | } | ||||
PSRETURN CXeromyces::CreateXMB(const xmlDocPtr doc, WriteBuffer& writeBuffer) | PSRETURN CXeromyces::CreateXMB(const DataTreeXML& doc, WriteBuffer& writeBuffer) | ||||
{ | { | ||||
// Header | // Header | ||||
writeBuffer.Append(UnfinishedHeaderMagicStr, 4); | writeBuffer.Append(UnfinishedHeaderMagicStr, 4); | ||||
// Version | // Version | ||||
writeBuffer.Append(&XMBVersion, 4); | writeBuffer.Append(&XMBVersion, 4); | ||||
const std::set<VfsPath>& deps = doc.GetDependencies(); | |||||
u32 ndep = deps.size(); | |||||
writeBuffer.Append(&ndep, 4); | |||||
// Output dependencies | |||||
for (const VfsPath& dep : deps) | |||||
{ | |||||
std::string st = dep.string8(); | |||||
u32 textLen = (u32)st.length()+1; | |||||
writeBuffer.Append(&textLen, 4); | |||||
writeBuffer.Append((void*)st.c_str(), textLen); | |||||
} | |||||
u32 i; | u32 i; | ||||
// Find the unique element/attribute names | // Find the unique element/attribute names | ||||
std::set<std::string> elementNames; | std::set<std::string> elementNames; | ||||
std::set<std::string> attributeNames; | std::set<std::string> attributeNames; | ||||
FindNames(xmlDocGetRootElement(doc), elementNames, attributeNames); | FindNames(xmlDocGetRootElement(doc), elementNames, attributeNames); | ||||
std::map<std::string, u32> elementIDs; | std::map<std::string, u32> elementIDs; | ||||
Show All 33 Lines |
Wildfire Games · Phabricator