Index: binaries/data/mods/public/simulation/templates/template_unit_infantry.xml =================================================================== --- binaries/data/mods/public/simulation/templates/template_unit_infantry.xml +++ binaries/data/mods/public/simulation/templates/template_unit_infantry.xml @@ -1,59 +1,30 @@ + - - 2 - 4 - 15 - - - - 2.5 - 4 - 1000 - Field Palisade Wall - - - - 50.0 - 0.0 - 0.0 - - 2 - - - - 1.0 - - structures/{civ}_civil_centre - structures/{civ}_crannog - structures/{civ}_military_colony - structures/{civ}_house - structures/{civ}_storehouse - structures/{civ}_farmstead - structures/{civ}_field - structures/{civ}_corral - structures/{civ}_outpost - structures/wallset_palisade - structures/{civ}_sentry_tower - structures/{civ}_dock - structures/{civ}_barracks - structures/{civ}_blacksmith - structures/{civ}_temple - structures/{civ}_market - structures/{civ}_defense_tower - structures/{civ}_wallset_stone - structures/{civ}_workshop - structures/{civ}_fortress - structures/{civ}_wonder - - - + + + 2.2 + 1 + 1.1 + + + + + 5 + -Field Jackson Five + + + + 10.0 + + + + + test + 10 - 50 - 0 - 0 - 0 + 10 Index: binaries/data/mods/public/simulation/templates/template_unit_support_female_citizen.xml =================================================================== --- binaries/data/mods/public/simulation/templates/template_unit_support_female_citizen.xml +++ binaries/data/mods/public/simulation/templates/template_unit_support_female_citizen.xml @@ -62,7 +62,7 @@ Citizen Worker Female Citizen template_unit_support_female_citizen - female + female Index: binaries/data/mods/public/simulation/templates/test.xml =================================================================== --- /dev/null +++ binaries/data/mods/public/simulation/templates/test.xml @@ -0,0 +1,26 @@ + + 1.0 + + structures/{civ}_civil_centre + structures/{civ}_crannog + structures/{civ}_military_colony + structures/{civ}_house + structures/{civ}_storehouse + structures/{civ}_farmstead + structures/{civ}_field + structures/{civ}_corral + structures/{civ}_outpost + structures/wallset_palisade + structures/{civ}_sentry_tower + structures/{civ}_dock + structures/{civ}_barracks + structures/{civ}_blacksmith + structures/{civ}_temple + structures/{civ}_market + structures/{civ}_defense_tower + structures/{civ}_wallset_stone + structures/{civ}_workshop + structures/{civ}_fortress + structures/{civ}_wonder + + Index: binaries/data/mods/public/simulation/templates/test_attarm.xml =================================================================== --- /dev/null +++ binaries/data/mods/public/simulation/templates/test_attarm.xml @@ -0,0 +1,23 @@ + + + 2 + 4 + 15 + + + + 2.5 + 4 + 1000 + Field Palisade Wall + + + + 50.0 + 0.0 + 0.0 + + 2 + + + Index: binaries/data/mods/public/simulation/templates/test_cost.xml =================================================================== --- /dev/null +++ binaries/data/mods/public/simulation/templates/test_cost.xml @@ -0,0 +1,11 @@ + + + 1 + 0 + + 40 + 0 + 0 + 0 + + Index: binaries/data/mods/public/simulation/templates/test_cost_champ.xml =================================================================== --- /dev/null +++ binaries/data/mods/public/simulation/templates/test_cost_champ.xml @@ -0,0 +1,9 @@ + + + + 2 + 2 + 2 + 2 + + Index: source/ps/DataTree.h =================================================================== --- /dev/null +++ source/ps/DataTree.h @@ -0,0 +1,187 @@ +/* Copyright (C) 2020 Wildfire Games. + * This file is part of 0 A.D. + * + * 0 A.D. is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * 0 A.D. is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with 0 A.D. If not, see . + */ + +#ifndef INCLUDED_DATATREE +#define INCLUDED_DATATREE + +#include "ps/Filesystem.h" + +template +class DataTree : public T +{ +// This defines the interface of T +private: + using doctype = typename T::doctype; + using nodetype = typename T::nodetype; + + bool LoadFile(const PIVFS& vfs, const VfsPath& filename) { return T::LoadFile(vfs, filename); } + bool CheckForInclude() const { return T::CheckForInclude(); } + + bool IsIncludeNode(const nodetype& node) const { return T::IsIncludeNode(node); } + std::vector GetIncludeFiles(const nodetype& node) const { return T::GetIncludeFiles(node); } + bool IsReplaceNode(const nodetype& node) const { return T::IsReplaceNode(node); } + + nodetype GetRootNode(const doctype& doc) const { return T::GetRootNode(doc); } + nodetype GetSameChild(const nodetype& node, const nodetype& similar_to) const { return T::GetSameChild(node, similar_to); } + + void UpdateNode(nodetype& node_to_update, const nodetype& reference) { T::UpdateNode(node_to_update, reference); } + void AddChild(nodetype& node, nodetype& child) { T::AddChild(node, child); } + void DeleteNode(nodetype& node) { T::DeleteNode(node); } + void PostMergeProcessing(nodetype& node) { T::PostMergeProcessing(node); } + +// Actual implementation +private: + PSRETURN MergeNodes(const PIVFS& vfs, nodetype& node, nodetype merge_from) + { + // Update node attributes + UpdateNode(node, merge_from); + + // Merge mode -> loop through our own nodes, if it exists in merge_from, modulate / replace / else + // If it doesn't, just use as-is. + // Then add all unused node from the target doc. + for (nodetype child = node.begin(); child != node.end(); ++child) + { + nodetype merge_child = GetSameChild(merge_from, child); + if (merge_child && !IsReplaceNode(child) && !IsIncludeNode(child)) + MergeNodes(vfs, child, merge_child); + else + ParseNode(vfs, child); + + if (merge_child) + DeleteNode(merge_child); + } + for (nodetype child : merge_from) + AddChild(node, child); + PostMergeProcessing(node); + return 0; + } + + PSRETURN ParseNode(const PIVFS& vfs, nodetype& node) + { + // Regular path -> just recurse. + if (!IsIncludeNode(node)) + // Cannot use a range-for loop as the iterator is the node and must be passed by reference below. + for (nodetype child = node.begin(); child != node.end(); ++child) + ParseNode(vfs, child); + else + { + // In this path, we'll load the included DataTree, then recursively merge us into them. + DataTree merge_from(m_IncludePaths); + std::vector include_files = GetIncludeFiles(node); + // Simple path: only one file. + if (include_files.size() == 1) + { + int err = merge_from.Load(vfs, include_files.front(), m_Filename); + m_Dependencies.insert(include_files.front()); + // Not sure if std::merge would be faster. + for (const VfsPath& path : merge_from.GetDependencies()) + m_Dependencies.insert(path); + if (err) + return err; + } + else + { + // Apply from end to start. + std::vector::const_reverse_iterator next = ++include_files.rbegin(); + DataTree current(m_IncludePaths); + current.Load(vfs, include_files.back(), m_Filename); + m_Dependencies.insert(include_files.back()); + while (next != include_files.rend()) + { + DataTree modulate(m_IncludePaths); + modulate.Load(vfs, *next, m_Filename); + m_Dependencies.insert(*next); + modulate.ApplyOver(vfs, current); // Handles dependencies + current = std::move(modulate); + ++next; + } + merge_from = std::move(current); + } + + MergeNodes(vfs, node, GetRootNode(merge_from)); + return 0; + } + return 0; + } +public: + DataTree() = default; + DataTree(const std::deque& includePaths) : m_IncludePaths(includePaths) {}; + + operator doctype() { return GetDoc(); } + operator doctype() const { return GetDoc(); } + doctype GetDoc() { return T::GetDoc(); } + const doctype GetDoc() const { return T::GetDoc(); } + + void AddIncludePath(const VfsPath& path) + { + // Trust the caller to not include the same path several time + // (it's not buggy, just inefficient). + m_IncludePaths.push_back(path); + } + + PSRETURN Load(const PIVFS& vfs, const VfsPath& filename, const VfsPath& include_from = "") + { + std::deque paths = m_IncludePaths; + paths.emplace_front(include_from); + for (const VfsPath& path : paths) + { + VfsPath file = path / filename; + if (vfs->GetFileInfo(file, 0) != INFO::OK) + continue; + if (!LoadFile(vfs, file)) + { + return 1; + } + m_Filename = file; + break; + } + + bool use_include = CheckForInclude(); + + if (!use_include) + return 0; + + // Iterate over nodes. + nodetype root = GetRootNode(GetDoc()); + ParseNode(vfs, root); + + return 0; + } + + PSRETURN ApplyOver(const PIVFS& vfs, DataTree& original) + { + // Not sure if std::merge would be faster. + for (const VfsPath& path : original.GetDependencies()) + m_Dependencies.insert(path); + + nodetype root = GetRootNode(GetDoc()); + nodetype original_root = GetRootNode(original.GetDoc()); + MergeNodes(vfs, root, original_root); + + return 0; + } + const std::set& GetDependencies() const { return m_Dependencies; } + +protected: + VfsPath m_Filename; + // Ordered list (decreasing priority) of include paths for new files. + std::deque m_IncludePaths; + // Set of all files this relies on (i.e. all inclusions). Excludes the original file. + std::set m_Dependencies; +}; + +#endif // INCLUDED_DATATREE Index: source/ps/DataTreeXML.h =================================================================== --- /dev/null +++ source/ps/DataTreeXML.h @@ -0,0 +1,121 @@ +/* Copyright (C) 2020 Wildfire Games. + * This file is part of 0 A.D. + * + * 0 A.D. is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * 0 A.D. is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with 0 A.D. If not, see . + */ + +#ifndef INCLUDED_DATATREEXML +#define INCLUDED_DATATREEXML + +#include "DataTree.h" + +#include + +class DataTreeXMLImpl +{ +protected: + struct NodeWrapper + { + NodeWrapper(xmlNodePtr p) : node(p) {} + + NodeWrapper begin() + { + if (!node->children || node->children->type == XML_ELEMENT_NODE) + return node->children; + return ++NodeWrapper(node->children); + } + NodeWrapper end() { return nullptr; }; + + NodeWrapper begin() const + { + if (!node->children || node->children->type == XML_ELEMENT_NODE) + return node->children; + return ++NodeWrapper(node->children); + } + NodeWrapper end() const { return nullptr; }; + + NodeWrapper& operator++() + { + do + node = node->next; + while (node && node->type != XML_ELEMENT_NODE); + return *this; + } + bool operator!=(const NodeWrapper& o) const { return node != o.node; } + + operator xmlNodePtr() const { return node; } + NodeWrapper& operator*() { return *this; } + xmlNodePtr operator->() const { return node; } + + xmlNodePtr node; + }; + using nodetype = NodeWrapper; + using doctype = xmlDocPtr; + +protected: + DataTreeXMLImpl() = default; + DataTreeXMLImpl(const DataTreeXMLImpl&) = delete; + DataTreeXMLImpl& operator=(const DataTreeXMLImpl&) = delete; + DataTreeXMLImpl(DataTreeXMLImpl&& o) + { + std::swap(doc, o.doc); + } + DataTreeXMLImpl& operator=(DataTreeXMLImpl&& o) + { + std::swap(doc, o.doc); + return *this; + } + + ~DataTreeXMLImpl() + { + //if (doc) + // xmlSaveFile(file.string8().c_str(), doc); + if (doc) + xmlFreeDoc(doc); + } + + bool LoadFile(const PIVFS& vfs, const VfsPath& filename); + bool CheckForInclude() const; + + bool IsIncludeNode(const nodetype& node) const; + std::vector GetIncludeFiles(const nodetype& node) const; + bool IsReplaceNode(const nodetype& node) const; + + nodetype GetRootNode(const doctype& doc) const; + nodetype GetSameChild(const nodetype& node, const nodetype& similar_to) const; + + void AddChild(nodetype& node, nodetype& child); + void DeleteNode(nodetype& node); + void UpdateNode(nodetype& node_to_update, const nodetype& reference); + + void PostMergeProcessing(nodetype& node); + + doctype GetDoc() { return doc; } + doctype GetDoc() const { return doc; } + +public: + void SetDoc(doctype d) + { + if (doc) + xmlFreeDoc(doc); + doc = d; + } +protected: + doctype doc = nullptr; + VfsPath file; +}; + +using DataTreeXML = DataTree; + +#endif // INCLUDED_DATATREEXML Index: source/ps/DataTreeXML.cpp =================================================================== --- /dev/null +++ source/ps/DataTreeXML.cpp @@ -0,0 +1,243 @@ +/* Copyright (C) 2020 Wildfire Games. + * This file is part of 0 A.D. + * + * 0 A.D. is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * 0 A.D. is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with 0 A.D. If not, see . + */ + +#include "precompiled.h" + +#include "DataTreeXML.h" + +#include "maths/Fixed.h" + +#include +#include // this isn't in string.hpp in old Boosts + +bool DataTreeXMLImpl::LoadFile(const PIVFS& vfs, const VfsPath& filename) +{ + CVFSFile input; + file = filename; + // Error reporting is handled in DataTree. + if (input.Load(vfs, filename)) + return false; + + doc = xmlReadMemory((const char*)input.GetBuffer(), input.GetBufferSize(), CStrW(filename.string()).ToUTF8().c_str(), NULL, + XML_PARSE_NONET|XML_PARSE_NOCDATA); + return true; +} + +bool DataTreeXMLImpl::CheckForInclude() const +{ + for (xmlNodePtr child = doc->children; child; child = child->next) + if (child->type == XML_PI_NODE && xmlStrEqual(child->name, (xmlChar*)"use_include")) + return true; + nodetype root = xmlDocGetRootElement(doc); + return IsIncludeNode(root); +} + +std::vector DataTreeXMLImpl::GetIncludeFiles(const nodetype& node) const +{ + xmlChar* xmlPath = nullptr; + if (xmlStrEqual(node->name, (xmlChar*)"include")) + xmlPath = xmlNodeGetContent(node); + else + xmlPath = xmlGetProp(node, (xmlChar*)"parent"); + + std::vector ret; + + std::string path((const char*)xmlPath); + std::string::const_iterator itStart = path.begin(), itEnd = path.begin(); + while (itEnd != path.end()) + { + // This won't detect a '|' in first position, but that's buggy so it's OK + // as the include path will contain it and fail to load (thus reporting the error). + while (*(++itEnd) != '|' && itEnd != path.end()) {} + + // Copy the sub-string, excluding '|' terminator. + VfsPath filename(std::string(&*itStart, itEnd-itStart)); + + // TODO: multiple include paths + ret.emplace_back(filename.ChangeExtension(".xml")); + if (itEnd != path.end()) + itStart = ++itEnd; + } + + if (xmlPath) + xmlFree(xmlPath); + + return ret; +} + +bool DataTreeXMLImpl::IsIncludeNode(const nodetype& node) const +{ + return node->type == XML_ELEMENT_NODE && + (xmlStrEqual(node->name, (xmlChar*)"include") || + xmlHasProp(node, (xmlChar*)"parent") != nullptr); +} + +bool DataTreeXMLImpl::IsReplaceNode(const nodetype& node) const +{ + return node->type == XML_ELEMENT_NODE && xmlHasProp(node, (xmlChar*)"replace") != nullptr; +} + +DataTreeXMLImpl::nodetype DataTreeXMLImpl::GetRootNode(const doctype& doc) const +{ + return xmlDocGetRootElement(doc); +} + +DataTreeXMLImpl::nodetype DataTreeXMLImpl::GetSameChild(const nodetype& node, const nodetype& similar_to) const +{ + for (xmlNodePtr child = node->children; child; child = child->next) + if (xmlStrEqual(child->name, similar_to->name)) + return child; + return nullptr; +} + + +void DataTreeXMLImpl::AddChild(nodetype& node, nodetype& child) +{ + xmlNodePtr add = xmlDocCopyNode(child, doc, 1); + xmlAddChild(node, add); +} + +void DataTreeXMLImpl::DeleteNode(nodetype& node) +{ + xmlUnlinkNode(node); + xmlFreeNode(node); +} + +void DataTreeXMLImpl::UpdateNode(nodetype& node_to_update, const nodetype& reference) +{ + if (xmlStrEqual(node_to_update->name, (xmlChar*)"include") || xmlHasProp(node_to_update, (xmlChar*)"parent") != nullptr) + { + // Copy the node and its prop, but not children. + xmlNodePtr ref = xmlDocCopyNode(reference, doc, 2); + // Swap exitsting children so they're preserved. + for (xmlNodePtr child = node_to_update->children; child;) + { + xmlNodePtr next = child->next; + if (child->type == XML_ELEMENT_NODE) + { + xmlUnlinkNode(child); + xmlAddChild(ref, child); + } + child = next; + } + // Replace the node. + xmlAddNextSibling(node_to_update, ref); + xmlUnlinkNode(node_to_update); + xmlFreeNode(node_to_update); + node_to_update = ref; + return; + } + if (xmlHasProp(node_to_update, (xmlChar*)"op") != nullptr) + { + xmlChar* op = xmlGetProp(node_to_update, (xmlChar*)"op"); + xmlChar* ref_val = xmlNodeGetContent(reference); + xmlChar* op_val = xmlNodeGetContent(node_to_update); + + fixed original_value = fixed::FromString(CStr((const char*)ref_val)); + fixed new_value = fixed::FromString(CStr((const char*)op_val)); + + if (xmlStrEqual(op, (xmlChar*)"add")) + new_value = original_value + new_value; + else if (xmlStrEqual(op, (xmlChar*)"mul")) + new_value = original_value.Multiply(new_value); + else if (xmlStrEqual(op, (xmlChar*)"mul_round")) + new_value = fixed::FromInt(original_value.Multiply(new_value).ToInt_RoundToNearest()); + else + LOGWARNING("Invalid op %s", (const char*)op_val); + + xmlNodeSetContent(node_to_update, (xmlChar*)new_value.ToString().c_str()); + + xmlFree(op_val); + xmlFree(ref_val); + xmlFree(op); + xmlUnsetProp(node_to_update, (xmlChar*)"op"); + return; + } + if (xmlHasProp(node_to_update, (xmlChar*)"datatype") != nullptr) + { + xmlChar* datatype = xmlGetProp(node_to_update, (xmlChar*)"datatype"); + if (xmlStrEqual(datatype, (xmlChar*)"tokens")) + { + // Split into tokens + std::vector oldTokens; + std::vector newTokens; + + xmlChar* reference_content = xmlNodeGetContent(reference); + xmlChar* node_content = xmlNodeGetContent(node_to_update); + std::string old_tokens = std::string((const char*)reference_content); + std::string new_tokens = std::string((const char*)node_content); + + boost::algorithm::split(oldTokens, old_tokens, boost::algorithm::is_space(), boost::algorithm::token_compress_on); + boost::algorithm::split(newTokens, new_tokens, boost::algorithm::is_space(), boost::algorithm::token_compress_on); + + // Merge the two lists + std::vector tokens = oldTokens; + for (const std::string& token : newTokens) + { + if (token[0] == L'-') + { + std::vector::iterator tokenIt = std::find(tokens.begin(), tokens.end(), token.substr(1)); + if (tokenIt != tokens.end()) + tokens.erase(tokenIt); + else + LOGWARNING("[DataTreeXML] Could not remove token '%s' from node '%s'; not present in list nor inherited (possible typo?)", + token.substr(1), (const char*)node_to_update->name/*, sourceIdentifier ? (" in '" + utf8_from_wstring(sourceIdentifier) + "'").c_str() : ""*/); + } + else + { + if (std::find(oldTokens.begin(), oldTokens.end(), token) == oldTokens.end()) + tokens.push_back(token); + } + } + + xmlNodeSetContent(node_to_update, (xmlChar*)boost::algorithm::join(tokens, L" ").c_str()); + + xmlFree(node_content); + xmlFree(reference_content); + } + xmlFree(datatype); + return; + } +} + +void DataTreeXMLImpl::PostMergeProcessing(nodetype& node) +{ + if (xmlHasProp(node, (xmlChar*)"replace") != nullptr) + xmlUnsetProp(node, (xmlChar*)"replace"); + + if (xmlHasProp(node, (xmlChar*)"disable") != nullptr) + { + // Disable node -> drop along with its children. + // We need to point node to the next sibling, if any. + nodetype next = node->next; + DeleteNode(node); + node = next; + } + if (xmlHasProp(node, (xmlChar*)"temporary") != nullptr) + { + // The node was a temporary node that needs to be dropped. + xmlNodePtr newnode = nullptr; + for (xmlNodePtr child = node->children; child;) + { + xmlNodePtr temp = child->next; + newnode = xmlAddPrevSibling(node, child); + child = temp; + } + DeleteNode(node); + node = newnode; + } +} Index: source/ps/TemplateLoader.cpp =================================================================== --- source/ps/TemplateLoader.cpp +++ source/ps/TemplateLoader.cpp @@ -70,7 +70,7 @@ } CXeromyces xero; - PSRETURN ok = xero.Load(g_VFS, path); + PSRETURN ok = xero.Load(g_VFS, path, "", "simulation/templates"); if (ok != PSRETURN_OK) return false; // (Xeromyces already logged an error with the full filename) @@ -83,7 +83,7 @@ VfsPath path = VfsPath(TEMPLATE_ROOT) / wstring_from_utf8(templateName + ".xml"); CXeromyces xero; - PSRETURN ok = xero.Load(g_VFS, path); + PSRETURN ok = xero.Load(g_VFS, path, "", "simulation/templates"); if (ok != PSRETURN_OK) return false; // (Xeromyces already logged an error with the full filename) Index: source/ps/XML/XeroXMB.h =================================================================== --- source/ps/XML/XeroXMB.h +++ source/ps/XML/XeroXMB.h @@ -37,6 +37,9 @@ char Header[4]; // because everyone has one; currently "XMB0" u32 Version; + u32 NumberOfDependencies; + ZStr8 DependenciesPath[]; + int ElementNameCount; ZStr8 ElementNames[]; @@ -123,7 +126,6 @@ // Returns the root element XMBElement GetRoot() const; - // Returns internal ID for a given element/attribute string. int GetElementID(const char* Name) const; int GetAttributeID(const char* Name) const; @@ -134,6 +136,9 @@ std::string GetElementString(const int ID) const; std::string GetAttributeString(const int ID) const; +protected: + std::set m_Dependencies; + private: const char* m_Pointer; Index: source/ps/XML/XeroXMB.cpp =================================================================== --- source/ps/XML/XeroXMB.cpp +++ source/ps/XML/XeroXMB.cpp @@ -26,7 +26,7 @@ const char* UnfinishedHeaderMagicStr = "XMBu"; // Arbitrary version number - change this if we update the code and // need to invalidate old users' caches -const u32 XMBVersion = 3; +const u32 XMBVersion = 4; template static inline T read(const void* ptr) @@ -52,6 +52,11 @@ if (Version != XMBVersion) return false; + u32 ndeps = read(m_Pointer); + m_Pointer += 4; + for (u32 u = 0; u < ndeps; ++u) + m_Dependencies.insert(ReadZStr8()); + int i; // FIXME Check that m_Pointer doesn't end up past the end of the buffer Index: source/ps/XML/Xeromyces.h =================================================================== --- source/ps/XML/Xeromyces.h +++ source/ps/XML/Xeromyces.h @@ -40,6 +40,11 @@ typedef struct _xmlDoc xmlDoc; typedef xmlDoc* xmlDocPtr; +template +class DataTree; +class DataTreeXMLImpl; +typedef DataTree DataTreeXML; + class CXeromyces : public XMBFile { friend class TestXeroXMB; @@ -47,7 +52,7 @@ /** * Load from an XML file (with invisible XMB caching). */ - PSRETURN Load(const PIVFS& vfs, const VfsPath& filename, const std::string& validatorName = ""); + PSRETURN Load(const PIVFS& vfs, const VfsPath& filename, const std::string& validatorName = "", const VfsPath& includePath = ""); /** * Load from an in-memory XML string (with no caching). @@ -79,11 +84,11 @@ private: static RelaxNGValidator& GetValidator(const std::string& name); - PSRETURN ConvertFile(const PIVFS& vfs, const VfsPath& filename, const VfsPath& xmbPath, const std::string& validatorName); + PSRETURN ConvertFile(const PIVFS& vfs, const VfsPath& filename, const VfsPath& xmbPath, const std::string& validatorName, const VfsPath& includePath); bool ReadXMBFile(const PIVFS& vfs, const VfsPath& filename); - static PSRETURN CreateXMB(const xmlDocPtr doc, WriteBuffer& writeBuffer); + static PSRETURN CreateXMB(const DataTreeXML& doc, WriteBuffer& writeBuffer); shared_ptr m_XMBBuffer; }; Index: source/ps/XML/Xeromyces.cpp =================================================================== --- source/ps/XML/Xeromyces.cpp +++ source/ps/XML/Xeromyces.cpp @@ -17,6 +17,8 @@ #include "precompiled.h" +#include "Xeromyces.h" + #include #include #include @@ -27,16 +29,19 @@ #include "maths/MD5.h" #include "ps/CacheLoader.h" #include "ps/CLogger.h" +#include "ps/DataTreeXML.h" #include "ps/Filesystem.h" #include "RelaxNG.h" -#include "Xeromyces.h" #include +#include static std::mutex g_ValidatorCacheLock; static std::map g_ValidatorCache; static bool g_XeromycesStarted = false; +static std::unordered_map m_ValidXMBFiles; + static void errorHandler(void* UNUSED(userData), xmlErrorPtr error) { // Strip a trailing newline @@ -58,6 +63,7 @@ xmlSetStructuredErrorFunc(NULL, &errorHandler); std::lock_guard lock(g_ValidatorCacheLock); g_ValidatorCache.insert(std::make_pair(std::string(), RelaxNGValidator())); + m_ValidXMBFiles.clear(); g_XeromycesStarted = true; } @@ -68,6 +74,7 @@ ClearSchemaCache(); std::lock_guard lock(g_ValidatorCacheLock); g_ValidatorCache.clear(); + m_ValidXMBFiles.clear(); xmlSetStructuredErrorFunc(NULL, NULL); xmlCleanupParser(); } @@ -108,7 +115,7 @@ return g_ValidatorCache.find(name)->second; } -PSRETURN CXeromyces::Load(const PIVFS& vfs, const VfsPath& filename, const std::string& validatorName /* = "" */) +PSRETURN CXeromyces::Load(const PIVFS& vfs, const VfsPath& filename, const std::string& validatorName /* = "" */, const VfsPath& includePath /* = "" */) { ENSURE(g_XeromycesStarted); @@ -126,7 +133,47 @@ { // Found a cached XMB - load it if (ReadXMBFile(vfs, xmbPath)) - return PSRETURN_OK; + { + // Check that dependencies aren't invalid themselves. + bool valid = true; + bool unknowns = false; + // Iterate through dependencies - first check for any explicitly invalid xmb files. + for (const std::string& dep : m_Dependencies) + { + std::unordered_map::const_iterator it = m_ValidXMBFiles.find(dep); + if (it != m_ValidXMBFiles.end() && !it->second) + { + // A dependency isn't valid - break + valid = false; + break; + } else if (it != m_ValidXMBFiles.end()) + unknowns = true; + } + // Then check for any unknowns - load them and verify. + // This is done in a 2nd pass as opening files is slower and might not be needed if we're lucky. + if (unknowns) + { + for (const std::string& dep : m_Dependencies) + { + std::unordered_map::const_iterator it = m_ValidXMBFiles.find(dep); + if (it != m_ValidXMBFiles.end()) + continue; + CXeromyces temp; + temp.Load(vfs, dep); + it = m_ValidXMBFiles.find(dep); + ENSURE(it != m_ValidXMBFiles.end()); + valid &= it->second; + } + } + + if (valid) + { + // TODO c++17: try_emplace + if (m_ValidXMBFiles.find(filename.string8()) == m_ValidXMBFiles.end()) + m_ValidXMBFiles.emplace(filename.string8(), true); + return PSRETURN_OK; + } + } // If this fails then we'll continue and (re)create the loose cache - // this failure legitimately happens due to partially-written XMB files. } @@ -138,14 +185,17 @@ { ENSURE(ret < 0); + m_ValidXMBFiles.emplace(filename.string8(), false); // No source file or archive cache was found, so we can't load the // XML file at all LOGERROR("CCacheLoader failed to find archived or source file for: \"%s\"", filename.string8()); return PSRETURN_Xeromyces_XMLOpenFailed; } + m_ValidXMBFiles.emplace(filename.string8(), false); + // XMB isn't up to date with the XML, so rebuild it - return ConvertFile(vfs, filename, xmbPath, validatorName); + return ConvertFile(vfs, filename, xmbPath, validatorName, includePath); } bool CXeromyces::GenerateCachedXMB(const PIVFS& vfs, const VfsPath& sourcePath, VfsPath& archiveCachePath, const std::string& validatorName /* = "" */) @@ -154,40 +204,36 @@ archiveCachePath = cacheLoader.ArchiveCachePath(sourcePath); - return (ConvertFile(vfs, sourcePath, VfsPath("cache") / archiveCachePath, validatorName) == PSRETURN_OK); + return (ConvertFile(vfs, sourcePath, VfsPath("cache") / archiveCachePath, validatorName, "") == PSRETURN_OK); } -PSRETURN CXeromyces::ConvertFile(const PIVFS& vfs, const VfsPath& filename, const VfsPath& xmbPath, const std::string& validatorName) +PSRETURN CXeromyces::ConvertFile(const PIVFS& vfs, const VfsPath& filename, const VfsPath& xmbPath, const std::string& validatorName, const VfsPath& includePath) { - CVFSFile input; - if (input.Load(vfs, filename)) - { - LOGERROR("CXeromyces: Failed to open XML file %s", filename.string8()); - return PSRETURN_Xeromyces_XMLOpenFailed; - } - - xmlDocPtr doc = xmlReadMemory((const char*)input.GetBuffer(), input.GetBufferSize(), CStrW(filename.string()).ToUTF8().c_str(), NULL, - XML_PARSE_NONET|XML_PARSE_NOCDATA); - if (!doc) - { - LOGERROR("CXeromyces: Failed to parse XML file %s", filename.string8()); - return PSRETURN_Xeromyces_XMLParseError; - } - - { - std::lock_guard lock(g_ValidatorCacheLock); - RelaxNGValidator& validator = GetValidator(validatorName); - if (validator.CanValidate() && !validator.ValidateEncoded(doc)) - { - LOGERROR("CXeromyces: failed to validate XML file %s", filename.string8()); - return PSRETURN_Xeromyces_XMLValidationFailed; - } - } - WriteBuffer writeBuffer; - CreateXMB(doc, writeBuffer); - xmlFreeDoc(doc); + { + DataTreeXML xmlDoc; + if (includePath.string8().size() != 0) + xmlDoc.AddIncludePath(includePath); + xmlDoc.Load(vfs, filename); + if (!xmlDoc) + { + LOGERROR("CXeromyces: Failed to parse XML file %s", filename.string8()); + return PSRETURN_Xeromyces_XMLParseError; + } + + { + std::lock_guard lock(g_ValidatorCacheLock); + RelaxNGValidator& validator = GetValidator(validatorName); + if (validator.CanValidate() && !validator.ValidateEncoded(xmlDoc)) + { + LOGERROR("CXeromyces: failed to validate XML file %s", filename.string8()); + return PSRETURN_Xeromyces_XMLValidationFailed; + } + } + + CreateXMB(xmlDoc, writeBuffer); + } // Save the file to disk, so it can be loaded quickly next time. // Don't save if invalid, because we want the syntax error every program start. @@ -243,9 +289,9 @@ } WriteBuffer writeBuffer; - CreateXMB(doc, writeBuffer); - - xmlFreeDoc(doc); + DataTreeXML tree; + tree.SetDoc(doc); + CreateXMB(tree, writeBuffer); m_XMBBuffer = writeBuffer.Data(); // add a reference @@ -376,13 +422,26 @@ writeBuffer.Overwrite(&length, 4, posLength); } -PSRETURN CXeromyces::CreateXMB(const xmlDocPtr doc, WriteBuffer& writeBuffer) +PSRETURN CXeromyces::CreateXMB(const DataTreeXML& doc, WriteBuffer& writeBuffer) { // Header writeBuffer.Append(UnfinishedHeaderMagicStr, 4); // Version writeBuffer.Append(&XMBVersion, 4); + const std::set& deps = doc.GetDependencies(); + u32 ndep = deps.size(); + writeBuffer.Append(&ndep, 4); + + // Output dependencies + for (const VfsPath& dep : deps) + { + std::string st = dep.string8(); + u32 textLen = (u32)st.length()+1; + writeBuffer.Append(&textLen, 4); + writeBuffer.Append((void*)st.c_str(), textLen); + } + u32 i; // Find the unique element/attribute names Index: source/ps/tests/test_DataTree.h =================================================================== --- /dev/null +++ source/ps/tests/test_DataTree.h @@ -0,0 +1,48 @@ +/* Copyright (C) 2019 Wildfire Games. + * This file is part of 0 A.D. + * + * 0 A.D. is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * 0 A.D. is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with 0 A.D. If not, see . + */ + +#include "lib/self_test.h" + +#include "ps/DataTree.h" +#include "ps/DataTreeXML.h" +#include "ps/Filesystem.h" + +#include + +class TestDataTree : public CxxTest::TestSuite +{ +public: + void setUp() + { + g_VFS = CreateVfs(); + TS_ASSERT_OK(g_VFS->Mount(L"", DataDir()/"mods"/"_test.datatree")); + TS_ASSERT_OK(g_VFS->Mount(L"cache", DataDir()/"_testcache")); + CXeromyces::Startup(); + } + + void tearDown() + { + CXeromyces::Terminate(); + g_VFS.reset(); + DeleteDirectory(DataDir()/"_test.datatree"); + DeleteDirectory(DataDir()/"_testcache"); + } + + void test_xml_tree() + { + } +}; Index: source/simulation2/system/ParamNode.cpp =================================================================== --- source/simulation2/system/ParamNode.cpp +++ source/simulation2/system/ParamNode.cpp @@ -45,7 +45,7 @@ void CParamNode::LoadXML(CParamNode& ret, const VfsPath& path, const std::string& validatorName) { CXeromyces xero; - PSRETURN ok = xero.Load(g_VFS, path, validatorName); + PSRETURN ok = xero.Load(g_VFS, path, validatorName, "simulation/templates"); if (ok != PSRETURN_OK) return; // (Xeromyces already logged an error)