Index: ps/trunk/build/premake/premake5.lua =================================================================== --- ps/trunk/build/premake/premake5.lua +++ ps/trunk/build/premake/premake5.lua @@ -731,6 +731,7 @@ "ps/scripting", "network/scripting", "ps/GameSetup", + "ps/XMB", "ps/XML", "soundmanager", "soundmanager/data", Index: ps/trunk/source/graphics/MapReader.cpp =================================================================== --- ps/trunk/source/graphics/MapReader.cpp +++ ps/trunk/source/graphics/MapReader.cpp @@ -491,7 +491,7 @@ #undef EL XMBElement root = xmb_file.GetRoot(); - ENSURE(xmb_file.GetElementString(root.GetNodeName()) == "Scenario"); + ENSURE(xmb_file.GetElementStringView(root.GetNodeName()) == "Scenario"); nodes = root.GetChildNodes(); // find out total number of entities+nonentities @@ -517,7 +517,7 @@ CStr CXMLReader::ReadScriptSettings() { XMBElement root = xmb_file.GetRoot(); - ENSURE(xmb_file.GetElementString(root.GetNodeName()) == "Scenario"); + ENSURE(xmb_file.GetElementStringView(root.GetNodeName()) == "Scenario"); nodes = root.GetChildNodes(); XMBElement settings = nodes.GetFirstNamedItem(xmb_file.GetElementID("ScriptSettings")); Index: ps/trunk/source/graphics/ObjectBase.cpp =================================================================== --- ps/trunk/source/graphics/ObjectBase.cpp +++ ps/trunk/source/graphics/ObjectBase.cpp @@ -195,7 +195,7 @@ if (variant.GetNodeName() != el_variant) { - LOGERROR("Invalid variant format (unrecognised root element '%s')", XeroFile.GetElementString(variant.GetNodeName()).c_str()); + LOGERROR("Invalid variant format (unrecognised root element '%s')", XeroFile.GetElementString(variant.GetNodeName())); return false; } @@ -821,7 +821,7 @@ if (root.GetNodeName() != el_actor && root.GetNodeName() != el_qualitylevels) { LOGERROR("Invalid actor format (actor '%s', unrecognised root element '%s')", - pathname.string8().c_str(), XeroFile.GetElementString(root.GetNodeName()).c_str()); + pathname.string8().c_str(), XeroFile.GetElementString(root.GetNodeName())); return false; } Index: ps/trunk/source/graphics/TerrainProperties.cpp =================================================================== --- ps/trunk/source/graphics/TerrainProperties.cpp +++ ps/trunk/source/graphics/TerrainProperties.cpp @@ -1,4 +1,4 @@ -/* Copyright (C) 2019 Wildfire Games. +/* Copyright (C) 2021 Wildfire Games. * This file is part of 0 A.D. * * 0 A.D. is free software: you can redistribute it and/or modify @@ -28,7 +28,6 @@ #include "maths/MathUtil.h" #include "ps/CLogger.h" #include "ps/Filesystem.h" -#include "ps/XML/XeroXMB.h" #include "ps/XML/Xeromyces.h" CTerrainProperties::CTerrainProperties(CTerrainPropertiesPtr parent): Index: ps/trunk/source/graphics/TerrainTextureEntry.cpp =================================================================== --- ps/trunk/source/graphics/TerrainTextureEntry.cpp +++ ps/trunk/source/graphics/TerrainTextureEntry.cpp @@ -1,4 +1,4 @@ -/* Copyright (C) 2020 Wildfire Games. +/* Copyright (C) 2021 Wildfire Games. * This file is part of 0 A.D. * * 0 A.D. is free software: you can redistribute it and/or modify @@ -71,7 +71,7 @@ if (root.GetNodeName() != el_terrain) { - LOGERROR("Invalid terrain format (unrecognised root element '%s')", XeroFile.GetElementString(root.GetNodeName()).c_str()); + LOGERROR("Invalid terrain format (unrecognised root element '%s')", XeroFile.GetElementString(root.GetNodeName())); return; } Index: ps/trunk/source/graphics/TextureConverter.cpp =================================================================== --- ps/trunk/source/graphics/TextureConverter.cpp +++ ps/trunk/source/graphics/TextureConverter.cpp @@ -227,7 +227,7 @@ } else { - LOGERROR("Invalid attribute name ", XeroFile.GetAttributeString(attr.Name).c_str()); + LOGERROR("Invalid attribute name ", XeroFile.GetAttributeString(attr.Name)); } } Index: ps/trunk/source/gui/CGUI.cpp =================================================================== --- ps/trunk/source/gui/CGUI.cpp +++ ps/trunk/source/gui/CGUI.cpp @@ -532,7 +532,7 @@ return; XMBElement node = XeroFile.GetRoot(); - CStr root_name(XeroFile.GetElementString(node.GetNodeName())); + std::string_view root_name(XeroFile.GetElementStringView(node.GetNodeName())); if (root_name == "objects") Xeromyces_ReadRootObjects(node, &XeroFile, Paths); @@ -543,7 +543,7 @@ else if (root_name == "setup") Xeromyces_ReadRootSetup(node, &XeroFile); else - LOGERROR("CGUI::LoadXmlFile encountered an unknown XML root node type: %s", root_name.c_str()); + LOGERROR("CGUI::LoadXmlFile encountered an unknown XML root node type: %s", root_name.data()); } void CGUI::LoadedXmlFiles() @@ -595,7 +595,7 @@ { for (XMBElement child : Element.GetChildNodes()) { - CStr name(pFile->GetElementString(child.GetNodeName())); + std::string_view name(pFile->GetElementStringView(child.GetNodeName())); if (name == "scrollbar") Xeromyces_ReadScrollBarStyle(child, pFile); @@ -989,7 +989,7 @@ for (XMBElement child : Element.GetChildNodes()) { - CStr ElementName(pFile->GetElementString(child.GetNodeName())); + std::string_view ElementName(pFile->GetElementStringView(child.GetNodeName())); if (ElementName == "image") Xeromyces_ReadImage(child, pFile, *Sprite); @@ -1026,7 +1026,7 @@ for (XMBAttribute attr : Element.GetAttributes()) { - CStr attr_name(pFile->GetAttributeString(attr.Name)); + std::string_view attr_name(pFile->GetAttributeStringView(attr.Name)); CStrW attr_value(attr.Value.FromUTF8()); if (attr_name == "texture") @@ -1109,7 +1109,7 @@ // Look for effects for (XMBElement child : Element.GetChildNodes()) { - CStr ElementName(pFile->GetElementString(child.GetNodeName())); + std::string_view ElementName(pFile->GetElementStringView(child.GetNodeName())); if (ElementName == "effect") { if (Image->m_Effects) @@ -1131,7 +1131,7 @@ { for (XMBAttribute attr : Element.GetAttributes()) { - CStr attr_name(pFile->GetAttributeString(attr.Name)); + std::string_view attr_name(pFile->GetAttributeStringView(attr.Name)); if (attr_name == "add_color") { @@ -1152,14 +1152,14 @@ for (XMBAttribute attr : Element.GetAttributes()) { - CStr attr_name(pFile->GetAttributeString(attr.Name)); + std::string_view attr_name(pFile->GetAttributeStringView(attr.Name)); // The "name" setting is actually the name of the style // and not a new default if (attr_name == "name") name = attr.Value; else - style.m_SettingsDefaults.emplace(attr_name, attr.Value.FromUTF8()); + style.m_SettingsDefaults.emplace(std::string(attr_name), attr.Value.FromUTF8()); } m_Styles.erase(name); @@ -1179,7 +1179,7 @@ for (XMBAttribute attr : Element.GetAttributes()) { - CStr attr_name = pFile->GetAttributeString(attr.Name); + std::string_view attr_name(pFile->GetAttributeStringView(attr.Name)); CStr attr_value(attr.Value); if (attr_value == "null") @@ -1256,7 +1256,7 @@ for (XMBAttribute attr : Element.GetAttributes()) { - CStr attr_name(pFile->GetAttributeString(attr.Name)); + std::string_view attr_name(pFile->GetAttributeStringView(attr.Name)); CStr attr_value(attr.Value); if (attr_value == "null") @@ -1288,13 +1288,13 @@ for (XMBAttribute attr : Element.GetAttributes()) { - CStr attr_name(pFile->GetAttributeString(attr.Name)); + std::string_view attr_name(pFile->GetAttributeStringView(attr.Name)); CStr attr_value(attr.Value); if (attr_name == "name") object->SetName("__tooltip_" + attr_value); else - object->SetSettingFromString(attr_name, attr_value.FromUTF8(), true); + object->SetSettingFromString(std::string(attr_name), attr_value.FromUTF8(), true); } if (!AddObject(*m_BaseObject, *object)) Index: ps/trunk/source/gui/ObjectTypes/COList.cpp =================================================================== --- ps/trunk/source/gui/ObjectTypes/COList.cpp +++ ps/trunk/source/gui/ObjectTypes/COList.cpp @@ -203,13 +203,13 @@ for (XMBAttribute attr : child.GetAttributes()) { - CStr attr_name(pFile->GetAttributeString(attr.Name)); + std::string_view attr_name(pFile->GetAttributeStringView(attr.Name)); CStr attr_value(attr.Value); if (attr_name == "color") { if (!CGUI::ParseString(&m_pGUI, attr_value.FromUTF8(), column.m_TextColor)) - LOGERROR("GUI: Error parsing '%s' (\"%s\")", attr_name.c_str(), attr_value.c_str()); + LOGERROR("GUI: Error parsing '%s' (\"%s\")", attr_name.data(), attr_value.c_str()); } else if (attr_name == "id") { @@ -219,7 +219,7 @@ { bool hidden = false; if (!CGUI::ParseString(&m_pGUI, attr_value.FromUTF8(), hidden)) - LOGERROR("GUI: Error parsing '%s' (\"%s\")", attr_name.c_str(), attr_value.c_str()); + LOGERROR("GUI: Error parsing '%s' (\"%s\")", attr_name.data(), attr_value.c_str()); else column.m_Hidden = hidden; } @@ -227,7 +227,7 @@ { float width; if (!CGUI::ParseString(&m_pGUI, attr_value.FromUTF8(), width)) - LOGERROR("GUI: Error parsing '%s' (\"%s\")", attr_name.c_str(), attr_value.c_str()); + LOGERROR("GUI: Error parsing '%s' (\"%s\")", attr_name.data(), attr_value.c_str()); else { // Check if it's a relative value, and save as decimal if so. Index: ps/trunk/source/ps/XMB/XMBData.h =================================================================== --- ps/trunk/source/ps/XMB/XMBData.h +++ ps/trunk/source/ps/XMB/XMBData.h @@ -0,0 +1,285 @@ +/* Copyright (C) 2021 Wildfire Games. + * This file is part of 0 A.D. + * + * 0 A.D. is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * 0 A.D. is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with 0 A.D. If not, see . + */ + +/* + Xeromyces - XMB reading library + +Brief outline: + +XMB originated as a binary representation of XML, with some limitations +but much more efficiency (particularly for loading simple data +classes that don't need much initialisation). + +Theoretical file structure: + +XMB_File { + char Header[4]; // because everyone has one; currently "XMB0" + u32 Version; + + int OffsetFromStartToElementNames; + int ElementNameCount; + + int OffsetFromStartToAttributeNames; + int AttributeNameCount; + + XMB_Node Root; + + ZStr8 ElementNames[]; + ZStr8 AttributeNames[]; + +} + +XMB_Node { +0) int Length; // of entire struct, so it can be skipped over + +4) int ElementName; + +8) int AttributeCount; +12) int ChildCount; + +16) int ChildrenOffset; // == sizeof(Text)+sizeof(Attributes) +20) XMB_Text Text; + XMB_Attribute Attributes[]; + XMB_Node Children[]; + +} + +XMB_Attribute { + int Name; + ZStr8 Value; +} + +ZStr8 { + int Length; // in bytes + char* Text; // null-terminated UTF8 +} + +XMB_Text { +20) int Length; // 0 if there's no text, else 4+sizeof(Text) in bytes including terminator + // If Length != 0: +24) int LineNumber; // for e.g. debugging scripts +28) char* Text; // null-terminated UTF8 +} + +*/ + +#ifndef INCLUDED_XEROXMB +#define INCLUDED_XEROXMB + +#include "ps/CStr.h" + +#include +#include + +class CXeromyces; +class XMBStorage; + +class XMBElement; +class XMBElementList; +class XMBAttributeList; + + +class XMBData +{ +public: + + XMBData() : m_Pointer(nullptr) {} + + /* + * Initialise from the contents of an XMBStorage. + * @param doc must remain allocated and unchanged while + * the XMBData is being used. + * @return indication of success; main cause for failure is attempting to + * load a partially valid XMB file (e.g. if the game was interrupted + * while writing it), which we detect by checking the magic string. + * It also fails when trying to load an XMB file with a different version. + */ + bool Initialise(const XMBStorage& doc); + + // Returns the root element + XMBElement GetRoot() const; + + // Returns internal ID for a given element/attribute string. + int GetElementID(const char* Name) const; + int GetAttributeID(const char* Name) const; + + // Returns element/attribute string for a given internal ID. + const char* GetElementString(const int ID) const; + const char* GetAttributeString(const int ID) const; + + std::string_view GetElementStringView(const int ID) const; + std::string_view GetAttributeStringView(const int ID) const; + +private: + const char* m_Pointer; + + int m_ElementNameCount; + int m_AttributeNameCount; + const char* m_ElementPointer; + const char* m_AttributePointer; +}; + +class XMBElement +{ +public: + XMBElement() + : m_Pointer(0) {} + + XMBElement(const char* offset) + : m_Pointer(offset) {} + + int GetNodeName() const; + XMBElementList GetChildNodes() const; + XMBAttributeList GetAttributes() const; + CStr8 GetText() const; + // Returns the line number of the text within this element, + // or -1 if there is no text + int GetLineNumber() const; + +private: + // Pointer to the start of the node + const char* m_Pointer; +}; + +class XMBElementList +{ +public: + XMBElementList(const char* offset, size_t count, const char* endoffset) + : m_Size(count), m_Pointer(offset), m_CurItemID(0), m_CurPointer(offset), m_EndPointer(endoffset) {} + + // Get first element in list with the given name. + // Performance is linear in the number of elements in the list. + XMBElement GetFirstNamedItem(const int ElementName) const; + + // Linear in the number of elements in the list + XMBElement operator[](size_t id); // returns Children[id] + + class iterator + { + public: + typedef ptrdiff_t difference_type; + typedef XMBElement value_type; + typedef XMBElement reference; // Because we need to construct the object + typedef XMBElement pointer; // Because we need to construct the object + typedef std::forward_iterator_tag iterator_category; + + iterator(size_t size, const char* ptr, const char* endptr = NULL) + : m_Size(size), m_CurItemID(endptr ? size : 0), m_CurPointer(endptr ? endptr : ptr) {} + XMBElement operator*() const { return XMBElement(m_CurPointer); } + XMBElement operator->() const { return **this; } + iterator& operator++(); + + bool operator==(const iterator& rhs) const + { + return m_Size == rhs.m_Size && + m_CurItemID == rhs.m_CurItemID && + m_CurPointer == rhs.m_CurPointer; + } + bool operator!=(const iterator& rhs) const { return !(*this == rhs); } + private: + size_t m_Size; + size_t m_CurItemID; + const char* m_CurPointer; + }; + iterator begin() { return iterator(m_Size, m_Pointer); } + iterator end() { return iterator(m_Size, m_Pointer, m_EndPointer); } + + size_t size() const { return m_Size; } + bool empty() const { return m_Size == 0; } + +private: + size_t m_Size; + + const char* m_Pointer; + + // For optimised sequential access: + size_t m_CurItemID; + const char* m_CurPointer; + + const char* m_EndPointer; +}; + + +struct XMBAttribute +{ + XMBAttribute() {} + XMBAttribute(int name, const CStr8& value) + : Name(name), Value(value) {}; + + int Name; + CStr8 Value; // UTF-8 encoded +}; + +class XMBAttributeList +{ +public: + XMBAttributeList(const char* offset, size_t count, const char* endoffset) + : m_Size(count), m_Pointer(offset), m_CurItemID(0), m_CurPointer(offset), m_EndPointer(endoffset) {} + + // Get the attribute value directly + CStr8 GetNamedItem(const int AttributeName) const; + + // Linear in the number of elements in the list + XMBAttribute operator[](size_t id); // returns Children[id] + + class iterator + { + public: + typedef ptrdiff_t difference_type; + typedef XMBAttribute value_type; + typedef XMBAttribute reference; // Because we need to construct the object + typedef XMBAttribute pointer; // Because we need to construct the object + typedef std::forward_iterator_tag iterator_category; + + iterator(size_t size, const char* ptr, const char* endptr = NULL) + : m_Size(size), m_CurItemID(endptr ? size : 0), m_CurPointer(endptr ? endptr : ptr) {} + XMBAttribute operator*() const; + XMBAttribute operator->() const { return **this; } + iterator& operator++(); + + bool operator==(const iterator& rhs) const + { + return m_Size == rhs.m_Size && + m_CurItemID == rhs.m_CurItemID && + m_CurPointer == rhs.m_CurPointer; + } + bool operator!=(const iterator& rhs) const { return !(*this == rhs); } + private: + size_t m_Size; + size_t m_CurItemID; + const char* m_CurPointer; + }; + iterator begin() const { return iterator(m_Size, m_Pointer); } + iterator end() const { return iterator(m_Size, m_Pointer, m_EndPointer); } + + size_t size() const { return m_Size; } + bool empty() const { return m_Size == 0; } + +private: + size_t m_Size; + + // Pointer to start of attribute list + const char* m_Pointer; + + // For optimised sequential access: + size_t m_CurItemID; + const char* m_CurPointer; + + const char* m_EndPointer; +}; + +#endif // INCLUDED_XEROXMB Index: ps/trunk/source/ps/XMB/XMBData.cpp =================================================================== --- ps/trunk/source/ps/XMB/XMBData.cpp +++ ps/trunk/source/ps/XMB/XMBData.cpp @@ -0,0 +1,290 @@ +/* Copyright (C) 2021 Wildfire Games. + * This file is part of 0 A.D. + * + * 0 A.D. is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * 0 A.D. is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with 0 A.D. If not, see . + */ + +#include "precompiled.h" + +#include "lib/byte_order.h" // FOURCC_LE +#include "ps/XMB/XMBStorage.h" +#include "ps/XML/Xeromyces.h" + +template +static inline T read(const void* ptr) +{ + T ret; + memcpy(&ret, ptr, sizeof(T)); + return ret; +} + +bool XMBData::Initialise(const XMBStorage& doc) +{ + const char* start = reinterpret_cast(doc.m_Buffer.get()); + m_Pointer = start; + char Header[5] = { 0 }; + strncpy_s(Header, 5, m_Pointer, 4); + m_Pointer += 4; + + if (strcmp(Header, XMBStorage::UnfinishedHeaderMagicStr) == 0) + return false; + ENSURE(strcmp(Header, XMBStorage::HeaderMagicStr) == 0 && "Invalid XMB header!"); + + u32 Version = read(m_Pointer); + m_Pointer += 4; + if (Version != XMBStorage::XMBVersion) + return false; + + // FIXME Check that m_Pointer doesn't end up past the end of the buffer + // (it shouldn't be all that dangerous since we're only doing read-only + // access, but it might crash on an invalid file, reading a couple of + // billion random element names from RAM) + + m_ElementPointer = start + read(m_Pointer); m_Pointer += 4; + m_ElementNameCount = read(m_Pointer); m_Pointer += 4; + m_AttributePointer = start + read(m_Pointer); m_Pointer += 4; + m_AttributeNameCount = read(m_Pointer); m_Pointer += 4; + // At this point m_Pointer points to the element start, as expected. + return true; // success +} + +XMBElement XMBData::GetRoot() const +{ + return XMBElement(m_Pointer); +} + +int XMBData::GetElementID(const char* Name) const +{ + const char* Pos = m_ElementPointer; + + int len = (int)strlen(Name)+1; // count bytes, including null terminator + + // Loop through each string to find a match + for (int i = 0; i < m_ElementNameCount; ++i) + { + // See if this could be the right string, checking its + // length and then its contents + if (read(Pos) == len && strncasecmp(Pos+4, Name, len) == 0) + return static_cast(Pos - m_ElementPointer); + // If not, jump to the next string + Pos += 4 + read(Pos); + } + // Failed + return -1; +} + +int XMBData::GetAttributeID(const char* Name) const +{ + const char* Pos = m_AttributePointer; + + int len = (int)strlen(Name)+1; // count bytes, including null terminator + + // Loop through each string to find a match + for (int i = 0; i < m_AttributeNameCount; ++i) + { + // See if this could be the right string, checking its + // length and then its contents + if (read(Pos) == len && strncasecmp(Pos+4, Name, len) == 0) + return static_cast(Pos - m_AttributePointer); + // If not, jump to the next string + Pos += 4 + read(Pos); + } + // Failed + return -1; +} + +const char* XMBData::GetElementString(const int ID) const +{ + return reinterpret_cast(m_ElementPointer + ID + 4); +} + +const char* XMBData::GetAttributeString(const int ID) const +{ + return reinterpret_cast(m_AttributePointer + ID + 4); +} + +std::string_view XMBData::GetElementStringView(const int ID) const +{ + return std::string_view(reinterpret_cast(m_ElementPointer + ID + 4), read(m_ElementPointer + ID) - 1); +} + +std::string_view XMBData::GetAttributeStringView(const int ID) const +{ + return std::string_view(reinterpret_cast(m_AttributePointer + ID + 4), read(m_AttributePointer + ID) - 1); +} + +int XMBElement::GetNodeName() const +{ + if (m_Pointer == NULL) + return -1; + + return read(m_Pointer + 4); // == ElementName +} + +XMBElementList XMBElement::GetChildNodes() const +{ + if (m_Pointer == NULL) + return XMBElementList(NULL, 0, NULL); + + return XMBElementList( + m_Pointer + 20 + read(m_Pointer + 16), // == Children[] + read(m_Pointer + 12), // == ChildCount + m_Pointer + read(m_Pointer) // == &Children[ChildCount] + ); +} + +XMBAttributeList XMBElement::GetAttributes() const +{ + if (m_Pointer == NULL) + return XMBAttributeList(NULL, 0, NULL); + + return XMBAttributeList( + m_Pointer + 24 + read(m_Pointer + 20), // == Attributes[] + read(m_Pointer + 8), // == AttributeCount + m_Pointer + 20 + read(m_Pointer + 16) // == &Attributes[AttributeCount] ( == &Children[]) + ); +} + +CStr8 XMBElement::GetText() const +{ + // Return empty string if there's no text + if (m_Pointer == NULL || read(m_Pointer + 20) == 0) + return CStr8(); + + return CStr8(m_Pointer + 28); +} + +int XMBElement::GetLineNumber() const +{ + // Make sure there actually was some text to record the line of + if (m_Pointer == NULL || read(m_Pointer + 20) == 0) + return -1; + else + return read(m_Pointer + 24); +} + +XMBElement XMBElementList::GetFirstNamedItem(const int ElementName) const +{ + const char* Pos = m_Pointer; + + // Maybe not the cleverest algorithm, but it should be + // fast enough with half a dozen attributes: + for (size_t i = 0; i < m_Size; ++i) + { + int Length = read(Pos); + int Name = read(Pos+4); + if (Name == ElementName) + return XMBElement(Pos); + Pos += Length; + } + + // Can't find element + return XMBElement(); +} + +XMBElementList::iterator& XMBElementList::iterator::operator++() +{ + m_CurPointer += read(m_CurPointer); + ++m_CurItemID; + return (*this); +} + +XMBElement XMBElementList::operator[](size_t id) +{ + ENSURE(id < m_Size && "Element ID out of range"); + const char* Pos; + size_t i; + + if (id < m_CurItemID) + { + Pos = m_Pointer; + i = 0; + } + else + { + // If access is sequential, don't bother scanning + // through all the nodes to find the next one + Pos = m_CurPointer; + i = m_CurItemID; + } + + // Skip over each preceding node + for (; i < id; ++i) + Pos += read(Pos); + + // Cache information about this node + m_CurItemID = id; + m_CurPointer = Pos; + + return XMBElement(Pos); +} + +CStr8 XMBAttributeList::GetNamedItem(const int AttributeName) const +{ + const char* Pos = m_Pointer; + + // Maybe not the cleverest algorithm, but it should be + // fast enough with half a dozen attributes: + for (size_t i = 0; i < m_Size; ++i) + { + if (read(Pos) == AttributeName) + return CStr8(Pos+8); + Pos += 8 + read(Pos+4); // Skip over the string + } + + // Can't find attribute + return CStr8(); +} + +XMBAttribute XMBAttributeList::iterator::operator*() const +{ + return XMBAttribute(read(m_CurPointer), CStr8(m_CurPointer+8)); +} + +XMBAttributeList::iterator& XMBAttributeList::iterator::operator++() +{ + m_CurPointer += 8 + read(m_CurPointer+4); // skip ID, length, and string data + ++m_CurItemID; + return (*this); +} + +XMBAttribute XMBAttributeList::operator[](size_t id) +{ + ENSURE(id < m_Size && "Attribute ID out of range"); + const char* Pos; + size_t i; + + if (id < m_CurItemID) + { + Pos = m_Pointer; + i = 0; + } + else + { + // If access is sequential, don't bother scanning + // through all the nodes to find the next one + Pos = m_CurPointer; + i = m_CurItemID; + } + + // Skip over each preceding attribute + for (; i < id; ++i) + Pos += 8 + read(Pos+4); // skip ID, length, and string data + + // Cache information about this attribute + m_CurItemID = id; + m_CurPointer = Pos; + + return XMBAttribute(read(Pos), CStr8(Pos+8)); +} Index: ps/trunk/source/ps/XMB/XMBStorage.h =================================================================== --- ps/trunk/source/ps/XMB/XMBStorage.h +++ ps/trunk/source/ps/XMB/XMBStorage.h @@ -0,0 +1,110 @@ +/* Copyright (C) 2021 Wildfire Games. + * This file is part of 0 A.D. + * + * 0 A.D. is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * 0 A.D. is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with 0 A.D. If not, see . + */ + +#ifndef INCLUDED_XMBSTORAGE +#define INCLUDED_XMBSTORAGE + +#include "scriptinterface/ScriptForward.h" + +#include + +typedef struct _xmlDoc xmlDoc; +typedef xmlDoc* xmlDocPtr; + +struct IVFS; +typedef std::shared_ptr PIVFS; + +class Path; +typedef Path VfsPath; + +/** + * Storage for XMBData + */ +class XMBStorage +{ +public: + // File headers, to make sure it doesn't try loading anything other than an XMB + static const char* HeaderMagicStr; + static const char* UnfinishedHeaderMagicStr; + static const u32 XMBVersion; + + XMBStorage() = default; + + /** + * Read an XMB file on disk. + */ + bool ReadFromFile(const PIVFS& vfs, const VfsPath& filename); + + /** + * Parse an XML document into XMB. + * + * Main limitations: + * - Can't correctly handle mixed text/elements inside elements - + * "
Text
" and "
Text
" are + * considered identical. + */ + bool LoadXMLDoc(const xmlDocPtr doc); + + /** + * Parse a Javascript value into XMB. + * The syntax is similar to ParamNode, but supports multiple children with the same name, to match XML. + * You need to pass the name of the root object, as unlike XML this cannot be recovered from the value. + * The following JS object: + * { + * "a": 5, + * "b": "test", + * "x": { + * // Like ParamNode, _string is used for the value. + * "_string": "value", + * // Like ParamNode, attributes are prefixed with @. + * "@param": "something", + * "y": 3 + * }, + * // Every array item is parsed as a child. + * "object": [ + * "a", + * "b", + * { "_string": "c" }, + * { "child": "value" }, + * ], + * // Same but without the array. + * "child@0@": 1, + * "child@1@": 2 + * } + * will parse like the following XML: + * 5 + * test + * value + * 3 + * + * a + * b + * c + * value + * 1 + * 2 + * + * See also tests for some other examples. + */ + bool LoadJSValue(const ScriptInterface& scriptInterface, JS::HandleValue value, const std::string& rootName); + + std::shared_ptr m_Buffer; + size_t m_Size = 0; +}; + + +#endif // INCLUDED_XMBSTORAGE Index: ps/trunk/source/ps/XMB/XMBStorage.cpp =================================================================== --- ps/trunk/source/ps/XMB/XMBStorage.cpp +++ ps/trunk/source/ps/XMB/XMBStorage.cpp @@ -0,0 +1,469 @@ +/* Copyright (C) 2021 Wildfire Games. + * This file is part of 0 A.D. + * + * 0 A.D. is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * 0 A.D. is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with 0 A.D. If not, see . + */ + +#include "precompiled.h" + +#include "XMBStorage.h" + +#include "lib/file/io/write_buffer.h" +#include "scriptinterface/ScriptExtraHeaders.h" +#include "scriptinterface/ScriptInterface.h" + +#include +#include + +const char* XMBStorage::HeaderMagicStr = "XMB0"; +const char* XMBStorage::UnfinishedHeaderMagicStr = "XMBu"; +// Arbitrary version number - change this if we update the code and +// need to invalidate old users' caches +const u32 XMBStorage::XMBVersion = 4; + +namespace +{ +class XMBStorageWriter +{ +public: + template + bool Load(WriteBuffer& writeBuffer, Args&&... args); + + int GetElementName(const std::string& name) { return GetName(m_ElementSize, m_ElementIDs, name); } + int GetAttributeName(const std::string& name) { return GetName(m_AttributeSize, m_AttributeIDs, name); } + +protected: + int GetName(int& totalSize, std::unordered_map& names, const std::string& name) + { + int nameIdx = totalSize; + auto [iterator, inserted] = names.try_emplace(name, nameIdx); + if (inserted) + totalSize += name.size() + 5; // Add 1 for the null terminator & 4 for the size int. + return iterator->second; + } + + void OutputNames(WriteBuffer& writeBuffer, const std::unordered_map& names) const; + + template + bool OutputElements(WriteBuffer&, Args...) + { + static_assert(sizeof...(Args) != sizeof...(Args), "OutputElements must be specialized."); + } + + int m_ElementSize = 0; + int m_AttributeSize = 0; + std::unordered_map m_ElementIDs; + std::unordered_map m_AttributeIDs; +}; + +// Output text, prefixed by length in bytes (including null-terminator) +void WriteStringAndLineNumber(WriteBuffer& writeBuffer, const std::string& text, int lineNumber) +{ + if (text.empty()) + { + // No text; don't write much + writeBuffer.Append("\0\0\0\0", 4); + } + else + { + // Write length and line number and null-terminated text + u32 nodeLen = u32(4 + text.length() + 1); + writeBuffer.Append(&nodeLen, 4); + writeBuffer.Append(&lineNumber, 4); + writeBuffer.Append((void*)text.c_str(), nodeLen-4); + } +} + +template +bool XMBStorageWriter::Load(WriteBuffer& writeBuffer, Args&&... args) +{ + // Header + writeBuffer.Append(XMBStorage::UnfinishedHeaderMagicStr, 4); + // Version + writeBuffer.Append(&XMBStorage::XMBVersion, 4); + + // Filled in below. + size_t elementPtr = writeBuffer.Size(); + writeBuffer.Append("????????", 8); + // Likewise with attributes. + size_t attributePtr = writeBuffer.Size(); + writeBuffer.Append("????????", 8); + + if (!OutputElements(writeBuffer, std::forward(args)...)) + return false; + + u32 data = writeBuffer.Size(); + writeBuffer.Overwrite(&data, 4, elementPtr); + data = m_ElementIDs.size(); + writeBuffer.Overwrite(&data, 4, elementPtr + 4); + OutputNames(writeBuffer, m_ElementIDs); + + data = writeBuffer.Size(); + writeBuffer.Overwrite(&data, 4, attributePtr); + data = m_AttributeIDs.size(); + writeBuffer.Overwrite(&data, 4, attributePtr + 4); + OutputNames(writeBuffer, m_AttributeIDs); + + // File is now valid, so insert correct magic string. + writeBuffer.Overwrite(XMBStorage::HeaderMagicStr, 4, 0); + + return true; +} + +void XMBStorageWriter::OutputNames(WriteBuffer& writeBuffer, const std::unordered_map& names) const +{ + std::vector> orderedElements; + for (const std::pair& n : names) + orderedElements.emplace_back(n); + std::sort(orderedElements.begin(), orderedElements.end(), [](const auto& a, const auto&b) { return a.second < b.second; }); + for (const std::pair& n : orderedElements) + { + u32 textLen = (u32)n.first.length() + 1; + writeBuffer.Append(&textLen, 4); + writeBuffer.Append((void*)n.first.c_str(), textLen); + } +} + +class JSNodeData +{ +public: + JSNodeData(const ScriptInterface& s) : scriptInterface(s), rq(s) {} + + bool Setup(XMBStorageWriter& xmb, JS::HandleValue value); + bool Output(WriteBuffer& writeBuffer, JS::HandleValue value) const; + + std::vector> m_Attributes; + std::vector>> m_Children; + + const ScriptInterface& scriptInterface; + const ScriptRequest rq; +}; + +template<> +bool XMBStorageWriter::OutputElements(WriteBuffer& writeBuffer, JSNodeData& data, const u32& nodeName, JS::HandleValue&& value) +{ + // Set up variables. + if (!data.Setup(*this, value)) + return false; + + size_t posLength = writeBuffer.Size(); + // Filled in later with the length of the element + writeBuffer.Append("????", 4); + + writeBuffer.Append(&nodeName, 4); + + u32 attrCount = static_cast(data.m_Attributes.size()); + writeBuffer.Append(&attrCount, 4); + + u32 childCount = data.m_Children.size(); + writeBuffer.Append(&childCount, 4); + + // Filled in later with the offset to the list of child elements + size_t posChildrenOffset = writeBuffer.Size(); + writeBuffer.Append("????", 4); + + data.Output(writeBuffer, value); + + // Output attributes + for (const std::pair attr : data.m_Attributes) + { + writeBuffer.Append(&attr.first, 4); + u32 attrLen = u32(attr.second.size())+1; + writeBuffer.Append(&attrLen, 4); + writeBuffer.Append((void*)attr.second.c_str(), attrLen); + } + + // Go back and fill in the child-element offset + u32 childrenOffset = (u32)(writeBuffer.Size() - (posChildrenOffset+4)); + writeBuffer.Overwrite(&childrenOffset, 4, posChildrenOffset); + + // Output all child elements, making a copy since data will be overwritten. + std::vector>> children = data.m_Children; + for (const std::pair>& child : children) + { + JS::RootedValue val(data.rq.cx, child.second); + if (!OutputElements(writeBuffer, data, child.first, val)) + return false; + } + + // Go back and fill in the length + u32 length = (u32)(writeBuffer.Size() - posLength); + writeBuffer.Overwrite(&length, 4, posLength); + + return true; +} + +bool JSNodeData::Setup(XMBStorageWriter& xmb, JS::HandleValue value) +{ + m_Attributes.clear(); + m_Children.clear(); + JSType valType = JS_TypeOfValue(rq.cx, value); + if (valType != JSTYPE_OBJECT) + return true; + + std::vector props; + if (!scriptInterface.EnumeratePropertyNames(value, true, props)) + { + LOGERROR("Failed to enumerate component properties."); + return false; + } + + for (const std::string& prop : props) + { + // Special 'value' key. + if (prop == "_string") + continue; + + bool attrib = !prop.empty() && prop.front() == '@'; + + std::string_view name = prop; + if (!attrib && !prop.empty() && prop.back() == '@') + { + size_t idx = prop.substr(0, prop.size()-1).find_last_of('@'); + if (idx == std::string::npos) + { + LOGERROR("Object key name cannot end with an '@' unless it is an index specifier."); + return false; + } + name = std::string_view(prop.c_str(), idx); + } + else if (attrib) + name = std::string_view(prop.c_str()+1, prop.length()-1); + + JS::RootedValue child(rq.cx); + if (!scriptInterface.GetProperty(value, prop.c_str(), &child)) + return false; + + if (attrib) + { + std::string attrVal; + if (!ScriptInterface::FromJSVal(rq, child, attrVal)) + { + LOGERROR("Attributes must be convertible to string"); + return false; + } + m_Attributes.emplace_back(xmb.GetAttributeName(std::string(name)), attrVal); + continue; + } + + bool isArray = false; + if (!JS::IsArrayObject(rq.cx, child, &isArray)) + return false; + if (!isArray) + { + m_Children.emplace_back(xmb.GetElementName(std::string(name)), child); + continue; + } + + // Parse each array object as a child. + JS::RootedObject obj(rq.cx); + JS_ValueToObject(rq.cx, child, &obj); + u32 length; + JS::GetArrayLength(rq.cx, obj, &length); + for (size_t i = 0; i < length; ++i) + { + JS::RootedValue arrayChild(rq.cx); + scriptInterface.GetPropertyInt(child, i, &arrayChild); + m_Children.emplace_back(xmb.GetElementName(std::string(name)), arrayChild); + } + } + return true; +} + +bool JSNodeData::Output(WriteBuffer& writeBuffer, JS::HandleValue value) const +{ + switch (JS_TypeOfValue(rq.cx, value)) + { + case JSTYPE_UNDEFINED: + case JSTYPE_NULL: + { + writeBuffer.Append("\0\0\0\0", 4); + break; + } + case JSTYPE_OBJECT: + { + if (!scriptInterface.HasProperty(value, "_string")) + { + writeBuffer.Append("\0\0\0\0", 4); + break; + } + JS::RootedValue actualValue(rq.cx); + if (!scriptInterface.GetProperty(value, "_string", &actualValue)) + return false; + std::string strVal; + if (!ScriptInterface::FromJSVal(rq, actualValue, strVal)) + { + LOGERROR("'_string' value must be convertible to string"); + return false; + } + WriteStringAndLineNumber(writeBuffer, strVal, 0); + break; + } + case JSTYPE_STRING: + case JSTYPE_NUMBER: + { + std::string strVal; + if (!ScriptInterface::FromJSVal(rq, value, strVal)) + return false; + + WriteStringAndLineNumber(writeBuffer, strVal, 0); + break; + } + default: + { + LOGERROR("Unsupported JS construct when parsing ParamNode"); + return false; + } + } + return true; +} + +template<> +bool XMBStorageWriter::OutputElements(WriteBuffer& writeBuffer, xmlNodePtr&& node) +{ + // Filled in later with the length of the element + size_t posLength = writeBuffer.Size(); + writeBuffer.Append("????", 4); + + u32 name = GetElementName((const char*)node->name); + writeBuffer.Append(&name, 4); + + u32 attrCount = 0; + for (xmlAttrPtr attr = node->properties; attr; attr = attr->next) + ++attrCount; + writeBuffer.Append(&attrCount, 4); + + u32 childCount = 0; + for (xmlNodePtr child = node->children; child; child = child->next) + if (child->type == XML_ELEMENT_NODE) + ++childCount; + writeBuffer.Append(&childCount, 4); + + // Filled in later with the offset to the list of child elements + size_t posChildrenOffset = writeBuffer.Size(); + writeBuffer.Append("????", 4); + + + // Trim excess whitespace in the entity's text, while counting + // the number of newlines trimmed (so that JS error reporting + // can give the correct line number within the script) + + std::string whitespace = " \t\r\n"; + std::string text; + for (xmlNodePtr child = node->children; child; child = child->next) + { + if (child->type == XML_TEXT_NODE) + { + xmlChar* content = xmlNodeGetContent(child); + text += std::string((const char*)content); + xmlFree(content); + } + } + + u32 linenum = xmlGetLineNo(node); + + // Find the start of the non-whitespace section + size_t first = text.find_first_not_of(whitespace); + + if (first == text.npos) + // Entirely whitespace - easy to handle + text = ""; + + else + { + // Count the number of \n being cut off, + // and add them to the line number + std::string trimmed (text.begin(), text.begin()+first); + linenum += std::count(trimmed.begin(), trimmed.end(), '\n'); + + // Find the end of the non-whitespace section, + // and trim off everything else + size_t last = text.find_last_not_of(whitespace); + text = text.substr(first, 1+last-first); + } + + + // Output text, prefixed by length in bytes + WriteStringAndLineNumber(writeBuffer, text, linenum); + + // Output attributes + for (xmlAttrPtr attr = node->properties; attr; attr = attr->next) + { + u32 attrName = GetAttributeName((const char*)attr->name); + writeBuffer.Append(&attrName, 4); + + xmlChar* value = xmlNodeGetContent(attr->children); + u32 attrLen = u32(xmlStrlen(value)+1); + writeBuffer.Append(&attrLen, 4); + writeBuffer.Append((void*)value, attrLen); + xmlFree(value); + } + + // Go back and fill in the child-element offset + u32 childrenOffset = (u32)(writeBuffer.Size() - (posChildrenOffset+4)); + writeBuffer.Overwrite(&childrenOffset, 4, posChildrenOffset); + + // Output all child elements + for (xmlNodePtr child = node->children; child; child = child->next) + if (child->type == XML_ELEMENT_NODE) + OutputElements(writeBuffer, std::move(child)); + + // Go back and fill in the length + u32 length = (u32)(writeBuffer.Size() - posLength); + writeBuffer.Overwrite(&length, 4, posLength); + + return true; +} +} // anonymous namespace + +bool XMBStorage::ReadFromFile(const PIVFS& vfs, const VfsPath& filename) +{ + if(vfs->LoadFile(filename, m_Buffer, m_Size) < 0) + return false; + // if the game crashes during loading, (e.g. due to driver bugs), + // it sometimes leaves empty XMB files in the cache. + // reporting failure will cause our caller to re-generate the XMB. + if (m_Size == 0) + return false; + ENSURE(m_Size >= 4); // make sure it's at least got the initial header + return true; +} + +bool XMBStorage::LoadXMLDoc(const xmlDocPtr doc) +{ + WriteBuffer writeBuffer; + + XMBStorageWriter writer; + if (!writer.Load(writeBuffer, std::move(xmlDocGetRootElement(doc)))) + return false; + + m_Buffer = writeBuffer.Data(); // add a reference + m_Size = writeBuffer.Size(); + return true; +} + +bool XMBStorage::LoadJSValue(const ScriptInterface& scriptInterface, JS::HandleValue value, const std::string& rootName) +{ + WriteBuffer writeBuffer; + + XMBStorageWriter writer; + const u32 name = writer.GetElementName(rootName); + JSNodeData data(scriptInterface); + if (!writer.Load(writeBuffer, data, name, std::move(value))) + return false; + + m_Buffer = writeBuffer.Data(); // add a reference + m_Size = writeBuffer.Size(); + return true; +} Index: ps/trunk/source/ps/XMB/tests/test_XMBData.h =================================================================== --- ps/trunk/source/ps/XMB/tests/test_XMBData.h +++ ps/trunk/source/ps/XMB/tests/test_XMBData.h @@ -0,0 +1,197 @@ +/* Copyright (C) 2021 Wildfire Games. + * This file is part of 0 A.D. + * + * 0 A.D. is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * 0 A.D. is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with 0 A.D. If not, see . + */ + +#include "lib/self_test.h" + +#include "ps/XML/Xeromyces.h" +#include "ps/XMB/XMBStorage.h" +#include "scriptinterface/ScriptInterface.h" + +#include +#include + +class TestXMBData : public CxxTest::TestSuite +{ +private: + shared_ptr m_Buffer; + + std::unique_ptr m_ScriptInterface; + + CXeromyces parseXML(const char* doc) + { + xmlDocPtr xmlDoc = xmlReadMemory(doc, int(strlen(doc)), "", NULL, + XML_PARSE_NONET|XML_PARSE_NOCDATA); + CXeromyces xmb; + bool ok = xmb.m_Data.LoadXMLDoc(xmlDoc); + xmlFreeDoc(xmlDoc); + TS_ASSERT_EQUALS(ok, true); + + TS_ASSERT(xmb.Initialise(xmb.m_Data)); + return xmb; + } + + CXeromyces parseJS(const std::string rootName, const char* code) + { + ScriptRequest rq(*m_ScriptInterface); + JS::RootedValue val(rq.cx); + m_ScriptInterface->Eval(code, &val); + CXeromyces xmb; + bool ok = xmb.m_Data.LoadJSValue(*m_ScriptInterface, val, rootName); + TS_ASSERT_EQUALS(ok, true); + + TS_ASSERT(xmb.Initialise(xmb.m_Data)); + return xmb; + } + + void setUp() + { + m_ScriptInterface = std::make_unique("Test", "Test", g_ScriptContext); + } + + void tearDown() + { + m_ScriptInterface.reset(); + m_Buffer.reset(); + } + +public: + void test_basic() + { + basic(parseXML("\n bar \n\n\nbar\n"), true); + // Array format for same-named elements. + basic(parseJS("test", "({ 'foo': [{ '@x': ' y ', '_string': 'bar' }, { '_string': '\\n\\n\\nbar' }] })"), false); + // Alternative format for same-named elements. + basic(parseJS("test", "({ 'foo@0@': { '@x': ' y ', '_string': 'bar' }, 'foo@1@': { '_string': '\\n\\n\\nbar' }})"), false); + } + + void basic(const CXeromyces& xmb, bool checkLines) + { + + TS_ASSERT_DIFFERS(xmb.GetElementID("test"), -1); + TS_ASSERT_DIFFERS(xmb.GetElementID("foo"), -1); + TS_ASSERT_EQUALS(xmb.GetElementID("bar"), -1); + + TS_ASSERT_DIFFERS(xmb.GetAttributeID("x"), -1); + TS_ASSERT_EQUALS(xmb.GetAttributeID("y"), -1); + TS_ASSERT_EQUALS(xmb.GetAttributeID("test"), -1); + + int el_test = xmb.GetElementID("test"); + int el_foo = xmb.GetElementID("foo"); + int at_x = xmb.GetAttributeID("x"); + + XMBElement root = xmb.GetRoot(); + TS_ASSERT_EQUALS(root.GetNodeName(), el_test); + if (checkLines) + TS_ASSERT_EQUALS(root.GetLineNumber(), -1); + TS_ASSERT_EQUALS(CStr(root.GetText()), ""); + + TS_ASSERT_EQUALS(root.GetChildNodes().size(), 2); + XMBElement child = root.GetChildNodes()[0]; + TS_ASSERT_EQUALS(child.GetNodeName(), el_foo); + if (checkLines) + TS_ASSERT_EQUALS(child.GetLineNumber(), 2); + TS_ASSERT_EQUALS(child.GetChildNodes().size(), 0); + TS_ASSERT_EQUALS(CStr(child.GetText()), "bar"); + + if (checkLines) + TS_ASSERT_EQUALS(root.GetChildNodes()[1].GetLineNumber(), 5); + + TS_ASSERT_EQUALS(child.GetAttributes().size(), 1); + XMBAttribute attr = child.GetAttributes()[0]; + TS_ASSERT_EQUALS(attr.Name, at_x); + TS_ASSERT_EQUALS(CStr(attr.Value), " y "); + } + + void test_GetFirstNamedItem() + { + GetFirstNamedItem(parseXML(" A B C D "), true); + GetFirstNamedItem(parseJS("test", "({ 'x': [{ '_string': 'A' }, 'B'], 'y': 'C', 'z': 'D' })"), false); + GetFirstNamedItem(parseJS("test", "({ 'x@0@': 'A', 'x@1@': 'B', 'y': 'C', 'z': 'D' })"), false); + } + + void GetFirstNamedItem(const CXeromyces& xmb, bool checkLines) + { + XMBElement root = xmb.GetRoot(); + TS_ASSERT_EQUALS(root.GetChildNodes().size(), 4); + + XMBElement x = root.GetChildNodes().GetFirstNamedItem(xmb.GetElementID("x")); + XMBElement y = root.GetChildNodes().GetFirstNamedItem(xmb.GetElementID("y")); + XMBElement w = root.GetChildNodes().GetFirstNamedItem(xmb.GetElementID("w")); + + TS_ASSERT_EQUALS(x.GetNodeName(), xmb.GetElementID("x")); + TS_ASSERT_EQUALS(CStr(x.GetText()), "A"); + + TS_ASSERT_EQUALS(y.GetNodeName(), xmb.GetElementID("y")); + TS_ASSERT_EQUALS(CStr(y.GetText()), "C"); + + TS_ASSERT_EQUALS(w.GetNodeName(), -1); + TS_ASSERT_EQUALS(CStr(w.GetText()), ""); + if (checkLines) + TS_ASSERT_EQUALS(w.GetLineNumber(), -1); + TS_ASSERT_EQUALS(w.GetChildNodes().size(), 0); + TS_ASSERT_EQUALS(w.GetAttributes().size(), 0); + } + + void test_doctype_ignored() + { + CXeromyces xmb (parseXML("")); + + TS_ASSERT_DIFFERS(xmb.GetElementID("foo"), -1); + } + + void test_complex_parse() + { + CXeromyces xmb (parseXML("\t\n \tx <>&"'bar\n\nbazqux")); + TS_ASSERT_EQUALS(CStr(xmb.GetRoot().GetText()), "x <>&\"'foobar\n\nbazqux"); + } + + void test_unicode() + { + CXeromyces xmb (parseXML("ሴ\xE1\x88\xB4")); + CStrW text; + + text = xmb.GetRoot().GetText().FromUTF8(); + TS_ASSERT_EQUALS((int)text.length(), 2); + TS_ASSERT_EQUALS(text[0], 0x1234); + TS_ASSERT_EQUALS(text[1], 0x1234); + + text = xmb.GetRoot().GetAttributes()[0].Value.FromUTF8(); + TS_ASSERT_EQUALS((int)text.length(), 2); + TS_ASSERT_EQUALS(text[0], 0x1234); + TS_ASSERT_EQUALS(text[1], 0x1234); + } + + void test_iso88591() + { + CXeromyces xmb (parseXML("ሴ\xE1\x88\xB4")); + CStrW text; + + text = xmb.GetRoot().GetText().FromUTF8(); + TS_ASSERT_EQUALS((int)text.length(), 4); + TS_ASSERT_EQUALS(text[0], 0x1234); + TS_ASSERT_EQUALS(text[1], 0x00E1); + TS_ASSERT_EQUALS(text[2], 0x0088); + TS_ASSERT_EQUALS(text[3], 0x00B4); + + text = xmb.GetRoot().GetAttributes()[0].Value.FromUTF8(); + TS_ASSERT_EQUALS((int)text.length(), 4); + TS_ASSERT_EQUALS(text[0], 0x1234); + TS_ASSERT_EQUALS(text[1], 0x00E1); + TS_ASSERT_EQUALS(text[2], 0x0088); + TS_ASSERT_EQUALS(text[3], 0x00B4); + } +}; Index: ps/trunk/source/ps/XML/XMLWriter.h =================================================================== --- ps/trunk/source/ps/XML/XMLWriter.h +++ ps/trunk/source/ps/XML/XMLWriter.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2019 Wildfire Games. +/* Copyright (C) 2021 Wildfire Games. * This file is part of 0 A.D. * * 0 A.D. is free software: you can redistribute it and/or modify @@ -65,7 +65,7 @@ #include "ps/CStr.h" class XMBElement; -class XMBFile; +class XMBData; class XMLWriter_Element; class XMLWriter_File @@ -77,7 +77,7 @@ void Comment(const char* text); - void XMB(const XMBFile& file); + void XMB(const XMBData& xmb); bool StoreVFS(const PIVFS& vfs, const VfsPath& pathname); const CStr8& GetOutput(); @@ -86,7 +86,7 @@ friend class XMLWriter_Element; - void ElementXMB(const XMBFile& file, XMBElement el); + void ElementXMB(const XMBData& xmb, XMBElement el); void ElementStart(XMLWriter_Element* element, const char* name); void ElementText(const char* text, bool cdata); Index: ps/trunk/source/ps/XML/XMLWriter.cpp =================================================================== --- ps/trunk/source/ps/XML/XMLWriter.cpp +++ ps/trunk/source/ps/XML/XMLWriter.cpp @@ -1,4 +1,4 @@ -/* Copyright (C) 2013 Wildfire Games. +/* Copyright (C) 2021 Wildfire Games. * This file is part of 0 A.D. * * 0 A.D. is free software: you can redistribute it and/or modify @@ -114,20 +114,20 @@ } -void XMLWriter_File::XMB(const XMBFile& file) +void XMLWriter_File::XMB(const XMBData& xmb) { - ElementXMB(file, file.GetRoot()); + ElementXMB(xmb, xmb.GetRoot()); } -void XMLWriter_File::ElementXMB(const XMBFile& file, XMBElement el) +void XMLWriter_File::ElementXMB(const XMBData& xmb, XMBElement el) { - XMLWriter_Element writer(*this, file.GetElementString(el.GetNodeName()).c_str()); + XMLWriter_Element writer(*this, xmb.GetElementString(el.GetNodeName())); XERO_ITER_ATTR(el, attr) - writer.Attribute(file.GetAttributeString(attr.Name).c_str(), attr.Value); + writer.Attribute(xmb.GetAttributeString(attr.Name), attr.Value); XERO_ITER_EL(el, child) - ElementXMB(file, child); + ElementXMB(xmb, child); } void XMLWriter_File::Comment(const char* text) Index: ps/trunk/source/ps/XML/XeroXMB.h =================================================================== --- ps/trunk/source/ps/XML/XeroXMB.h +++ ps/trunk/source/ps/XML/XeroXMB.h @@ -1,302 +0,0 @@ -/* Copyright (C) 2020 Wildfire Games. - * This file is part of 0 A.D. - * - * 0 A.D. is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 2 of the License, or - * (at your option) any later version. - * - * 0 A.D. is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with 0 A.D. If not, see . - */ - -/* - Xeromyces - XMB reading library - -Brief outline: - -XMB is a binary representation of XML, with some limitations -but much more efficiency (particularly for loading simple data -classes that don't need much initialisation). - -Main limitations: - * Can't correctly handle mixed text/elements inside elements - - "
Text
" and "
Text
" are - considered identical. - * Tries to avoid using strings - you usually have to load the - numeric IDs and use them instead. - -Theoretical file structure: - -XMB_File { - char Header[4]; // because everyone has one; currently "XMB0" - u32 Version; - - int ElementNameCount; - ZStr8 ElementNames[]; - - int AttributeNameCount; - ZStr8 AttributeNames[]; - - XMB_Node Root; -} - -XMB_Node { -0) int Length; // of entire struct, so it can be skipped over - -4) int ElementName; - -8) int AttributeCount; -12) int ChildCount; - -16) int ChildrenOffset; // == sizeof(Text)+sizeof(Attributes) -20) XMB_Text Text; - XMB_Attribute Attributes[]; - XMB_Node Children[]; - -} - -XMB_Attribute { - int Name; - ZStr8 Value; -} - -ZStr8 { - int Length; // in bytes - char* Text; // null-terminated UTF8 -} - -XMB_Text { -20) int Length; // 0 if there's no text, else 4+sizeof(Text) in bytes including terminator - // If Length != 0: -24) int LineNumber; // for e.g. debugging scripts -28) char* Text; // null-terminated UTF8 -} - -*/ - -#ifndef INCLUDED_XEROXMB -#define INCLUDED_XEROXMB - -// Define to use a std::map for name lookups rather than a linear search. -// (The map is usually slower.) -//#define XERO_USEMAP - -#include - -#ifdef XERO_USEMAP -# include -#endif - -#include "ps/CStr.h" - -// File headers, to make sure it doesn't try loading anything other than an XMB -extern const char* HeaderMagicStr; -extern const char* UnfinishedHeaderMagicStr; -extern const u32 XMBVersion; - -class XMBElement; -class XMBElementList; -class XMBAttributeList; - - -class XMBFile -{ -public: - - XMBFile() : m_Pointer(NULL) {} - - // Initialise from the contents of an XMB file. - // FileData must remain allocated and unchanged while - // the XMBFile is being used. - // @return indication of success; main cause for failure is attempting to - // load a partially valid XMB file (e.g. if the game was interrupted - // while writing it), which we detect by checking the magic string. - // It also fails when trying to load an XMB file with a different version. - bool Initialise(const char* FileData); - - // Returns the root element - XMBElement GetRoot() const; - - - // Returns internal ID for a given element/attribute string. - int GetElementID(const char* Name) const; - int GetAttributeID(const char* Name) const; - - // For lazy people (e.g. me) when speed isn't vital: - - // Returns element/attribute string for a given internal ID - std::string GetElementString(const int ID) const; - std::string GetAttributeString(const int ID) const; - -private: - const char* m_Pointer; - -#ifdef XERO_USEMAP - std::map m_ElementNames; - std::map m_AttributeNames; -#else - int m_ElementNameCount; - int m_AttributeNameCount; - const char* m_ElementPointer; - const char* m_AttributePointer; -#endif - - std::string ReadZStr8(); -}; - -class XMBElement -{ -public: - XMBElement() - : m_Pointer(0) {} - - XMBElement(const char* offset) - : m_Pointer(offset) {} - - int GetNodeName() const; - XMBElementList GetChildNodes() const; - XMBAttributeList GetAttributes() const; - CStr8 GetText() const; - // Returns the line number of the text within this element, - // or -1 if there is no text - int GetLineNumber() const; - -private: - // Pointer to the start of the node - const char* m_Pointer; -}; - -class XMBElementList -{ -public: - XMBElementList(const char* offset, size_t count, const char* endoffset) - : m_Size(count), m_Pointer(offset), m_CurItemID(0), m_CurPointer(offset), m_EndPointer(endoffset) {} - - // Get first element in list with the given name. - // Performance is linear in the number of elements in the list. - XMBElement GetFirstNamedItem(const int ElementName) const; - - // Linear in the number of elements in the list - XMBElement operator[](size_t id); // returns Children[id] - - class iterator - { - public: - typedef ptrdiff_t difference_type; - typedef XMBElement value_type; - typedef XMBElement reference; // Because we need to construct the object - typedef XMBElement pointer; // Because we need to construct the object - typedef std::forward_iterator_tag iterator_category; - - iterator(size_t size, const char* ptr, const char* endptr = NULL) - : m_Size(size), m_CurItemID(endptr ? size : 0), m_CurPointer(endptr ? endptr : ptr) {} - XMBElement operator*() const { return XMBElement(m_CurPointer); } - XMBElement operator->() const { return **this; } - iterator& operator++(); - - bool operator==(const iterator& rhs) const - { - return m_Size == rhs.m_Size && - m_CurItemID == rhs.m_CurItemID && - m_CurPointer == rhs.m_CurPointer; - } - bool operator!=(const iterator& rhs) const { return !(*this == rhs); } - private: - size_t m_Size; - size_t m_CurItemID; - const char* m_CurPointer; - }; - iterator begin() { return iterator(m_Size, m_Pointer); } - iterator end() { return iterator(m_Size, m_Pointer, m_EndPointer); } - - size_t size() const { return m_Size; } - bool empty() const { return m_Size == 0; } - -private: - size_t m_Size; - - const char* m_Pointer; - - // For optimised sequential access: - size_t m_CurItemID; - const char* m_CurPointer; - - const char* m_EndPointer; -}; - - -struct XMBAttribute -{ - XMBAttribute() {} - XMBAttribute(int name, const CStr8& value) - : Name(name), Value(value) {}; - - int Name; - CStr8 Value; // UTF-8 encoded -}; - -class XMBAttributeList -{ -public: - XMBAttributeList(const char* offset, size_t count, const char* endoffset) - : m_Size(count), m_Pointer(offset), m_CurItemID(0), m_CurPointer(offset), m_EndPointer(endoffset) {} - - // Get the attribute value directly - CStr8 GetNamedItem(const int AttributeName) const; - - // Linear in the number of elements in the list - XMBAttribute operator[](size_t id); // returns Children[id] - - class iterator - { - public: - typedef ptrdiff_t difference_type; - typedef XMBAttribute value_type; - typedef XMBAttribute reference; // Because we need to construct the object - typedef XMBAttribute pointer; // Because we need to construct the object - typedef std::forward_iterator_tag iterator_category; - - iterator(size_t size, const char* ptr, const char* endptr = NULL) - : m_Size(size), m_CurItemID(endptr ? size : 0), m_CurPointer(endptr ? endptr : ptr) {} - XMBAttribute operator*() const; - XMBAttribute operator->() const { return **this; } - iterator& operator++(); - - bool operator==(const iterator& rhs) const - { - return m_Size == rhs.m_Size && - m_CurItemID == rhs.m_CurItemID && - m_CurPointer == rhs.m_CurPointer; - } - bool operator!=(const iterator& rhs) const { return !(*this == rhs); } - private: - size_t m_Size; - size_t m_CurItemID; - const char* m_CurPointer; - }; - iterator begin() const { return iterator(m_Size, m_Pointer); } - iterator end() const { return iterator(m_Size, m_Pointer, m_EndPointer); } - - size_t size() const { return m_Size; } - bool empty() const { return m_Size == 0; } - -private: - size_t m_Size; - - // Pointer to start of attribute list - const char* m_Pointer; - - // For optimised sequential access: - size_t m_CurItemID; - const char* m_CurPointer; - - const char* m_EndPointer; -}; - -#endif // INCLUDED_XEROXMB Index: ps/trunk/source/ps/XML/XeroXMB.cpp =================================================================== --- ps/trunk/source/ps/XML/XeroXMB.cpp +++ ps/trunk/source/ps/XML/XeroXMB.cpp @@ -1,342 +0,0 @@ -/* Copyright (C) 2015 Wildfire Games. - * This file is part of 0 A.D. - * - * 0 A.D. is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 2 of the License, or - * (at your option) any later version. - * - * 0 A.D. is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with 0 A.D. If not, see . - */ - -#include "precompiled.h" - -#include "Xeromyces.h" - -#include "lib/byte_order.h" // FOURCC_LE - -// external linkage (also used by Xeromyces.cpp) -const char* HeaderMagicStr = "XMB0"; -const char* UnfinishedHeaderMagicStr = "XMBu"; -// Arbitrary version number - change this if we update the code and -// need to invalidate old users' caches -const u32 XMBVersion = 3; - -template -static inline T read(const void* ptr) -{ - T ret; - memcpy(&ret, ptr, sizeof(T)); - return ret; -} - -bool XMBFile::Initialise(const char* FileData) -{ - m_Pointer = FileData; - char Header[5] = { 0 }; - strncpy_s(Header, 5, m_Pointer, 4); - m_Pointer += 4; - // (c.f. @return documentation of this function) - if(!strcmp(Header, UnfinishedHeaderMagicStr)) - return false; - ENSURE(!strcmp(Header, HeaderMagicStr) && "Invalid XMB header!"); - - u32 Version = read(m_Pointer); - m_Pointer += 4; - if (Version != XMBVersion) - return false; - - int i; - - // FIXME Check that m_Pointer doesn't end up past the end of the buffer - // (it shouldn't be all that dangerous since we're only doing read-only - // access, but it might crash on an invalid file, reading a couple of - // billion random element names from RAM) - -#ifdef XERO_USEMAP - // Build a std::map of all the names->ids - u32 ElementNameCount = read(m_Pointer); m_Pointer += 4; - for (i = 0; i < ElementNameCount; ++i) - m_ElementNames[ReadZStr8()] = i; - - u32 AttributeNameCount = read(m_Pointer); m_Pointer += 4; - for (i = 0; i < AttributeNameCount; ++i) - m_AttributeNames[ReadZStr8()] = i; -#else - // Ignore all the names for now, and skip over them - // (remembering the position of the first) - m_ElementNameCount = read(m_Pointer); m_Pointer += 4; - m_ElementPointer = m_Pointer; - for (i = 0; i < m_ElementNameCount; ++i) - m_Pointer += 4 + read(m_Pointer); // skip over the string - - m_AttributeNameCount = read(m_Pointer); m_Pointer += 4; - m_AttributePointer = m_Pointer; - for (i = 0; i < m_AttributeNameCount; ++i) - m_Pointer += 4 + read(m_Pointer); // skip over the string -#endif - - return true; // success -} - -std::string XMBFile::ReadZStr8() -{ - int Length = read(m_Pointer); - m_Pointer += 4; - std::string String (m_Pointer); // reads up until the first NULL - m_Pointer += Length; - return String; -} - -XMBElement XMBFile::GetRoot() const -{ - return XMBElement(m_Pointer); -} - - -#ifdef XERO_USEMAP - -int XMBFile::GetElementID(const char* Name) const -{ - return m_ElementNames[Name]; -} - -int XMBFile::GetAttributeID(const char* Name) const -{ - return m_AttributeNames[Name]; -} - -#else // #ifdef XERO_USEMAP - -int XMBFile::GetElementID(const char* Name) const -{ - const char* Pos = m_ElementPointer; - - int len = (int)strlen(Name)+1; // count bytes, including null terminator - - // Loop through each string to find a match - for (int i = 0; i < m_ElementNameCount; ++i) - { - // See if this could be the right string, checking its - // length and then its contents - if (read(Pos) == len && strncasecmp(Pos+4, Name, len) == 0) - return i; - // If not, jump to the next string - Pos += 4 + read(Pos); - } - // Failed - return -1; -} - -int XMBFile::GetAttributeID(const char* Name) const -{ - const char* Pos = m_AttributePointer; - - int len = (int)strlen(Name)+1; // count bytes, including null terminator - - // Loop through each string to find a match - for (int i = 0; i < m_AttributeNameCount; ++i) - { - // See if this could be the right string, checking its - // length and then its contents - if (read(Pos) == len && strncasecmp(Pos+4, Name, len) == 0) - return i; - // If not, jump to the next string - Pos += 4 + read(Pos); - } - // Failed - return -1; -} -#endif // #ifdef XERO_USEMAP / #else - - -// Relatively inefficient, so only use when -// laziness overcomes the need for speed -std::string XMBFile::GetElementString(const int ID) const -{ - const char* Pos = m_ElementPointer; - for (int i = 0; i < ID; ++i) - Pos += 4 + read(Pos); - return std::string(Pos+4); -} - -std::string XMBFile::GetAttributeString(const int ID) const -{ - const char* Pos = m_AttributePointer; - for (int i = 0; i < ID; ++i) - Pos += 4 + read(Pos); - return std::string(Pos+4); -} - - - -int XMBElement::GetNodeName() const -{ - if (m_Pointer == NULL) - return -1; - - return read(m_Pointer + 4); // == ElementName -} - -XMBElementList XMBElement::GetChildNodes() const -{ - if (m_Pointer == NULL) - return XMBElementList(NULL, 0, NULL); - - return XMBElementList( - m_Pointer + 20 + read(m_Pointer + 16), // == Children[] - read(m_Pointer + 12), // == ChildCount - m_Pointer + read(m_Pointer) // == &Children[ChildCount] - ); -} - -XMBAttributeList XMBElement::GetAttributes() const -{ - if (m_Pointer == NULL) - return XMBAttributeList(NULL, 0, NULL); - - return XMBAttributeList( - m_Pointer + 24 + read(m_Pointer + 20), // == Attributes[] - read(m_Pointer + 8), // == AttributeCount - m_Pointer + 20 + read(m_Pointer + 16) // == &Attributes[AttributeCount] ( == &Children[]) - ); -} - -CStr8 XMBElement::GetText() const -{ - // Return empty string if there's no text - if (m_Pointer == NULL || read(m_Pointer + 20) == 0) - return CStr8(); - - return CStr8(m_Pointer + 28); -} - -int XMBElement::GetLineNumber() const -{ - // Make sure there actually was some text to record the line of - if (m_Pointer == NULL || read(m_Pointer + 20) == 0) - return -1; - else - return read(m_Pointer + 24); -} - -XMBElement XMBElementList::GetFirstNamedItem(const int ElementName) const -{ - const char* Pos = m_Pointer; - - // Maybe not the cleverest algorithm, but it should be - // fast enough with half a dozen attributes: - for (size_t i = 0; i < m_Size; ++i) - { - int Length = read(Pos); - int Name = read(Pos+4); - if (Name == ElementName) - return XMBElement(Pos); - Pos += Length; - } - - // Can't find element - return XMBElement(); -} - -XMBElementList::iterator& XMBElementList::iterator::operator++() -{ - m_CurPointer += read(m_CurPointer); - ++m_CurItemID; - return (*this); -} - -XMBElement XMBElementList::operator[](size_t id) -{ - ENSURE(id < m_Size && "Element ID out of range"); - const char* Pos; - size_t i; - - if (id < m_CurItemID) - { - Pos = m_Pointer; - i = 0; - } - else - { - // If access is sequential, don't bother scanning - // through all the nodes to find the next one - Pos = m_CurPointer; - i = m_CurItemID; - } - - // Skip over each preceding node - for (; i < id; ++i) - Pos += read(Pos); - - // Cache information about this node - m_CurItemID = id; - m_CurPointer = Pos; - - return XMBElement(Pos); -} - -CStr8 XMBAttributeList::GetNamedItem(const int AttributeName) const -{ - const char* Pos = m_Pointer; - - // Maybe not the cleverest algorithm, but it should be - // fast enough with half a dozen attributes: - for (size_t i = 0; i < m_Size; ++i) - { - if (read(Pos) == AttributeName) - return CStr8(Pos+8); - Pos += 8 + read(Pos+4); // Skip over the string - } - - // Can't find attribute - return CStr8(); -} - -XMBAttribute XMBAttributeList::iterator::operator*() const -{ - return XMBAttribute(read(m_CurPointer), CStr8(m_CurPointer+8)); -} - -XMBAttributeList::iterator& XMBAttributeList::iterator::operator++() -{ - m_CurPointer += 8 + read(m_CurPointer+4); // skip ID, length, and string data - ++m_CurItemID; - return (*this); -} - -XMBAttribute XMBAttributeList::operator[](size_t id) -{ - ENSURE(id < m_Size && "Attribute ID out of range"); - const char* Pos; - size_t i; - - if (id < m_CurItemID) - { - Pos = m_Pointer; - i = 0; - } - else - { - // If access is sequential, don't bother scanning - // through all the nodes to find the next one - Pos = m_CurPointer; - i = m_CurItemID; - } - - // Skip over each preceding attribute - for (; i < id; ++i) - Pos += 8 + read(Pos+4); // skip ID, length, and string data - - // Cache information about this attribute - m_CurItemID = id; - m_CurPointer = Pos; - - return XMBAttribute(read(Pos), CStr8(Pos+8)); -} Index: ps/trunk/source/ps/XML/Xeromyces.h =================================================================== --- ps/trunk/source/ps/XML/Xeromyces.h +++ ps/trunk/source/ps/XML/Xeromyces.h @@ -30,19 +30,17 @@ ERROR_TYPE(Xeromyces, XMLParseError); ERROR_TYPE(Xeromyces, XMLValidationFailed); -#include "XeroXMB.h" +#include "ps/XMB/XMBData.h" +#include "ps/XMB/XMBStorage.h" #include "lib/file/vfs/vfs.h" class RelaxNGValidator; -class WriteBuffer; -typedef struct _xmlDoc xmlDoc; -typedef xmlDoc* xmlDocPtr; - -class CXeromyces : public XMBFile +class CXeromyces : public XMBData { - friend class TestXeroXMB; + friend class TestXMBData; + friend class XMBData; public: /** * Load from an XML file (with invisible XMB caching). @@ -81,11 +79,7 @@ PSRETURN ConvertFile(const PIVFS& vfs, const VfsPath& filename, const VfsPath& xmbPath, const std::string& validatorName); - bool ReadXMBFile(const PIVFS& vfs, const VfsPath& filename); - - static PSRETURN CreateXMB(const xmlDocPtr doc, WriteBuffer& writeBuffer); - - shared_ptr m_XMBBuffer; + XMBStorage m_Data; }; Index: ps/trunk/source/ps/XML/Xeromyces.cpp =================================================================== --- ps/trunk/source/ps/XML/Xeromyces.cpp +++ ps/trunk/source/ps/XML/Xeromyces.cpp @@ -28,6 +28,7 @@ #include "ps/CacheLoader.h" #include "ps/CLogger.h" #include "ps/Filesystem.h" + #include "RelaxNG.h" #include "Xeromyces.h" @@ -120,13 +121,17 @@ validatorGrammarHash = GetValidator(validatorName).GetGrammarHash(); } VfsPath xmbPath; - Status ret = cacheLoader.TryLoadingCached(filename, validatorGrammarHash, XMBVersion, xmbPath); + Status ret = cacheLoader.TryLoadingCached(filename, validatorGrammarHash, XMBStorage::XMBVersion, xmbPath); if (ret == INFO::OK) { // Found a cached XMB - load it - if (ReadXMBFile(vfs, xmbPath)) + if (m_Data.ReadFromFile(vfs, xmbPath)) + { + if(!Initialise(m_Data)) + return PSRETURN_Xeromyces_XMLParseError; return PSRETURN_OK; + } // If this fails then we'll continue and (re)create the loose cache - // this failure legitimately happens due to partially-written XMB files. } @@ -184,43 +189,20 @@ } } - WriteBuffer writeBuffer; - CreateXMB(doc, writeBuffer); - + m_Data.LoadXMLDoc(doc); xmlFreeDoc(doc); // Save the file to disk, so it can be loaded quickly next time. // Don't save if invalid, because we want the syntax error every program start. - vfs->CreateFile(xmbPath, writeBuffer.Data(), writeBuffer.Size()); - - m_XMBBuffer = writeBuffer.Data(); // add a reference + vfs->CreateFile(xmbPath, m_Data.m_Buffer, m_Data.m_Size); - // Set up the XMBFile - const bool ok = Initialise((const char*)m_XMBBuffer.get()); + // Set up the XMBData + const bool ok = Initialise(m_Data); ENSURE(ok); return PSRETURN_OK; } -bool CXeromyces::ReadXMBFile(const PIVFS& vfs, const VfsPath& filename) -{ - size_t size; - if(vfs->LoadFile(filename, m_XMBBuffer, size) < 0) - return false; - // if the game crashes during loading, (e.g. due to driver bugs), - // it sometimes leaves empty XMB files in the cache. - // reporting failure will cause our caller to re-generate the XMB. - if(size == 0) - return false; - ENSURE(size >= 4); // make sure it's at least got the initial header - - // Set up the XMBFile - if(!Initialise((const char*)m_XMBBuffer.get())) - return false; - - return true; -} - PSRETURN CXeromyces::LoadString(const char* xml, const std::string& validatorName /* = "" */) { ENSURE(g_XeromycesStarted); @@ -242,185 +224,12 @@ } } - WriteBuffer writeBuffer; - CreateXMB(doc, writeBuffer); - + m_Data.LoadXMLDoc(doc); xmlFreeDoc(doc); - m_XMBBuffer = writeBuffer.Data(); // add a reference - - // Set up the XMBFile - const bool ok = Initialise((const char*)m_XMBBuffer.get()); + // Set up the XMBData + const bool ok = Initialise(m_Data); ENSURE(ok); return PSRETURN_OK; } - - -static void FindNames(const xmlNodePtr node, std::set& elementNames, std::set& attributeNames) -{ - elementNames.insert((const char*)node->name); - - for (xmlAttrPtr attr = node->properties; attr; attr = attr->next) - attributeNames.insert((const char*)attr->name); - - for (xmlNodePtr child = node->children; child; child = child->next) - if (child->type == XML_ELEMENT_NODE) - FindNames(child, elementNames, attributeNames); -} - -static void OutputElement(const xmlNodePtr node, WriteBuffer& writeBuffer, - std::map& elementIDs, - std::map& attributeIDs -) -{ - // Filled in later with the length of the element - size_t posLength = writeBuffer.Size(); - writeBuffer.Append("????", 4); - - writeBuffer.Append(&elementIDs[(const char*)node->name], 4); - - u32 attrCount = 0; - for (xmlAttrPtr attr = node->properties; attr; attr = attr->next) - ++attrCount; - writeBuffer.Append(&attrCount, 4); - - u32 childCount = 0; - for (xmlNodePtr child = node->children; child; child = child->next) - if (child->type == XML_ELEMENT_NODE) - ++childCount; - writeBuffer.Append(&childCount, 4); - - // Filled in later with the offset to the list of child elements - size_t posChildrenOffset = writeBuffer.Size(); - writeBuffer.Append("????", 4); - - - // Trim excess whitespace in the entity's text, while counting - // the number of newlines trimmed (so that JS error reporting - // can give the correct line number within the script) - - std::string whitespace = " \t\r\n"; - std::string text; - for (xmlNodePtr child = node->children; child; child = child->next) - { - if (child->type == XML_TEXT_NODE) - { - xmlChar* content = xmlNodeGetContent(child); - text += std::string((const char*)content); - xmlFree(content); - } - } - - u32 linenum = xmlGetLineNo(node); - - // Find the start of the non-whitespace section - size_t first = text.find_first_not_of(whitespace); - - if (first == text.npos) - // Entirely whitespace - easy to handle - text = ""; - - else - { - // Count the number of \n being cut off, - // and add them to the line number - std::string trimmed (text.begin(), text.begin()+first); - linenum += std::count(trimmed.begin(), trimmed.end(), '\n'); - - // Find the end of the non-whitespace section, - // and trim off everything else - size_t last = text.find_last_not_of(whitespace); - text = text.substr(first, 1+last-first); - } - - - // Output text, prefixed by length in bytes - if (text.length() == 0) - { - // No text; don't write much - writeBuffer.Append("\0\0\0\0", 4); - } - else - { - // Write length and line number and null-terminated text - u32 nodeLen = u32(4 + text.length()+1); - writeBuffer.Append(&nodeLen, 4); - writeBuffer.Append(&linenum, 4); - writeBuffer.Append((void*)text.c_str(), nodeLen-4); - } - - // Output attributes - for (xmlAttrPtr attr = node->properties; attr; attr = attr->next) - { - writeBuffer.Append(&attributeIDs[(const char*)attr->name], 4); - - xmlChar* value = xmlNodeGetContent(attr->children); - u32 attrLen = u32(xmlStrlen(value)+1); - writeBuffer.Append(&attrLen, 4); - writeBuffer.Append((void*)value, attrLen); - xmlFree(value); - } - - // Go back and fill in the child-element offset - u32 childrenOffset = (u32)(writeBuffer.Size() - (posChildrenOffset+4)); - writeBuffer.Overwrite(&childrenOffset, 4, posChildrenOffset); - - // Output all child elements - for (xmlNodePtr child = node->children; child; child = child->next) - if (child->type == XML_ELEMENT_NODE) - OutputElement(child, writeBuffer, elementIDs, attributeIDs); - - // Go back and fill in the length - u32 length = (u32)(writeBuffer.Size() - posLength); - writeBuffer.Overwrite(&length, 4, posLength); -} - -PSRETURN CXeromyces::CreateXMB(const xmlDocPtr doc, WriteBuffer& writeBuffer) -{ - // Header - writeBuffer.Append(UnfinishedHeaderMagicStr, 4); - // Version - writeBuffer.Append(&XMBVersion, 4); - - u32 i; - - // Find the unique element/attribute names - std::set elementNames; - std::set attributeNames; - FindNames(xmlDocGetRootElement(doc), elementNames, attributeNames); - - std::map elementIDs; - std::map attributeIDs; - - // Output element names - i = 0; - u32 elementCount = (u32)elementNames.size(); - writeBuffer.Append(&elementCount, 4); - for (const std::string& n : elementNames) - { - u32 textLen = (u32)n.length()+1; - writeBuffer.Append(&textLen, 4); - writeBuffer.Append((void*)n.c_str(), textLen); - elementIDs[n] = i++; - } - - // Output attribute names - i = 0; - u32 attributeCount = (u32)attributeNames.size(); - writeBuffer.Append(&attributeCount, 4); - for (const std::string& n : attributeNames) - { - u32 textLen = (u32)n.length()+1; - writeBuffer.Append(&textLen, 4); - writeBuffer.Append((void*)n.c_str(), textLen); - attributeIDs[n] = i++; - } - - OutputElement(xmlDocGetRootElement(doc), writeBuffer, elementIDs, attributeIDs); - - // file is now valid, so insert correct magic string - writeBuffer.Overwrite(HeaderMagicStr, 4, 0); - - return PSRETURN_OK; -} Index: ps/trunk/source/ps/XML/tests/test_XeroXMB.h =================================================================== --- ps/trunk/source/ps/XML/tests/test_XeroXMB.h +++ ps/trunk/source/ps/XML/tests/test_XeroXMB.h @@ -1,161 +0,0 @@ -/* Copyright (C) 2015 Wildfire Games. - * This file is part of 0 A.D. - * - * 0 A.D. is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 2 of the License, or - * (at your option) any later version. - * - * 0 A.D. is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with 0 A.D. If not, see . - */ - -#include "lib/self_test.h" - -#include "ps/XML/Xeromyces.h" - -#include "lib/file/io/write_buffer.h" - -#include - -class TestXeroXMB : public CxxTest::TestSuite -{ -private: - shared_ptr m_Buffer; - - XMBFile parse(const char* doc) - { - xmlDocPtr xmlDoc = xmlReadMemory(doc, int(strlen(doc)), "", NULL, - XML_PARSE_NONET|XML_PARSE_NOCDATA); - WriteBuffer buffer; - PSRETURN ret = CXeromyces::CreateXMB(xmlDoc, buffer); - xmlFreeDoc(xmlDoc); - - TS_ASSERT_EQUALS(ret, PSRETURN_OK); - - XMBFile xmb; - m_Buffer = buffer.Data(); // hold a reference - TS_ASSERT(xmb.Initialise((const char*)m_Buffer.get())); - return xmb; - } - - void tearDown() - { - m_Buffer.reset(); - } - -public: - void test_basic() - { - XMBFile xmb (parse("\n bar \n\n\nbar\n")); - - TS_ASSERT_DIFFERS(xmb.GetElementID("test"), -1); - TS_ASSERT_DIFFERS(xmb.GetElementID("foo"), -1); - TS_ASSERT_EQUALS(xmb.GetElementID("bar"), -1); - - TS_ASSERT_DIFFERS(xmb.GetAttributeID("x"), -1); - TS_ASSERT_EQUALS(xmb.GetAttributeID("y"), -1); - TS_ASSERT_EQUALS(xmb.GetAttributeID("test"), -1); - - int el_test = xmb.GetElementID("test"); - int el_foo = xmb.GetElementID("foo"); - int at_x = xmb.GetAttributeID("x"); - - XMBElement root = xmb.GetRoot(); - TS_ASSERT_EQUALS(root.GetNodeName(), el_test); - TS_ASSERT_EQUALS(root.GetLineNumber(), -1); - TS_ASSERT_EQUALS(CStr(root.GetText()), ""); - - TS_ASSERT_EQUALS(root.GetChildNodes().size(), 2); - XMBElement child = root.GetChildNodes()[0]; - TS_ASSERT_EQUALS(child.GetNodeName(), el_foo); - TS_ASSERT_EQUALS(child.GetLineNumber(), 2); - TS_ASSERT_EQUALS(child.GetChildNodes().size(), 0); - TS_ASSERT_EQUALS(CStr(child.GetText()), "bar"); - - TS_ASSERT_EQUALS(root.GetChildNodes()[1].GetLineNumber(), 5); - - TS_ASSERT_EQUALS(child.GetAttributes().size(), 1); - XMBAttribute attr = child.GetAttributes()[0]; - TS_ASSERT_EQUALS(attr.Name, at_x); - TS_ASSERT_EQUALS(CStr(attr.Value), " y "); - } - - void test_GetFirstNamedItem() - { - XMBFile xmb (parse(" A B C D ")); - - XMBElement root = xmb.GetRoot(); - TS_ASSERT_EQUALS(root.GetChildNodes().size(), 4); - - XMBElement x = root.GetChildNodes().GetFirstNamedItem(xmb.GetElementID("x")); - XMBElement y = root.GetChildNodes().GetFirstNamedItem(xmb.GetElementID("y")); - XMBElement w = root.GetChildNodes().GetFirstNamedItem(xmb.GetElementID("w")); - - TS_ASSERT_EQUALS(x.GetNodeName(), xmb.GetElementID("x")); - TS_ASSERT_EQUALS(CStr(x.GetText()), "A"); - - TS_ASSERT_EQUALS(y.GetNodeName(), xmb.GetElementID("y")); - TS_ASSERT_EQUALS(CStr(y.GetText()), "C"); - - TS_ASSERT_EQUALS(w.GetNodeName(), -1); - TS_ASSERT_EQUALS(CStr(w.GetText()), ""); - TS_ASSERT_EQUALS(w.GetLineNumber(), -1); - TS_ASSERT_EQUALS(w.GetChildNodes().size(), 0); - TS_ASSERT_EQUALS(w.GetAttributes().size(), 0); - } - - void test_doctype_ignored() - { - XMBFile xmb (parse("")); - - TS_ASSERT_DIFFERS(xmb.GetElementID("foo"), -1); - } - - void test_complex_parse() - { - XMBFile xmb (parse("\t\n \tx <>&"'bar\n\nbazqux")); - TS_ASSERT_EQUALS(CStr(xmb.GetRoot().GetText()), "x <>&\"'foobar\n\nbazqux"); - } - - void test_unicode() - { - XMBFile xmb (parse("ሴ\xE1\x88\xB4")); - CStrW text; - - text = xmb.GetRoot().GetText().FromUTF8(); - TS_ASSERT_EQUALS((int)text.length(), 2); - TS_ASSERT_EQUALS(text[0], 0x1234); - TS_ASSERT_EQUALS(text[1], 0x1234); - - text = xmb.GetRoot().GetAttributes()[0].Value.FromUTF8(); - TS_ASSERT_EQUALS((int)text.length(), 2); - TS_ASSERT_EQUALS(text[0], 0x1234); - TS_ASSERT_EQUALS(text[1], 0x1234); - } - - void test_iso88591() - { - XMBFile xmb (parse("ሴ\xE1\x88\xB4")); - CStrW text; - - text = xmb.GetRoot().GetText().FromUTF8(); - TS_ASSERT_EQUALS((int)text.length(), 4); - TS_ASSERT_EQUALS(text[0], 0x1234); - TS_ASSERT_EQUALS(text[1], 0x00E1); - TS_ASSERT_EQUALS(text[2], 0x0088); - TS_ASSERT_EQUALS(text[3], 0x00B4); - - text = xmb.GetRoot().GetAttributes()[0].Value.FromUTF8(); - TS_ASSERT_EQUALS((int)text.length(), 4); - TS_ASSERT_EQUALS(text[0], 0x1234); - TS_ASSERT_EQUALS(text[1], 0x00E1); - TS_ASSERT_EQUALS(text[2], 0x0088); - TS_ASSERT_EQUALS(text[3], 0x00B4); - } -}; Index: ps/trunk/source/simulation2/system/ParamNode.h =================================================================== --- ps/trunk/source/simulation2/system/ParamNode.h +++ ps/trunk/source/simulation2/system/ParamNode.h @@ -27,7 +27,7 @@ #include #include -class XMBFile; +class XMBData; class XMBElement; class ScriptRequest; @@ -165,7 +165,7 @@ * @param sourceIdentifier Optional; string you can pass along to indicate the source of * the data getting loaded. Used for output to log messages if an error occurs. */ - static void LoadXML(CParamNode& ret, const XMBFile& file, const wchar_t* sourceIdentifier = NULL); + static void LoadXML(CParamNode& ret, const XMBData& xmb, const wchar_t* sourceIdentifier = NULL); /** * Loads the XML data specified by @a path into the node @a ret. @@ -276,7 +276,7 @@ * @param sourceIdentifier Optional; string you can pass along to indicate the source of * the data getting applied. Used for output to log messages if an error occurs. */ - void ApplyLayer(const XMBFile& xmb, const XMBElement& element, const wchar_t* sourceIdentifier = NULL); + void ApplyLayer(const XMBData& xmb, const XMBElement& element, const wchar_t* sourceIdentifier = NULL); void ResetScriptVal(); Index: ps/trunk/source/simulation2/system/ParamNode.cpp =================================================================== --- ps/trunk/source/simulation2/system/ParamNode.cpp +++ ps/trunk/source/simulation2/system/ParamNode.cpp @@ -37,7 +37,7 @@ { } -void CParamNode::LoadXML(CParamNode& ret, const XMBFile& xmb, const wchar_t* sourceIdentifier /*= NULL*/) +void CParamNode::LoadXML(CParamNode& ret, const XMBData& xmb, const wchar_t* sourceIdentifier /*= NULL*/) { ret.ApplyLayer(xmb, xmb.GetRoot(), sourceIdentifier); } @@ -64,11 +64,11 @@ return PSRETURN_OK; } -void CParamNode::ApplyLayer(const XMBFile& xmb, const XMBElement& element, const wchar_t* sourceIdentifier /*= NULL*/) +void CParamNode::ApplyLayer(const XMBData& xmb, const XMBElement& element, const wchar_t* sourceIdentifier /*= NULL*/) { ResetScriptVal(); - std::string name = xmb.GetElementString(element.GetNodeName()); // TODO: is GetElementString inefficient? + std::string name = xmb.GetElementString(element.GetNodeName()); CStr value = element.GetText(); bool hasSetValue = false; @@ -224,8 +224,8 @@ if (attr.Name == at_replace || attr.Name == at_op || attr.Name == at_merge || attr.Name == at_filtered) continue; // Add any others - std::string attrName = xmb.GetAttributeString(attr.Name); - node.m_Childs["@" + attrName].m_Value = attr.Value; + const char* attrName(xmb.GetAttributeString(attr.Name)); + node.m_Childs[CStr("@") + attrName].m_Value = attr.Value; } } Index: ps/trunk/source/soundmanager/scripting/SoundGroup.cpp =================================================================== --- ps/trunk/source/soundmanager/scripting/SoundGroup.cpp +++ ps/trunk/source/soundmanager/scripting/SoundGroup.cpp @@ -329,7 +329,7 @@ if (root.GetNodeName() != el_soundgroup) { - LOGERROR("Invalid SoundGroup format (unrecognised root element '%s')", XeroFile.GetElementString(root.GetNodeName()).c_str()); + LOGERROR("Invalid SoundGroup format (unrecognised root element '%s')", XeroFile.GetElementString(root.GetNodeName())); return false; }