Index: ps/trunk/source/lib/file/common/trace.cpp =================================================================== --- ps/trunk/source/lib/file/common/trace.cpp (revision 17132) +++ ps/trunk/source/lib/file/common/trace.cpp (revision 17133) @@ -1,234 +1,235 @@ -/* Copyright (c) 2010 Wildfire Games +/* Copyright (c) 2015 Wildfire Games * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sublicense, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be included * in all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /* * IO event recording */ #include "precompiled.h" #include "lib/file/common/trace.h" #include #include #include "lib/allocators/pool.h" #include "lib/timer.h" // timer_Time #include "lib/sysdep/sysdep.h" // sys_OpenFile /*virtual*/ ITrace::~ITrace() { } //----------------------------------------------------------------------------- TraceEntry::TraceEntry(EAction action, const Path& pathname, size_t size) : m_timestamp((float)timer_Time()) , m_action(action) , m_pathname(pathname) , m_size(size) { } TraceEntry::TraceEntry(const std::wstring& text) { // swscanf is far too awkward to get working cross-platform, // so use iostreams here instead wchar_t dummy; wchar_t action; std::wstringstream stream(text); stream >> m_timestamp; stream >> dummy; ENSURE(dummy == ':'); stream >> action; ENSURE(action == 'L' || action == 'S'); m_action = (EAction)action; stream >> dummy; ENSURE(dummy == '"'); Path::String pathname; std::getline(stream, pathname, L'"'); m_pathname = Path(pathname); stream >> m_size; ENSURE(stream.get() == '\n'); - ENSURE(stream.good()); + // NOTE: Don't use good() here - it fails due to a bug in older libc++ versions + ENSURE(!stream.bad() && !stream.fail()); ENSURE(stream.get() == WEOF); } std::wstring TraceEntry::EncodeAsText() const { const wchar_t action = (wchar_t)m_action; wchar_t buf[1000]; swprintf_s(buf, ARRAY_SIZE(buf), L"%#010f: %c \"%ls\" %lu\n", m_timestamp, action, m_pathname.string().c_str(), (unsigned long)m_size); return buf; } //----------------------------------------------------------------------------- class Trace_Dummy : public ITrace { public: Trace_Dummy(size_t UNUSED(maxSize)) { } virtual void NotifyLoad(const Path& UNUSED(pathname), size_t UNUSED(size)) { } virtual void NotifyStore(const Path& UNUSED(pathname), size_t UNUSED(size)) { } virtual Status Load(const OsPath& UNUSED(pathname)) { return INFO::OK; } virtual Status Store(const OsPath& UNUSED(pathname)) const { return INFO::OK; } virtual const TraceEntry* Entries() const { return 0; } virtual size_t NumEntries() const { return 0; } }; //----------------------------------------------------------------------------- class Trace : public ITrace { public: Trace(size_t maxSize) { (void)pool_create(&m_pool, maxSize, sizeof(TraceEntry)); } virtual ~Trace() { for(size_t i = 0; i < NumEntries(); i++) { TraceEntry* entry = (TraceEntry*)(uintptr_t(m_pool.da.base) + i*m_pool.el_size); entry->~TraceEntry(); } (void)pool_destroy(&m_pool); } virtual void NotifyLoad(const Path& pathname, size_t size) { new(Allocate()) TraceEntry(TraceEntry::Load, pathname, size); } virtual void NotifyStore(const Path& pathname, size_t size) { new(Allocate()) TraceEntry(TraceEntry::Store, pathname, size); } virtual Status Load(const OsPath& pathname) { pool_free_all(&m_pool); errno = 0; FILE* file = sys_OpenFile(pathname, "rt"); if(!file) WARN_RETURN(StatusFromErrno()); for(;;) { wchar_t text[500]; if(!fgetws(text, ARRAY_SIZE(text)-1, file)) break; new(Allocate()) TraceEntry(text); } fclose(file); return INFO::OK; } virtual Status Store(const OsPath& pathname) const { errno = 0; FILE* file = sys_OpenFile(pathname, "at"); if(!file) WARN_RETURN(StatusFromErrno()); for(size_t i = 0; i < NumEntries(); i++) { std::wstring text = Entries()[i].EncodeAsText(); fputws(text.c_str(), file); } (void)fclose(file); return INFO::OK; } virtual const TraceEntry* Entries() const { return (const TraceEntry*)m_pool.da.base; } virtual size_t NumEntries() const { return m_pool.da.pos / m_pool.el_size; } private: void* Allocate() { void* p = pool_alloc(&m_pool, 0); ENSURE(p); return p; } Pool m_pool; }; PITrace CreateDummyTrace(size_t maxSize) { return PITrace(new Trace_Dummy(maxSize)); } PITrace CreateTrace(size_t maxSize) { return PITrace(new Trace(maxSize)); } Index: ps/trunk/source/simulation2/serialization/BinarySerializer.h =================================================================== --- ps/trunk/source/simulation2/serialization/BinarySerializer.h (revision 17132) +++ ps/trunk/source/simulation2/serialization/BinarySerializer.h (revision 17133) @@ -1,206 +1,234 @@ /* Copyright (C) 2015 Wildfire Games. * This file is part of 0 A.D. * * 0 A.D. is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 2 of the License, or * (at your option) any later version. * * 0 A.D. is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with 0 A.D. If not, see . */ #ifndef INCLUDED_BINARYSERIALIZER #define INCLUDED_BINARYSERIALIZER #include "ISerializer.h" #include "scriptinterface/third_party/ObjectToIDMap.h" #include "lib/byte_order.h" #include "lib/allocators/arena.h" #include /** * Wrapper for redirecting ostream writes to CBinarySerializer's impl */ template class CSerializerStreamBuf : public std::streambuf { NONCOPYABLE(CSerializerStreamBuf); T& m_SerializerImpl; + char m_Buffer[2048]; public: CSerializerStreamBuf(T& impl) : m_SerializerImpl(impl) { + setp(m_Buffer, m_Buffer + ARRAY_SIZE(m_Buffer) - 1); + } + +protected: + // Override overflow and sync, because older versions of libc++ streams + // write strings as individual characters, then xsputn is never called + int overflow(int ch) + { + if (ch == traits_type::eof()) + return traits_type::not_eof(ch); + + ENSURE(pptr() <= epptr()); + *pptr() = ch; + pbump(1); + sync(); + return ch; + } + + int sync() + { + std::ptrdiff_t n = pptr() - pbase(); + if (n != 0) + { + pbump(-n); + m_SerializerImpl.Put("stream", reinterpret_cast (pbase()), n); + } + return 0; } std::streamsize xsputn(const char* s, std::streamsize n) { m_SerializerImpl.Put("stream", reinterpret_cast (s), n); return n; } }; /** * PutScriptVal implementation details. * (Split out from the main class because it's too big to be inlined.) */ class CBinarySerializerScriptImpl { public: CBinarySerializerScriptImpl(ScriptInterface& scriptInterface, ISerializer& serializer); void ScriptString(const char* name, JS::HandleString string); void HandleScriptVal(JS::HandleValue val); void SetSerializablePrototypes(shared_ptr > prototypes); private: ScriptInterface& m_ScriptInterface; ISerializer& m_Serializer; ObjectIdCache m_ScriptBackrefs; u32 m_ScriptBackrefsNext; u32 GetScriptBackrefTag(JS::HandleObject obj); shared_ptr > m_SerializablePrototypes; bool IsSerializablePrototype(JS::HandleObject prototype); std::wstring GetPrototypeName(JS::HandleObject prototype); }; /** * Serialize to a binary stream. T must just implement the Put() method. * (We use this templated approach to allow compiler inlining.) */ template class CBinarySerializer : public ISerializer { NONCOPYABLE(CBinarySerializer); public: CBinarySerializer(ScriptInterface& scriptInterface) : m_ScriptImpl(new CBinarySerializerScriptImpl(scriptInterface, *this)), m_RawStreamBuf(m_Impl), m_RawStream(&m_RawStreamBuf) { } template CBinarySerializer(ScriptInterface& scriptInterface, A& a) : m_ScriptImpl(new CBinarySerializerScriptImpl(scriptInterface, *this)), m_Impl(a), m_RawStreamBuf(m_Impl), m_RawStream(&m_RawStreamBuf) { } virtual void SetSerializablePrototypes(shared_ptr >& prototypes) { m_ScriptImpl->SetSerializablePrototypes(prototypes); } protected: /* The Put* implementations here are designed for subclasses that want an efficient, portable, deserializable representation. (Subclasses with different requirements should override these methods.) Numbers are converted to little-endian byte strings, for portability and efficiency. Data is not aligned, for storage efficiency. */ virtual void PutNumber(const char* name, uint8_t value) { m_Impl.Put(name, (const u8*)&value, sizeof(uint8_t)); } virtual void PutNumber(const char* name, int8_t value) { m_Impl.Put(name, (const u8*)&value, sizeof(int8_t)); } virtual void PutNumber(const char* name, uint16_t value) { uint16_t v = to_le16(value); m_Impl.Put(name, (const u8*)&v, sizeof(uint16_t)); } virtual void PutNumber(const char* name, int16_t value) { int16_t v = (i16)to_le16((u16)value); m_Impl.Put(name, (const u8*)&v, sizeof(int16_t)); } virtual void PutNumber(const char* name, uint32_t value) { uint32_t v = to_le32(value); m_Impl.Put(name, (const u8*)&v, sizeof(uint32_t)); } virtual void PutNumber(const char* name, int32_t value) { int32_t v = (i32)to_le32((u32)value); m_Impl.Put(name, (const u8*)&v, sizeof(int32_t)); } virtual void PutNumber(const char* name, float value) { m_Impl.Put(name, (const u8*)&value, sizeof(float)); } virtual void PutNumber(const char* name, double value) { m_Impl.Put(name, (const u8*)&value, sizeof(double)); } virtual void PutNumber(const char* name, fixed value) { int32_t v = (i32)to_le32((u32)value.GetInternalValue()); m_Impl.Put(name, (const u8*)&v, sizeof(int32_t)); } virtual void PutBool(const char* name, bool value) { NumberU8(name, value ? 1 : 0, 0, 1); } virtual void PutString(const char* name, const std::string& value) { // TODO: maybe should intern strings, particularly to save space with script property names PutNumber("string length", (uint32_t)value.length()); m_Impl.Put(name, (u8*)value.data(), value.length()); } virtual void PutScriptVal(const char* UNUSED(name), JS::MutableHandleValue value) { m_ScriptImpl->HandleScriptVal(value); } virtual void PutRaw(const char* name, const u8* data, size_t len) { m_Impl.Put(name, data, len); } virtual std::ostream& GetStream() { return m_RawStream; } protected: T m_Impl; private: std::unique_ptr m_ScriptImpl; CSerializerStreamBuf m_RawStreamBuf; std::ostream m_RawStream; }; #endif // INCLUDED_BINARYSERIALIZER Index: ps/trunk/source/simulation2/serialization/StdDeserializer.cpp =================================================================== --- ps/trunk/source/simulation2/serialization/StdDeserializer.cpp (revision 17132) +++ ps/trunk/source/simulation2/serialization/StdDeserializer.cpp (revision 17133) @@ -1,508 +1,508 @@ /* Copyright (C) 2015 Wildfire Games. * This file is part of 0 A.D. * * 0 A.D. is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 2 of the License, or * (at your option) any later version. * * 0 A.D. is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with 0 A.D. If not, see . */ #include "precompiled.h" #include "StdDeserializer.h" #include "SerializedScriptTypes.h" #include "StdSerializer.h" // for DEBUG_SERIALIZER_ANNOTATE #include "scriptinterface/ScriptInterface.h" #include "scriptinterface/ScriptExtraHeaders.h" // for typed arrays #include "lib/byte_order.h" CStdDeserializer::CStdDeserializer(ScriptInterface& scriptInterface, std::istream& stream) : m_ScriptInterface(scriptInterface), m_Stream(stream), m_dummyObject(scriptInterface.GetJSRuntime()) { JSContext* cx = m_ScriptInterface.GetContext(); JSAutoRequest rq(cx); JS_AddExtraGCRootsTracer(m_ScriptInterface.GetJSRuntime(), CStdDeserializer::Trace, this); // Add a dummy tag because the serializer uses the tag 0 to indicate that a value // needs to be serialized and then tagged m_dummyObject.set(JS_NewObject(cx, nullptr, JS::NullPtr(), JS::NullPtr())); m_ScriptBackrefs.push_back(JS::Heap(m_dummyObject)); } CStdDeserializer::~CStdDeserializer() { FreeScriptBackrefs(); JS_RemoveExtraGCRootsTracer(m_ScriptInterface.GetJSRuntime(), CStdDeserializer::Trace, this); } void CStdDeserializer::Trace(JSTracer *trc, void *data) { reinterpret_cast(data)->TraceMember(trc); } void CStdDeserializer::TraceMember(JSTracer *trc) { for (size_t i=0; i') break; else strName += c; } ENSURE(strName == name); #else UNUSED2(name); #endif m_Stream.read((char*)data, (std::streamsize)len); if (!m_Stream.good()) { // hit eof before len, or other errors - // Note: older libc++ versions incorrectly set eofbit on the last char; test gcount as a workaround + // NOTE: older libc++ versions incorrectly set eofbit on the last char; test gcount as a workaround // see https://llvm.org/bugs/show_bug.cgi?id=9335 if (m_Stream.bad() || m_Stream.fail() || (m_Stream.eof() && m_Stream.gcount() != (std::streamsize)len)) throw PSERROR_Deserialize_ReadFailed(); } } std::istream& CStdDeserializer::GetStream() { return m_Stream; } void CStdDeserializer::RequireBytesInStream(size_t numBytes) { // It would be nice to do: // if (numBytes > (size_t)m_Stream.rdbuf()->in_avail()) // throw PSERROR_Deserialize_OutOfBounds("RequireBytesInStream"); // but that doesn't work (at least on MSVC) since in_avail isn't // guaranteed to return the actual number of bytes available; see e.g. // http://social.msdn.microsoft.com/Forums/en/vclanguage/thread/13009a88-933f-4be7-bf3d-150e425e66a6#70ea562d-8605-4742-8851-1bae431ce6ce // Instead we'll just verify that it's not an extremely large number: if (numBytes > 64*MiB) throw PSERROR_Deserialize_OutOfBounds("RequireBytesInStream"); } void CStdDeserializer::AddScriptBackref(JS::HandleObject obj) { m_ScriptBackrefs.push_back(JS::Heap(obj)); } void CStdDeserializer::GetScriptBackref(u32 tag, JS::MutableHandleObject ret) { ENSURE(m_ScriptBackrefs.size() > tag); ret.set(m_ScriptBackrefs[tag]); } u32 CStdDeserializer::ReserveScriptBackref() { m_ScriptBackrefs.push_back(JS::Heap(m_dummyObject)); return m_ScriptBackrefs.size()-1; } void CStdDeserializer::SetReservedScriptBackref(u32 tag, JS::HandleObject obj) { ENSURE(m_ScriptBackrefs[tag] == m_dummyObject); m_ScriptBackrefs[tag] = JS::Heap(obj); } void CStdDeserializer::FreeScriptBackrefs() { m_ScriptBackrefs.clear(); } //////////////////////////////////////////////////////////////// jsval CStdDeserializer::ReadScriptVal(const char* UNUSED(name), JS::HandleObject appendParent) { JSContext* cx = m_ScriptInterface.GetContext(); JSAutoRequest rq(cx); uint8_t type; NumberU8_Unbounded("type", type); switch (type) { case SCRIPT_TYPE_VOID: return JS::UndefinedValue(); case SCRIPT_TYPE_NULL: return JS::NullValue(); case SCRIPT_TYPE_ARRAY: case SCRIPT_TYPE_OBJECT: case SCRIPT_TYPE_OBJECT_PROTOTYPE: { JS::RootedObject obj(cx); if (appendParent) { obj.set(appendParent); } else if (type == SCRIPT_TYPE_ARRAY) { u32 length; NumberU32_Unbounded("array length", length); obj.set(JS_NewArrayObject(cx, length)); } else if (type == SCRIPT_TYPE_OBJECT) { obj.set(JS_NewObject(cx, nullptr, JS::NullPtr(), JS::NullPtr())); } else // SCRIPT_TYPE_OBJECT_PROTOTYPE { std::wstring prototypeName; String("proto name", prototypeName, 0, 256); // Get constructor object JS::RootedObject proto(cx); GetSerializablePrototype(prototypeName, &proto); if (!proto) throw PSERROR_Deserialize_ScriptError("Failed to find serializable prototype for object"); JS::RootedObject parent(cx, JS_GetParent(proto)); if (!proto || !parent) throw PSERROR_Deserialize_ScriptError(); obj.set(JS_NewObject(cx, nullptr, proto, parent)); if (!obj) throw PSERROR_Deserialize_ScriptError("JS_NewObject failed"); // Does it have custom Deserialize function? // if so, we let it handle the deserialized data, rather than adding properties directly bool hasCustomDeserialize, hasCustomSerialize; if (!JS_HasProperty(cx, obj, "Serialize", &hasCustomSerialize) || !JS_HasProperty(cx, obj, "Deserialize", &hasCustomDeserialize)) throw PSERROR_Serialize_ScriptError("JS_HasProperty failed"); if (hasCustomDeserialize) { JS::RootedValue serialize(cx); if (!JS_LookupProperty(cx, obj, "Serialize", &serialize)) throw PSERROR_Serialize_ScriptError("JS_LookupProperty failed"); bool hasNullSerialize = hasCustomSerialize && serialize.isNull(); // If Serialize is null, we'll still call Deserialize but with undefined argument JS::RootedValue data(cx); if (!hasNullSerialize) ScriptVal("data", &data); JS::RootedValue objVal(cx, JS::ObjectValue(*obj)); m_ScriptInterface.CallFunctionVoid(objVal, "Deserialize", data); AddScriptBackref(obj); return JS::ObjectValue(*obj); } } if (!obj) throw PSERROR_Deserialize_ScriptError("Deserializer failed to create new object"); AddScriptBackref(obj); uint32_t numProps; NumberU32_Unbounded("num props", numProps); for (uint32_t i = 0; i < numProps; ++i) { utf16string propname; ReadStringUTF16("prop name", propname); JS::RootedValue propval(cx, ReadScriptVal("prop value", JS::NullPtr())); if (!JS_SetUCProperty(cx, obj, (const jschar*)propname.data(), propname.length(), propval)) throw PSERROR_Deserialize_ScriptError(); } return JS::ObjectValue(*obj); } case SCRIPT_TYPE_STRING: { JS::RootedString str(cx); ScriptString("string", &str); return JS::StringValue(str); } case SCRIPT_TYPE_INT: { int32_t value; NumberI32("value", value, JSVAL_INT_MIN, JSVAL_INT_MAX); return JS::NumberValue(value); } case SCRIPT_TYPE_DOUBLE: { double value; NumberDouble_Unbounded("value", value); JS::RootedValue rval(cx, JS::NumberValue(value)); if (rval.isNull()) throw PSERROR_Deserialize_ScriptError("JS_NewNumberValue failed"); return rval; } case SCRIPT_TYPE_BOOLEAN: { uint8_t value; NumberU8("value", value, 0, 1); return JS::BooleanValue(value ? true : false); } case SCRIPT_TYPE_BACKREF: { u32 tag; NumberU32_Unbounded("tag", tag); JS::RootedObject obj(cx); GetScriptBackref(tag, &obj); if (!obj) throw PSERROR_Deserialize_ScriptError("Invalid backref tag"); return JS::ObjectValue(*obj); } case SCRIPT_TYPE_OBJECT_NUMBER: { double value; NumberDouble_Unbounded("value", value); JS::RootedValue val(cx, JS::NumberValue(value)); JS::RootedObject ctorobj(cx); if (!JS_GetClassObject(cx, JSProto_Number, &ctorobj)) throw PSERROR_Deserialize_ScriptError("JS_GetClassObject failed"); JS::RootedObject obj(cx, JS_New(cx, ctorobj, val)); if (!obj) throw PSERROR_Deserialize_ScriptError("JS_New failed"); AddScriptBackref(obj); return JS::ObjectValue(*obj); } case SCRIPT_TYPE_OBJECT_STRING: { JS::RootedString str(cx); ScriptString("value", &str); if (!str) throw PSERROR_Deserialize_ScriptError(); JS::RootedValue val(cx, JS::StringValue(str)); JS::RootedObject ctorobj(cx); if (!JS_GetClassObject(cx, JSProto_String, &ctorobj)) throw PSERROR_Deserialize_ScriptError("JS_GetClassObject failed"); JS::RootedObject obj(cx, JS_New(cx, ctorobj, val)); if (!obj) throw PSERROR_Deserialize_ScriptError("JS_New failed"); AddScriptBackref(obj); return JS::ObjectValue(*obj); } case SCRIPT_TYPE_OBJECT_BOOLEAN: { bool value; Bool("value", value); JS::RootedValue val(cx, JS::BooleanValue(value)); JS::RootedObject ctorobj(cx); if (!JS_GetClassObject(cx, JSProto_Boolean, &ctorobj)) throw PSERROR_Deserialize_ScriptError("JS_GetClassObject failed"); JS::RootedObject obj(cx, JS_New(cx, ctorobj, val)); if (!obj) throw PSERROR_Deserialize_ScriptError("JS_New failed"); AddScriptBackref(obj); return JS::ObjectValue(*obj); } case SCRIPT_TYPE_TYPED_ARRAY: { u8 arrayType; u32 byteOffset, length; NumberU8_Unbounded("array type", arrayType); NumberU32_Unbounded("byte offset", byteOffset); NumberU32_Unbounded("length", length); // To match the serializer order, we reserve the typed array's backref tag here u32 arrayTag = ReserveScriptBackref(); // Get buffer object JS::RootedValue bufferVal(cx, ReadScriptVal("buffer", JS::NullPtr())); if (!bufferVal.isObject()) throw PSERROR_Deserialize_ScriptError(); JS::RootedObject bufferObj(cx, &bufferVal.toObject()); if (!JS_IsArrayBufferObject(bufferObj)) throw PSERROR_Deserialize_ScriptError("js_IsArrayBuffer failed"); JS::RootedObject arrayObj(cx); switch(arrayType) { case SCRIPT_TYPED_ARRAY_INT8: arrayObj = JS_NewInt8ArrayWithBuffer(cx, bufferObj, byteOffset, length); break; case SCRIPT_TYPED_ARRAY_UINT8: arrayObj = JS_NewUint8ArrayWithBuffer(cx, bufferObj, byteOffset, length); break; case SCRIPT_TYPED_ARRAY_INT16: arrayObj = JS_NewInt16ArrayWithBuffer(cx, bufferObj, byteOffset, length); break; case SCRIPT_TYPED_ARRAY_UINT16: arrayObj = JS_NewUint16ArrayWithBuffer(cx, bufferObj, byteOffset, length); break; case SCRIPT_TYPED_ARRAY_INT32: arrayObj = JS_NewInt32ArrayWithBuffer(cx, bufferObj, byteOffset, length); break; case SCRIPT_TYPED_ARRAY_UINT32: arrayObj = JS_NewUint32ArrayWithBuffer(cx, bufferObj, byteOffset, length); break; case SCRIPT_TYPED_ARRAY_FLOAT32: arrayObj = JS_NewFloat32ArrayWithBuffer(cx, bufferObj, byteOffset, length); break; case SCRIPT_TYPED_ARRAY_FLOAT64: arrayObj = JS_NewFloat64ArrayWithBuffer(cx, bufferObj, byteOffset, length); break; case SCRIPT_TYPED_ARRAY_UINT8_CLAMPED: arrayObj = JS_NewUint8ClampedArrayWithBuffer(cx, bufferObj, byteOffset, length); break; default: throw PSERROR_Deserialize_ScriptError("Failed to deserialize unrecognized typed array view"); } if (!arrayObj) throw PSERROR_Deserialize_ScriptError("js_CreateTypedArrayWithBuffer failed"); SetReservedScriptBackref(arrayTag, arrayObj); return JS::ObjectValue(*arrayObj); } case SCRIPT_TYPE_ARRAY_BUFFER: { u32 length; NumberU32_Unbounded("buffer length", length); #if BYTE_ORDER != LITTLE_ENDIAN #error TODO: need to convert JS ArrayBuffer data from little-endian #endif void* contents = NULL; contents = JS_AllocateArrayBufferContents(cx, length); RawBytes("buffer data", (u8*)contents, length); JS::RootedObject bufferObj(cx, JS_NewArrayBufferWithContents(cx, length, contents)); AddScriptBackref(bufferObj); return JS::ObjectValue(*bufferObj); } case SCRIPT_TYPE_OBJECT_MAP: { u32 mapSize; NumberU32_Unbounded("map size", mapSize); JS::RootedValue mapVal(cx); m_ScriptInterface.Eval("(new Map())", &mapVal); // To match the serializer order, we reserve the map's backref tag here u32 mapTag = ReserveScriptBackref(); for (u32 i=0; i >& prototypes) { m_SerializablePrototypes = prototypes; } bool CStdDeserializer::IsSerializablePrototype(const std::wstring& name) { return m_SerializablePrototypes.find(name) != m_SerializablePrototypes.end(); } void CStdDeserializer::GetSerializablePrototype(const std::wstring& name, JS::MutableHandleObject ret) { std::map >::iterator it = m_SerializablePrototypes.find(name); if (it != m_SerializablePrototypes.end()) ret.set(it->second); else ret.set(NULL); } Index: ps/trunk/source/simulation2/serialization/StdSerializer.h =================================================================== --- ps/trunk/source/simulation2/serialization/StdSerializer.h (revision 17132) +++ ps/trunk/source/simulation2/serialization/StdSerializer.h (revision 17133) @@ -1,62 +1,67 @@ -/* Copyright (C) 2011 Wildfire Games. +/* Copyright (C) 2015 Wildfire Games. * This file is part of 0 A.D. * * 0 A.D. is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 2 of the License, or * (at your option) any later version. * * 0 A.D. is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with 0 A.D. If not, see . */ #ifndef INCLUDED_STDSERIALIZER #define INCLUDED_STDSERIALIZER #include "BinarySerializer.h" #include #define DEBUG_SERIALIZER_ANNOTATE 0 // annotate the stream to help debugging if you're reading the output in a hex editor class CStdSerializerImpl { NONCOPYABLE(CStdSerializerImpl); public: CStdSerializerImpl(std::ostream& stream); + ~CStdSerializerImpl() + { + m_Stream.flush(); + } + void Put(const char* name, const u8* data, size_t len) { #if DEBUG_SERIALIZER_ANNOTATE m_Stream.put('<'); m_Stream.write(name, strlen(name)); m_Stream.put('>'); #else UNUSED2(name); #endif m_Stream.write((const char*)data, (std::streamsize)len); } std::ostream& GetStream() { return m_Stream; } private: std::ostream& m_Stream; }; class CStdSerializer : public CBinarySerializer { public: CStdSerializer(ScriptInterface& scriptInterface, std::ostream& stream); virtual std::ostream& GetStream(); }; #endif // INCLUDED_STDSERIALIZER Index: ps/trunk/source/simulation2/tests/test_Serializer.h =================================================================== --- ps/trunk/source/simulation2/tests/test_Serializer.h (revision 17132) +++ ps/trunk/source/simulation2/tests/test_Serializer.h (revision 17133) @@ -1,846 +1,873 @@ /* Copyright (C) 2015 Wildfire Games. * This file is part of 0 A.D. * * 0 A.D. is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 2 of the License, or * (at your option) any later version. * * 0 A.D. is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with 0 A.D. If not, see . */ #include "lib/self_test.h" #include "simulation2/serialization/DebugSerializer.h" #include "simulation2/serialization/HashSerializer.h" #include "simulation2/serialization/StdSerializer.h" #include "simulation2/serialization/StdDeserializer.h" #include "scriptinterface/ScriptInterface.h" #include "graphics/MapReader.h" #include "graphics/Terrain.h" #include "graphics/TerrainTextureManager.h" #include "lib/timer.h" #include "ps/CLogger.h" #include "ps/Filesystem.h" #include "ps/Loader.h" #include "ps/XML/Xeromyces.h" #include "simulation2/Simulation2.h" #include "callgrind.h" #include #define TS_ASSERT_STREAM(stream, len, buffer) \ TS_ASSERT_EQUALS(stream.str().length(), (size_t)len); \ TS_ASSERT_SAME_DATA(stream.str().data(), buffer, len) #define TSM_ASSERT_STREAM(m, stream, len, buffer) \ TSM_ASSERT_EQUALS(m, stream.str().length(), (size_t)len); \ TSM_ASSERT_SAME_DATA(m, stream.str().data(), buffer, len) class TestSerializer : public CxxTest::TestSuite { public: void serialize_types(ISerializer& serialize) { serialize.NumberI8_Unbounded("i8", (signed char)-123); serialize.NumberU8_Unbounded("u8", (unsigned char)255); serialize.NumberI16_Unbounded("i16", -12345); serialize.NumberU16_Unbounded("u16", 56789); serialize.NumberI32_Unbounded("i32", -123); serialize.NumberU32_Unbounded("u32", (unsigned)-123); serialize.NumberFloat_Unbounded("float", 1e+30f); serialize.NumberDouble_Unbounded("double", 1e+300); serialize.NumberFixed_Unbounded("fixed", fixed::FromFloat(1234.5f)); serialize.Bool("t", true); serialize.Bool("f", false); serialize.StringASCII("string", "example", 0, 255); serialize.StringASCII("string 2", "example\"\\\"", 0, 255); serialize.StringASCII("string 3", "example\n\ntest", 0, 255); wchar_t testw[] = { 't', 0xEA, 's', 't', 0 }; serialize.String("string 4", testw, 0, 255); serialize.RawBytes("raw bytes", (const u8*)"\0\1\2\3\x0f\x10", 6); } void test_Debug_basic() { ScriptInterface script("Test", "Test", g_ScriptRuntime); std::stringstream stream; CDebugSerializer serialize(script, stream); serialize.NumberI32_Unbounded("x", -123); serialize.NumberU32_Unbounded("y", 1234); serialize.NumberI32("z", 12345, 0, 65535); TS_ASSERT_STR_EQUALS(stream.str(), "x: -123\ny: 1234\nz: 12345\n"); } void test_Debug_floats() { ScriptInterface script("Test", "Test", g_ScriptRuntime); std::stringstream stream; CDebugSerializer serialize(script, stream); serialize.NumberFloat_Unbounded("x", 1e4f); serialize.NumberFloat_Unbounded("x", 1e-4f); serialize.NumberFloat_Unbounded("x", 1e5f); serialize.NumberFloat_Unbounded("x", 1e-5f); serialize.NumberFloat_Unbounded("x", 1e6f); serialize.NumberFloat_Unbounded("x", 1e-6f); serialize.NumberFloat_Unbounded("x", 1e10f); serialize.NumberFloat_Unbounded("x", 1e-10f); serialize.NumberDouble_Unbounded("x", 1e4); serialize.NumberDouble_Unbounded("x", 1e-4); serialize.NumberDouble_Unbounded("x", 1e5); serialize.NumberDouble_Unbounded("x", 1e-5); serialize.NumberDouble_Unbounded("x", 1e6); serialize.NumberDouble_Unbounded("x", 1e-6); serialize.NumberDouble_Unbounded("x", 1e10); serialize.NumberDouble_Unbounded("x", 1e-10); serialize.NumberDouble_Unbounded("x", 1e100); serialize.NumberDouble_Unbounded("x", 1e-100); serialize.NumberFixed_Unbounded("x", fixed::FromDouble(1e4)); TS_ASSERT_STR_EQUALS(stream.str(), "x: 10000\nx: 9.9999997e-05\nx: 100000\nx: 9.9999997e-06\nx: 1000000\nx: 1e-06\nx: 1e+10\nx: 1e-10\n" "x: 10000\nx: 0.0001\nx: 100000\nx: 1.0000000000000001e-05\nx: 1000000\nx: 9.9999999999999995e-07\nx: 10000000000\nx: 1e-10\nx: 1e+100\nx: 1e-100\n" "x: 10000\n" ); } void test_Debug_types() { ScriptInterface script("Test", "Test", g_ScriptRuntime); std::stringstream stream; CDebugSerializer serialize(script, stream); serialize.Comment("comment"); serialize_types(serialize); TS_ASSERT_STR_EQUALS(stream.str(), "# comment\n" "i8: -123\n" "u8: 255\n" "i16: -12345\n" "u16: 56789\n" "i32: -123\n" "u32: 4294967173\n" "float: 1e+30\n" "double: 1.0000000000000001e+300\n" "fixed: 1234.5\n" "t: true\n" "f: false\n" "string: \"example\"\n" "string 2: \"example\\\"\\\\\\\"\"\n" // C-escaped form of: "example\"\\\"" "string 3: \"example\\n\\ntest\"\n" "string 4: \"t\xC3\xAAst\"\n" "raw bytes: (6 bytes) 00 01 02 03 0f 10\n" ); } void test_Std_basic() { ScriptInterface script("Test", "Test", g_ScriptRuntime); std::stringstream stream; CStdSerializer serialize(script, stream); serialize.NumberI32_Unbounded("x", -123); serialize.NumberU32_Unbounded("y", 1234); serialize.NumberI32("z", 12345, 0, 65535); TS_ASSERT_STREAM(stream, 12, "\x85\xff\xff\xff" "\xd2\x04\x00\x00" "\x39\x30\x00\x00"); CStdDeserializer deserialize(script, stream); int32_t n; deserialize.NumberI32_Unbounded("x", n); TS_ASSERT_EQUALS(n, -123); deserialize.NumberI32_Unbounded("y", n); TS_ASSERT_EQUALS(n, 1234); deserialize.NumberI32("z", n, 0, 65535); TS_ASSERT_EQUALS(n, 12345); - TS_ASSERT(stream.good()); + // NOTE: Don't use good() here - it fails due to a bug in older libc++ versions + TS_ASSERT(!stream.bad() && !stream.fail()); TS_ASSERT_EQUALS(stream.peek(), EOF); } void test_Std_types() { ScriptInterface script("Test", "Test", g_ScriptRuntime); std::stringstream stream; CStdSerializer serialize(script, stream); serialize_types(serialize); CStdDeserializer deserialize(script, stream); int8_t i8v; uint8_t u8v; int16_t i16v; uint16_t u16v; int32_t i32v; uint32_t u32v; float flt; double dbl; fixed fxd; bool bl; std::string str; std::wstring wstr; u8 cbuf[256]; deserialize.NumberI8_Unbounded("i8", i8v); TS_ASSERT_EQUALS(i8v, -123); deserialize.NumberU8_Unbounded("u8", u8v); TS_ASSERT_EQUALS(u8v, 255); deserialize.NumberI16_Unbounded("i16", i16v); TS_ASSERT_EQUALS(i16v, -12345); deserialize.NumberU16_Unbounded("u16", u16v); TS_ASSERT_EQUALS(u16v, 56789); deserialize.NumberI32_Unbounded("i32", i32v); TS_ASSERT_EQUALS(i32v, -123); deserialize.NumberU32_Unbounded("u32", u32v); TS_ASSERT_EQUALS(u32v, 4294967173u); deserialize.NumberFloat_Unbounded("float", flt); TS_ASSERT_EQUALS(flt, 1e+30f); deserialize.NumberDouble_Unbounded("double", dbl); TS_ASSERT_EQUALS(dbl, 1e+300); deserialize.NumberFixed_Unbounded("fixed", fxd); TS_ASSERT_EQUALS(fxd.ToDouble(), 1234.5); deserialize.Bool("t", bl); TS_ASSERT_EQUALS(bl, true); deserialize.Bool("f", bl); TS_ASSERT_EQUALS(bl, false); deserialize.StringASCII("string", str, 0, 255); TS_ASSERT_STR_EQUALS(str, "example"); deserialize.StringASCII("string 2", str, 0, 255); TS_ASSERT_STR_EQUALS(str, "example\"\\\""); deserialize.StringASCII("string 3", str, 0, 255); TS_ASSERT_STR_EQUALS(str, "example\n\ntest"); wchar_t testw[] = { 't', 0xEA, 's', 't', 0 }; deserialize.String("string 4", wstr, 0, 255); TS_ASSERT_WSTR_EQUALS(wstr, testw); cbuf[6] = 0x42; // sentinel deserialize.RawBytes("raw bytes", cbuf, 6); TS_ASSERT_SAME_DATA(cbuf, (const u8*)"\0\1\2\3\x0f\x10\x42", 7); - TS_ASSERT(stream.good()); + // NOTE: Don't use good() here - it fails due to a bug in older libc++ versions + TS_ASSERT(!stream.bad() && !stream.fail()); TS_ASSERT_EQUALS(stream.peek(), EOF); } void test_Hash_basic() { ScriptInterface script("Test", "Test", g_ScriptRuntime); CHashSerializer serialize(script); serialize.NumberI32_Unbounded("x", -123); serialize.NumberU32_Unbounded("y", 1234); serialize.NumberI32("z", 12345, 0, 65535); TS_ASSERT_EQUALS(serialize.GetHashLength(), (size_t)16); TS_ASSERT_SAME_DATA(serialize.ComputeHash(), "\xa0\x3a\xe5\x3e\x9b\xd7\xfb\x11\x88\x35\xc6\xfb\xb9\x94\xa9\x72", 16); - // echo -en "\x85\xff\xff\xff\xd2\x04\x00\x00\x39\x30\x00\x00" | openssl md5 | perl -pe 's/(..)/\\x$1/g' + // echo -en "\x85\xff\xff\xff\xd2\x04\x00\x00\x39\x30\x00\x00" | openssl md5 -binary | xxd -p | perl -pe 's/(..)/\\x$1/g' + } + + void test_Hash_stream() + { + ScriptInterface script("Test", "Test", g_ScriptRuntime); + CHashSerializer hashSerialize(script); + + hashSerialize.NumberI32_Unbounded("x", -123); + hashSerialize.NumberU32_Unbounded("y", 1234); + hashSerialize.NumberI32("z", 12345, 0, 65535); + + ISerializer& serialize = hashSerialize; + + { + CStdSerializer streamSerialize(script, serialize.GetStream()); + streamSerialize.NumberI32_Unbounded("x2", -456); + streamSerialize.NumberU32_Unbounded("y2", 5678); + streamSerialize.NumberI32("z2", 45678, 0, 65535); + } + + TS_ASSERT_EQUALS(hashSerialize.GetHashLength(), (size_t)16); + TS_ASSERT_SAME_DATA(hashSerialize.ComputeHash(), "\x5c\xff\x33\xd1\x72\xdd\x6d\x77\xa8\xd4\xa1\xf6\x84\xcc\xaa\x10", 16); + // echo -en "\x85\xff\xff\xff\xd2\x04\x00\x00\x39\x30\x00\x00\x38\xfe\xff\xff\x2e\x16\x00\x00\x6e\xb2\x00\x00" | openssl md5 -binary | xxd -p | perl -pe 's/(..)/\\x$1/g' } void test_bounds() { ScriptInterface script("Test", "Test", g_ScriptRuntime); std::stringstream stream; CDebugSerializer serialize(script, stream); serialize.NumberI32("x", 16, -16, 16); serialize.NumberI32("x", -16, -16, 16); TS_ASSERT_THROWS(serialize.NumberI32("x", 17, -16, 16), PSERROR_Serialize_OutOfBounds); TS_ASSERT_THROWS(serialize.NumberI32("x", -17, -16, 16), PSERROR_Serialize_OutOfBounds); } // TODO: test exceptions more thoroughly void helper_script_roundtrip(const char* msg, const char* input, const char* expected, size_t expstreamlen = 0, const char* expstream = NULL, const char* debug = NULL) { ScriptInterface script("Test", "Test", g_ScriptRuntime); JSContext* cx = script.GetContext(); JSAutoRequest rq(cx); JS::RootedValue obj(cx); TSM_ASSERT(msg, script.Eval(input, &obj)); if (debug) { std::stringstream dbgstream; CDebugSerializer serialize(script, dbgstream); serialize.ScriptVal("script", &obj); TS_ASSERT_STR_EQUALS(dbgstream.str(), debug); } std::stringstream stream; CStdSerializer serialize(script, stream); serialize.ScriptVal("script", &obj); if (expstream) { TSM_ASSERT_STREAM(msg, stream, expstreamlen, expstream); } CStdDeserializer deserialize(script, stream); JS::RootedValue newobj(cx); deserialize.ScriptVal("script", &newobj); - TSM_ASSERT(msg, stream.good()); + // NOTE: Don't use good() here - it fails due to a bug in older libc++ versions + TSM_ASSERT(msg, !stream.bad() && !stream.fail()); TSM_ASSERT_EQUALS(msg, stream.peek(), EOF); std::string source; TSM_ASSERT(msg, script.CallFunction(newobj, "toSource", source)); TS_ASSERT_STR_EQUALS(source, expected); } void test_script_basic() { helper_script_roundtrip("Object", "({'x': 123, 'y': [1, 1.5, '2', 'test', undefined, null, true, false]})", /* expected: */ "({x:123, y:[1, 1.5, \"2\", \"test\", (void 0), null, true, false]})", /* expected stream: */ 119, "\x03" // SCRIPT_TYPE_OBJECT "\x02\0\0\0" // num props "\x01\0\0\0" "x\0" // "x" "\x05" // SCRIPT_TYPE_INT "\x7b\0\0\0" // 123 "\x01\0\0\0" "y\0" // "y" "\x02" // SCRIPT_TYPE_ARRAY "\x08\0\0\0" // array length "\x08\0\0\0" // num props "\x01\0\0\0" "0\0" // "0" "\x05" "\x01\0\0\0" // SCRIPT_TYPE_INT 1 "\x01\0\0\0" "1\0" // "1" "\x06" "\0\0\0\0\0\0\xf8\x3f" // SCRIPT_TYPE_DOUBLE 1.5 "\x01\0\0\0" "2\0" // "2" "\x04" "\x01\0\0\0" "2\0" // SCRIPT_TYPE_STRING "2" "\x01\0\0\0" "3\0" // "3" "\x04" "\x04\0\0\0" "t\0e\0s\0t\0" // SCRIPT_TYPE_STRING "test" "\x01\0\0\0" "4\0" // "4" "\x00" // SCRIPT_TYPE_VOID "\x01\0\0\0" "5\0" // "5" "\x01" // SCRIPT_TYPE_NULL "\x01\0\0\0" "6\0" // "6" "\x07" "\x01" // SCRIPT_TYPE_BOOLEAN true "\x01\0\0\0" "7\0" // "7" "\x07" "\x00", // SCRIPT_TYPE_BOOLEAN false /* expected debug: */ "script: {\n" " \"x\": 123,\n" " \"y\": [\n" " 1,\n" " 1.5,\n" " \"2\",\n" " \"test\",\n" " null,\n" " null,\n" " true,\n" " false\n" " ]\n" "}\n" ); } void test_script_unicode() { helper_script_roundtrip("unicode", "({" "'x': \"\\x01\\x80\\xff\\u0100\\ud7ff\", " "'y': \"\\ue000\\ufffd\"" "})", /* expected: */ "({" "x:\"\\x01\\x80\\xFF\\u0100\\uD7FF\", " "y:\"\\uE000\\uFFFD\"" "})"); // Disabled since we no longer do the UTF-8 conversion that rejects invalid characters // TS_ASSERT_THROWS(helper_script_roundtrip("invalid chars 1", "(\"\\ud7ff\\ud800\")", "..."), PSERROR_Serialize_InvalidCharInString); // TS_ASSERT_THROWS(helper_script_roundtrip("invalid chars 2", "(\"\\udfff\")", "..."), PSERROR_Serialize_InvalidCharInString); // TS_ASSERT_THROWS(helper_script_roundtrip("invalid chars 3", "(\"\\uffff\")", "..."), PSERROR_Serialize_InvalidCharInString); // TS_ASSERT_THROWS(helper_script_roundtrip("invalid chars 4", "(\"\\ud800\\udc00\")" /* U+10000 */, "..."), PSERROR_Serialize_InvalidCharInString); helper_script_roundtrip("unicode", "\"\\ud800\\uffff\"", "(new String(\"\\uD800\\uFFFF\"))"); } void test_script_objects() { helper_script_roundtrip("Number", "[1, new Number('2.0'), 3]", "[1, (new Number(2)), 3]"); helper_script_roundtrip("Number with props", "var n=new Number('2.0'); n.foo='bar'; n", "(new Number(2))"); helper_script_roundtrip("String", "['test1', new String('test2'), 'test3']", "[\"test1\", (new String(\"test2\")), \"test3\"]"); helper_script_roundtrip("String with props", "var s=new String('test'); s.foo='bar'; s", "(new String(\"test\"))"); helper_script_roundtrip("Boolean", "[new Boolean('true'), false]", "[(new Boolean(true)), false]"); helper_script_roundtrip("Boolean with props", "var b=new Boolean('true'); b.foo='bar'; b", "(new Boolean(true))"); } void test_script_typed_arrays_simple() { helper_script_roundtrip("Int8Array", "var arr=new Int8Array(8);" "for(var i=0; iMount(L"", DataDir()/"mods"/"public", VFS_MOUNT_MUST_EXIST)); TS_ASSERT_OK(g_VFS->Mount(L"cache/", DataDir()/"cache")); // Need some stuff for terrain movement costs: // (TODO: this ought to be independent of any graphics code) new CTerrainTextureManager; g_TexMan.LoadTerrainTextures(); CTerrain terrain; CSimulation2 sim2(NULL, g_ScriptRuntime, &terrain); sim2.LoadDefaultScripts(); sim2.ResetState(); CMapReader* mapReader = new CMapReader(); // it'll call "delete this" itself LDR_BeginRegistering(); mapReader->LoadMap(L"maps/skirmishes/Greek Acropolis (2).pmp", sim2.GetScriptInterface().GetJSRuntime(), JS::UndefinedHandleValue, &terrain, NULL, NULL, NULL, NULL, NULL, NULL, NULL, &sim2, &sim2.GetSimContext(), -1, false); LDR_EndRegistering(); TS_ASSERT_OK(LDR_NonprogressiveLoad()); sim2.Update(0); { std::stringstream str; std::string hash; sim2.SerializeState(str); sim2.ComputeStateHash(hash, false); debug_printf("\n"); debug_printf("# size = %d\n", (int)str.str().length()); debug_printf("# hash = "); for (size_t i = 0; i < hash.size(); ++i) debug_printf("%02x", (unsigned int)(u8)hash[i]); debug_printf("\n"); } double t = timer_Time(); CALLGRIND_START_INSTRUMENTATION; size_t reps = 128; for (size_t i = 0; i < reps; ++i) { std::string hash; sim2.ComputeStateHash(hash, false); } CALLGRIND_STOP_INSTRUMENTATION; t = timer_Time() - t; debug_printf("# time = %f (%f/%d)\n", t/reps, t, (int)reps); // Shut down the world delete &g_TexMan; g_VFS.reset(); CXeromyces::Terminate(); } };