Index: binaries/data/config/default.cfg
===================================================================
--- binaries/data/config/default.cfg
+++ binaries/data/config/default.cfg
@@ -397,6 +397,11 @@
[mod]
enabledmods = "mod public"
+[modio.v1]
+baseurl = "https://api.test.mod.io/v1"
+api_key = "acf8fc07e3a8e9228ef4d5704c1659a1"
+name_id = "0ad"
+
[network]
duplicateplayernames = false ; Rename joining player to "User (2)" if "User" is already connected, otherwise prohibit join.
lateobservers = everyone ; Allow observers to join the game after it started. Possible values: everyone, buddies, disabled.
Index: binaries/data/mods/mod/gui/modmod/modio.js
===================================================================
--- /dev/null
+++ binaries/data/mods/mod/gui/modmod/modio.js
@@ -0,0 +1,65 @@
+let g_Mods = [];
+
+function init()
+{
+ g_Mods = Engine.ModIoGetMods();
+
+ generateModsList(g_Mods);
+}
+
+function generateModsList(mods)
+{
+ var [keys, names, name_ids, versions, filenames, filesizes, filehash_md5s, downloads] = [[],[],[],[],[],[],[],[]];
+
+ let i = 0;
+ for (let mod of mods)
+ {
+ keys.push(i++);
+ names.push(mod.name);
+ name_ids.push(mod.name_id);
+ versions.push(mod.version);
+ filenames.push(mod.filename);
+ filesizes.push(mod.filesize);
+ filehash_md5s.push(mod.filehash_md5);
+ downloads.push(mod.binary_url);
+ }
+
+ var obj = Engine.GetGUIObjectByName("modsAvailableList");
+ obj.list_name = names;
+ obj.list_modVersion = versions;
+ obj.list_modname_id = name_ids;
+ obj.list_modfilename = filenames;
+ obj.list_modfilesize = filesizes;
+ obj.list_modfilehash_md5 = filehash_md5s;
+ obj.list_moddownload = downloads;
+
+ obj.list = keys;
+}
+
+function showModDescription()
+{
+ let listObject = Engine.GetGUIObjectByName("modsAvailableList");
+ let desc = "No mod has been selected.";
+ if (listObject.selected != -1)
+ desc = g_Mods[listObject.selected].summary;
+
+ Engine.GetGUIObjectByName("globalModDescription").caption = desc;
+}
+
+function downloadMod()
+{
+ let listObject = Engine.GetGUIObjectByName("modsAvailableList");
+ if (listObject.selected == -1)
+ {
+ warn("Select something first.");
+ return;
+ }
+
+ Engine.ModIoDownloadMod(listObject.selected); // TODO in case we support sorting or filtering this should be the keys value.
+ // TODO: pass index to engine to download that. We cannot be trusted with anything but an index (and even that just barely).
+}
+
+function closePage()
+{
+ Engine.SwitchGuiPage("page_pregame.xml", {});
+}
Index: binaries/data/mods/mod/gui/modmod/modio.xml
===================================================================
--- /dev/null
+++ binaries/data/mods/mod/gui/modmod/modio.xml
@@ -0,0 +1,81 @@
+
+
+
+
+
+
+
Index: binaries/data/mods/mod/gui/modmod/modmod.xml
===================================================================
--- binaries/data/mods/mod/gui/modmod/modmod.xml
+++ binaries/data/mods/mod/gui/modmod/modmod.xml
@@ -193,8 +193,8 @@
- Cancel
- closePage();
+ mod.io test
+ Engine.SwitchGuiPage("page_modio.xml");
Index: binaries/data/mods/mod/gui/page_modio.xml
===================================================================
--- /dev/null
+++ binaries/data/mods/mod/gui/page_modio.xml
@@ -0,0 +1,9 @@
+
+
+ common/modern/setup.xml
+ common/modern/styles.xml
+ common/modern/sprites.xml
+
+ modmod/styles.xml
+ modmod/modio.xml
+
Index: build/premake/extern_libs5.lua
===================================================================
--- build/premake/extern_libs5.lua
+++ build/premake/extern_libs5.lua
@@ -393,6 +393,15 @@
})
end,
},
+ libsodium = {
+ -- TODO check other OSes
+ compile_settings = function()
+ pkgconfig.add_includes("libsodium")
+ end,
+ link_settings = function()
+ pkgconfig.add_links("libsodium")
+ end,
+ },
libxml2 = {
compile_settings = function()
if os.istarget("windows") then
Index: build/premake/premake5.lua
===================================================================
--- build/premake/premake5.lua
+++ build/premake/premake5.lua
@@ -909,6 +909,7 @@
"tinygettext",
"icu",
"iconv",
+ "libsodium",
"valgrind",
}
Index: source/ps/GameSetup/GameSetup.cpp
===================================================================
--- source/ps/GameSetup/GameSetup.cpp
+++ source/ps/GameSetup/GameSetup.cpp
@@ -68,6 +68,7 @@
#include "ps/Joystick.h"
#include "ps/Loader.h"
#include "ps/Mod.h"
+#include "ps/ModIo.h"
#include "ps/Profile.h"
#include "ps/ProfileViewer.h"
#include "ps/Profiler2.h"
@@ -734,6 +735,8 @@
SAFE_DELETE(g_XmppClient);
+ SAFE_DELETE(g_ModIo);
+
ShutdownPs();
TIMER_BEGIN(L"shutdown TexMan");
Index: source/ps/ModIo.h
===================================================================
--- /dev/null
+++ source/ps/ModIo.h
@@ -0,0 +1,70 @@
+/* Copyright (C) 2017 Wildfire Games.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef INCLUDED_MODIO
+#define INCLUDED_MODIO
+
+#include
+
+#include "lib/external_libraries/curl.h"
+
+#include "scriptinterface/ScriptInterface.h"
+
+class ModIo
+{
+ NONCOPYABLE(ModIo);
+public:
+ ModIo();
+ ~ModIo();
+
+ const std::vector>& GetMods(const ScriptInterface& scriptinterface);
+
+ void DownloadMod(size_t idx);
+
+private:
+ static size_t ReceiveCallback(void* buffer, size_t size, size_t nmemb, void* userp);
+
+ bool ParseGameIdResponse(const ScriptInterface& scriptInterface);
+ bool ParseModsResponse(const ScriptInterface& scriptInterface);
+
+ void PrintMods();
+
+ // Url parts
+ std::string m_BaseUrl;
+ std::string m_GamesRequest;
+ std::string m_GameId;
+
+ // Query parameters
+ std::string m_ApiKey;
+ std::string m_IdQuery;
+
+ CURL* m_Curl;
+ curl_slist* m_Headers;
+ char m_ErrorBuffer[CURL_ERROR_SIZE];
+ std::string m_ResponseData;
+
+ std::vector> m_ModData;
+};
+
+extern ModIo* g_ModIo;
+
+#endif // INCLUDED_MODIO
Index: source/ps/ModIo.cpp
===================================================================
--- /dev/null
+++ source/ps/ModIo.cpp
@@ -0,0 +1,644 @@
+/* Copyright (C) 2018 Wildfire Games.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+// TODO: Comment on why we handle things this way (not relying on user input, not trusting the remote too much, etc.)
+
+// TODO: Better error messages. Also ask upstream about some special queries with partially malformed replies
+
+#include "precompiled.h"
+
+#include "ModIo.h"
+
+#include "lib/file/file_system.h"
+#include "lib/sysdep/sysdep.h"
+#include "lib/sysdep/filesystem.h"
+#include "maths/MD5.h"
+#include "ps/CLogger.h"
+#include "ps/ConfigDB.h"
+#include "ps/GameSetup/Paths.h"
+#include "ps/Mod.h"
+#include "scriptinterface/ScriptConversions.h"
+
+#include
+
+#include
+
+ModIo* g_ModIo = nullptr;
+// TODO: add a compile-time config so we can build without this
+
+struct DownloadCallbackData {
+ DownloadCallbackData(FILE* _fp)
+ : fp(_fp), md5()
+ {
+ crypto_sign_init(&sign_state);
+ }
+ FILE* fp;
+ MD5 md5;
+ crypto_sign_state sign_state;
+};
+
+static bool VerifyDownload(const OsPath& filePath, DownloadCallbackData& callbackData, const std::map& modData);
+
+// TODO move this comment somewhere nicer?
+/**
+ * mod.io interfacing code.
+ * This both distrusts the loaded JS mods, and the API as much as possible.
+ * We do not want a malicious mod to use this to download arbitrary files, nor do we want the API
+ * to serve us anything we have not verified.
+ * Therefore we only allow mods to download one of the mods returned by the API (using indices).
+ * Everything downloaded from the API has its signature verified against our public key. (XXX TODO)
+ * The mod.io settings are also locked down such that only mods that have been authorized by us
+ * show up in API queries. This is both done so that all required information (dependencies)
+ * are stored for the files, and that only mods that have been checked for being ok are acutally
+ * shown to users.
+ * This (mostly) necessitates parsing the API responses here, as opposed to in JS.
+ * One could alternatively parse the responses in a locked down JS context, but that would require
+ * storing that code in here, or making sure nobody can overwrite it. Also this would possibly make
+ * some of the needed accesses for downloading and verifying files a bit more complicated.
+ */
+
+ModIo::ModIo()
+ : m_GamesRequest("/games")
+{
+ // Get config values from the sytem namespace, or below (default); this can be overridden on the command line.
+ // We do this so a malicious mod cannot change the base url and get the user to make connections
+ // to someone else's endpoint.
+ g_ConfigDB.GetValue(CFG_SYSTEM, "modio.v1.baseurl", m_BaseUrl);
+ // TODO: Should we allow mods to actually change the two settings below? (Might be nice for total conversions.)
+ {
+ std::string api_key;
+ g_ConfigDB.GetValue(CFG_SYSTEM, "modio.v1.api_key", api_key);
+ m_ApiKey = "api_key=" + api_key;
+ }
+ {
+ std::string nameid;
+ g_ConfigDB.GetValue(CFG_SYSTEM, "modio.v1.name_id", nameid);
+ m_IdQuery = "name_id="+nameid;
+ }
+
+ // Initialise everything except Win32 sockets (because our networking
+ // system already inits those)
+ curl_global_init(CURL_GLOBAL_ALL & ~CURL_GLOBAL_WIN32);
+// TODO we should only do this in one single place (same for shutdown, currently we do it here and in the userreporter)
+
+ m_Curl = curl_easy_init();
+ ENSURE(m_Curl);
+
+ // Capture error messages
+ curl_easy_setopt(m_Curl, CURLOPT_ERRORBUFFER, m_ErrorBuffer);
+
+ // Fail if the server did
+ curl_easy_setopt(m_Curl, CURLOPT_FAILONERROR, 1L);
+
+ // Disable signal handlers (required for multithreaded applications)
+ curl_easy_setopt(m_Curl, CURLOPT_NOSIGNAL, 1L);
+
+ // To minimise security risks, don't support redirects
+ curl_easy_setopt(m_Curl, CURLOPT_FOLLOWLOCATION, 0L);
+
+ m_Headers = NULL;
+ std::string ua = "User-Agent: pyrogenesis ";
+ ua += curl_version();
+ ua += " (https://play0ad.com/)";
+ m_Headers = curl_slist_append(m_Headers, ua.c_str());
+ // TODO more?
+ curl_easy_setopt(m_Curl, CURLOPT_HTTPHEADER, m_Headers);
+
+
+ // we only accept indices from JS when downloading something later on (security)
+
+ if (sodium_init() < 0) {
+ ENSURE(0); // TODO
+ }
+}
+
+ModIo::~ModIo()
+{
+ curl_slist_free_all(m_Headers);
+ curl_easy_cleanup(m_Curl);
+ curl_global_cleanup(); // TODO
+}
+
+size_t ModIo::ReceiveCallback(void* buffer, size_t size, size_t nmemb, void* userp)
+{
+ ModIo* self = static_cast(userp);
+
+ self->m_ResponseData += std::string((char*)buffer, (char*)buffer+size*nmemb);
+
+ return size*nmemb;
+}
+
+size_t DownloadCallback(void* buffer, size_t size, size_t nmemb, void* userp)
+{
+ DownloadCallbackData* data = static_cast(userp);
+
+ size_t len = fwrite(buffer, size, nmemb, data->fp);
+
+ // Only update the hash with data we actually managed to write.
+ // In case we did not write all of it we will fail the download,
+ // but we do not want to have a possibly valid hash in that case.
+ data->md5.Update((const u8*)buffer, len*size);
+
+ crypto_sign_update(&data->sign_state, (const u8*)buffer, len*size);
+
+ return len*size;
+}
+
+static CURLcode PerformRequest(CURL* curl, const std::string& url, char* errorBuffer)
+{
+ errorBuffer[0] = '\0';
+ curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+ return curl_easy_perform(curl);
+}
+
+const std::vector>& ModIo::GetMods(const ScriptInterface& scriptInterface)
+{
+ m_ModData.clear();
+
+ // Set IO callbacks
+ curl_easy_setopt(m_Curl, CURLOPT_WRITEFUNCTION, ReceiveCallback);
+ curl_easy_setopt(m_Curl, CURLOPT_WRITEDATA, this);
+
+ // Get the game id if we didn't fetch it already
+ if (m_GameId.empty())
+ {
+ CURLcode err = PerformRequest(m_Curl, m_BaseUrl+m_GamesRequest+"?"+m_ApiKey+"&"+m_IdQuery, m_ErrorBuffer);
+
+ if (err != CURLE_OK)
+ {
+ LOGERROR("Failure while querying for game id. Server response: %s; %s", curl_easy_strerror(err), m_ErrorBuffer);
+
+ return m_ModData;
+ }
+
+ if (!ParseGameIdResponse(scriptInterface))
+ return m_ModData;
+ }
+
+ // Get the actual mods
+ CURLcode err = PerformRequest(m_Curl, m_BaseUrl+m_GamesRequest+m_GameId+"/mods?"+m_ApiKey, m_ErrorBuffer);
+ if (err != CURLE_OK)
+ {
+ LOGERROR("Failure while querying for mods. Server response: %s; %s", curl_easy_strerror(err), m_ErrorBuffer);
+ return m_ModData;
+ }
+
+ if (!ParseModsResponse(scriptInterface))
+ m_ModData.clear(); // Failed during parsing, make sure we don't provide partial data
+
+PrintMods();
+
+ return m_ModData;
+}
+
+// TODO make localizing errors easier by having #define CTXT __func__? or just "Parsing game id. " and LOGERROR(CTXT __VA_ARGS__) below
+#define FAIL(...) STMT(LOGERROR(__VA_ARGS__); return false;)
+
+/**
+ * Parses the current content of m_ResponseData to extract m_GameId.
+ *
+ * The JSON data is expected to look like
+ * { "data": [{"id": 42, ...}, ...], ... }
+ * where we are only interested in the value of the id property.
+ *
+ * @returns true iff it successfully parsed the id.
+ */
+bool ModIo::ParseGameIdResponse(const ScriptInterface& scriptInterface)
+{
+ JSContext* cx = scriptInterface.GetContext();
+ JS::RootedValue gameResponse(cx);
+
+// XXX testing TODO add this to some testing file? possibly make this take m_ResponseData as a param? then we could check/fuzz this nicely without having to either fake curl responses, or be-friend-ing this class to change m_ResponseData
+/*
+m_ResponseData = ""; // "Failed to parse response as JSON."
+m_ResponseData = "()"; // "Failed to parse response as JSON."
+m_ResponseData = "[]"; // "data property not an object"
+m_ResponseData = "null"; // "response not an object"
+m_ResponseData = "{\"data\":null}"; // "data property not an object"
+m_ResponseData = "{}"; // "data property not an object"
+m_ResponseData = "{\"data\":{}}"; // "data property not an array with at least one element"
+m_ResponseData = "{\"data\":[]}"; // "data property not an array with at least one element"
+m_ResponseData = "{\"data\":[null]}"; // "couldn't get id"
+m_ResponseData = "{\"data\":[false]}"; // "couldn't get id"
+m_ResponseData = "{\"data\":[{}]}"; // "couldn't get id"
+m_ResponseData = "{\"data\":[[]]}"; // "couldn't get id"
+m_ResponseData = "{\"data\":[{\"id\": null}]}"; // "couldn't get id", but only because we check for == 0 see TODOs above the check
+m_ResponseData = "{\"data\":[{\"id\": {}}]}"; // "couldn't get id", but only because we check for == 0 see TODOs above the check
+m_ResponseData = "{\"data\":[{\"id\": -12}]}"; // "couldn't get id"
+m_ResponseData = "{\"data\":[{\"id\": 0}]}"; // "couldn't get id"
+*/
+//m_ResponseData = "{\"data\":[{\"id\": true}]}"; // TODO XXX this should fail with "couldn't get id", but does not due to only warning on type mismatches instead of failing
+
+ if (!scriptInterface.ParseJSON(m_ResponseData, &gameResponse))
+ FAIL("Failed to parse response as JSON.");
+
+//LOGWARNING("%s", m_ResponseData);
+ m_ResponseData.clear();
+
+ if (!gameResponse.isObject())
+ FAIL("response not an object");
+
+ JS::RootedObject gameResponseObj(cx, gameResponse.toObjectOrNull());
+ JS::RootedValue dataVal(cx);
+ if (!JS_GetProperty(cx, gameResponseObj, "data", &dataVal))
+ FAIL("data property not in response.");
+
+ // [{"id": 42, ...}, ...]
+ if (!dataVal.isObject())
+ FAIL("data property not an object");
+
+ JS::RootedObject data(cx, dataVal.toObjectOrNull());
+ u32 length;
+ if (!JS_IsArrayObject(cx, data) || !JS_GetArrayLength(cx, data, &length) || !length)
+ FAIL("data property not an array with at least one element");
+
+ // {"id": 42, ...}
+ JS::RootedValue first(cx);
+ if (!JS_GetElement(cx, data, 0, &first))
+ FAIL("couldn't get first element.");
+
+ int id = -1;
+// TODO: This check currently does not really do anything if "id" is present...
+// meaning we can set id to be null and we get id=0.
+// There is a script value conversion check failed warning, but that does not change anything.
+// TODO: We should probably make those fail (hard), if that isn't done by default (which it probably should)
+// the we should add a templated variant that does.
+// TODO check if id < 0 (<=?) and if yes also fail here
+// NOTE: FromJSProperty does set things to probably 0 even if there is some stupid type conversion
+// So we check for <= 0, so we actually get proper results here...
+ // Valid ids are always > 0.
+ if (!ScriptInterface::FromJSProperty(cx, first, "id", id) || id <= 0)
+ FAIL("couldn't get id");
+
+LOGWARNING("id=%d", id);
+
+ m_GameId = "/" + std::to_string(id);
+ return true;
+}
+
+/**
+ * Parses the current content of m_ResponseData into m_ModData.
+ *
+ * The JSON data is expected to look like
+ * { data: [modobj1, modobj2, ...], ... (including result_count) }
+ * where modobjN has the following structure
+ * { homepage: "url", name: "displayname", nameid: "short-non-whitespace-name",
+ * summary: "short desc.", modfile: { version: "1.2.4", filename: "asdf.zip",
+ * filehash: { md5: "deadbeef" }, filesize: 1234, download: { binary_url: "someurl", ... } }, ... }.
+ * Only the listed properties are of interest to consumers, and we flatten
+ * the modfile structure as that simplifies handling and there are no conflicts.
+ */
+bool ModIo::ParseModsResponse(const ScriptInterface& scriptInterface)
+{
+ JSContext* cx = scriptInterface.GetContext();
+ JS::RootedValue modResponse(cx);
+
+// XXX testing
+// TODO same as for parsing the game id, this should be done in a test
+/*
+m_ResponseData = ""; // "Failed to parse response as JSON."
+m_ResponseData = "()"; // "Failed to parse response as JSON."
+m_ResponseData = "[]"; // "data property not an object"
+m_ResponseData = "null"; // "response not an object"
+m_ResponseData = "{\"data\":null}"; // "data property not an object"
+m_ResponseData = "{}"; // "data property not an object"
+m_ResponseData = "{\"data\":{}}"; // "data property not an array with at least one element"
+m_ResponseData = "{\"data\":[]}"; // "data property not an array with at least one element"
+m_ResponseData = "{\"data\":[null]}"; // "Failed to get array element object"
+m_ResponseData = "{\"data\":[false]}"; // "Failed to get array element object"
+m_ResponseData = "{\"data\":[{}]}"; // "Failed to get array element object"
+m_ResponseData = "{\"data\":[[]]}"; // "failed to get name from el"
+m_ResponseData = "{\"data\":[{\"name\":\"foobar\"}]}"; // "failed to get name_id from el"
+m_ResponseData = "{\"data\":[{\"name\":null}]}"; // "failed to get name_id from el" after some Script value conversion check failed warning
+m_ResponseData = "{\"data\":[{\"name\":42}]}"; // "failed to get name_id from el" (I guess converting numbers to strings by default is ok)
+m_ResponseData = "{\"data\":[{\"name\":false}]}"; // "failed to get name_id from el" after some Script value conversion check failed warning
+m_ResponseData = "{\"data\":[{\"name\":{}}]}"; // "failed to get name_id from el" after some Script value conversion check failed warning
+m_ResponseData = "{\"data\":[{\"name\":[]}]}"; // "failed to get name_id from el" after some Script value conversion check failed warning
+
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\"}]}"; // "modfile not an object"
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":null}]}"; // "modfile not an object"
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":[]}]}"; // "failed to get version from modFile"
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":{}}]}"; // "failed to get version from modFile"
+
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":{\"version\":null}}]}"; // "failed to get filename from modFile" after some Script value conversion check failed warning
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":{\"version\":\"\",\"filename\":\"\",\"filesize\":1234}}]}"; // "failed to get metadata_blob from modFile"
+// TODO if we parse metadata_blob specially then we need to fix this here (better testing for that)
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":{\"version\":\"\",\"filename\":\"\",\"filesize\":1234,\"metadata_blob\":\"\"}}]}"; // "failed to get md5 from filehash"
+
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":{\"version\":\"\",\"filename\":\"\",\"filesize\":1234,\"metadata_blob\":\"\",\"filehash\":null}}]}"; // "failed to get md5 from filehash"
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":{\"version\":\"\",\"filename\":\"\",\"filesize\":1234,\"metadata_blob\":\"\",\"filehash\":{}}}]}"; // "failed to get md5 from filehash"
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":{\"version\":\"\",\"filename\":\"\",\"filesize\":1234,\"metadata_blob\":\"\",\"filehash\":{\"md5\":null}}}]}"; // "failed to get binary_url from download" after some Script value conversion check failed warning
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":{\"version\":\"\",\"filename\":\"\",\"filesize\":1234,\"metadata_blob\":\"\",\"filehash\":{\"md5\":\"abc\"}}}]}"; // "failed to get binary_url from download"
+
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":{\"version\":\"\",\"filename\":\"\",\"filesize\":1234,\"metadata_blob\":\"\",\"filehash\":{\"md5\":\"abc\"},\"download\":null}}]}"; // "failed to get binary_url from download"
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":{\"version\":\"\",\"filename\":\"\",\"filesize\":1234,\"metadata_blob\":\"\",\"filehash\":{\"md5\":\"abc\"},\"download\":{\"binary_url\":null}}}]}"; // some Script value conversion check failed warning
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":{\"version\":\"\",\"filename\":\"\",\"filesize\":1234,\"metadata_blob\":\"\",\"filehash\":{\"md5\":\"abc\"},\"download\":{\"binary_url\":\"\"}}}]}"; // success
+*/
+
+ if (!scriptInterface.ParseJSON(m_ResponseData, &modResponse))
+ FAIL("Failed to parse response as JSON.");
+
+//LOGWARNING("%s", m_ResponseData);
+ m_ResponseData.clear();
+
+ if (!modResponse.isObject())
+ FAIL("response not an object");
+
+ JS::RootedObject modResponseObj(cx, modResponse.toObjectOrNull());
+ JS::RootedValue dataVal(cx);
+ if (!JS_GetProperty(cx, modResponseObj, "data", &dataVal))
+ FAIL("data property not in response.");
+
+ // [modobj1, modobj2, ... ]
+ if (!dataVal.isObject())
+ FAIL("data property not an object");
+
+ JS::RootedObject data(cx, dataVal.toObjectOrNull());
+ u32 length;
+ if (!JS_IsArrayObject(cx, data) || !JS_GetArrayLength(cx, data, &length) || !length)
+ FAIL("data property not an array with at least one element");
+
+ m_ModData.clear();
+ m_ModData.reserve(length);
+
+ for (u32 i = 0; i < length; ++i)
+ {
+ JS::RootedValue el(cx);
+ if (!JS_GetElement(cx, data, i, &el) || !el.isObject())
+ FAIL("Failed to get array element object");
+
+ m_ModData.emplace_back();
+
+#define COPY_STRINGS(prefix, obj, ...) \
+ for (const std::string& prop : { __VA_ARGS__ }) \
+ { \
+ std::string val; \
+ if (!ScriptInterface::FromJSProperty(cx, obj, prop.c_str(), val)) \
+ FAIL("failed to get %s from %s", prop, #obj);\
+ m_ModData.back().emplace(prefix+prop, val); \
+ }
+
+ // TODO: Currently the homepage field does not contain a non-null value for any entry.
+ COPY_STRINGS("", el, "name", "name_id", "summary");
+
+ // Now copy over the modfile part, but without the pointless substructure
+ JS::RootedObject elObj(cx, el.toObjectOrNull());
+ // TODO handle it being null
+ JS::RootedValue modFile(cx);
+ if (!JS_GetProperty(cx, elObj, "modfile", &modFile))
+ FAIL("Failed to get modfile data");
+
+ if (!modFile.isObject())
+ FAIL("modfile not an object");
+ // TODO: look into storing the remaining mod.json things we need in metatdata_blob (deps, etc?)
+ // TODO: Actually split up metadata_blob into something we can use (TODO: this might need some
+ // adjustment to how we store data?
+ // TODO: actually kill filename? (we ignore it)
+ COPY_STRINGS("", modFile, "version", "filename", "filesize", "metadata_blob");
+
+ JS::RootedObject modFileObj(cx, modFile.toObjectOrNull());
+ JS::RootedValue filehash(cx);
+ if (!JS_GetProperty(cx, modFileObj, "filehash", &filehash))
+ FAIL("Failed to get filehash data");
+
+ // TODO: Actually copy all elements of this over? (So we get them automatically instead of having to do this manually)
+ COPY_STRINGS("filehash_", filehash, "md5");
+
+ JS::RootedValue download(cx);
+ if (!JS_GetProperty(cx, modFileObj, "download", &download))
+ FAIL("Failed to get download data");
+
+ COPY_STRINGS("", download, "binary_url");
+
+#undef COPY_STRINGS
+ }
+
+ return true;
+}
+
+#undef FAIL
+
+void ModIo::DownloadMod(size_t idx)
+{
+ if (idx >= m_ModData.size())
+ return;
+
+ // TODO: do this asynchronously? or at least tell the gui that we are doing something -> progress bar (or something spinning)
+
+ const Paths paths(g_args);
+ const OsPath modUserPath = paths.UserData()/"mods";
+ const OsPath modPath = modUserPath/m_ModData[idx]["name_id"];
+ // TODO: Always call create?
+ if (!DirectoryExists(modPath) && INFO::OK != CreateDirectories(modPath, 0700, false))
+ return; // TODO complain?
+
+ // Name the file after the name_id, since using the filename would mean that
+ // we could end up with multiple zip files in the folder that might not work
+ // as expected for a user (since a later version might remove some files
+ // that aren't compatible anymore with the engine version).
+ // So we ignore the filename provided by the API and assume that we do not
+ // care about handling update.zip files. If that is the case we would need
+ // a way to find out what files are required by the current one and which
+ // should be removed for everything to work. This seems to be too complicated
+ // so we just do not support that usage.
+ const OsPath filePath = modPath/m_ModData[idx]["name_id"];
+ DownloadCallbackData callbackData(sys_OpenFile(filePath, "w"));
+ if (!callbackData.fp)
+ return; // TODO error handling
+
+ // Set IO callbacks
+ curl_easy_setopt(m_Curl, CURLOPT_WRITEFUNCTION, DownloadCallback);
+ curl_easy_setopt(m_Curl, CURLOPT_WRITEDATA, (void*)&callbackData);
+
+ LOGERROR("Trying to download %s", m_ModData[idx]["binary_url"].c_str());
+
+ // The download link will most likely redirect elsewhere, so allow that.
+ // We verify the validity of the file below.
+ curl_easy_setopt(m_Curl, CURLOPT_FOLLOWLOCATION, 1L);
+ // TODO: Restrict how often we can be redirected?
+
+ // Download the file
+ CURLcode err = PerformRequest(m_Curl, m_ModData[idx]["binary_url"], m_ErrorBuffer);
+ fclose(callbackData.fp);
+ callbackData.fp = NULL;
+
+ // To minimise security risks, don't support redirects for queries
+ curl_easy_setopt(m_Curl, CURLOPT_FOLLOWLOCATION, 0L);
+
+ if (err != CURLE_OK)
+ {
+ LOGERROR("Failure while downloading mod file. Server response: %s; %s", curl_easy_strerror(err), m_ErrorBuffer);
+ if (wunlink(filePath) != 0)
+ LOGERROR("Failed to delete file.");
+ return;
+ }
+
+ // TODO: verify that the file is what we expect it to be (hash, filesize, signature)
+ // XXX
+ if (!VerifyDownload(filePath, callbackData, m_ModData[idx]))
+ {
+ LOGERROR("File %s failed to verify.", filePath.string8());
+ // delete the file again as it does not match
+ if (wunlink(filePath) != 0)
+ LOGERROR("Failed to delete file.");
+
+ return;
+ }
+
+
+ LOGERROR("downloaded something.");
+
+ // TODO: hook into .pyromod code to do the win32 specific thing because win32...
+ // everything else should already work, given that we placed the file in the correct location already.
+}
+
+static bool VerifyDownload(const OsPath& UNUSED(filePath), DownloadCallbackData& callbackData, const std::map& modData)
+{
+ // TODO: Should we also check that the filesize is what was returned by the API?
+ // This seems slightly useless, given that the filesize is already contained in the hash.
+ // However we currently only have md5 which is easy to collide...
+
+ u8 digest[MD5::DIGESTSIZE];
+ callbackData.md5.Final(digest);
+ std::stringstream md5digest;
+ md5digest << std::hex << std::setfill('0');
+ for (size_t i = 0; i < MD5::DIGESTSIZE; ++i)
+ md5digest << std::setw(2) << (int)digest[i];
+
+ if (modData.at("filehash_md5") != md5digest.str())
+ {
+ LOGERROR("Invalid file. Expected md5 %s, got %s.", modData.at("filehash_md5").c_str(), md5digest.str());
+ return false;
+ }
+
+ // TODO XXX: Get upstream to actually have proper hashes that are collision-resistant.
+ // (someone might want to fix the hashes we provide for our downloads)
+ // (that someone might also want to add those hashes in other places than right next to the downloads)
+
+
+ // XXX check signature, this requires upstream to allow adding that (unless we shove it into the metadata blob) (which would be ugly but doable)
+
+ // TODO: possibly support more than a single key; checking the message_sig+trusted_comment sig would help with figuring out what key works (in case verifying the key would require having multiple hash states)
+
+ // TODO: get this from the config.
+ std::string wfg_pk = "RWTA6VIoth2Q1PFLsRILr3G7NB+mwwO8BSGoXs63X6TQgNGM4cE8Pvd6";
+
+ struct PKStruct {
+ unsigned char sig_alg[2] = {}; // == "Ed" (or actually "ED" since we are using the hashed on)
+ unsigned char keynum[8] = {};
+ unsigned char pk[crypto_sign_PUBLICKEYBYTES] = {};
+ };
+
+ // TODO possibly allocate this using sodium_malloc (and then sodium_free) so possible decoding failures are taken care of by the heap memory protection there?
+ PKStruct pk;
+
+ size_t bin_len = 0;
+ if (!sodium_base642bin((unsigned char*)&pk, sizeof(pk), wfg_pk.c_str(), wfg_pk.size(), NULL, &bin_len, NULL, sodium_base64_VARIANT_ORIGINAL_NO_PADDING)) // TODO is this the right variant?
+ {
+ // TODO invalid pk thingy
+ LOGERROR("failed to decode base64 key");
+ return false;
+ }
+
+ // TODO format of a .minisig file: untrusted comment: .*\nb64sign_of_file\ntrusted comment: .*\nb64sign_of_sign_of_file_and_trusted_comment
+ // Created using minisign(1) with -SHm tm.zip (using -H so we do not have to keep the whole file in memory (which might be a bit much for some users, and does not restrict bigger mods in the future))
+ // (this does make it incompatible with signify, but that seems like an ok tradeoff)
+ std::string file_sig = "untrusted comment: signature from minisign secret key\nRUTA6VIoth2Q1HUg5bwwbCUZPcqbQ/reLXqxiaWARH5PNcwxX5vBv/mLPLgdxGsIrOyK90763+rCVTmjeYx5BDz8C0CIbGZTNQs=\ntrusted comment: timestamp:1517285433 file:tm.zip\nTHwNMhK4Ogj6XA4305p1K9/ouP/DrxPcDFrPaiu+Ke6/WGlHIzBZHvmHWUedvsK6dzL31Gk8YNzscKWnZqWNCw==\n";
+
+ // We only _really_ care about the second line which is the signature of the file (b64-encoded)
+ // Also handling the other signature is nice, but not really required.
+ size_t first_line = file_sig.find('\n')+1;
+ size_t second_line = file_sig.find('\n', first_line);
+ // TODO: Check what happens if we don't have any, or only one \n
+ std::string msg_sig = file_sig.substr(first_line, second_line-first_line);
+
+ LOGWARNING("sig extracted: '%s'", msg_sig);
+
+
+ struct SigStruct {
+ unsigned char sig_alg[2] = {};
+ unsigned char keynum[8] = {};
+ unsigned char sig[crypto_sign_BYTES] = {};
+ };
+
+ // TODO possibly allocate this using sodium_malloc (see above)
+ SigStruct sig;
+
+ if (!sodium_base642bin((unsigned char*)&sig, sizeof(sig), msg_sig.c_str(), msg_sig.size(), NULL, &bin_len, NULL, sodium_base64_VARIANT_ORIGINAL_NO_PADDING)) // TODO padding, see above
+ {
+ LOGERROR("failed to decode base64 sig");
+ return false;
+ }
+
+ // TODO check keynum to be the same for sig and pk (else a match is very unlikely)
+ // TODO also check the sig alg to be what we expect
+
+ if (!crypto_sign_final_verify(&callbackData.sign_state, sig.sig, pk.pk))
+ {
+ LOGERROR("failed to verify signature");
+ return false;
+ }
+
+ // TODO: also check the trusted comment? (seems slightly pointless, but since we have it, why not check that nobody even tried tampering with it)
+
+ // TODO decide on what to use.
+ // Options:
+ // * gpgme
+ // - seems quite unwieldy to use
+ // - would allow key revocation
+ // - could use the different trust levels to allow some trusted people to sign things
+ // * signify (possibly asignify, since that can be used as a library)
+ // - seems quite simple to use
+ // - shorter keys
+ // - simple command line tool (harder to mess things up)
+
+ // TODO: libsodium? Seems quite nice for this
+
+ // Having signatures that _we_ create for the files one can retrieve from the API is
+ // a requirement (XXX). Else the API could just feed the users whatever in case of a compromise.
+ // We need signatures so we do not have to trust the API to provide files we actually want
+ // to be distributed. This means mods checked by us to be non-malicious only.
+ // If things are signed a compromised API can just continue to feed old files that might have
+ // issues that nobody noticed when we signed the files.
+
+ // Note that this does not prevent all possible attacks a package manager/update system should
+ // defend against. This is not an update system, however other possible attack vectors should
+ // be evalutated if they apply and how to fix those. (See https://github.com/theupdateframework/specification/blob/master/tuf-spec.md for some ideas.)
+
+ return true;
+}
+
+// TODO: nuke this at some point
+void ModIo::PrintMods()
+{
+ printf("[\n");
+ for (const auto& a : m_ModData)
+ {
+ printf(" {\n");
+ for (const auto& b : a)
+ {
+ printf(" \"%s\": \"%s\",\n", b.first.c_str(), b.second.c_str());
+ }
+ printf(" },\n");
+ }
+ printf("]\n");
+}
Index: source/ps/scripting/JSInterface_Mod.h
===================================================================
--- source/ps/scripting/JSInterface_Mod.h
+++ source/ps/scripting/JSInterface_Mod.h
@@ -26,6 +26,9 @@
JS::Value GetAvailableMods(ScriptInterface::CxPrivate* pCxPrivate);
void RestartEngine(ScriptInterface::CxPrivate* pCxPrivate);
void SetMods(ScriptInterface::CxPrivate* pCxPrivate, const std::vector& mods);
+
+ JS::Value ModIoGetMods(ScriptInterface::CxPrivate* pCxPrivate);
+ void ModIoDownloadMod(ScriptInterface::CxPrivate* pCxPrivate, uint32_t idx);
}
#endif
Index: source/ps/scripting/JSInterface_Mod.cpp
===================================================================
--- source/ps/scripting/JSInterface_Mod.cpp
+++ source/ps/scripting/JSInterface_Mod.cpp
@@ -1,4 +1,4 @@
-/* Copyright (C) 2017 Wildfire Games.
+/* Copyright (C) 2018 Wildfire Games.
* This file is part of 0 A.D.
*
* 0 A.D. is free software: you can redistribute it and/or modify
@@ -27,6 +27,7 @@
#include "ps/GameSetup/GameSetup.h"
#include "ps/GameSetup/Paths.h"
#include "ps/Mod.h"
+#include "ps/ModIo.h"
#include "ps/Profile.h"
#include "ps/scripting/JSInterface_Mod.h"
@@ -123,9 +124,51 @@
g_modsLoaded = mods;
}
+JS::Value JSI_Mod::ModIoGetMods(ScriptInterface::CxPrivate* pCxPrivate)
+{
+ ScriptInterface* scriptInterface = pCxPrivate->pScriptInterface;
+ JSContext* cx = scriptInterface->GetContext();
+ JSAutoRequest rq(cx);
+
+ if (!g_ModIo)
+ g_ModIo = new ModIo();
+
+ if (!g_ModIo)
+ return JS::NullValue(); // TODO: error/warning?
+
+ const std::vector>& availableMods = g_ModIo->GetMods(*scriptInterface);
+
+ JS::RootedObject mods(cx, JS_NewArrayObject(cx, availableMods.size()));
+ if (!mods)
+ return JS::NullValue(); // TODO: error?
+
+ u32 i = 0;
+ for (const std::map& mod : availableMods)
+ {
+ JS::RootedValue m(cx, JS::ObjectValue(*JS_NewPlainObject(cx)));
+ for (const std::pair& prop : mod)
+ scriptInterface->SetProperty(m, prop.first.c_str(), prop.second, true);
+
+ JS_SetElement(cx, mods, i++, m);
+ }
+
+ return JS::ObjectValue(*mods);
+}
+
+void JSI_Mod::ModIoDownloadMod(ScriptInterface::CxPrivate* UNUSED(pCxPrivate), uint32_t idx)
+{
+ if (!g_ModIo)
+ return; // TODO: Warn and tell users to fix their code?
+
+ g_ModIo->DownloadMod(idx);
+}
+
void JSI_Mod::RegisterScriptFunctions(const ScriptInterface& scriptInterface)
{
scriptInterface.RegisterFunction("GetAvailableMods");
scriptInterface.RegisterFunction("RestartEngine");
scriptInterface.RegisterFunction, &JSI_Mod::SetMods>("SetMods");
+
+ scriptInterface.RegisterFunction("ModIoGetMods");
+ scriptInterface.RegisterFunction("ModIoDownloadMod");
}