Index: binaries/data/config/default.cfg
===================================================================
--- binaries/data/config/default.cfg
+++ binaries/data/config/default.cfg
@@ -397,6 +397,11 @@
[mod]
enabledmods = "mod public"
+[modio.v1]
+baseurl = "https://api.test.mod.io/v1"
+api_key = "acf8fc07e3a8e9228ef4d5704c1659a1"
+name_id = "0ad"
+
[network]
duplicateplayernames = false ; Rename joining player to "User (2)" if "User" is already connected, otherwise prohibit join.
lateobservers = everyone ; Allow observers to join the game after it started. Possible values: everyone, buddies, disabled.
Index: binaries/data/mods/mod/gui/modmod/modio.js
===================================================================
--- /dev/null
+++ binaries/data/mods/mod/gui/modmod/modio.js
@@ -0,0 +1,66 @@
+let g_Mods = [];
+
+function init()
+{
+ g_Mods = Engine.ModIoGetMods();
+
+ warn(uneval(g_Mods));
+
+ generateModsList(g_Mods);
+}
+
+function generateModsList(mods)
+{
+ var [keys, names, name_ids, versions, filenames, filesizes, filehash_md5s, downloads] = [[],[],[],[],[],[],[],[]];
+
+ let i = 0;
+ for (let mod of mods)
+ {
+ keys.push(i++);
+ names.push(mod.name);
+ name_ids.push(mod.name_id);
+ versions.push(mod.version);
+ filenames.push(mod.filename);
+ filesizes.push(mod.filesize);
+ filehash_md5s.push(mod.filehash_md5);
+ downloads.push(mod.binary_url);
+ }
+
+ var obj = Engine.GetGUIObjectByName("modsAvailableList");
+ obj.list_name = names;
+ obj.list_modVersion = versions;
+ obj.list_modname_id = name_ids;
+ obj.list_modfilename = filenames;
+ obj.list_modfilesize = filesizes;
+ obj.list_modfilehash_md5 = filehash_md5s;
+ obj.list_moddownload = downloads;
+
+ obj.list = keys;
+}
+
+function showModDescription()
+{
+ let listObject = Engine.GetGUIObjectByName("modsAvailableList");
+ let desc = "No mod has been selected.";
+ if (listObject.selected != -1)
+ desc = g_Mods[listObject.selected].summary;
+
+ Engine.GetGUIObjectByName("globalModDescription").caption = desc;
+}
+
+function downloadMod()
+{
+ let listObject = Engine.GetGUIObjectByName("modsAvailableList");
+ if (listObject.selected == -1)
+ {
+ warn("Select something first.");
+ return;
+ }
+
+ Engine.ModIoDownloadMod(listObject.selected);
+}
+
+function closePage()
+{
+ Engine.SwitchGuiPage("page_pregame.xml", {});
+}
Index: binaries/data/mods/mod/gui/modmod/modio.xml
===================================================================
--- /dev/null
+++ binaries/data/mods/mod/gui/modmod/modio.xml
@@ -0,0 +1,81 @@
+
+
+
+
+
+
+
Index: binaries/data/mods/mod/gui/modmod/modmod.xml
===================================================================
--- binaries/data/mods/mod/gui/modmod/modmod.xml
+++ binaries/data/mods/mod/gui/modmod/modmod.xml
@@ -193,8 +193,8 @@
- Cancel
- closePage();
+ mod.io test
+ Engine.SwitchGuiPage("page_modio.xml");
Index: binaries/data/mods/mod/gui/page_modio.xml
===================================================================
--- /dev/null
+++ binaries/data/mods/mod/gui/page_modio.xml
@@ -0,0 +1,9 @@
+
+
+ common/modern/setup.xml
+ common/modern/styles.xml
+ common/modern/sprites.xml
+
+ modmod/styles.xml
+ modmod/modio.xml
+
Index: build/premake/extern_libs5.lua
===================================================================
--- build/premake/extern_libs5.lua
+++ build/premake/extern_libs5.lua
@@ -393,6 +393,15 @@
})
end,
},
+ libsodium = {
+ -- TODO check other OSes
+ compile_settings = function()
+ pkgconfig.add_includes("libsodium")
+ end,
+ link_settings = function()
+ pkgconfig.add_links("libsodium")
+ end,
+ },
libxml2 = {
compile_settings = function()
if os.istarget("windows") then
Index: build/premake/premake5.lua
===================================================================
--- build/premake/premake5.lua
+++ build/premake/premake5.lua
@@ -909,6 +909,7 @@
"tinygettext",
"icu",
"iconv",
+ "libsodium",
"valgrind",
}
Index: source/ps/GameSetup/GameSetup.cpp
===================================================================
--- source/ps/GameSetup/GameSetup.cpp
+++ source/ps/GameSetup/GameSetup.cpp
@@ -68,6 +68,7 @@
#include "ps/Joystick.h"
#include "ps/Loader.h"
#include "ps/Mod.h"
+#include "ps/ModIo.h"
#include "ps/Profile.h"
#include "ps/ProfileViewer.h"
#include "ps/Profiler2.h"
@@ -734,6 +735,8 @@
SAFE_DELETE(g_XmppClient);
+ SAFE_DELETE(g_ModIo);
+
ShutdownPs();
TIMER_BEGIN(L"shutdown TexMan");
Index: source/ps/ModIo.h
===================================================================
--- /dev/null
+++ source/ps/ModIo.h
@@ -0,0 +1,83 @@
+/* Copyright (C) 2017 Wildfire Games.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef INCLUDED_MODIO
+#define INCLUDED_MODIO
+
+#include
+
+#include "lib/external_libraries/curl.h"
+
+class DownloadCallbackData;
+class ScriptInterface;
+
+struct ModIoModData
+{
+ std::map properties;
+ std::vector dependencies;
+ std::vector minisigs;
+};
+
+class ModIo
+{
+ NONCOPYABLE(ModIo);
+public:
+ ModIo();
+ ~ModIo();
+
+ const std::vector& GetMods(const ScriptInterface& scriptinterface);
+
+ void DownloadMod(size_t idx);
+
+private:
+ static size_t ReceiveCallback(void* buffer, size_t size, size_t nmemb, void* userp);
+ static size_t DownloadCallback(void* buffer, size_t size, size_t nmemb, void* userp);
+
+ CURLcode PerformRequest(const std::string& url);
+
+ bool ParseGameIdResponse(const ScriptInterface& scriptInterface);
+ bool ParseModsResponse(const ScriptInterface& scriptInterface);
+
+ bool VerifyDownload(const OsPath& filePath, DownloadCallbackData& callbackData, const ModIoModData& modData);
+
+ void PrintMods();
+
+ // Url parts
+ std::string m_BaseUrl;
+ std::string m_GamesRequest;
+ std::string m_GameId;
+
+ // Query parameters
+ std::string m_ApiKey;
+ std::string m_IdQuery;
+
+ CURL* m_Curl;
+ curl_slist* m_Headers;
+ char m_ErrorBuffer[CURL_ERROR_SIZE];
+ std::string m_ResponseData;
+
+ std::vector m_ModData;
+};
+
+extern ModIo* g_ModIo;
+
+#endif // INCLUDED_MODIO
Index: source/ps/ModIo.cpp
===================================================================
--- /dev/null
+++ source/ps/ModIo.cpp
@@ -0,0 +1,711 @@
+/* Copyright (C) 2018 Wildfire Games.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+// TODO: Comment on why we handle things this way (not relying on user input, not trusting the remote too much, etc.)
+
+// TODO: Better error messages. Also ask upstream about some special queries with partially malformed replies
+
+#include "precompiled.h"
+
+#include "ModIo.h"
+
+#include "lib/file/file_system.h"
+#include "lib/sysdep/sysdep.h"
+#include "lib/sysdep/filesystem.h"
+#include "maths/MD5.h"
+#include "ps/CLogger.h"
+#include "ps/ConfigDB.h"
+#include "ps/GameSetup/Paths.h"
+#include "ps/Mod.h"
+#include "scriptinterface/ScriptConversions.h"
+#include "scriptinterface/ScriptInterface.h"
+
+#include
+
+#include
+
+#include
+#include
+
+ModIo* g_ModIo = nullptr;
+// TODO: add a compile-time config so we can build without this
+
+struct DownloadCallbackData {
+ DownloadCallbackData(FILE* _fp)
+ : fp(_fp), md5()
+ {
+ crypto_generichash_init(&hash_state, NULL, 0U, crypto_generichash_BYTES_MAX);
+ }
+ FILE* fp;
+ MD5 md5;
+ crypto_generichash_state hash_state;
+};
+
+// TODO move this comment somewhere nicer?
+/**
+ * mod.io interfacing code.
+ * This both distrusts the loaded JS mods, and the API as much as possible.
+ * We do not want a malicious mod to use this to download arbitrary files, nor do we want the API
+ * to serve us anything we have not verified.
+ * Therefore we only allow mods to download one of the mods returned by the API (using indices).
+ * Everything downloaded from the API has its signature verified against our public key.
+ * The mod.io settings are also locked down such that only mods that have been authorized by us
+ * show up in API queries. This is both done so that all required information (dependencies)
+ * are stored for the files, and that only mods that have been checked for being ok are acutally
+ * shown to users.
+ * This (mostly) necessitates parsing the API responses here, as opposed to in JS.
+ * One could alternatively parse the responses in a locked down JS context, but that would require
+ * storing that code in here, or making sure nobody can overwrite it. Also this would possibly make
+ * some of the needed accesses for downloading and verifying files a bit more complicated.
+ */
+
+ModIo::ModIo()
+ : m_GamesRequest("/games")
+{
+ // Get config values from the sytem namespace, or below (default); this can be overridden on the command line.
+ // We do this so a malicious mod cannot change the base url and get the user to make connections
+ // to someone else's endpoint.
+ g_ConfigDB.GetValue(CFG_SYSTEM, "modio.v1.baseurl", m_BaseUrl);
+ // TODO: Should we allow mods to actually change the two settings below? (Might be nice for total conversions.)
+ {
+ std::string api_key;
+ g_ConfigDB.GetValue(CFG_SYSTEM, "modio.v1.api_key", api_key);
+ m_ApiKey = "api_key=" + api_key;
+ }
+ {
+ std::string nameid;
+ g_ConfigDB.GetValue(CFG_SYSTEM, "modio.v1.name_id", nameid);
+ m_IdQuery = "name_id="+nameid;
+ }
+
+ // Initialise everything except Win32 sockets (because our networking
+ // system already inits those)
+ curl_global_init(CURL_GLOBAL_ALL & ~CURL_GLOBAL_WIN32);
+// TODO we should only do this in one single place (same for shutdown, currently we do it here and in the userreporter)
+
+ m_Curl = curl_easy_init();
+ ENSURE(m_Curl);
+
+ // Capture error messages
+ curl_easy_setopt(m_Curl, CURLOPT_ERRORBUFFER, m_ErrorBuffer);
+
+ // Fail if the server did
+ curl_easy_setopt(m_Curl, CURLOPT_FAILONERROR, 1L);
+
+ // Disable signal handlers (required for multithreaded applications)
+ curl_easy_setopt(m_Curl, CURLOPT_NOSIGNAL, 1L);
+
+ // To minimise security risks, don't support redirects
+ curl_easy_setopt(m_Curl, CURLOPT_FOLLOWLOCATION, 0L);
+
+ m_Headers = NULL;
+ std::string ua = "User-Agent: pyrogenesis ";
+ ua += curl_version();
+ ua += " (https://play0ad.com/)";
+ m_Headers = curl_slist_append(m_Headers, ua.c_str());
+ // TODO more?
+ curl_easy_setopt(m_Curl, CURLOPT_HTTPHEADER, m_Headers);
+
+ // TODO move elsewhere (comment)
+ // we only accept indices from JS when downloading something later on (security)
+
+ if (sodium_init() < 0) {
+ ENSURE(0); // TODO
+ }
+}
+
+ModIo::~ModIo()
+{
+ curl_slist_free_all(m_Headers);
+ curl_easy_cleanup(m_Curl);
+ curl_global_cleanup(); // TODO
+}
+
+size_t ModIo::ReceiveCallback(void* buffer, size_t size, size_t nmemb, void* userp)
+{
+ ModIo* self = static_cast(userp);
+
+ self->m_ResponseData += std::string((char*)buffer, (char*)buffer+size*nmemb);
+
+ return size*nmemb;
+}
+
+size_t ModIo::DownloadCallback(void* buffer, size_t size, size_t nmemb, void* userp)
+{
+ DownloadCallbackData* data = static_cast(userp);
+
+ size_t len = fwrite(buffer, size, nmemb, data->fp);
+
+ // Only update the hash with data we actually managed to write.
+ // In case we did not write all of it we will fail the download,
+ // but we do not want to have a possibly valid hash in that case.
+ data->md5.Update((const u8*)buffer, len*size);
+
+ crypto_generichash_update(&data->hash_state, (const u8*)buffer, len*size);
+
+ return len*size;
+}
+
+CURLcode ModIo::PerformRequest(const std::string& url)
+{
+ m_ErrorBuffer[0] = '\0';
+ curl_easy_setopt(m_Curl, CURLOPT_URL, url.c_str());
+ return curl_easy_perform(m_Curl);
+}
+
+const std::vector& ModIo::GetMods(const ScriptInterface& scriptInterface)
+{
+ m_ModData.clear();
+
+ // Set IO callbacks
+ curl_easy_setopt(m_Curl, CURLOPT_WRITEFUNCTION, ReceiveCallback);
+ curl_easy_setopt(m_Curl, CURLOPT_WRITEDATA, this);
+
+ // Get the game id if we didn't fetch it already
+ if (m_GameId.empty())
+ {
+ CURLcode err = PerformRequest(m_BaseUrl+m_GamesRequest+"?"+m_ApiKey+"&"+m_IdQuery);
+
+ if (err != CURLE_OK)
+ {
+ LOGERROR("Failure while querying for game id. Server response: %s; %s", curl_easy_strerror(err), m_ErrorBuffer);
+
+ return m_ModData;
+ }
+
+ if (!ParseGameIdResponse(scriptInterface))
+ return m_ModData;
+ }
+
+ // Get the actual mods
+ CURLcode err = PerformRequest(m_BaseUrl+m_GamesRequest+m_GameId+"/mods?"+m_ApiKey);
+ if (err != CURLE_OK)
+ {
+ LOGERROR("Failure while querying for mods. Server response: %s; %s", curl_easy_strerror(err), m_ErrorBuffer);
+ return m_ModData;
+ }
+
+ if (!ParseModsResponse(scriptInterface))
+ m_ModData.clear(); // Failed during parsing, make sure we don't provide partial data
+
+PrintMods();
+
+ return m_ModData;
+}
+
+// TODO make localizing errors easier by having #define CTXT __func__? or just "Parsing game id. " and LOGERROR(CTXT __VA_ARGS__) below
+#define FAIL(...) STMT(LOGERROR(__VA_ARGS__); return false;)
+
+/**
+ * Parses the current content of m_ResponseData to extract m_GameId.
+ *
+ * The JSON data is expected to look like
+ * { "data": [{"id": 42, ...}, ...], ... }
+ * where we are only interested in the value of the id property.
+ *
+ * @returns true iff it successfully parsed the id.
+ */
+bool ModIo::ParseGameIdResponse(const ScriptInterface& scriptInterface)
+{
+ JSContext* cx = scriptInterface.GetContext();
+ JS::RootedValue gameResponse(cx);
+
+// XXX testing TODO add this to some testing file? possibly make this take m_ResponseData as a param? then we could check/fuzz this nicely without having to either fake curl responses, or be-friend-ing this class to change m_ResponseData
+/*
+m_ResponseData = ""; // "Failed to parse response as JSON."
+m_ResponseData = "()"; // "Failed to parse response as JSON."
+m_ResponseData = "[]"; // "data property not an object"
+m_ResponseData = "null"; // "response not an object"
+m_ResponseData = "{\"data\":null}"; // "data property not an object"
+m_ResponseData = "{}"; // "data property not an object"
+m_ResponseData = "{\"data\":{}}"; // "data property not an array with at least one element"
+m_ResponseData = "{\"data\":[]}"; // "data property not an array with at least one element"
+m_ResponseData = "{\"data\":[null]}"; // "couldn't get id"
+m_ResponseData = "{\"data\":[false]}"; // "couldn't get id"
+m_ResponseData = "{\"data\":[{}]}"; // "couldn't get id"
+m_ResponseData = "{\"data\":[[]]}"; // "couldn't get id"
+m_ResponseData = "{\"data\":[{\"id\": null}]}"; // "couldn't get id", but only because we check for == 0 see TODOs above the check
+m_ResponseData = "{\"data\":[{\"id\": {}}]}"; // "couldn't get id", but only because we check for == 0 see TODOs above the check
+m_ResponseData = "{\"data\":[{\"id\": -12}]}"; // "couldn't get id"
+m_ResponseData = "{\"data\":[{\"id\": 0}]}"; // "couldn't get id"
+*/
+//m_ResponseData = "{\"data\":[{\"id\": true}]}"; // TODO XXX this should fail with "couldn't get id", but does not due to only warning on type mismatches instead of failing
+
+ if (!scriptInterface.ParseJSON(m_ResponseData, &gameResponse))
+ FAIL("Failed to parse response as JSON.");
+
+//LOGWARNING("%s", m_ResponseData);
+ m_ResponseData.clear();
+
+ if (!gameResponse.isObject())
+ FAIL("response not an object");
+
+ JS::RootedObject gameResponseObj(cx, gameResponse.toObjectOrNull());
+ JS::RootedValue dataVal(cx);
+ if (!JS_GetProperty(cx, gameResponseObj, "data", &dataVal))
+ FAIL("data property not in response.");
+
+ // [{"id": 42, ...}, ...]
+ if (!dataVal.isObject())
+ FAIL("data property not an object");
+
+ JS::RootedObject data(cx, dataVal.toObjectOrNull());
+ u32 length;
+ if (!JS_IsArrayObject(cx, data) || !JS_GetArrayLength(cx, data, &length) || !length)
+ FAIL("data property not an array with at least one element");
+
+ // {"id": 42, ...}
+ JS::RootedValue first(cx);
+ if (!JS_GetElement(cx, data, 0, &first))
+ FAIL("couldn't get first element.");
+
+ int id = -1;
+// TODO: This check currently does not really do anything if "id" is present...
+// meaning we can set id to be null and we get id=0.
+// There is a script value conversion check failed warning, but that does not change anything.
+// TODO: We should probably make those fail (hard), if that isn't done by default (which it probably should)
+// the we should add a templated variant that does.
+// TODO check if id < 0 (<=?) and if yes also fail here
+// NOTE: FromJSProperty does set things to probably 0 even if there is some stupid type conversion
+// So we check for <= 0, so we actually get proper results here...
+ // Valid ids are always > 0.
+ if (!ScriptInterface::FromJSProperty(cx, first, "id", id) || id <= 0)
+ FAIL("couldn't get id");
+
+LOGWARNING("id=%d", id);
+
+ m_GameId = "/" + std::to_string(id);
+ return true;
+}
+
+/**
+ * Parses the current content of m_ResponseData into m_ModData.
+ *
+ * The JSON data is expected to look like
+ * { data: [modobj1, modobj2, ...], ... (including result_count) }
+ * where modobjN has the following structure
+ * { homepage: "url", name: "displayname", nameid: "short-non-whitespace-name",
+ * summary: "short desc.", modfile: { version: "1.2.4", filename: "asdf.zip",
+ * filehash: { md5: "deadbeef" }, filesize: 1234, download: { binary_url: "someurl", ... } }, ... }.
+ * Only the listed properties are of interest to consumers, and we flatten
+ * the modfile structure as that simplifies handling and there are no conflicts.
+ */
+bool ModIo::ParseModsResponse(const ScriptInterface& scriptInterface)
+{
+ JSContext* cx = scriptInterface.GetContext();
+ JS::RootedValue modResponse(cx);
+
+// XXX testing
+// TODO same as for parsing the game id, this should be done in a test
+/*
+m_ResponseData = ""; // "Failed to parse response as JSON."
+m_ResponseData = "()"; // "Failed to parse response as JSON."
+m_ResponseData = "[]"; // "data property not an object"
+m_ResponseData = "null"; // "response not an object"
+m_ResponseData = "{\"data\":null}"; // "data property not an object"
+m_ResponseData = "{}"; // "data property not an object"
+m_ResponseData = "{\"data\":{}}"; // "data property not an array with at least one element"
+m_ResponseData = "{\"data\":[]}"; // "data property not an array with at least one element"
+m_ResponseData = "{\"data\":[null]}"; // "Failed to get array element object"
+m_ResponseData = "{\"data\":[false]}"; // "Failed to get array element object"
+m_ResponseData = "{\"data\":[{}]}"; // "Failed to get array element object"
+m_ResponseData = "{\"data\":[[]]}"; // "failed to get name from el"
+m_ResponseData = "{\"data\":[{\"name\":\"foobar\"}]}"; // "failed to get name_id from el"
+m_ResponseData = "{\"data\":[{\"name\":null}]}"; // "failed to get name_id from el" after some Script value conversion check failed warning
+m_ResponseData = "{\"data\":[{\"name\":42}]}"; // "failed to get name_id from el" (I guess converting numbers to strings by default is ok)
+m_ResponseData = "{\"data\":[{\"name\":false}]}"; // "failed to get name_id from el" after some Script value conversion check failed warning
+m_ResponseData = "{\"data\":[{\"name\":{}}]}"; // "failed to get name_id from el" after some Script value conversion check failed warning
+m_ResponseData = "{\"data\":[{\"name\":[]}]}"; // "failed to get name_id from el" after some Script value conversion check failed warning
+
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\"}]}"; // "modfile not an object"
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":null}]}"; // "modfile not an object"
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":[]}]}"; // "failed to get version from modFile"
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":{}}]}"; // "failed to get version from modFile"
+
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":{\"version\":null}}]}"; // "failed to get filename from modFile" after some Script value conversion check failed warning
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":{\"version\":\"\",\"filename\":\"\",\"filesize\":1234}}]}"; // "failed to get metadata_blob from modFile"
+// TODO if we parse metadata_blob specially then we need to fix this here (better testing for that)
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":{\"version\":\"\",\"filename\":\"\",\"filesize\":1234,\"metadata_blob\":\"\"}}]}"; // "failed to get md5 from filehash"
+
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":{\"version\":\"\",\"filename\":\"\",\"filesize\":1234,\"metadata_blob\":\"\",\"filehash\":null}}]}"; // "failed to get md5 from filehash"
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":{\"version\":\"\",\"filename\":\"\",\"filesize\":1234,\"metadata_blob\":\"\",\"filehash\":{}}}]}"; // "failed to get md5 from filehash"
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":{\"version\":\"\",\"filename\":\"\",\"filesize\":1234,\"metadata_blob\":\"\",\"filehash\":{\"md5\":null}}}]}"; // "failed to get binary_url from download" after some Script value conversion check failed warning
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":{\"version\":\"\",\"filename\":\"\",\"filesize\":1234,\"metadata_blob\":\"\",\"filehash\":{\"md5\":\"abc\"}}}]}"; // "failed to get binary_url from download"
+
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":{\"version\":\"\",\"filename\":\"\",\"filesize\":1234,\"metadata_blob\":\"\",\"filehash\":{\"md5\":\"abc\"},\"download\":null}}]}"; // "failed to get binary_url from download"
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":{\"version\":\"\",\"filename\":\"\",\"filesize\":1234,\"metadata_blob\":\"\",\"filehash\":{\"md5\":\"abc\"},\"download\":{\"binary_url\":null}}}]}"; // some Script value conversion check failed warning
+m_ResponseData = "{\"data\":[{\"name\":\"\",\"name_id\":\"\",\"summary\":\"\",\"modfile\":{\"version\":\"\",\"filename\":\"\",\"filesize\":1234,\"metadata_blob\":\"\",\"filehash\":{\"md5\":\"abc\"},\"download\":{\"binary_url\":\"\"}}}]}"; // success
+
+
+// TODO metadata_blob minisigs and dependencies parsing
+*/
+
+ if (!scriptInterface.ParseJSON(m_ResponseData, &modResponse))
+ FAIL("Failed to parse response as JSON.");
+
+//LOGWARNING("%s", m_ResponseData);
+ m_ResponseData.clear();
+
+ if (!modResponse.isObject())
+ FAIL("response not an object");
+
+ JS::RootedObject modResponseObj(cx, modResponse.toObjectOrNull());
+ JS::RootedValue dataVal(cx);
+ if (!JS_GetProperty(cx, modResponseObj, "data", &dataVal))
+ FAIL("data property not in response.");
+
+ // [modobj1, modobj2, ... ]
+ if (!dataVal.isObject())
+ FAIL("data property not an object");
+
+ JS::RootedObject data(cx, dataVal.toObjectOrNull());
+ u32 length;
+ if (!JS_IsArrayObject(cx, data) || !JS_GetArrayLength(cx, data, &length) || !length)
+ FAIL("data property not an array with at least one element");
+
+ m_ModData.clear();
+ m_ModData.reserve(length);
+
+// TODO: Should we give partial results in case only one returned mod contains bogus data? (that would mean popping the last thing added (in most FAIL cases below)) No, seems like trouble if the remote cannot even provide valid data
+ for (u32 i = 0; i < length; ++i)
+ {
+ JS::RootedValue el(cx);
+ if (!JS_GetElement(cx, data, i, &el) || !el.isObject())
+ FAIL("Failed to get array element object");
+
+ m_ModData.emplace_back();
+
+#define COPY_STRINGS(prefix, obj, ...) \
+ for (const std::string& prop : { __VA_ARGS__ }) \
+ { \
+ std::string val; \
+ if (!ScriptInterface::FromJSProperty(cx, obj, prop.c_str(), val)) \
+ FAIL("failed to get %s from %s", prop, #obj);\
+ m_ModData.back().properties.emplace(prefix+prop, val); \
+ }
+
+ // TODO: Currently the homepage field does not contain a non-null value for any entry.
+ COPY_STRINGS("", el, "name", "name_id", "summary");
+
+ // Now copy over the modfile part, but without the pointless substructure
+ JS::RootedObject elObj(cx, el.toObjectOrNull());
+ // TODO handle it being null
+ JS::RootedValue modFile(cx);
+ if (!JS_GetProperty(cx, elObj, "modfile", &modFile))
+ FAIL("Failed to get modfile data");
+
+ if (!modFile.isObject())
+ FAIL("modfile not an object");
+
+ // TODO: actually kill filename? (we ignore it)
+ COPY_STRINGS("", modFile, "version", "filename", "filesize");
+
+ JS::RootedObject modFileObj(cx, modFile.toObjectOrNull());
+ JS::RootedValue filehash(cx);
+ if (!JS_GetProperty(cx, modFileObj, "filehash", &filehash))
+ FAIL("Failed to get filehash data");
+
+ COPY_STRINGS("filehash_", filehash, "md5");
+
+ JS::RootedValue download(cx);
+ if (!JS_GetProperty(cx, modFileObj, "download", &download))
+ FAIL("Failed to get download data");
+
+ COPY_STRINGS("", download, "binary_url");
+
+ // Parse metadata_blob (sig+deps)
+ std::string metadata_blob;
+ if (!ScriptInterface::FromJSProperty(cx, modFile, "metadata_blob", metadata_blob))
+ FAIL("failed to get metadata_blob from modFile");
+
+ JS::RootedValue metadata(cx);
+ if (!scriptInterface.ParseJSON(metadata_blob, &metadata))
+ FAIL("Failed to parse metadata_blob as JSON.");
+
+ if (!metadata.isObject())
+ FAIL("response not an object");
+
+ if (!ScriptInterface::FromJSProperty(cx, metadata, "minisigs", m_ModData.back().minisigs))
+ FAIL("failed to get minisigs from metadata");
+
+ if (!ScriptInterface::FromJSProperty(cx, metadata, "dependencies", m_ModData.back().dependencies))
+ FAIL("failed to get dependencies from metadata");
+
+#undef COPY_STRINGS
+ }
+
+ return true;
+}
+
+#undef FAIL
+
+void ModIo::DownloadMod(size_t idx)
+{
+ if (idx >= m_ModData.size())
+ return;
+
+ // TODO: do this asynchronously? or at least tell the gui that we are doing something -> progress bar (or something spinning)
+
+ const Paths paths(g_args);
+ const OsPath modUserPath = paths.UserData()/"mods";
+ const OsPath modPath = modUserPath/m_ModData[idx].properties["name_id"];
+ // TODO: Always call create?
+ if (!DirectoryExists(modPath) && INFO::OK != CreateDirectories(modPath, 0700, false))
+ return; // TODO complain?
+
+ // Name the file after the name_id, since using the filename would mean that
+ // we could end up with multiple zip files in the folder that might not work
+ // as expected for a user (since a later version might remove some files
+ // that aren't compatible anymore with the engine version).
+ // So we ignore the filename provided by the API and assume that we do not
+ // care about handling update.zip files. If that is the case we would need
+ // a way to find out what files are required by the current one and which
+ // should be removed for everything to work. This seems to be too complicated
+ // so we just do not support that usage.
+ // NOTE: We do save the file under a slightly different name from the final
+ // one, to ensure that in case a download aborts and the file stays
+ // around, the game will not attempt to open the file which has not
+ // been verified.
+ const OsPath filePath = modPath/(m_ModData[idx].properties["name_id"]+".zip.temp");
+ DownloadCallbackData callbackData(sys_OpenFile(filePath, "w"));
+ if (!callbackData.fp)
+ return; // TODO error handling
+
+ // Set IO callbacks
+ curl_easy_setopt(m_Curl, CURLOPT_WRITEFUNCTION, DownloadCallback);
+ curl_easy_setopt(m_Curl, CURLOPT_WRITEDATA, (void*)&callbackData);
+
+ LOGERROR("Trying to download %s", m_ModData[idx].properties["binary_url"].c_str());
+
+ // The download link will most likely redirect elsewhere, so allow that.
+ // We verify the validity of the file below.
+ curl_easy_setopt(m_Curl, CURLOPT_FOLLOWLOCATION, 1L);
+ // TODO: Restrict how often we can be redirected?
+
+ // Download the file
+ CURLcode err = PerformRequest(m_ModData[idx].properties["binary_url"]);
+ fclose(callbackData.fp);
+ callbackData.fp = NULL;
+
+ // To minimise security risks, don't support redirects for queries
+ curl_easy_setopt(m_Curl, CURLOPT_FOLLOWLOCATION, 0L);
+
+ if (err != CURLE_OK)
+ {
+ LOGERROR("Failure while downloading mod file. Server response: %s; %s", curl_easy_strerror(err), m_ErrorBuffer);
+ if (wunlink(filePath) != 0)
+ LOGERROR("Failed to delete file.");
+ return;
+ }
+
+ if (!VerifyDownload(filePath, callbackData, m_ModData[idx]))
+ {
+ LOGERROR("File %s failed to verify.", filePath.string8());
+ // delete the file again as it does not match
+ if (wunlink(filePath) != 0)
+ LOGERROR("Failed to delete file.");
+
+ return;
+ }
+
+ const OsPath finalFilePath = modPath/(m_ModData[idx].properties["name_id"]+".zip");
+ if (wrename(filePath, finalFilePath) != 0)
+ {
+ LOGERROR("failed to rename file.");
+ wunlink(filePath);
+ return;
+ }
+
+ LOGERROR("downloaded something and verified it successfully.");
+
+ // TODO: hook into .pyromod code to do the win32 specific thing because win32...
+ // everything else should already work, given that we placed the file in the correct location already.
+}
+
+bool ModIo::VerifyDownload(const OsPath& filePath, DownloadCallbackData& callbackData, const ModIoModData& modData)
+{
+ {
+ u64 filesize = std::stoull(modData.properties.at("filesize"));
+ if (filesize != FileSize(filePath))
+ {
+ LOGERROR("Invalid filesize.");
+ return false;
+ }
+ }
+
+ // MD5 (because upstream provides it)
+ // Just used to make sure there was no obvious corruption during transfer.
+ {
+ u8 digest[MD5::DIGESTSIZE];
+ callbackData.md5.Final(digest);
+ std::stringstream md5digest;
+ md5digest << std::hex << std::setfill('0');
+ for (size_t i = 0; i < MD5::DIGESTSIZE; ++i)
+ md5digest << std::setw(2) << (int)digest[i];
+
+ if (modData.properties.at("filehash_md5") != md5digest.str())
+ {
+ LOGERROR("Invalid file. Expected md5 %s, got %s.", modData.properties.at("filehash_md5").c_str(), md5digest.str());
+ return false;
+ }
+ }
+
+ // TODO XXX: Get upstream to actually have proper hashes that are collision-resistant.
+ // (someone might want to fix the hashes we provide for our downloads)
+ // (that someone might also want to add those hashes in other places than right next to the downloads)
+ // TODO: This comment above can be ignored, we have signing.
+
+
+ // Verify file signature.
+ // Used to make sure that the downloaded file was actually checked and signed
+ // by Wildfire Games.
+
+
+// TODO: move rationale to a slightly better location
+ // Having signatures that _we_ create for the files one can retrieve from the API is
+ // a requirement. Else the API could just feed the users whatever in case of a compromise.
+ // We need signatures so we do not have to trust the API to provide files we actually want
+ // to be distributed. This means mods checked by us to be non-malicious only.
+ // If things are signed a compromised API can just continue to feed old files that might have
+ // issues that nobody noticed when we signed the files.
+
+ // Note that this does not prevent all possible attacks a package manager/update system should
+ // defend against. This is not an update system, however other possible attack vectors should
+ // be evalutated if they apply and how to fix those. (See https://github.com/theupdateframework/specification/blob/master/tuf-spec.md for some ideas.)
+
+ unsigned char hash_fin[crypto_generichash_BYTES_MAX] = {};
+ if (crypto_generichash_final(&callbackData.hash_state, hash_fin, sizeof hash_fin) != 0)
+ {
+ LOGERROR("failed to compute final hash");
+ return false;
+ }
+
+ struct PKStruct {
+ unsigned char sig_alg[2] = {}; // == "Ed"
+ unsigned char keynum[8] = {}; // should match the keynum in the sigstruct, else this is the wrong key
+ unsigned char pk[crypto_sign_PUBLICKEYBYTES] = {};
+ };
+
+ struct SigStruct {
+ unsigned char sig_alg[2] = {}; // "ED" (since we only support the hashed mode)
+ unsigned char keynum[8] = {}; // should match the keynum in the PKStruct
+ unsigned char sig[crypto_sign_BYTES] = {};
+ };
+
+ // TODO: is there actually a use-case where a client needs multiple keys?
+ // having some files signed by more than one key is nice since we can rotate keys
+ // and some mods might be compatible with more than a single version.
+ // TODO: related to that, what if one signature is valid, and another is invalid (in the actual check)? (probably consider it invalid since well wtf)
+ // TODO: get this from the config?
+ for (const std::string& pk_str : {"RWTA6VIoth2Q1PFLsRILr3G7NB+mwwO8BSGoXs63X6TQgNGM4cE8Pvd6"})
+ {
+ // TODO possibly allocate this using sodium_malloc (and then sodium_free) so possible decoding failures are taken care of by the heap memory protection there?
+ PKStruct pk;
+
+ size_t bin_len = 0;
+ if (sodium_base642bin((unsigned char*)&pk, sizeof pk, pk_str.c_str(), pk_str.size(), NULL, &bin_len, NULL, sodium_base64_VARIANT_ORIGINAL) != 0 || bin_len != sizeof pk)
+ {
+ LOGERROR("failed to decode base64 key");
+ continue;
+ }
+
+//LOGWARNING("sig_alg='%c%c'", pk.sig_alg[0], pk.sig_alg[1]);
+//LOGWARNING("keynum='%01x%01x%01x%01x%01x%01x%01x%01x'", pk.keynum[0], pk.keynum[1], pk.keynum[2], pk.keynum[3], pk.keynum[4], pk.keynum[5], pk.keynum[6], pk.keynum[7]);
+
+ for (const std::string& file_sig : modData.minisigs)
+ {
+ // Format of a .minisig file (created using minisign(1) with -SHm file.zip)
+ // untrusted comment: .*\nb64sign_of_file\ntrusted comment: .*\nb64sign_of_sign_of_file_and_trusted_comment
+ std::vector sig_lines;
+ boost::split(sig_lines, file_sig, boost::is_any_of("\n"));
+ if (sig_lines.size() < 4)
+ {
+ LOGERROR("invalid (too short) sig");
+ continue;
+ }
+
+ // We only _really_ care about the second line which is the signature of the file (b64-encoded)
+ // Also handling the other signature is nice, but not really required.
+ const std::string& msg_sig = sig_lines[1];
+
+//LOGWARNING("sig extracted: '%s'", msg_sig);
+
+ // TODO possibly allocate this using sodium_malloc (see above)
+ SigStruct sig;
+
+ if (sodium_base642bin((unsigned char*)&sig, sizeof sig, msg_sig.c_str(), msg_sig.size(), NULL, &bin_len, NULL, sodium_base64_VARIANT_ORIGINAL) != 0 || bin_len != sizeof sig)
+ {
+ LOGERROR("failed to decode base64 sig");
+ continue;
+ }
+
+//LOGWARNING("sig_alg='%c%c'", sig.sig_alg[0], sig.sig_alg[1]);
+//LOGWARNING("keynum='%01x%01x%01x%01x%01x%01x%01x%01x'", sig.keynum[0], sig.keynum[1], sig.keynum[2], sig.keynum[3], sig.keynum[4], sig.keynum[5], sig.keynum[6], sig.keynum[7]);
+
+ cassert(sizeof pk.keynum == sizeof sig.keynum);
+
+ if (memcmp(&sig.sig_alg, "ED", 2) != 0)
+ {
+ LOGERROR("only hashed minisign signatures are supported");
+ continue;
+ }
+
+ if (memcmp(&pk.keynum, &sig.keynum, sizeof sig.keynum) != 0)
+ continue; // mismatched key, try another one
+
+ if (crypto_sign_verify_detached(sig.sig, hash_fin, sizeof hash_fin, pk.pk) != 0)
+ {
+ LOGERROR("failed to verify signature");
+ return false;
+ }
+
+ // TODO: also check the trusted comment? (seems slightly pointless, but since we have it, why not check that nobody even tried tampering with it)
+
+ return true;
+ }
+ }
+
+ LOGERROR("did not find a matching valid signature");
+ return false;
+}
+
+// TODO: nuke this at some point
+void ModIo::PrintMods()
+{
+ printf("[\n");
+ for (const auto& a : m_ModData)
+ {
+ printf(" {\n");
+ for (const auto& b : a.properties)
+ printf(" \"%s\": \"%s\",\n", b.first.c_str(), b.second.c_str());
+ printf(" \"dependencies\": [");
+ for (const auto& c : a.dependencies)
+ printf("\"%s\", ", c.c_str());
+ printf("]\n");
+ printf(" \"minisigs\": [");
+ for (const auto& d : a.minisigs)
+ printf("\"%s\", ", d.c_str());
+ printf("]\n");
+ printf(" },\n");
+ }
+ printf("]\n");
+}
Index: source/ps/scripting/JSInterface_Mod.h
===================================================================
--- source/ps/scripting/JSInterface_Mod.h
+++ source/ps/scripting/JSInterface_Mod.h
@@ -26,6 +26,9 @@
JS::Value GetAvailableMods(ScriptInterface::CxPrivate* pCxPrivate);
void RestartEngine(ScriptInterface::CxPrivate* pCxPrivate);
void SetMods(ScriptInterface::CxPrivate* pCxPrivate, const std::vector& mods);
+
+ JS::Value ModIoGetMods(ScriptInterface::CxPrivate* pCxPrivate);
+ void ModIoDownloadMod(ScriptInterface::CxPrivate* pCxPrivate, uint32_t idx);
}
#endif
Index: source/ps/scripting/JSInterface_Mod.cpp
===================================================================
--- source/ps/scripting/JSInterface_Mod.cpp
+++ source/ps/scripting/JSInterface_Mod.cpp
@@ -1,4 +1,4 @@
-/* Copyright (C) 2017 Wildfire Games.
+/* Copyright (C) 2018 Wildfire Games.
* This file is part of 0 A.D.
*
* 0 A.D. is free software: you can redistribute it and/or modify
@@ -27,6 +27,7 @@
#include "ps/GameSetup/GameSetup.h"
#include "ps/GameSetup/Paths.h"
#include "ps/Mod.h"
+#include "ps/ModIo.h"
#include "ps/Profile.h"
#include "ps/scripting/JSInterface_Mod.h"
@@ -123,9 +124,53 @@
g_modsLoaded = mods;
}
+JS::Value JSI_Mod::ModIoGetMods(ScriptInterface::CxPrivate* pCxPrivate)
+{
+ ScriptInterface* scriptInterface = pCxPrivate->pScriptInterface;
+ JSContext* cx = scriptInterface->GetContext();
+ JSAutoRequest rq(cx);
+
+ if (!g_ModIo)
+ g_ModIo = new ModIo();
+
+ if (!g_ModIo)
+ return JS::NullValue(); // TODO: error/warning?
+
+ const std::vector& availableMods = g_ModIo->GetMods(*scriptInterface);
+
+ JS::RootedObject mods(cx, JS_NewArrayObject(cx, availableMods.size()));
+ if (!mods)
+ return JS::NullValue(); // TODO: error?
+
+ u32 i = 0;
+ for (const ModIoModData& mod : availableMods)
+ {
+ JS::RootedValue m(cx, JS::ObjectValue(*JS_NewPlainObject(cx)));
+ for (const std::pair& prop : mod.properties)
+ scriptInterface->SetProperty(m, prop.first.c_str(), prop.second, true);
+
+ scriptInterface->SetProperty(m, "dependencies", mod.dependencies, true);
+
+ JS_SetElement(cx, mods, i++, m);
+ }
+
+ return JS::ObjectValue(*mods);
+}
+
+void JSI_Mod::ModIoDownloadMod(ScriptInterface::CxPrivate* UNUSED(pCxPrivate), uint32_t idx)
+{
+ if (!g_ModIo)
+ return; // TODO: Warn and tell users to fix their code?
+
+ g_ModIo->DownloadMod(idx);
+}
+
void JSI_Mod::RegisterScriptFunctions(const ScriptInterface& scriptInterface)
{
scriptInterface.RegisterFunction("GetAvailableMods");
scriptInterface.RegisterFunction("RestartEngine");
scriptInterface.RegisterFunction, &JSI_Mod::SetMods>("SetMods");
+
+ scriptInterface.RegisterFunction("ModIoGetMods");
+ scriptInterface.RegisterFunction("ModIoDownloadMod");
}