Index: ps/trunk/binaries/data/mods/public/gui/replaymenu/replay_actions.js =================================================================== --- ps/trunk/binaries/data/mods/public/gui/replaymenu/replay_actions.js +++ ps/trunk/binaries/data/mods/public/gui/replaymenu/replay_actions.js @@ -127,6 +127,12 @@ }); } +function reloadCache() +{ + let selected = Engine.GetGUIObjectByName("replaySelection").selected; + loadReplays(selected > -1 ? createReplaySelectionData(g_ReplaysFiltered[selected].directory) : "", true); +} + /** * Callback. */ Index: ps/trunk/binaries/data/mods/public/gui/replaymenu/replay_menu.js =================================================================== --- ps/trunk/binaries/data/mods/public/gui/replaymenu/replay_menu.js +++ ps/trunk/binaries/data/mods/public/gui/replaymenu/replay_menu.js @@ -59,7 +59,7 @@ return; } - loadReplays(data && data.replaySelectionData); + loadReplays(data && data.replaySelectionData, false); if (!g_Replays) { @@ -75,10 +75,13 @@ * Store the list of replays loaded in C++ in g_Replays. * Check timestamp and compatibility and extract g_Playernames, g_MapNames, g_VictoryConditions. * Restore selected filters and item. + * @param replaySelectionData - Currently selected filters and item to be restored after the loading. + * @param compareFiles - If true, compares files briefly (which might be slow with optical harddrives), + * otherwise blindly trusts the replay cache. */ -function loadReplays(replaySelectionData) +function loadReplays(replaySelectionData, compareFiles) { - g_Replays = Engine.GetReplays(); + g_Replays = Engine.GetReplays(compareFiles); if (!g_Replays) return; Index: ps/trunk/binaries/data/mods/public/gui/replaymenu/replay_menu.xml =================================================================== --- ps/trunk/binaries/data/mods/public/gui/replaymenu/replay_menu.xml +++ ps/trunk/binaries/data/mods/public/gui/replaymenu/replay_menu.xml @@ -249,6 +249,13 @@ deleteReplayButtonPressed(); + + + Reload Cache + Rebuild the replay cache from scratch. Potentially slow! + reloadCache(); + + Summary Index: ps/trunk/binaries/data/mods/public/gui/session/session.js =================================================================== --- ps/trunk/binaries/data/mods/public/gui/session/session.js +++ ps/trunk/binaries/data/mods/public/gui/session/session.js @@ -671,6 +671,11 @@ Engine.EndGame(); + // After the replay file was closed in EndGame + // Done here to keep EndGame small + if (!g_IsReplay) + Engine.AddReplayToCache(replayDirectory); + if (g_IsController && Engine.HasXmppClient()) Engine.SendUnregisterGame(); Index: ps/trunk/source/ps/VisualReplay.h =================================================================== --- ps/trunk/source/ps/VisualReplay.h +++ ps/trunk/source/ps/VisualReplay.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2016 Wildfire Games. +/* Copyright (C) 2017 Wildfire Games. * This file is part of 0 A.D. * * 0 A.D. is free software: you can redistribute it and/or modify @@ -31,7 +31,7 @@ /** * Returns the path to the sim-log directory (that contains the directories with the replay files. * - * @param scriptInterface the ScriptInterface in which to create the return data. + * @param scriptInterface - the ScriptInterface in which to create the return data. * @return OsPath the absolute file path */ OsPath GetDirectoryName(); @@ -42,23 +42,51 @@ void StartVisualReplay(const CStrW& directory); /** + * Reads the replay Cache file and parses it into a jsObject + * + * @param scriptInterface - the ScriptInterface in which to create the return data. + * @param cachedReplaysObject - the cached replays. + * @return true on succes + */ +bool ReadCacheFile(ScriptInterface& scriptInterface, JS::MutableHandleObject cachedReplaysObject); + +/** + * Stores the replay list in the replay cache file + * + * @param scriptInterface - the ScriptInterface in which to create the return data. + * @param replays - the replay list to store. + */ +void StoreCacheFile(ScriptInterface& scriptInterface, JS::HandleObject replays); + +/** + * Load the replay cache and check if there are new/deleted replays. If so, update the cache. + * + * @param scriptInterface - the ScriptInterface in which to create the return data. + * @param compareFiles - compare the directory name and the FileSize of the replays and the cache. + * @return cache entries + */ +JS::HandleObject ReloadReplayCache(ScriptInterface& scriptInterface, bool compareFiles); + +/** * Get a list of replays to display in the GUI. * - * @param scriptInterface the ScriptInterface in which to create the return data. + * @param scriptInterface - the ScriptInterface in which to create the return data. + * @param compareFiles - reload the cache, which takes more time, + * but nearly ensures, that no changed replay is missed. * @return array of objects containing replay data */ -JS::Value GetReplays(ScriptInterface& scriptInterface); +JS::Value GetReplays(ScriptInterface& scriptInterface, bool compareFiles); /** * Parses a commands.txt file and extracts metadata. * Works similarly to CGame::LoadReplayData(). */ -JS::Value LoadReplayData(ScriptInterface& scriptInterface, OsPath& directory); +JS::Value LoadReplayData(ScriptInterface& scriptInterface, const OsPath& directory); /** * Permanently deletes the visual replay (including the parent directory) * - * @param replayFile path to commands.txt, whose parent directory will be deleted + * @param replayFile - path to commands.txt, whose parent directory will be deleted. * @return true if deletion was successful, false on error */ bool DeleteReplay(const CStrW& replayFile); @@ -79,10 +107,14 @@ JS::Value GetReplayMetadata(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directoryName); /** - * Saves the metadata from the session to metadata.json + * Saves the metadata from the session to metadata.json. */ void SaveReplayMetadata(ScriptInterface* scriptInterface); +/** +* Adds a replay to the replayCache. +*/ +void AddReplayToCache(ScriptInterface& scriptInterface, const CStrW& directoryName); } #endif Index: ps/trunk/source/ps/VisualReplay.cpp =================================================================== --- ps/trunk/source/ps/VisualReplay.cpp +++ ps/trunk/source/ps/VisualReplay.cpp @@ -40,6 +40,9 @@ */ const u8 minimumReplayDuration = 3; +static const OsPath tempCacheFileName = VisualReplay::GetDirectoryName() / L"replayCache_temp.json"; +static const OsPath cacheFileName = VisualReplay::GetDirectoryName() / L"replayCache.json"; + OsPath VisualReplay::GetDirectoryName() { const Paths paths(g_args); @@ -61,33 +64,171 @@ g_Game->StartVisualReplay(replayFile.string8()); } -/** - * Load all replays found in the directory. - * - * Since files are spread across the harddisk, - * loading hundreds of them can consume a lot of time. - */ -JS::Value VisualReplay::GetReplays(ScriptInterface& scriptInterface) +bool VisualReplay::ReadCacheFile(ScriptInterface& scriptInterface, JS::MutableHandleObject cachedReplaysObject) { - TIMER(L"GetReplays"); JSContext* cx = scriptInterface.GetContext(); JSAutoRequest rq(cx); - u32 i = 0; - DirectoryNames directories; + if (!FileExists(cacheFileName)) + return false; + + std::ifstream cacheStream(cacheFileName.string8().c_str()); + CStr cacheStr((std::istreambuf_iterator(cacheStream)), std::istreambuf_iterator()); + cacheStream.close(); + + JS::RootedValue cachedReplays(cx); + if (scriptInterface.ParseJSON(cacheStr, &cachedReplays)) + { + cachedReplaysObject.set(&cachedReplays.toObject()); + if (JS_IsArrayObject(cx, cachedReplaysObject)) + return true; + } + + LOGWARNING("The replay cache file is corrupted, it will be deleted"); + wunlink(cacheFileName); + return false; +} + +void VisualReplay::StoreCacheFile(ScriptInterface& scriptInterface, JS::HandleObject replays) +{ + JSContext* cx = scriptInterface.GetContext(); + JSAutoRequest rq(cx); + + JS::RootedValue replaysRooted(cx, JS::ObjectValue(*replays)); + std::ofstream cacheStream(tempCacheFileName.string8().c_str(), std::ofstream::out | std::ofstream::trunc); + cacheStream << scriptInterface.StringifyJSON(&replaysRooted); + cacheStream.close(); + + wunlink(cacheFileName); + if (wrename(tempCacheFileName, cacheFileName)) + LOGERROR("Could not store the replay cache"); +} + +JS::HandleObject VisualReplay::ReloadReplayCache(ScriptInterface& scriptInterface, bool compareFiles) +{ + TIMER(L"ReloadReplayCache"); + JSContext* cx = scriptInterface.GetContext(); + JSAutoRequest rq(cx); + + // Maps the filename onto the index and size + typedef std::map> replayCacheMap; + + replayCacheMap fileList; + + JS::RootedObject cachedReplaysObject(cx); + if (ReadCacheFile(scriptInterface, &cachedReplaysObject)) + { + // Create list of files included in the cache + u32 cacheLength = 0; + JS_GetArrayLength(cx, cachedReplaysObject, &cacheLength); + for (u32 j = 0; j < cacheLength; ++j) + { + JS::RootedValue replay(cx); + JS_GetElement(cx, cachedReplaysObject, j, &replay); + + JS::RootedValue file(cx); + CStr fileName; + double fileSize; + scriptInterface.GetProperty(replay, "directory", fileName); + scriptInterface.GetProperty(replay, "fileSize", fileSize); + + fileList[fileName] = std::make_pair(j, fileSize); + } + } + JS::RootedObject replays(cx, JS_NewArrayObject(cx, 0)); + DirectoryNames directories; + + if (GetDirectoryEntries(GetDirectoryName(), nullptr, &directories) != INFO::OK) + return replays; - if (GetDirectoryEntries(GetDirectoryName(), NULL, &directories) == INFO::OK) - for (OsPath& directory : directories) + bool newReplays = false; + std::vector copyFromOldCache; + // Specifies where the next replay should be kept + u32 i = 0; + + for (const OsPath& directory : directories) + { + if (SDL_QuitRequested()) + // We want to save our progress in searching through the replays + break; + + bool isNew = true; + replayCacheMap::iterator it = fileList.find(directory.string8()); + if (it != fileList.end()) { - if (SDL_QuitRequested()) - return JSVAL_NULL; + if (compareFiles) + { + CFileInfo fileInfo; + GetFileInfo(GetDirectoryName() / directory / L"commands.txt", &fileInfo); + if (fileInfo.Size() == it->second.second) + isNew = false; + } + else + isNew = false; + } + if (isNew) + { JS::RootedValue replayData(cx, LoadReplayData(scriptInterface, directory)); - if (!replayData.isNull()) - JS_SetElement(cx, replays, i++, replayData); + if (replayData.isNull()) + { + CFileInfo fileInfo; + GetFileInfo(GetDirectoryName() / directory / L"commands.txt", &fileInfo); + scriptInterface.Eval("({})", &replayData); + scriptInterface.SetProperty(replayData, "directory", directory); + scriptInterface.SetProperty(replayData, "fileSize", (double)fileInfo.Size()); + } + JS_SetElement(cx, replays, i++, replayData); + newReplays = true; } - return JS::ObjectValue(*replays); + else + copyFromOldCache.push_back(it->second.first); + } + + debug_printf( + "Loading %lu cached replays, removed %lu outdated entries, loaded %i new entries\n", + (unsigned long)fileList.size(), (unsigned long)(fileList.size() - copyFromOldCache.size()), i); + + if (!newReplays && fileList.empty()) + return replays; + + // No replay was changed, so just return the cache + if (!newReplays && fileList.size() == copyFromOldCache.size()) + return cachedReplaysObject; + + { + // Copy the replays from the old cache that are not deleted + if (!copyFromOldCache.empty()) + for (u32 j : copyFromOldCache) + { + JS::RootedValue replay(cx); + JS_GetElement(cx, cachedReplaysObject, j, &replay); + JS_SetElement(cx, replays, i++, replay); + } + } + StoreCacheFile(scriptInterface, replays); + return replays; +} + +JS::Value VisualReplay::GetReplays(ScriptInterface& scriptInterface, bool compareFiles) +{ + TIMER(L"GetReplays"); + JSContext* cx = scriptInterface.GetContext(); + JSAutoRequest rq(cx); + JS::RootedObject replays(cx, ReloadReplayCache(scriptInterface, compareFiles)); + // Only take entries with data + JS::RootedObject replaysWithoutNullEntries(cx, JS_NewArrayObject(cx, 0)); + u32 replaysLength = 0; + JS_GetArrayLength(cx, replays, &replaysLength); + for (u32 j = 0, i = 0; j < replaysLength; ++j) + { + JS::RootedValue replay(cx); + JS_GetElement(cx, replays, j, &replay); + if (scriptInterface.HasProperty(replay, "attribs")) + JS_SetElement(cx, replaysWithoutNullEntries, i++, replay); + } + return JS::ObjectValue(*replaysWithoutNullEntries); } /** @@ -173,7 +314,7 @@ return -1; } -JS::Value VisualReplay::LoadReplayData(ScriptInterface& scriptInterface, OsPath& directory) +JS::Value VisualReplay::LoadReplayData(ScriptInterface& scriptInterface, const OsPath& directory) { // The directory argument must not be constant, otherwise concatenating will fail const OsPath replayFile = GetDirectoryName() / directory / L"commands.txt"; @@ -250,6 +391,7 @@ scriptInterface.Eval("({})", &replayData); scriptInterface.SetProperty(replayData, "file", replayFile); scriptInterface.SetProperty(replayData, "directory", directory); + scriptInterface.SetProperty(replayData, "fileSize", (double)fileSize); scriptInterface.SetProperty(replayData, "attribs", attribs); scriptInterface.SetProperty(replayData, "duration", duration); return replayData; @@ -264,7 +406,6 @@ return DirectoryExists(directory) && DeleteDirectory(directory) == INFO::OK; } - JS::Value VisualReplay::GetReplayAttributes(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directoryName) { // Create empty JS object @@ -290,6 +431,27 @@ return attribs; } +void VisualReplay::AddReplayToCache(ScriptInterface& scriptInterface, const CStrW& directoryName) +{ + TIMER(L"AddReplayToCache"); + JSContext* cx = scriptInterface.GetContext(); + JSAutoRequest rq(cx); + + JS::RootedValue replayData(cx, LoadReplayData(scriptInterface, OsPath(directoryName))); + if (replayData.isNull()) + return; + + JS::RootedObject cachedReplaysObject(cx); + if (!ReadCacheFile(scriptInterface, &cachedReplaysObject)) + cachedReplaysObject = JS_NewArrayObject(cx, 0); + + u32 cacheLength = 0; + JS_GetArrayLength(cx, cachedReplaysObject, &cacheLength); + JS_SetElement(cx, cachedReplaysObject, cacheLength, replayData); + + StoreCacheFile(scriptInterface, cachedReplaysObject); +} + void VisualReplay::SaveReplayMetadata(ScriptInterface* scriptInterface) { JSContext* cx = scriptInterface->GetContext(); Index: ps/trunk/source/ps/scripting/JSInterface_VisualReplay.h =================================================================== --- ps/trunk/source/ps/scripting/JSInterface_VisualReplay.h +++ ps/trunk/source/ps/scripting/JSInterface_VisualReplay.h @@ -25,10 +25,11 @@ { void StartVisualReplay(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directory); bool DeleteReplay(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& replayFile); - JS::Value GetReplays(ScriptInterface::CxPrivate* pCxPrivate); + JS::Value GetReplays(ScriptInterface::CxPrivate* pCxPrivate, bool compareFiles); JS::Value GetReplayAttributes(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directoryName); bool HasReplayMetadata(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directoryName); JS::Value GetReplayMetadata(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directoryName); + void AddReplayToCache(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directoryName); void RegisterScriptFunctions(ScriptInterface& scriptInterface); CStrW GetReplayDirectoryName(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directoryName); } Index: ps/trunk/source/ps/scripting/JSInterface_VisualReplay.cpp =================================================================== --- ps/trunk/source/ps/scripting/JSInterface_VisualReplay.cpp +++ ps/trunk/source/ps/scripting/JSInterface_VisualReplay.cpp @@ -33,9 +33,9 @@ return VisualReplay::DeleteReplay(replayFile); } -JS::Value JSI_VisualReplay::GetReplays(ScriptInterface::CxPrivate* pCxPrivate) +JS::Value JSI_VisualReplay::GetReplays(ScriptInterface::CxPrivate* pCxPrivate, bool compareFiles) { - return VisualReplay::GetReplays(*(pCxPrivate->pScriptInterface)); + return VisualReplay::GetReplays(*(pCxPrivate->pScriptInterface), compareFiles); } JS::Value JSI_VisualReplay::GetReplayAttributes(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directoryName) @@ -53,6 +53,11 @@ return VisualReplay::GetReplayMetadata(pCxPrivate, directoryName); } +void JSI_VisualReplay::AddReplayToCache(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directoryName) +{ + VisualReplay::AddReplayToCache(*(pCxPrivate->pScriptInterface), directoryName); +} + CStrW JSI_VisualReplay::GetReplayDirectoryName(ScriptInterface::CxPrivate* UNUSED(pCxPrivate), const CStrW& directoryName) { return OsPath(VisualReplay::GetDirectoryName() / directoryName).string(); @@ -60,11 +65,12 @@ void JSI_VisualReplay::RegisterScriptFunctions(ScriptInterface& scriptInterface) { - scriptInterface.RegisterFunction("GetReplays"); + scriptInterface.RegisterFunction("GetReplays"); scriptInterface.RegisterFunction("DeleteReplay"); scriptInterface.RegisterFunction("StartVisualReplay"); scriptInterface.RegisterFunction("GetReplayAttributes"); scriptInterface.RegisterFunction("GetReplayMetadata"); scriptInterface.RegisterFunction("HasReplayMetadata"); + scriptInterface.RegisterFunction("AddReplayToCache"); scriptInterface.RegisterFunction("GetReplayDirectoryName"); }