Index: ps/trunk/source/main.cpp =================================================================== --- ps/trunk/source/main.cpp +++ ps/trunk/source/main.cpp @@ -74,7 +74,7 @@ #include "graphics/TextureManager.h" #include "gui/GUIManager.h" #include "renderer/Renderer.h" -#include "rlinterface/RLInterface.cpp" +#include "rlinterface/RLInterface.h" #include "scriptinterface/ScriptEngine.h" #include "simulation2/Simulation2.h" #include "simulation2/system/TurnManager.h" @@ -492,57 +492,10 @@ if (!args.Get("rl-interface").empty()) server_address = args.Get("rl-interface"); - g_RLInterface = new RLInterface(); - g_RLInterface->EnableHTTP(server_address.c_str()); + g_RLInterface = std::make_unique(server_address.c_str()); debug_printf("RL interface listening on %s\n", server_address.c_str()); } -static void RunRLServer(const bool isNonVisual, const std::vector modsToInstall, const CmdLineArgs args) -{ - int flags = INIT_MODS; - while (!Init(args, flags)) - { - flags &= ~INIT_MODS; - Shutdown(SHUTDOWN_FROM_CONFIG); - } - g_Shutdown = ShutdownType::None; - - std::vector installedMods; - if (!modsToInstall.empty()) - { - Paths paths(args); - CModInstaller installer(paths.UserData() / "mods", paths.Cache()); - - // Install the mods without deleting the pyromod files - for (const OsPath& modPath : modsToInstall) - installer.Install(modPath, g_ScriptContext, true); - - installedMods = installer.GetInstalledMods(); - } - - if (isNonVisual) - { - InitNonVisual(args); - StartRLInterface(args); - while (g_Shutdown == ShutdownType::None) - g_RLInterface->TryApplyMessage(); - QuitEngine(); - } - else - { - InitGraphics(args, 0, installedMods); - MainControllerInit(); - StartRLInterface(args); - while (g_Shutdown == ShutdownType::None) - Frame(); - } - - Shutdown(0); - MainControllerShutdown(); - CXeromyces::Terminate(); - delete g_RLInterface; -} - // moved into a helper function to ensure args is destroyed before // exit(), which may result in a memory leak. static void RunGameOrAtlas(int argc, const char* argv[]) @@ -569,6 +522,7 @@ const bool isVisualReplay = args.Has("replay-visual"); const bool isNonVisualReplay = args.Has("replay"); const bool isNonVisual = args.Has("autostart-nonvisual"); + const bool isUsingRLInterface = args.Has("rl-interface"); const OsPath replayFile( isVisualReplay ? args.Get("replay-visual") : @@ -681,12 +635,6 @@ const double res = timer_Resolution(); g_frequencyFilter = CreateFrequencyFilter(res, 30.0); - if (args.Has("rl-interface")) - { - RunRLServer(isNonVisual, modsToInstall, args); - return; - } - // run the game int flags = INIT_MODS; do @@ -716,13 +664,23 @@ if (isNonVisual) { InitNonVisual(args); + if (isUsingRLInterface) + StartRLInterface(args); + while (g_Shutdown == ShutdownType::None) - NonVisualFrame(); + { + if (isUsingRLInterface) + g_RLInterface->TryApplyMessage(); + else + NonVisualFrame(); + } } else { InitGraphics(args, 0, installedMods); MainControllerInit(); + if (isUsingRLInterface) + StartRLInterface(args); while (g_Shutdown == ShutdownType::None) Frame(); } Index: ps/trunk/source/ps/GameSetup/GameSetup.h =================================================================== --- ps/trunk/source/ps/GameSetup/GameSetup.h +++ ps/trunk/source/ps/GameSetup/GameSetup.h @@ -35,6 +35,8 @@ **/ extern void EarlyInit(); +extern void EndGame(); + enum InitFlags { // avoid setting a video mode / initializing OpenGL; assume that has Index: ps/trunk/source/ps/scripting/JSInterface_Game.cpp =================================================================== --- ps/trunk/source/ps/scripting/JSInterface_Game.cpp +++ ps/trunk/source/ps/scripting/JSInterface_Game.cpp @@ -26,13 +26,12 @@ #include "ps/Game.h" #include "ps/Replay.h" #include "ps/World.h" +#include "ps/GameSetup/GameSetup.h" #include "scriptinterface/ScriptInterface.h" #include "simulation2/system/TurnManager.h" #include "simulation2/Simulation2.h" #include "soundmanager/SoundManager.h" -extern void EndGame(); - bool JSI_Game::IsGameStarted(ScriptInterface::CmptPrivate* UNUSED(pCmptPrivate)) { return g_Game; Index: ps/trunk/source/rlinterface/RLInterface.h =================================================================== --- ps/trunk/source/rlinterface/RLInterface.h +++ ps/trunk/source/rlinterface/RLInterface.h @@ -14,64 +14,145 @@ * You should have received a copy of the GNU General Public License * along with 0 A.D. If not, see . */ + #ifndef INCLUDED_RLINTERFACE #define INCLUDED_RLINTERFACE #include "simulation2/helpers/Player.h" +#include "third_party/mongoose/mongoose.h" #include #include -#include #include -struct ScenarioConfig { +namespace RL +{ +struct ScenarioConfig +{ bool saveReplay; player_id_t playerID; std::string content; }; -struct Command { + +struct GameCommand +{ int playerID; std::string json_cmd; }; -enum GameMessageType { Reset, Commands }; -struct GameMessage { - GameMessageType type; - std::vector commands; +enum class GameMessageType +{ + None, + Reset, + Commands, }; -extern void EndGame(); - -struct mg_context; -const static std::string EMPTY_STATE; - -class RLInterface +/** + * Holds messages from the RL client to the game. + */ +struct GameMessage { + GameMessageType type; + std::vector commands; +}; - public: - - std::string Step(const std::vector commands); - std::string Reset(const ScenarioConfig* scenario); - std::vector GetTemplates(const std::vector names) const; - - void EnableHTTP(const char* server_address); - std::string SendGameMessage(const GameMessage msg); - bool TryGetGameMessage(GameMessage& msg); - void TryApplyMessage(); - std::string GetGameState(); - bool IsGameRunning(); - - private: - mg_context* m_MgContext = nullptr; - const GameMessage* m_GameMessage = nullptr; - std::string m_GameState; - bool m_NeedsGameState = false; - mutable std::mutex m_lock; - std::mutex m_msgLock; - std::condition_variable m_msgApplied; - ScenarioConfig m_ScenarioConfig; +/** + * Implements an interface providing fundamental capabilities required for reinforcement + * learning (over HTTP). + * + * This consists of enabling an external script to configure the scenario (via Reset) and + * then step the game engine manually and apply player actions (via Step). The interface + * also supports querying unit templates to provide information about max health and other + * potentially relevant game state information. + * + * See source/tools/rlclient/ for the external client code. + * + * The HTTP server is threaded. + * Flow of data (with the interface active): + * 0. The game/main thread calls TryApplyMessage() + * - If no messages are pending, GOTO 0 (the simulation is not advanced). + * 1. TryApplyMessage locks m_MsgLock, pulls the message, processes it, advances the simulation, and sets m_GameState. + * 2. TryApplyMessage notifies the RL thread that it can carry on and unlocks m_MsgLock. The main thread carries on frame rendering and goes back to 0. + * 3. The RL thread locks m_MsgLock, reads m_GameState, unlocks m_MsgLock, and sends the gamestate as HTTP Response to the RL client. + * 4. The client processes the response and ultimately sends a new HTTP message to the RL Interface. + * 5. The RL thread locks m_MsgLock, pushes the message, and starts waiting on the game/main thread to notify it (step 2). + * - GOTO 0. + */ +class Interface +{ + NONCOPYABLE(Interface); +public: + Interface(const char* server_address); + + /** + * Non-blocking call to process any pending messages from the RL client. + * Updates m_GameState to the gamestate after messages have been processed. + */ + void TryApplyMessage(); + +private: + static void* MgCallback(mg_event event, struct mg_connection *conn, const struct mg_request_info *request_info); + + /** + * Process commands, update the simulation by one turn. + * @return the gamestate after processing commands. + */ + std::string Step(std::vector&& commands); + + /** + * Reset the game state according to scenario, cleaning up existing games if required. + * @return the gamestate after resetting. + */ + std::string Reset(ScenarioConfig&& scenario); + + /** + * @return template data for all templates of @param names. + */ + std::vector GetTemplates(const std::vector& names) const; + + /** + * @return true if a game is currently running. + */ + bool IsGameRunning() const; + + /** + * Internal helper. Move @param msg into m_GameMessage, wait until it has been processed by the main thread, + * and @return the gamestate after that message is processed. + * It is invalid to call this if m_GameMessage is not currently empty. + */ + std::string SendGameMessage(GameMessage&& msg); + + /** + * Internal helper. + * @return true if m_GameMessage is not empty, and updates @param msg, false otherwise (msg is then unchanged). + */ + bool TryGetGameMessage(GameMessage& msg); + + /** + * Process any pending messages from the RL client. + * Updates m_GameState to the gamestate after messages have been processed. + */ + void ApplyMessage(const GameMessage& msg); + + /** + * @return the full gamestate as a JSON strong. + * This uses the AI representation since it is readily available in the JS Engine. + */ + std::string GetGameState() const; + +private: + GameMessage m_GameMessage; + ScenarioConfig m_ScenarioConfig; + std::string m_GameState; + bool m_NeedsGameState = false; + + mutable std::mutex m_Lock; + std::mutex m_MsgLock; + std::condition_variable m_MsgApplied; }; -extern RLInterface* g_RLInterface; +} + +extern std::unique_ptr g_RLInterface; #endif // INCLUDED_RLINTERFACE Index: ps/trunk/source/rlinterface/RLInterface.cpp =================================================================== --- ps/trunk/source/rlinterface/RLInterface.cpp +++ ps/trunk/source/rlinterface/RLInterface.cpp @@ -22,49 +22,65 @@ #include "rlinterface/RLInterface.h" #include "gui/GUIManager.h" +#include "ps/CLogger.h" #include "ps/Game.h" #include "ps/Loader.h" -#include "ps/CLogger.h" +#include "ps/GameSetup/GameSetup.h" +#include "simulation2/Simulation2.h" #include "simulation2/components/ICmpAIInterface.h" #include "simulation2/components/ICmpTemplateManager.h" -#include "simulation2/Simulation2.h" #include "simulation2/system/LocalTurnManager.h" -#include "third_party/mongoose/mongoose.h" +#include #include +#include // Globally accessible pointer to the RL Interface. -RLInterface* g_RLInterface = nullptr; +std::unique_ptr g_RLInterface; + +namespace RL +{ +Interface::Interface(const char* server_address) : m_GameMessage({GameMessageType::None}) +{ + LOGMESSAGERENDER("Starting RL interface HTTP server"); + + const char *options[] = { + "listening_ports", server_address, + "num_threads", "1", + nullptr + }; + mg_context* mgContext = mg_start(MgCallback, this, options); + ENSURE(mgContext); +} // Interactions with the game engine (g_Game) must be done in the main -// thread as there are specific checks for this. We will pass our commands -// to the main thread to be applied -std::string RLInterface::SendGameMessage(const GameMessage msg) -{ - std::unique_lock msgLock(m_msgLock); - m_GameMessage = &msg; - m_msgApplied.wait(msgLock); +// thread as there are specific checks for this. We will pass messages +// to the main thread to be applied (ie, "GameMessage"s). +std::string Interface::SendGameMessage(GameMessage&& msg) +{ + std::unique_lock msgLock(m_MsgLock); + ENSURE(m_GameMessage.type == GameMessageType::None); + m_GameMessage = std::move(msg); + m_MsgApplied.wait(msgLock, [this]() { return m_GameMessage.type == GameMessageType::None; }); return m_GameState; } -std::string RLInterface::Step(const std::vector commands) +std::string Interface::Step(std::vector&& commands) { - std::lock_guard lock(m_lock); - GameMessage msg = { GameMessageType::Commands, commands }; - return SendGameMessage(msg); + std::lock_guard lock(m_Lock); + return SendGameMessage({ GameMessageType::Commands, std::move(commands) }); } -std::string RLInterface::Reset(const ScenarioConfig* scenario) +std::string Interface::Reset(ScenarioConfig&& scenario) { - std::lock_guard lock(m_lock); - m_ScenarioConfig = *scenario; - struct GameMessage msg = { GameMessageType::Reset }; - return SendGameMessage(msg); + std::lock_guard lock(m_Lock); + m_ScenarioConfig = std::move(scenario); + return SendGameMessage({ GameMessageType::Reset }); } -std::vector RLInterface::GetTemplates(const std::vector names) const +std::vector Interface::GetTemplates(const std::vector& names) const { - std::lock_guard lock(m_lock); + std::lock_guard lock(m_Lock); CSimulation2& simulation = *g_Game->GetSimulation2(); CmpPtr cmpTemplateManager(simulation.GetSimContext().GetSystemEntity()); @@ -74,18 +90,15 @@ const CParamNode* node = cmpTemplateManager->GetTemplate(templateName); if (node != nullptr) - { - std::string content = utf8_from_wstring(node->ToXML()); - templates.push_back(content); - } + templates.push_back(utf8_from_wstring(node->ToXML())); } return templates; } -static void* RLMgCallback(mg_event event, struct mg_connection *conn, const struct mg_request_info *request_info) +void* Interface::MgCallback(mg_event event, struct mg_connection *conn, const struct mg_request_info *request_info) { - RLInterface* interface = (RLInterface*)request_info->user_data; + Interface* interface = (Interface*)request_info->user_data; ENSURE(interface); void* handled = (void*)""; // arbitrary non-NULL pointer to indicate successful handling @@ -116,7 +129,7 @@ { std::stringstream stream; - std::string uri = request_info->uri; + const std::string uri = request_info->uri; if (uri == "/reset") { @@ -127,22 +140,22 @@ return handled; } ScenarioConfig scenario; - std::string qs(request_info->query_string); + const std::string qs(request_info->query_string); scenario.saveReplay = qs.find("saveReplay") != std::string::npos; scenario.playerID = 1; char playerID[1]; - int len = mg_get_var(request_info->query_string, qs.length(), "playerID", playerID, 1); + const int len = mg_get_var(request_info->query_string, qs.length(), "playerID", playerID, 1); if (len != -1) scenario.playerID = std::stoi(playerID); - int bufSize = std::atoi(val); - std::unique_ptr buf = std::unique_ptr(new char[bufSize]); + const int bufSize = std::atoi(val); + std::unique_ptr buf = std::unique_ptr(new char[bufSize]); mg_read(conn, buf.get(), bufSize); - std::string content(buf.get(), bufSize); + const std::string content(buf.get(), bufSize); scenario.content = content; - std::string gameState = interface->Reset(&scenario); + const std::string gameState = interface->Reset(std::move(scenario)); stream << gameState.c_str(); } @@ -161,16 +174,16 @@ return handled; } int bufSize = std::atoi(val); - std::unique_ptr buf = std::unique_ptr(new char[bufSize]); + std::unique_ptr buf = std::unique_ptr(new char[bufSize]); mg_read(conn, buf.get(), bufSize); - std::string postData(buf.get(), bufSize); + const std::string postData(buf.get(), bufSize); std::stringstream postStream(postData); std::string line; - std::vector commands; + std::vector commands; while (std::getline(postStream, line, '\n')) { - Command cmd; + GameCommand cmd; const std::size_t splitPos = line.find(";"); if (splitPos != std::string::npos) { @@ -179,7 +192,7 @@ commands.push_back(cmd); } } - std::string gameState = interface->Step(commands); + const std::string gameState = interface->Step(std::move(commands)); if (gameState.empty()) { mg_printf(conn, "%s", notRunningResponse); @@ -200,10 +213,10 @@ mg_printf(conn, "%s", noPostData); return handled; } - int bufSize = std::atoi(val); - std::unique_ptr buf = std::unique_ptr(new char[bufSize]); + const int bufSize = std::atoi(val); + std::unique_ptr buf = std::unique_ptr(new char[bufSize]); mg_read(conn, buf.get(), bufSize); - std::string postData(buf.get(), bufSize); + const std::string postData(buf.get(), bufSize); std::stringstream postStream(postData); std::string line; std::vector templateNames; @@ -221,7 +234,7 @@ } mg_printf(conn, "%s", header200); - std::string str = stream.str(); + const std::string str = stream.str(); mg_write(conn, str.c_str(), str.length()); return handled; } @@ -243,146 +256,137 @@ } }; -void RLInterface::EnableHTTP(const char* server_address) -{ - LOGMESSAGERENDER("Starting RL interface HTTP server"); - - // Ignore multiple enablings - if (m_MgContext) - return; - - const char *options[] = { - "listening_ports", server_address, - "num_threads", "6", // enough for the browser's parallel connection limit - nullptr - }; - m_MgContext = mg_start(RLMgCallback, this, options); - ENSURE(m_MgContext); -} - -bool RLInterface::TryGetGameMessage(GameMessage& msg) +bool Interface::TryGetGameMessage(GameMessage& msg) { - if (m_GameMessage != nullptr) { - msg = *m_GameMessage; - m_GameMessage = nullptr; + if (m_GameMessage.type != GameMessageType::None) + { + msg = m_GameMessage; + m_GameMessage = {GameMessageType::None}; return true; } return false; } -void RLInterface::TryApplyMessage() +void Interface::TryApplyMessage() { - const bool nonVisual = !g_GUI; const bool isGameStarted = g_Game && g_Game->IsGameStarted(); if (m_NeedsGameState && isGameStarted) { m_GameState = GetGameState(); - m_msgApplied.notify_one(); - m_msgLock.unlock(); + m_MsgApplied.notify_one(); + m_MsgLock.unlock(); m_NeedsGameState = false; } - if (m_msgLock.try_lock()) + if (!m_MsgLock.try_lock()) + return; + + GameMessage msg; + if (!TryGetGameMessage(msg)) { - GameMessage msg; - if (TryGetGameMessage(msg)) { - switch (msg.type) + m_MsgLock.unlock(); + return; + } + + ApplyMessage(msg); +} + +void Interface::ApplyMessage(const GameMessage& msg) +{ + const static std::string EMPTY_STATE; + const bool nonVisual = !g_GUI; + const bool isGameStarted = g_Game && g_Game->IsGameStarted(); + switch (msg.type) + { + case GameMessageType::Reset: + { + if (isGameStarted) + EndGame(); + + g_Game = new CGame(m_ScenarioConfig.saveReplay); + ScriptInterface& scriptInterface = g_Game->GetSimulation2()->GetScriptInterface(); + ScriptRequest rq(scriptInterface); + JS::RootedValue attrs(rq.cx); + scriptInterface.ParseJSON(m_ScenarioConfig.content, &attrs); + + g_Game->SetPlayerID(m_ScenarioConfig.playerID); + g_Game->StartGame(&attrs, ""); + + if (nonVisual) { - case GameMessageType::Reset: - { - if (isGameStarted) - EndGame(); + LDR_NonprogressiveLoad(); + ENSURE(g_Game->ReallyStartGame() == PSRETURN_OK); + m_GameState = GetGameState(); + m_MsgApplied.notify_one(); + m_MsgLock.unlock(); + } + else + { + JS::RootedValue initData(rq.cx); + scriptInterface.CreateObject(rq, &initData); + scriptInterface.SetProperty(initData, "attribs", attrs); + + JS::RootedValue playerAssignments(rq.cx); + scriptInterface.CreateObject(rq, &playerAssignments); + scriptInterface.SetProperty(initData, "playerAssignments", playerAssignments); - g_Game = new CGame(m_ScenarioConfig.saveReplay); - ScriptInterface& scriptInterface = g_Game->GetSimulation2()->GetScriptInterface(); - ScriptRequest rq(scriptInterface); - - JS::RootedValue attrs(rq.cx); - scriptInterface.ParseJSON(m_ScenarioConfig.content, &attrs); - - g_Game->SetPlayerID(m_ScenarioConfig.playerID); - g_Game->StartGame(&attrs, ""); - - if (nonVisual) - { - LDR_NonprogressiveLoad(); - ENSURE(g_Game->ReallyStartGame() == PSRETURN_OK); - m_GameState = GetGameState(); - m_msgApplied.notify_one(); - m_msgLock.unlock(); - } - else - { - JS::RootedValue initData(rq.cx); - scriptInterface.CreateObject(rq, &initData); - scriptInterface.SetProperty(initData, "attribs", attrs); - - JS::RootedValue playerAssignments(rq.cx); - scriptInterface.CreateObject(rq, &playerAssignments); - scriptInterface.SetProperty(initData, "playerAssignments", playerAssignments); - - g_GUI->SwitchPage(L"page_loading.xml", &scriptInterface, initData); - m_NeedsGameState = true; - } - break; - } + g_GUI->SwitchPage(L"page_loading.xml", &scriptInterface, initData); + m_NeedsGameState = true; + } + break; + } - case GameMessageType::Commands: - { - if (!g_Game) - { - m_GameState = EMPTY_STATE; - m_msgApplied.notify_one(); - m_msgLock.unlock(); - return; - } - - CLocalTurnManager* turnMgr = static_cast(g_Game->GetTurnManager()); - - const ScriptInterface& scriptInterface = g_Game->GetSimulation2()->GetScriptInterface(); - ScriptRequest rq(scriptInterface); - for (Command command : msg.commands) - { - JS::RootedValue commandJSON(rq.cx); - scriptInterface.ParseJSON(command.json_cmd, &commandJSON); - turnMgr->PostCommand(command.playerID, commandJSON); - } - - const double deltaRealTime = DEFAULT_TURN_LENGTH_SP; - if (nonVisual) - { - const double deltaSimTime = deltaRealTime * g_Game->GetSimRate(); - size_t maxTurns = static_cast(g_Game->GetSimRate()); - g_Game->GetTurnManager()->Update(deltaSimTime, maxTurns); - } - else - g_Game->Update(deltaRealTime); - - m_GameState = GetGameState(); - m_msgApplied.notify_one(); - m_msgLock.unlock(); - break; - } + case GameMessageType::Commands: + { + if (!g_Game) + { + m_GameState = EMPTY_STATE; + m_MsgApplied.notify_one(); + m_MsgLock.unlock(); + return; } + const ScriptInterface& scriptInterface = g_Game->GetSimulation2()->GetScriptInterface(); + CLocalTurnManager* turnMgr = static_cast(g_Game->GetTurnManager()); + + for (const GameCommand& command : msg.commands) + { + ScriptRequest rq(scriptInterface); + JS::RootedValue commandJSON(rq.cx); + scriptInterface.ParseJSON(command.json_cmd, &commandJSON); + turnMgr->PostCommand(command.playerID, commandJSON); + } + + const u32 deltaRealTime = DEFAULT_TURN_LENGTH_SP; + if (nonVisual) + { + const double deltaSimTime = deltaRealTime * g_Game->GetSimRate(); + const size_t maxTurns = static_cast(g_Game->GetSimRate()); + g_Game->GetTurnManager()->Update(deltaSimTime, maxTurns); + } + else + g_Game->Update(deltaRealTime); + + m_GameState = GetGameState(); + m_MsgApplied.notify_one(); + m_MsgLock.unlock(); + break; } - else - m_msgLock.unlock(); } } -std::string RLInterface::GetGameState() +std::string Interface::GetGameState() const { const ScriptInterface& scriptInterface = g_Game->GetSimulation2()->GetScriptInterface(); - ScriptRequest rq(scriptInterface); - const CSimContext simContext = g_Game->GetSimulation2()->GetSimContext(); CmpPtr cmpAIInterface(simContext.GetSystemEntity()); + ScriptRequest rq(scriptInterface); JS::RootedValue state(rq.cx); cmpAIInterface->GetFullRepresentation(&state, true); return scriptInterface.StringifyJSON(&state, false); } -bool RLInterface::IsGameRunning() +bool Interface::IsGameRunning() const { - return !!g_Game; + return g_Game != nullptr; +} } Index: ps/trunk/source/scriptinterface/ScriptEngine.h =================================================================== --- ps/trunk/source/scriptinterface/ScriptEngine.h +++ ps/trunk/source/scriptinterface/ScriptEngine.h @@ -23,6 +23,8 @@ #include "js/Initialization.h" +#include + /** * A class using the RAII (Resource Acquisition Is Initialization) idiom to manage initialization * and shutdown of the SpiderMonkey script engine. It also keeps a count of active script contexts