Index: ps/trunk/source/main.cpp =================================================================== --- ps/trunk/source/main.cpp +++ ps/trunk/source/main.cpp @@ -468,9 +468,9 @@ PROFILE2_ATTR("%d", g_Profiler2.GetFrameNumber()); static u32 turn = 0; - debug_printf("Turn %u (%u)...\n", turn++, DEFAULT_TURN_LENGTH_SP); + debug_printf("Turn %u (%u)...\n", turn++, DEFAULT_TURN_LENGTH); - g_Game->GetSimulation2()->Update(DEFAULT_TURN_LENGTH_SP); + g_Game->GetSimulation2()->Update(DEFAULT_TURN_LENGTH); g_Profiler.Frame(); Index: ps/trunk/source/network/NetClient.cpp =================================================================== --- ps/trunk/source/network/NetClient.cpp +++ ps/trunk/source/network/NetClient.cpp @@ -40,6 +40,14 @@ #include "simulation2/Simulation2.h" #include "network/StunClient.h" +/** + * Once ping goes above turn length * command delay, + * the game will start 'freezing' for other clients while we catch up. + * Since commands are sent client -> server -> client, divide by 2. + * (duplicated in NetServer.cpp to avoid having to fetch the constants in a header file) + */ +constexpr u32 NETWORK_BAD_PING = DEFAULT_TURN_LENGTH * COMMAND_DELAY_MP / 2; + CNetClient *g_NetClient = NULL; /** @@ -334,9 +342,9 @@ return; } - // Report if we have a bad ping to the server + // Report if we have a bad ping to the server. u32 meanRTT = m_Session->GetMeanRTT(); - if (meanRTT > DEFAULT_TURN_LENGTH_MP) + if (meanRTT > NETWORK_BAD_PING) { PushGuiMessage( "type", "netwarn", @@ -857,7 +865,7 @@ // Display warnings for other clients with bad ping for (size_t i = 0; i < message->m_Clients.size(); ++i) { - if (message->m_Clients[i].m_MeanRTT < DEFAULT_TURN_LENGTH_MP || message->m_Clients[i].m_GUID == client->m_GUID) + if (message->m_Clients[i].m_MeanRTT < NETWORK_BAD_PING || message->m_Clients[i].m_GUID == client->m_GUID) continue; client->PushGuiMessage( Index: ps/trunk/source/network/NetClientTurnManager.cpp =================================================================== --- ps/trunk/source/network/NetClientTurnManager.cpp +++ ps/trunk/source/network/NetClientTurnManager.cpp @@ -36,7 +36,7 @@ extern CStrW g_UniqueLogPostfix; CNetClientTurnManager::CNetClientTurnManager(CSimulation2& simulation, CNetClient& client, int clientId, IReplayLogger& replay) - : CTurnManager(simulation, DEFAULT_TURN_LENGTH_MP, clientId, replay), m_NetClient(client) + : CTurnManager(simulation, DEFAULT_TURN_LENGTH, COMMAND_DELAY_MP, clientId, replay), m_NetClient(client) { } @@ -45,11 +45,11 @@ NETCLIENTTURN_LOG("PostCommand()\n"); // Transmit command to server - CSimulationMessage msg(m_Simulation2.GetScriptInterface(), m_ClientId, m_PlayerId, m_CurrentTurn + COMMAND_DELAY, data); + CSimulationMessage msg(m_Simulation2.GetScriptInterface(), m_ClientId, m_PlayerId, m_CurrentTurn + m_CommandDelay, data); m_NetClient.SendMessage(&msg); // Add to our local queue - //AddCommand(m_ClientId, m_PlayerId, data, m_CurrentTurn + COMMAND_DELAY); + //AddCommand(m_ClientId, m_PlayerId, data, m_CurrentTurn + m_CommandDelay); // TODO: we should do this when the server stops sending our commands back to us } @@ -98,7 +98,7 @@ // Attempt to flush messages before leaving. // Notice the sending is not reliable and rarely makes it to the Server. if (m_NetClient.GetCurrState() == NCS_INGAME) - NotifyFinishedOwnCommands(m_CurrentTurn + COMMAND_DELAY); + NotifyFinishedOwnCommands(m_CurrentTurn + m_CommandDelay); } void CNetClientTurnManager::OnSimulationMessage(CSimulationMessage* msg) Index: ps/trunk/source/network/NetServer.cpp =================================================================== --- ps/trunk/source/network/NetServer.cpp +++ ps/trunk/source/network/NetServer.cpp @@ -67,6 +67,14 @@ */ static const int HOST_SERVICE_TIMEOUT = 50; +/** + * Once ping goes above turn length * command delay, + * the game will start 'freezing' for other clients while we catch up. + * Since commands are sent client -> server -> client, divide by 2. + * (duplicated in NetServer.cpp to avoid having to fetch the constants in a header file) + */ +constexpr u32 NETWORK_BAD_PING = DEFAULT_TURN_LENGTH * COMMAND_DELAY_MP / 2; + CNetServer* g_NetServer = NULL; static CStr DebugName(CNetServerSession* session) @@ -604,7 +612,7 @@ message = msg; } // Report if the client has bad ping - else if (meanRTT > DEFAULT_TURN_LENGTH_MP) + else if (meanRTT > NETWORK_BAD_PING) { CClientPerformanceMessage* msg = new CClientPerformanceMessage(); CClientPerformanceMessage::S_m_Clients client; Index: ps/trunk/source/network/NetServerTurnManager.cpp =================================================================== --- ps/trunk/source/network/NetServerTurnManager.cpp +++ ps/trunk/source/network/NetServerTurnManager.cpp @@ -27,19 +27,21 @@ #include "simulation2/system/TurnManager.h" #if 0 +#include "ps/Util.h" #define NETSERVERTURN_LOG(...) debug_printf(__VA_ARGS__) #else #define NETSERVERTURN_LOG(...) #endif CNetServerTurnManager::CNetServerTurnManager(CNetServerWorker& server) - : m_NetServer(server), m_ReadyTurn(1), m_TurnLength(DEFAULT_TURN_LENGTH_MP), m_HasSyncError(false) + : m_NetServer(server), m_ReadyTurn(COMMAND_DELAY_MP - 1), m_TurnLength(DEFAULT_TURN_LENGTH), m_HasSyncError(false) { // Turn 0 is not actually executed, store a dummy value. m_SavedTurnLengths.push_back(0); - // Turn 1 is special: all clients run it without waiting on a server command batch. - // Because of this, it is always run with the default MP turn length. - m_SavedTurnLengths.push_back(m_TurnLength); + // Turns [1..COMMAND_DELAY - 1] are special: all clients run them without waiting on a server command batch. + // Because of this, they are always run with the default MP turn length. + for (u32 i = 1; i < COMMAND_DELAY_MP; ++i) + m_SavedTurnLengths.push_back(m_TurnLength); } void CNetServerTurnManager::NotifyFinishedClientCommands(CNetServerSession& session, u32 turn) @@ -139,7 +141,7 @@ std::vector OOSPlayerNames; for (const std::pair& hashPair : clientStateHash.second) { - NETSERVERTURN_LOG("sync check %d: %d = %hs\n", it->first, cit->first, Hexify(cit->second).c_str()); + NETSERVERTURN_LOG("sync check %d: %d = %hs\n", clientStateHash.first, hashPair.first, Hexify(hashPair.second).c_str()); if (hashPair.second != expected) { // Oh no, out of sync @@ -174,7 +176,7 @@ NETSERVERTURN_LOG("InitialiseClient(client=%d, turn=%d)\n", client, turn); ENSURE(m_ClientsReady.find(client) == m_ClientsReady.end()); - m_ClientsReady[client] = turn + 1; + m_ClientsReady[client] = turn + COMMAND_DELAY_MP - 1; m_ClientsSimulated[client] = turn; } Index: ps/trunk/source/rlinterface/RLInterface.cpp =================================================================== --- ps/trunk/source/rlinterface/RLInterface.cpp +++ ps/trunk/source/rlinterface/RLInterface.cpp @@ -389,7 +389,7 @@ turnMgr->PostCommand(command.playerID, commandJSON); } - const u32 deltaRealTime = DEFAULT_TURN_LENGTH_SP; + const u32 deltaRealTime = DEFAULT_TURN_LENGTH; if (nonVisual) { const double deltaSimTime = deltaRealTime * g_Game->GetSimRate(); Index: ps/trunk/source/simulation2/system/LocalTurnManager.cpp =================================================================== --- ps/trunk/source/simulation2/system/LocalTurnManager.cpp +++ ps/trunk/source/simulation2/system/LocalTurnManager.cpp @@ -20,20 +20,18 @@ #include "LocalTurnManager.h" CLocalTurnManager::CLocalTurnManager(CSimulation2& simulation, IReplayLogger& replay) - : CTurnManager(simulation, DEFAULT_TURN_LENGTH_SP, 0, replay) + : CTurnManager(simulation, DEFAULT_TURN_LENGTH, COMMAND_DELAY_SP, 0, replay) { } void CLocalTurnManager::PostCommand(player_id_t playerid, JS::HandleValue data) { - AddCommand(m_ClientId, playerid, data, m_CurrentTurn + 1); + AddCommand(m_ClientId, playerid, data, m_CurrentTurn + m_CommandDelay); } void CLocalTurnManager::PostCommand(JS::HandleValue data) { - // Add directly to the next turn, ignoring COMMAND_DELAY, - // because we don't need to compensate for network latency - AddCommand(m_ClientId, m_PlayerId, data, m_CurrentTurn + 1); + AddCommand(m_ClientId, m_PlayerId, data, m_CurrentTurn + m_CommandDelay); } void CLocalTurnManager::NotifyFinishedOwnCommands(u32 turn) Index: ps/trunk/source/simulation2/system/TurnManager.h =================================================================== --- ps/trunk/source/simulation2/system/TurnManager.h +++ ps/trunk/source/simulation2/system/TurnManager.h @@ -30,11 +30,6 @@ class CSimulation2; class IReplayLogger; -extern const u32 DEFAULT_TURN_LENGTH_SP; -extern const u32 DEFAULT_TURN_LENGTH_MP; - -extern const int COMMAND_DELAY; - /** * This file defines the base class of the turn managers for clients, local games and replays. * The basic idea of our turn managing system across a network is as in this article: @@ -54,6 +49,31 @@ */ /** + * Default turn length in SP & MP. + * This value should be as low as possible, while not introducing un-necessary lag. + */ +inline constexpr u32 DEFAULT_TURN_LENGTH = 200; + +/** + * In single-player, commands are directly scheduled for the next turn. + */ +inline constexpr u32 COMMAND_DELAY_SP = 1; + +/** + * In multi-player, clients can only compute turn N if all clients have finished sending commands for it, + * i.e. N < CurrentTurn + COMMAND_DELAY for all clients. + * Commands are sent from client to server to client, and both client and network can lag. + * If a client reaches turn CURRENT_TURN + COMMAND_DELAY - 1, it'll freeze while waiting for commands. + * To avoid that, we increase the command-delay to make sure that in general players will have received all commands + * by the time they reach a given turn. Keep in mind the minimum delay is one turn. + * This value should be as low as possible while avoiding 'freezing' in general usage. + * TODO: + * - this command-delay could vary based on server-client pings + * - it ought be possible to send commands in a P2P fashion (with server verification), which would lower the ping. + */ +inline constexpr u32 COMMAND_DELAY_MP = 4; + +/** * Common turn system (used by clients and offline games). */ class CTurnManager @@ -63,7 +83,7 @@ /** * Construct for a given network session ID. */ - CTurnManager(CSimulation2& simulation, u32 defaultTurnLength, int clientId, IReplayLogger& replay); + CTurnManager(CSimulation2& simulation, u32 defaultTurnLength, u32 commandDelay, int clientId, IReplayLogger& replay); virtual ~CTurnManager() { } @@ -164,6 +184,9 @@ /// The turn that we have most recently executed u32 m_CurrentTurn; + // Current command delay (commands are scheduled for m_CurrentTurn + m_CommandDelay) + u32 m_CommandDelay; + /// The latest turn for which we have received all commands from all clients u32 m_ReadyTurn; Index: ps/trunk/source/simulation2/system/TurnManager.cpp =================================================================== --- ps/trunk/source/simulation2/system/TurnManager.cpp +++ ps/trunk/source/simulation2/system/TurnManager.cpp @@ -30,32 +30,36 @@ #include "scriptinterface/ScriptInterface.h" #include "simulation2/Simulation2.h" -const u32 DEFAULT_TURN_LENGTH_MP = 500; -const u32 DEFAULT_TURN_LENGTH_SP = 200; - -const int COMMAND_DELAY = 2; - #if 0 #define NETTURN_LOG(...) debug_printf(__VA_ARGS__) #else #define NETTURN_LOG(...) #endif +/** + * Maximum number of turns between two clients. + * When we are on turn n, we schedule new commands for n+COMMAND_DELAY. + * We know that all other clients have finished scheduling commands for n, + * else we couldn't have got here, which means they're at least on turn n-COMMAND_DELAY+1. + * We know we have not yet finished scheduling commands for n+COMMAND_DELAY, so no client can be there. + * Hence other clients can be on turns [n-COMMAND_DELAY+1, ..., n+COMMAND_DELAY-1], and no other, + * hence any two clients can only be this many turns apart. + */ +constexpr int MaxClientTurnDelta(int commandDelay) +{ + return 2 * (commandDelay - 1); +} + const CStr CTurnManager::EventNameSavegameLoaded = "SavegameLoaded"; -CTurnManager::CTurnManager(CSimulation2& simulation, u32 defaultTurnLength, int clientId, IReplayLogger& replay) - : m_Simulation2(simulation), m_CurrentTurn(0), m_ReadyTurn(1), m_TurnLength(defaultTurnLength), +CTurnManager::CTurnManager(CSimulation2& simulation, u32 defaultTurnLength, u32 commandDelay, int clientId, IReplayLogger& replay) + : m_Simulation2(simulation), m_CurrentTurn(0), m_CommandDelay(commandDelay), m_ReadyTurn(commandDelay - 1), m_TurnLength(defaultTurnLength), m_PlayerId(-1), m_ClientId(clientId), m_DeltaSimTime(0), m_HasSyncError(false), m_Replay(replay), m_FinalTurn(std::numeric_limits::max()), m_TimeWarpNumTurns(0), m_QuickSaveMetadata(m_Simulation2.GetScriptInterface().GetGeneralJSContext()) { - // When we are on turn n, we schedule new commands for n+2. - // We know that all other clients have finished scheduling commands for n (else we couldn't have got here). - // We know we have not yet finished scheduling commands for n+2. - // Hence other clients can be on turn n-1, n, n+1, and no other. - // So they can be sending us commands scheduled for n+1, n+2, n+3. - // So we need a 3-element buffer: - m_QueuedCommands.resize(COMMAND_DELAY + 1); + // Lag between any two clients is bounded. Add 1 for inclusive bounds. + m_QueuedCommands.resize(MaxClientTurnDelta(m_CommandDelay) + 1); } void CTurnManager::ResetState(u32 newCurrentTurn, u32 newReadyTurn) @@ -108,7 +112,7 @@ NETTURN_LOG("Update current=%d ready=%d\n", m_CurrentTurn, m_ReadyTurn); // Check that the next turn is ready for execution - if (m_ReadyTurn <= m_CurrentTurn) + if (m_ReadyTurn <= m_CurrentTurn && m_CommandDelay > 1) { // Oops, we wanted to start the next turn but it's not ready yet - // there must be too much network lag. @@ -132,10 +136,10 @@ break; // Check that the i'th next turn is still ready - if (m_ReadyTurn <= m_CurrentTurn) + if (m_ReadyTurn <= m_CurrentTurn && m_CommandDelay > 1) break; - NotifyFinishedOwnCommands(m_CurrentTurn + COMMAND_DELAY); + NotifyFinishedOwnCommands(m_CurrentTurn + m_CommandDelay); // Increase now, so Update can send new commands for a subsequent turn ++m_CurrentTurn; @@ -231,9 +235,10 @@ void CTurnManager::AddCommand(int client, int player, JS::HandleValue data, u32 turn) { - NETTURN_LOG("AddCommand(client=%d player=%d turn=%d)\n", client, player, turn); + NETTURN_LOG("AddCommand(client=%d player=%d turn=%d current=%d, ready=%d)\n", client, player, turn, m_CurrentTurn, m_ReadyTurn); - if (!(m_CurrentTurn < turn && turn <= m_CurrentTurn + COMMAND_DELAY + 1)) + // Reject commands for turns that we should not be able to compute (in the past or too far future). + if (m_CurrentTurn >= turn || turn > m_CurrentTurn + MaxClientTurnDelta(m_CommandDelay) + 1) { debug_warn(L"Received command for invalid turn"); return;