feature: NetworkTime.predictedTime to prepare for Prediction (#3599)

* feature: NetworkTime.predictedTime to prepare for prediction

* disable log

* NetworkTime.predictedTime simplified: client timeline based on Time.time to fix first 5s being way ahead, history being too old, etc.

Assets/Mirror/Core/Messages.cs

@@ -105,11 +105,17 @@ public struct EntityStateMessage : NetworkMessage
     // whoever wants to measure rtt, sends this to the other end.
     public struct NetworkPingMessage : NetworkMessage
     {
+        // local time is used to calculate round trip time,
+        // and to calculate the predicted time offset.
         public double localTime;
 
-        public NetworkPingMessage(double value)
+        // predicted time is sent to compare the final error, for debugging only
+        public double predictedTimeAdjusted;
+
+        public NetworkPingMessage(double localTime, double predictedTimeAdjusted)
         {
-            localTime = value;
+            this.localTime = localTime;
+            this.predictedTimeAdjusted = predictedTimeAdjusted;
         }
     }
 
@@ -117,6 +123,18 @@ public NetworkPingMessage(double value)
     // we can use this to calculate rtt.
     public struct NetworkPongMessage : NetworkMessage
     {
+        // local time is used to calculate round trip time.
         public double localTime;
+
+        // predicted error is used to adjust the predicted timeline.
+        public double predictionErrorUnadjusted;
+        public double predictionErrorAdjusted; // for debug purposes
+
+        public NetworkPongMessage(double localTime, double predictionErrorUnadjusted, double predictionErrorAdjusted)
+        {
+            this.localTime = localTime;
+            this.predictionErrorUnadjusted = predictionErrorUnadjusted;
+            this.predictionErrorAdjusted = predictionErrorAdjusted;
+        }
     }
 }

Assets/Mirror/Core/NetworkClient.cs

@@ -1702,7 +1702,7 @@ public static void OnGUI()
             // only if in world
             if (!ready) return;
 
-            GUILayout.BeginArea(new Rect(10, 5, 800, 50));
+            GUILayout.BeginArea(new Rect(10, 5, 1000, 50));
 
             GUILayout.BeginHorizontal("Box");
             GUILayout.Label("Snapshot Interp.:");
@@ -1717,6 +1717,8 @@ public static void OnGUI()
             GUILayout.Box($"timescale: {localTimescale:F2}");
             GUILayout.Box($"BTM: {NetworkClient.bufferTimeMultiplier:F2}"); // current dynamically adjusted multiplier
             GUILayout.Box($"RTT: {NetworkTime.rtt * 1000:F0}ms");
+            GUILayout.Box($"PredErrUNADJ: {NetworkTime.predictionErrorUnadjusted * 1000:F0}ms");
+            GUILayout.Box($"PredErrADJ: {NetworkTime.predictionErrorAdjusted * 1000:F0}ms");
             GUILayout.EndHorizontal();
 
             GUILayout.EndArea();

Assets/Mirror/Core/NetworkConnectionToClient.cs

@@ -132,7 +132,8 @@ protected virtual void UpdatePing()
                 // TODO it would be safer for the server to store the last N
                 // messages' timestamp and only send a message number.
                 // This way client's can't just modify the timestamp.
-                NetworkPingMessage pingMessage = new NetworkPingMessage(NetworkTime.localTime);
+                // predictedTime parameter is 0 because the server doesn't predict.
+                NetworkPingMessage pingMessage = new NetworkPingMessage(NetworkTime.localTime, 0);
                 Send(pingMessage, Channels.Unreliable);
                 lastPingTime = NetworkTime.localTime;
             }

Assets/Mirror/Core/NetworkTime.cs

@@ -16,8 +16,11 @@ namespace Mirror
     /// <summary>Synchronizes server time to clients.</summary>
     public static class NetworkTime
     {
-        /// <summary>Ping message interval, used to calculate network time and RTT</summary>
+        /// <summary>Ping message interval, used to calculate latency / RTT and predicted time.</summary>
-        public static float PingInterval = 2;
+        // 2s was enough to get a good average RTT.
+        // for prediction, we want to react to latency changes more rapidly.
+        const float DefaultPingInterval = 0.1f; // for resets
+        public static float PingInterval = DefaultPingInterval;
 
         // DEPRECATED 2023-07-06
         [Obsolete("NetworkTime.PingFrequency was renamed to PingInterval, because we use it as seconds, not as Hz. Please rename all usages, but keep using it just as before.")]
@@ -28,7 +31,8 @@ public static float PingFrequency
         }
 
         /// <summary>Average out the last few results from Ping</summary>
-        public static int PingWindowSize = 6;
+        // const because it's used immediately in _rtt constructor.
+        public const int PingWindowSize = 50; // average over 50 * 100ms = 5s
 
         static double lastPingTime;
 
@@ -73,6 +77,45 @@ public static double time
                 : NetworkClient.localTimeline;
         }
 
+        // prediction //////////////////////////////////////////////////////////
+        // NetworkTime.time is server time, behind by bufferTime.
+        // for prediction, we want server time, ahead by latency.
+        // so that client inputs at predictedTime=2 arrive on server at time=2.
+        // the more accurate this is, the more closesly will corrections be
+        // be applied and the less jitter we will see.
+        //
+        // we'll use a two step process to calculate predicted time:
+        // 1. move snapshot interpolated time to server time, without being behind by bufferTime
+        // 2. constantly send this time to server (included in ping message)
+        //    server replies with how far off it was.
+        //    client averages that offset and applies it to predictedTime to get ever closer.
+        //
+        // this is also very easy to test & verify:
+        // - add LatencySimulation with 50ms latency
+        // - log predictionError on server in OnServerPing, see if it gets closer to 0
+        //
+        // credits: FakeByte, imer, NinjaKickja, mischa
+        // const because it's used immediately in _predictionError constructor.
+
+        static int PredictionErrorWindowSize = 20; // average over 20 * 100ms = 2s
+        static ExponentialMovingAverage _predictionErrorUnadjusted = new ExponentialMovingAverage(PredictionErrorWindowSize);
+        public static double predictionErrorUnadjusted => _predictionErrorUnadjusted.Value;
+        public static double predictionErrorAdjusted { get; private set; } // for debugging
+
+        /// <summary>Predicted timeline in order for client inputs to be timestamped with the exact time when they will most likely arrive on the server. This is the basis for all prediction like PredictedRigidbody.</summary>
+        // on client, this is based on localTime (aka Time.time) instead of the snapshot interpolated timeline.
+        // this gives much better and immediately accurate results.
+        // -> snapshot interpolation timeline tries to emulate a server timeline without hard offset corrections.
+        // -> predictedTime does have hard offset corrections, so might as well use Time.time directly for this.
+        public static double predictedTime
+        {
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get => NetworkServer.active
+                ? localTime // server always uses it's own timeline
+                : localTime + predictionErrorUnadjusted; // add the offset that the server told us we are off by
+        }
+        ////////////////////////////////////////////////////////////////////////
+
         /// <summary>Clock difference in seconds between the client and the server. Always 0 on server.</summary>
         // original implementation used 'client - server' time. keep it this way.
         // TODO obsolete later. people shouldn't worry about this.
@@ -89,8 +132,7 @@ public static double time
         [RuntimeInitializeOnLoadMethod]
         public static void ResetStatics()
         {
-            PingInterval = 2;
+            PingInterval = DefaultPingInterval;
-            PingWindowSize = 6;
             lastPingTime = 0;
             _rtt = new ExponentialMovingAverage(PingWindowSize);
 #if !UNITY_2020_3_OR_NEWER
@@ -103,7 +145,13 @@ internal static void UpdateClient()
             // localTime (double) instead of Time.time for accuracy over days
             if (localTime >= lastPingTime + PingInterval)
             {
-                NetworkPingMessage pingMessage = new NetworkPingMessage(localTime);
+                // send raw predicted time without the offset applied yet.
+                // we then apply the offset to it after.
+                NetworkPingMessage pingMessage = new NetworkPingMessage
+                (
+                    localTime,
+                    predictedTime
+                );
                 NetworkClient.Send(pingMessage, Channels.Unreliable);
                 lastPingTime = localTime;
             }
@@ -115,17 +163,28 @@ internal static void UpdateClient()
         // and time from the server
         internal static void OnServerPing(NetworkConnectionToClient conn, NetworkPingMessage message)
         {
+            // calculate the prediction offset that the client needs to apply to unadjusted time to reach server time.
+            // this will be sent back to client for corrections.
+            double unadjustedError = localTime - message.localTime;
+
+            // to see how well the client's final prediction worked, compare with adjusted time.
+            // this is purely for debugging.
+            double adjustedError = localTime - message.predictedTimeAdjusted;
+            // Debug.Log($"[Server] unadjustedError:{(unadjustedError*1000):F1}ms adjustedError:{(adjustedError*1000):F1}ms");
+
             // Debug.Log($"OnServerPing conn:{conn}");
             NetworkPongMessage pongMessage = new NetworkPongMessage
-            {
+            (
-                localTime = message.localTime,
+                message.localTime,
-            };
+                unadjustedError,
+                adjustedError
+            );
             conn.Send(pongMessage, Channels.Unreliable);
         }
 
         // Executed at the client when we receive a Pong message
         // find out how long it took since we sent the Ping
-        // and update time offset
+        // and update time offset & prediction offset.
         internal static void OnClientPong(NetworkPongMessage message)
         {
             // prevent attackers from sending timestamps which are in the future
@@ -134,6 +193,12 @@ internal static void OnClientPong(NetworkPongMessage message)
             // how long did this message take to come back
             double newRtt = localTime - message.localTime;
             _rtt.Add(newRtt);
+
+            // feed unadjusted prediction error into our exponential moving average
+            // store adjusted prediction error for debug / GUI purposes
+            _predictionErrorUnadjusted.Add(message.predictionErrorUnadjusted);
+            predictionErrorAdjusted = message.predictionErrorAdjusted;
+            // Debug.Log($"[Client] predictionError avg={(_predictionErrorUnadjusted.Value*1000):F1} ms");
         }
 
         // server rtt calculation //////////////////////////////////////////////
@@ -144,9 +209,10 @@ internal static void OnClientPing(NetworkPingMessage message)
         {
             // Debug.Log($"OnClientPing conn:{conn}");
             NetworkPongMessage pongMessage = new NetworkPongMessage
-            {
+            (
-                localTime = message.localTime,
+                message.localTime,
-            };
+                0, 0 // server doesn't predict
+            );
             NetworkClient.Send(pongMessage, Channels.Unreliable);
         }