diff --git a/Assets/Mirror/Core/LagCompensation/HistoryBounds.cs b/Assets/Mirror/Core/LagCompensation/HistoryBounds.cs
index e7e6aec79..4800d9503 100644
--- a/Assets/Mirror/Core/LagCompensation/HistoryBounds.cs
+++ b/Assets/Mirror/Core/LagCompensation/HistoryBounds.cs
@@ -23,28 +23,36 @@ public class HistoryBounds
         public HistoryBounds(int limit)
         {
             // initialize queue with maximum capacity to avoid runtime resizing
+            // +1 because it makes the code easier if we insert first, and then remove.
             this.limit = limit;
-            history = new Queue<Bounds>(limit);
+            history = new Queue<Bounds>(limit + 1);
         }
 
         // insert new bounds into history. calculates new total bounds.
         // Queue.Dequeue() always has the oldest bounds.
         public void Insert(Bounds bounds)
         {
-            // remove oldest if limit reached
-            if (history.Count >= limit)
+            // initialize 'total' if not initialized yet.
+            // we don't want to call (0,0).Encapsulate(bounds).
+            if (history.Count == 0)
+                total = bounds;
+
+            // insert and encapsulate the new bounds
+            history.Enqueue(bounds);
+            total.Encapsulate(bounds);
+
+            // ensure history stays within limit
+            if (history.Count > limit)
+            {
+                // remove oldest
                 history.Dequeue();
 
-            // insert the new bounds
-            history.Enqueue(bounds);
-
-            // summarize total bounds.
-            // starting at latest bounds, not at 'new Bounds' because that would
-            // encapsulate (0,0) too.
-            // TODO make this not be O(N)
-            total = bounds;
-            foreach (Bounds b in history)
-                total.Encapsulate(b);
+                // recalculate total bounds
+                // (only needed after removing the oldest)
+                total = bounds;
+                foreach (Bounds b in history)
+                    total.Encapsulate(b);
+            }
         }
 
         public void Reset()
diff --git a/Assets/Mirror/Tests/Editor/LagCompensation/HistoryBoundsTests.cs b/Assets/Mirror/Tests/Editor/LagCompensation/HistoryBoundsTests.cs
index 62c5fb706..94cfbae67 100644
--- a/Assets/Mirror/Tests/Editor/LagCompensation/HistoryBoundsTests.cs
+++ b/Assets/Mirror/Tests/Editor/LagCompensation/HistoryBoundsTests.cs
@@ -22,8 +22,8 @@ public static Bounds MinMax(float min, float max) =>
         // simple benchmark to compare some optimizations later.
         // 64 entries are much more than we would usually use.
         //
-        // Unity 2021.3 LTS, release mode, 10_000 x 64 x 8:
-        //   native O(N) Queue<Bounds> implementation: 1005 ms
+        // Unity 2021.3 LTS, release mode: 10x000 x 65; limit=8
+        //   native O(N) Queue<Bounds> implementation: 1045 ms
         [Test]
         [TestCase(10_000, 64, 8)]
         public void Benchmark(int iterations, int insertions, int limit)
@@ -158,7 +158,9 @@ public void InsertFar()
         [Test]
         public void Reset()
         {
-            HistoryBounds history = new HistoryBounds(3);
+            const int limit = 3;
+            HistoryBounds history = new HistoryBounds(limit);
+
             history.Insert(MinMax(1, 2));
             history.Insert(MinMax(2, 3));
             history.Insert(MinMax(3, 4));