diff --git a/Assets/Mirror/Runtime/NetworkConnection.cs b/Assets/Mirror/Runtime/NetworkConnection.cs
index 2f4ad10ef..958d7aa94 100644
--- a/Assets/Mirror/Runtime/NetworkConnection.cs
+++ b/Assets/Mirror/Runtime/NetworkConnection.cs
@@ -9,11 +9,17 @@ public abstract class NetworkConnection
     {
         public const int LocalConnectionId = 0;
 
-        // NetworkIdentities that this connection can see
-        // sorted by distance to the main player object,
-        // which is necessary for priority based WorldState sync
+        // NetworkIdentities that this connection can see.
+        // IMPORTANT: priority WorldState needs them sorted, but we can NOT
+        //            store them sorted for two reasons:
+        //            - they are only sorted during insertion, but
+        //              entities still move around after
+        //            - SortedSet uses comparer as key so entities with the same
+        //              distance would be dropped
+        // => keep this unsorted. sort when needed.
         // TODO move to server's NetworkConnectionToClient?
-        internal readonly SortedSet<NetworkIdentity> observing;
+        internal readonly HashSet<NetworkIdentity> observing =
+            new HashSet<NetworkIdentity>();
 
         // TODO this is NetworkServer.handlers on server and NetworkClient.handlers on client.
         //      maybe use them directly. avoid extra state.
@@ -54,9 +60,6 @@ public abstract class NetworkConnection
 
         internal NetworkConnection()
         {
-            // initialize 'observing' with the custom comparer
-            observing = new SortedSet<NetworkIdentity>(new ObservingComparer(this));
-
             // set lastTime to current time when creating connection to make
             // sure it isn't instantly kicked for inactivity
             lastMessageTime = Time.time;
diff --git a/Assets/Mirror/Runtime/NetworkServer.cs b/Assets/Mirror/Runtime/NetworkServer.cs
index 27a96f0fe..0233b4595 100644
--- a/Assets/Mirror/Runtime/NetworkServer.cs
+++ b/Assets/Mirror/Runtime/NetworkServer.cs
@@ -1467,6 +1467,15 @@ internal static void NetworkLateUpdate()
                         PartialWorldStateMessage world = new PartialWorldStateMessage();
                         using (PooledNetworkWriter writer = NetworkWriterPool.GetWriter())
                         {
+                            // we want to fill WorldState by priority.
+                            // we can't store .observing as sorted because that
+                            // guarantee would only exist during insertion since
+                            // entities still move around after.
+                            // => so let's sorted here
+                            // TODO nonalloc
+                            List<NetworkIdentity> sorted = connection.observing.ToList();
+                            sorted.Sort(new ObservingComparer(connection));
+
                             // for each entity that this connection is seeing,
                             // sorted by priority (see NetworkConnection.observing)
                             foreach (NetworkIdentity identity in connection.observing)