perf: kcp2k V1.11 - where-allocation - 25x reduction in Socket.SendTo/ReceiveFrom allocations (#2759)

Assets/Mirror/Runtime/Transport/KCP/kcp2k/VERSION

@@ -1,3 +1,30 @@
+V1.11 [2021-06-01]
+- perf: where-allocation (https://github.com/vis2k/where-allocation):
+  nearly removes all socket.SendTo/ReceiveFrom allocations
+- perf: Segment MemoryStream initial capacity set to MTU to avoid early runtime
+  resizing/allocations
+
+V1.10 [2021-05-28]
+- feature: configurable Timeout
+- allocations explained with comments (C# ReceiveFrom / IPEndPoint.GetHashCode)
+- fix: #17 KcpConnection.ReceiveNextReliable now assigns message default so it
+  works in .net too
+- fix: Segment pool is not static anymore. Each kcp instance now has it's own
+  Pool<Segment>. fixes #18 concurrency issues
+
+V1.9 [2021-03-02]
+- Tick() split into TickIncoming()/TickOutgoing() to use in Mirror's new update
+  functions. allows to minimize latency.
+  => original Tick() is still supported for convenience. simply processes both!
+
+V1.8 [2021-02-14]
+- fix: Unity IPv6 errors on Nintendo Switch
+- fix: KcpConnection now disconnects if data message was received without content.
+  previously it would call OnData with an empty ArraySegment, causing all kinds of
+  weird behaviour in Mirror/DOTSNET. Added tests too.
+- fix: KcpConnection.SendData: don't allow sending empty messages anymore. disconnect
+  and log a warning to make it completely obvious.
+
 V1.7 [2021-01-13]
 - fix: unreliable messages reset timeout now too
 - perf: KcpConnection OnCheckEnabled callback changed to a simple 'paused' boolean.

Assets/Mirror/Runtime/Transport/KCP/kcp2k/highlevel/KcpClientConnection.cs

@@ -1,5 +1,6 @@
 using System.Net;
 using System.Net.Sockets;
+using WhereAllocation;
 
 namespace kcp2k
 {
@@ -12,6 +13,9 @@ public class KcpClientConnection : KcpConnection
         //            => we need the MTU to fit channel + message!
         readonly byte[] rawReceiveBuffer = new byte[Kcp.MTU_DEF];
 
+        // where-allocation
+        IPEndPointNonAlloc reusableEP;
+
         public void Connect(string host, ushort port, bool noDelay, uint interval = Kcp.INTERVAL, int fastResend = 0, bool congestionWindow = true, uint sendWindowSize = Kcp.WND_SND, uint receiveWindowSize = Kcp.WND_RCV, int timeout = DEFAULT_TIMEOUT)
         {
             Log.Info($"KcpClient: connect to {host}:{port}");
@@ -21,6 +25,11 @@ public void Connect(string host, ushort port, bool noDelay, uint interval = Kcp.
 
             remoteEndpoint = new IPEndPoint(ipAddress[0], port);
             socket = new Socket(remoteEndpoint.AddressFamily, SocketType.Dgram, ProtocolType.Udp);
+
+            // create reusableEP with same address family as remoteEndPoint.
+            // otherwise ReceiveFrom_NonAlloc couldn't use it.
+            reusableEP = new IPEndPointNonAlloc(ipAddress[0], port);
+
             socket.Connect(remoteEndpoint);
             SetupKcp(noDelay, interval, fastResend, congestionWindow, sendWindowSize, receiveWindowSize, timeout);
 
@@ -39,7 +48,8 @@ public void RawReceive()
                 {
                     while (socket.Poll(0, SelectMode.SelectRead))
                     {
-                        int msgLength = socket.ReceiveFrom(rawReceiveBuffer, ref remoteEndpoint);
+                        // where-allocation: receive nonalloc.
+                        int msgLength = socket.ReceiveFrom_NonAlloc(rawReceiveBuffer, reusableEP);
                         // IMPORTANT: detect if buffer was too small for the
                         //            received msgLength. otherwise the excess
                         //            data would be silently lost.

Assets/Mirror/Runtime/Transport/KCP/kcp2k/highlevel/KcpServer.cs

@@ -4,6 +4,7 @@
 using System.Collections.Generic;
 using System.Net;
 using System.Net.Sockets;
+using WhereAllocation;
 
 namespace kcp2k
 {
@@ -43,9 +44,11 @@ public class KcpServer
         Socket socket;
 #if UNITY_SWITCH
         // switch does not support ipv6
-        EndPoint newClientEP = new IPEndPoint(IPAddress.Any, 0);
+        //EndPoint newClientEP = new IPEndPoint(IPAddress.Any, 0);
+        IPEndPointNonAlloc reusableClientEP = new IPEndPointNonAlloc(IPAddress.Any, 0); // where-allocation
 #else
-        EndPoint newClientEP = new IPEndPoint(IPAddress.IPv6Any, 0);
+        //EndPoint newClientEP = new IPEndPoint(IPAddress.IPv6Any, 0);
+        IPEndPointNonAlloc reusableClientEP = new IPEndPointNonAlloc(IPAddress.IPv6Any, 0); // where-allocation
 #endif
         // IMPORTANT: raw receive buffer always needs to be of 'MTU' size, even
         //            if MaxMessageSize is larger. kcp always sends in MTU
@@ -140,9 +143,13 @@ public void TickIncoming()
                     //   receive from calls newClientEP.Create(socketAddr).
                     //   IPEndPoint.Create always returns a new IPEndPoint.
                     //   https://github.com/mono/mono/blob/f74eed4b09790a0929889ad7fc2cf96c9b6e3757/mcs/class/System/System.Net.Sockets/Socket.cs#L1761
-                    int msgLength = socket.ReceiveFrom(rawReceiveBuffer, 0, rawReceiveBuffer.Length, SocketFlags.None, ref newClientEP);
+                    //int msgLength = socket.ReceiveFrom(rawReceiveBuffer, 0, rawReceiveBuffer.Length, SocketFlags.None, ref newClientEP);
                     //Log.Info($"KCP: server raw recv {msgLength} bytes = {BitConverter.ToString(buffer, 0, msgLength)}");
 
+                    // where-allocation nonalloc ReceiveFrom.
+                    int msgLength = socket.ReceiveFrom_NonAlloc(rawReceiveBuffer, 0, rawReceiveBuffer.Length, SocketFlags.None, reusableClientEP);
+                    SocketAddress remoteAddress = reusableClientEP.temp;
+
                     // calculate connectionId from endpoint
                     // NOTE: IPEndPoint.GetHashCode() allocates.
                     //  it calls m_Address.GetHashCode().
@@ -152,7 +159,10 @@ public void TickIncoming()
                     //
                     // => using only newClientEP.Port wouldn't work, because
                     //    different connections can have the same port.
-                    int connectionId = newClientEP.GetHashCode();
+                    //int connectionId = newClientEP.GetHashCode();
+
+                    // where-allocation nonalloc GetHashCode
+                    int connectionId = remoteAddress.GetHashCode();
 
                     // IMPORTANT: detect if buffer was too small for the received
                     //            msgLength. otherwise the excess data would be
@@ -163,8 +173,19 @@ public void TickIncoming()
                         // is this a new connection?
                         if (!connections.TryGetValue(connectionId, out KcpServerConnection connection))
                         {
+                            // IPEndPointNonAlloc is reused all the time.
+                            // we can't store that as the connection's endpoint.
+                            // we need a new copy!
+                            IPEndPoint newClientEP = reusableClientEP.DeepCopyIPEndPoint();
+
+                            // for allocation free sending, we also need another
+                            // IPEndPointNonAlloc...
+                            IPEndPointNonAlloc reusableSendEP = new IPEndPointNonAlloc(newClientEP.Address, newClientEP.Port);
+
                             // create a new KcpConnection
-                            connection = new KcpServerConnection(socket, newClientEP, NoDelay, Interval, FastResend, CongestionWindow, SendWindowSize, ReceiveWindowSize, Timeout);
+                            // -> where-allocation IPEndPointNonAlloc is reused.
+                            //    need to create a new one from the temp address.
+                            connection = new KcpServerConnection(socket, newClientEP, reusableSendEP, NoDelay, Interval, FastResend, CongestionWindow, SendWindowSize, ReceiveWindowSize, Timeout);
 
                             // DO NOT add to connections yet. only if the first message
                             // is actually the kcp handshake. otherwise it's either:

Assets/Mirror/Runtime/Transport/KCP/kcp2k/highlevel/KcpServerConnection.cs

@@ -1,20 +1,25 @@
 using System.Net;
 using System.Net.Sockets;
+using WhereAllocation;
 
 namespace kcp2k
 {
     public class KcpServerConnection : KcpConnection
     {
-        public KcpServerConnection(Socket socket, EndPoint remoteEndpoint, bool noDelay, uint interval = Kcp.INTERVAL, int fastResend = 0, bool congestionWindow = true, uint sendWindowSize = Kcp.WND_SND, uint receiveWindowSize = Kcp.WND_RCV, int timeout = DEFAULT_TIMEOUT)
+        IPEndPointNonAlloc reusableSendEndPoint;
+
+        public KcpServerConnection(Socket socket, EndPoint remoteEndPoint, IPEndPointNonAlloc reusableSendEndPoint, bool noDelay, uint interval = Kcp.INTERVAL, int fastResend = 0, bool congestionWindow = true, uint sendWindowSize = Kcp.WND_SND, uint receiveWindowSize = Kcp.WND_RCV, int timeout = DEFAULT_TIMEOUT)
         {
             this.socket = socket;
-            this.remoteEndpoint = remoteEndpoint;
+            this.remoteEndpoint = remoteEndPoint;
+            this.reusableSendEndPoint = reusableSendEndPoint;
             SetupKcp(noDelay, interval, fastResend, congestionWindow, sendWindowSize, receiveWindowSize, timeout);
         }
 
         protected override void RawSend(byte[] data, int length)
         {
-            socket.SendTo(data, 0, length, SocketFlags.None, remoteEndpoint);
+            // where-allocation nonalloc
+            socket.SendTo_NonAlloc(data, 0, length, SocketFlags.None, reusableSendEndPoint);
         }
     }
 }

Assets/Mirror/Runtime/Transport/KCP/kcp2k/kcp/Kcp.cs

@@ -747,10 +747,15 @@ void FlushBuffer()
 
             // calculate window size
             uint cwnd_ = Math.Min(snd_wnd, rmt_wnd);
+            // if congestion window:
             if (!nocwnd) cwnd_ = Math.Min(cwnd, cwnd_);
 
             // move data from snd_queue to snd_buf
             // sliding window, controlled by snd_nxt && sna_una+cwnd
+            //
+            // ELI5: 'snd_nxt' is what we want to send.
+            //       'snd_una' is what hasn't been acked yet.
+            //       copy up to 'cwnd_' difference between them (sliding window)
             while (Utils.TimeDiff(snd_nxt, snd_una + cwnd_) < 0)
             {
                 if (snd_queue.Count == 0) break;

Assets/Mirror/Runtime/Transport/KCP/kcp2k/kcp/Segment.cs

@@ -16,10 +16,12 @@ internal class Segment
         internal int rto;
         internal uint fastack;
         internal uint xmit;
-        // we need a auto scaling byte[] with a WriteBytes function.
+
+        // we need an auto scaling byte[] with a WriteBytes function.
         // MemoryStream does that perfectly, no need to reinvent the wheel.
         // note: no need to pool it, because Segment is already pooled.
-        internal MemoryStream data = new MemoryStream();
+        // -> MTU as initial capacity to avoid most runtime resizing/allocations
+        internal MemoryStream data = new MemoryStream(Kcp.MTU_DEF);
 
         // ikcp_encode_seg
         // encode a segment into buffer

Assets/Mirror/Runtime/Transport/KCP/kcp2k/kcp2k.asmdef

@@ -1,12 +1,15 @@
 {
     "name": "kcp2k",
-    "references": [],
+    "references": [
-    "optionalUnityReferences": [],
+        "GUID:63c380d6dae6946209ed0832388a657c"
+    ],
     "includePlatforms": [],
     "excludePlatforms": [],
     "allowUnsafeCode": true,
     "overrideReferences": false,
     "precompiledReferences": [],
     "autoReferenced": true,
-    "defineConstraints": []
+    "defineConstraints": [],
+    "versionDefines": [],
+    "noEngineReferences": false
 }

Assets/Mirror/Runtime/Transport/KCP/kcp2k/where-allocation.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: e9de45e025f26411bbb52d1aefc8d5a5
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Mirror/Runtime/Transport/KCP/kcp2k/where-allocation/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Mirror Networking (vis2k, FakeByte)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

Assets/Mirror/Runtime/Transport/KCP/kcp2k/where-allocation/LICENSE.meta

@@ -0,0 +1,7 @@
+fileFormatVersion: 2
+guid: a857d4e863bbf4a7dba70bc2cd1b5949
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Mirror/Runtime/Transport/KCP/kcp2k/where-allocation/Scripts.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 6b7f3f8e8fa16475bbe48a8e9fbe800b
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Mirror/Runtime/Transport/KCP/kcp2k/where-allocation/Scripts/AssemblyInfo.cs

@@ -0,0 +1,3 @@
+using System.Runtime.CompilerServices;
+
+[assembly: InternalsVisibleTo("where-allocations.Tests")]

Assets/Mirror/Runtime/Transport/KCP/kcp2k/where-allocation/Scripts/AssemblyInfo.cs.meta

@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: 158a96a7489b450485a8b06a13328871
+timeCreated: 1622356221

Assets/Mirror/Runtime/Transport/KCP/kcp2k/where-allocation/Scripts/Extensions.cs

@@ -0,0 +1,58 @@
+using System.Net;
+using System.Net.Sockets;
+
+namespace WhereAllocation
+{
+    public static class Extensions
+    {
+        // always pass the same IPEndPointNonAlloc instead of allocating a new
+        // one each time.
+        //
+        // use IPEndPointNonAlloc.temp to get the latest SocketAdddress written
+        // by ReceiveFrom_Internal!
+        //
+        // IMPORTANT: .temp will be overwritten in next call!
+        //            hash or manually copy it if you need to store it, e.g.
+        //            when adding a new connection.
+        public static int ReceiveFrom_NonAlloc(
+            this Socket socket,
+            byte[] buffer,
+            int offset,
+            int size,
+            SocketFlags socketFlags,
+            IPEndPointNonAlloc remoteEndPoint)
+        {
+            // call ReceiveFrom with IPEndPointNonAlloc.
+            // need to wrap this in ReceiveFrom_NonAlloc because it's not
+            // obvious that IPEndPointNonAlloc.Create does NOT create a new
+            // IPEndPoint. it saves the result in IPEndPointNonAlloc.temp!
+            EndPoint casted = remoteEndPoint;
+            return  socket.ReceiveFrom(buffer, offset, size, socketFlags, ref casted);
+        }
+
+        // same as above, different parameters
+        public static int ReceiveFrom_NonAlloc(this Socket socket, byte[] buffer, IPEndPointNonAlloc remoteEndPoint)
+        {
+            EndPoint casted = remoteEndPoint;
+            return socket.ReceiveFrom(buffer, ref casted);
+        }
+
+        // SendTo allocates too:
+        // https://github.com/mono/mono/blob/f74eed4b09790a0929889ad7fc2cf96c9b6e3757/mcs/class/System/System.Net.Sockets/Socket.cs#L2240
+        // -> the allocation is in EndPoint.Serialize()
+        // NOTE: technically this function isn't necessary.
+        //       could just pass IPEndPointNonAlloc.
+        //       still good for strong typing.
+        public static int SendTo_NonAlloc(
+            this Socket socket,
+            byte[] buffer,
+            int offset,
+            int size,
+            SocketFlags socketFlags,
+            IPEndPointNonAlloc remoteEndPoint)
+        {
+            EndPoint casted = remoteEndPoint;
+            return socket.SendTo(buffer, offset, size, socketFlags, casted);
+        }
+    }
+}

Assets/Mirror/Runtime/Transport/KCP/kcp2k/where-allocation/Scripts/Extensions.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 9e801942544d44d65808fb250623fe25
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Mirror/Runtime/Transport/KCP/kcp2k/where-allocation/Scripts/IPEndPointNonAlloc.cs

@@ -0,0 +1,208 @@
+using System;
+using System.Net;
+using System.Net.Sockets;
+
+namespace WhereAllocation
+{
+    public class IPEndPointNonAlloc : IPEndPoint
+    {
+        // Two steps to remove allocations in ReceiveFrom_Internal:
+        //
+        // 1.) remoteEndPoint.Serialize():
+        //     https://github.com/mono/mono/blob/f74eed4b09790a0929889ad7fc2cf96c9b6e3757/mcs/class/System/System.Net.Sockets/Socket.cs#L1733
+        //     -> creates an EndPoint for ReceiveFrom_Internal to write into
+        //     -> it's never read from:
+        //        ReceiveFrom_Internal passes it to native:
+        //          https://github.com/mono/mono/blob/f74eed4b09790a0929889ad7fc2cf96c9b6e3757/mcs/class/System/System.Net.Sockets/Socket.cs#L1885
+        //        native recv populates 'sockaddr* from' with the remote address:
+        //          https://docs.microsoft.com/en-us/windows/win32/api/winsock/nf-winsock-recvfrom
+        //     -> can NOT be null. bricks both Unity and Unity Hub otherwise.
+        //     -> it seems as if Serialize() is only called to avoid allocating
+        //        a 'new SocketAddress' in ReceiveFrom. it's up to the EndPoint.
+        //
+        // 2.) EndPoint.Create(SocketAddress):
+        //     https://github.com/mono/mono/blob/f74eed4b09790a0929889ad7fc2cf96c9b6e3757/mcs/class/System/System.Net.Sockets/Socket.cs#L1761
+        //     -> SocketAddress is the remote's address that we want to return
+        //     -> to avoid 'new EndPoint(SocketAddress), it seems up to the user
+        //        to decide how to create a new EndPoint via .Create
+        //     -> SocketAddress is the object that was returned by Serialize()
+        //
+        // in other words, all we need is an extra SocketAddress field that we
+        // can pass to ReceiveFrom_Internal to write the result into.
+        // => callers can then get the result from the extra field!
+        // => no allocations
+        //
+        // IMPORTANT: remember that IPEndPointNonAlloc is always the same object
+        //            and never changes. only the helper field is changed.
+        public SocketAddress temp;
+
+        // constructors simply create the field once by calling the base method.
+        // (our overwritten method would create anything new)
+        public IPEndPointNonAlloc(long address, int port) : base(address, port)
+        {
+            temp = base.Serialize();
+        }
+        public IPEndPointNonAlloc(IPAddress address, int port) : base(address, port)
+        {
+            temp = base.Serialize();
+        }
+
+        // Serialize simply returns it
+        public override SocketAddress Serialize() => temp;
+
+        // Create doesn't need to create anything.
+        // SocketAddress object is already the one we returned in Serialize().
+        // ReceiveFrom_Internal simply wrote into it.
+        public override EndPoint Create(SocketAddress socketAddress)
+        {
+            // original IPEndPoint.Create validates:
+            if (socketAddress.Family != AddressFamily)
+                throw new ArgumentException($"Unsupported socketAddress.AddressFamily: {socketAddress.Family}. Expected: {AddressFamily}");
+            if (socketAddress.Size < 8)
+                throw new ArgumentException($"Unsupported socketAddress.Size: {socketAddress.Size}. Expected: <8");
+
+            // double check to guarantee that ReceiveFrom actually did write
+            // into our 'temp' field. just in case that's ever changed.
+            if (socketAddress != temp)
+            {
+                // well this is fun.
+                // in the latest mono from the above github links,
+                // the result of Serialize() is passed as 'ref' so ReceiveFrom
+                // does in fact write into it.
+                //
+                // in Unity 2019 LTS's mono version, it does create a new one
+                // each time. this is from ILSpy Receive_From:
+                //
+                //     SocketPal.CheckDualModeReceiveSupport(this);
+                //     ValidateBlockingMode();
+                //     if (NetEventSource.IsEnabled)
+                //     {
+                //         NetEventSource.Info(this, $"SRC{LocalEndPoint} size:{size} remoteEP:{remoteEP}", "ReceiveFrom");
+                //     }
+                //     EndPoint remoteEP2 = remoteEP;
+                //     System.Net.Internals.SocketAddress socketAddress = SnapshotAndSerialize(ref remoteEP2);
+                //     System.Net.Internals.SocketAddress socketAddress2 = IPEndPointExtensions.Serialize(remoteEP2);
+                //     int bytesTransferred;
+                //     SocketError socketError = SocketPal.ReceiveFrom(_handle, buffer, offset, size, socketFlags, socketAddress.Buffer, ref socketAddress.InternalSize, out bytesTransferred);
+                //     SocketException ex = null;
+                //     if (socketError != 0)
+                //     {
+                //         ex = new SocketException((int)socketError);
+                //         UpdateStatusAfterSocketError(ex);
+                //         if (NetEventSource.IsEnabled)
+                //         {
+                //             NetEventSource.Error(this, ex, "ReceiveFrom");
+                //         }
+                //         if (ex.SocketErrorCode != SocketError.MessageSize)
+                //         {
+                //             throw ex;
+                //         }
+                //     }
+                //     if (!socketAddress2.Equals(socketAddress))
+                //     {
+                //         try
+                //         {
+                //             remoteEP = remoteEP2.Create(socketAddress);
+                //         }
+                //         catch
+                //         {
+                //         }
+                //         if (_rightEndPoint == null)
+                //         {
+                //             _rightEndPoint = remoteEP2;
+                //         }
+                //     }
+                //     if (ex != null)
+                //     {
+                //         throw ex;
+                //     }
+                //     if (NetEventSource.IsEnabled)
+                //     {
+                //         NetEventSource.DumpBuffer(this, buffer, offset, size, "ReceiveFrom");
+                //         NetEventSource.Exit(this, bytesTransferred, "ReceiveFrom");
+                //     }
+                //     return bytesTransferred;
+                //
+
+                // so until they upgrade their mono version, we are stuck with
+                // some allocations.
+                //
+                // for now, let's pass the newly created on to our temp so at
+                // least we reuse it next time.
+                temp = socketAddress;
+
+                // SocketAddress.GetHashCode() depends on SocketAddress.m_changed.
+                // ReceiveFrom only sets the buffer, it does not seem to set m_changed.
+                // we need to reset m_changed for two reasons:
+                // * if m_changed is false, GetHashCode() returns the cahced m_hash
+                //   which is '0'. that would be a problem.
+                //   https://github.com/mono/mono/blob/bdd772531d379b4e78593587d15113c37edd4a64/mcs/class/referencesource/System/net/System/Net/SocketAddress.cs#L262
+                // * if we have a cached m_hash, but ReceiveFrom modified the buffer
+                //   then the GetHashCode() should change too. so we need to reset
+                //   either way.
+                //
+                // the only way to do that is by _actually_ modifying the buffer:
+                // https://github.com/mono/mono/blob/bdd772531d379b4e78593587d15113c37edd4a64/mcs/class/referencesource/System/net/System/Net/SocketAddress.cs#L99
+                // so let's do that.
+                // -> unchecked in case it's byte.Max
+                unchecked
+                {
+                    temp[0] += 1;
+                    temp[0] -= 1;
+                }
+
+                // make sure this worked.
+                // at least throw an Exception to make it obvious if the trick does
+                // not work anymore, in case ReceiveFrom is ever changed.
+                if (temp.GetHashCode() == 0)
+                    throw new Exception($"SocketAddress GetHashCode() is 0 after ReceiveFrom. Does the m_changed trick not work anymore?");
+
+                // in the future, enable this again:
+                //throw new Exception($"Socket.ReceiveFrom(): passed SocketAddress={socketAddress} but expected {temp}. This should never happen. Did ReceiveFrom() change?");
+            }
+
+            // ReceiveFrom sets seed_endpoint to the result of Create():
+            // https://github.com/mono/mono/blob/f74eed4b09790a0929889ad7fc2cf96c9b6e3757/mcs/class/System/System.Net.Sockets/Socket.cs#L1764
+            // so let's return ourselves at least.
+            // (seed_endpoint only seems to matter for BeginSend etc.)
+            return this;
+        }
+
+        // we need to overwrite GetHashCode() for two reasons.
+        // https://github.com/mono/mono/blob/bdd772531d379b4e78593587d15113c37edd4a64/mcs/class/referencesource/System/net/System/Net/IPEndPoint.cs#L160
+        // * it uses m_Address. but our true SocketAddress is in m_temp.
+        //   m_Address might not be set at all.
+        // * m_Address.GetHashCode() allocates:
+        //   https://github.com/mono/mono/blob/bdd772531d379b4e78593587d15113c37edd4a64/mcs/class/referencesource/System/net/System/Net/IPAddress.cs#L699
+        public override int GetHashCode() => temp.GetHashCode();
+
+        // helper function to create an ACTUAL new IPEndPoint from this.
+        // server needs it to store new connections as unique IPEndPoints.
+        public IPEndPoint DeepCopyIPEndPoint()
+        {
+            // we need to create a new IPEndPoint from 'temp' SocketAddress.
+            // there is no 'new IPEndPoint(SocketAddress) constructor.
+            // so we need to be a bit creative...
+
+            // allocate a placeholder IPAddress to copy
+            // our SocketAddress into.
+            // -> needs to be the same address family.
+            IPAddress ipAddress;
+            if (temp.Family == AddressFamily.InterNetworkV6)
+                ipAddress = IPAddress.IPv6Any;
+            else if (temp.Family == AddressFamily.InterNetwork)
+                ipAddress = IPAddress.Any;
+            else
+                throw new Exception($"Unexpected SocketAddress family: {temp.Family}");
+
+            // allocate a placeholder IPEndPoint
+            // with the needed size form IPAddress.
+            // (the real class. not NonAlloc)
+            IPEndPoint placeholder = new IPEndPoint(ipAddress, 0);
+
+            // the real IPEndPoint's .Create function can create a new IPEndPoint
+            // copy from a SocketAddress.
+            return (IPEndPoint)placeholder.Create(temp);
+        }
+    }
+}

Assets/Mirror/Runtime/Transport/KCP/kcp2k/where-allocation/Scripts/IPEndPointNonAlloc.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: af0279d15e39b484792394f1d3cad4d9
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Mirror/Runtime/Transport/KCP/kcp2k/where-allocation/Scripts/where-allocations.asmdef

@@ -0,0 +1,13 @@
+{
+    "name": "where-allocations",
+    "references": [],
+    "includePlatforms": [],
+    "excludePlatforms": [],
+    "allowUnsafeCode": false,
+    "overrideReferences": false,
+    "precompiledReferences": [],
+    "autoReferenced": true,
+    "defineConstraints": [],
+    "versionDefines": [],
+    "noEngineReferences": false
+}

Assets/Mirror/Runtime/Transport/KCP/kcp2k/where-allocation/Scripts/where-allocations.asmdef.meta

@@ -0,0 +1,7 @@
+fileFormatVersion: 2
+guid: 63c380d6dae6946209ed0832388a657c
+AssemblyDefinitionImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Mirror/Runtime/Transport/KCP/kcp2k/where-allocation/VERSION

@@ -0,0 +1,2 @@
+V0.1 [2021-06-01]
+- initial release

Assets/Mirror/Runtime/Transport/KCP/kcp2k/where-allocation/VERSION.meta

@@ -0,0 +1,7 @@
+fileFormatVersion: 2
+guid: f1256cadc037546ccb66071784fce137
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: