mirror of
https://github.com/MirrorNetworking/Mirror.git
synced 2024-11-18 02:50:32 +00:00
fix: kcp2p V1.30 (#3391)
- fix: set send/recv buffer sizes directly instead of iterating to find the limit. fixes: https://github.com/MirrorNetworking/Mirror/issues/3390 - fix: server & client sockets are now always non-blocking to ensure main thread never blocks on socket.recv/send. Send() now also handles WouldBlock. - fix: socket.Receive/From directly with non-blocking sockets and handle WouldBlock, instead of socket.Poll. faster, more obvious, and fixes Poll() looping forever while socket is in error state. fixes: https://github.com/MirrorNetworking/Mirror/issues/2733
This commit is contained in:
mischa
GitHub
parent
af787b8f06
commit
228a577683
@ -27,16 +27,20 @@ public class KcpTransport : Transport
|
||||
public uint Interval = 10;
|
||||
[Tooltip("KCP timeout in milliseconds. Note that KCP sends a ping automatically.")]
|
||||
public int Timeout = 10000;
|
||||
[Tooltip("Socket receive buffer size. Large buffer helps support more connections. Increase operating system socket buffer size limits if needed.")]
|
||||
public int RecvBufferSize = 1024 * 1027 * 7;
|
||||
[Tooltip("Socket send buffer size. Large buffer helps support more connections. Increase operating system socket buffer size limits if needed.")]
|
||||
public int SendBufferSize = 1024 * 1027 * 7;
|
||||
|
||||
[Header("Advanced")]
|
||||
[Tooltip("KCP fastresend parameter. Faster resend for the cost of higher bandwidth. 0 in normal mode, 2 in turbo mode.")]
|
||||
public int FastResend = 2;
|
||||
[Tooltip("KCP congestion window. Restricts window size to reduce congestion. Results in only 2-3 MTU messages per Flush even on loopback. Best to keept his disabled.")]
|
||||
/*public*/ bool CongestionWindow = false; // KCP 'NoCongestionWindow' is false by default. here we negate it for ease of use.
|
||||
[Tooltip("KCP window size can be modified to support higher loads.")]
|
||||
public uint SendWindowSize = 4096; //Kcp.WND_SND; 32 by default. Mirror sends a lot, so we need a lot more.
|
||||
[Tooltip("KCP window size can be modified to support higher loads. This also increases max message size.")]
|
||||
public uint ReceiveWindowSize = 4096; //Kcp.WND_RCV; 128 by default. Mirror sends a lot, so we need a lot more.
|
||||
[Tooltip("KCP window size can be modified to support higher loads.")]
|
||||
public uint SendWindowSize = 4096; //Kcp.WND_SND; 32 by default. Mirror sends a lot, so we need a lot more.
|
||||
[Tooltip("KCP will try to retransmit lost messages up to MaxRetransmit (aka dead_link) before disconnecting.")]
|
||||
public uint MaxRetransmit = Kcp.DEADLINK * 2; // default prematurely disconnects a lot of people (#3022). use 2x.
|
||||
[Tooltip("Enable to automatically set client & server send/recv buffers to OS limit. Avoids issues with too small buffers under heavy load, potentially dropping connections. Increase the OS limit if this is still too small.")]
|
||||
@ -101,7 +105,7 @@ void Awake()
|
||||
Log.Error = Debug.LogError;
|
||||
|
||||
// create config from serialized settings
|
||||
config = new KcpConfig(DualMode, MaximizeSocketBuffers, NoDelay, Interval, FastResend, CongestionWindow, SendWindowSize, ReceiveWindowSize, Timeout, MaxRetransmit);
|
||||
config = new KcpConfig(DualMode, RecvBufferSize, SendBufferSize, NoDelay, Interval, FastResend, CongestionWindow, SendWindowSize, ReceiveWindowSize, Timeout, MaxRetransmit);
|
||||
|
||||
// client (NonAlloc version is not necessary anymore)
|
||||
client = new KcpClient(
|
||||
|
||||
@ -1,3 +1,12 @@
|
||||
V1.30 [2023-02-20]
|
||||
- fix: set send/recv buffer sizes directly instead of iterating to find the limit.
|
||||
fixes: https://github.com/MirrorNetworking/Mirror/issues/3390
|
||||
- fix: server & client sockets are now always non-blocking to ensure main thread never
|
||||
blocks on socket.recv/send. Send() now also handles WouldBlock.
|
||||
- fix: socket.Receive/From directly with non-blocking sockets and handle WouldBlock,
|
||||
instead of socket.Poll. faster, more obvious, and fixes Poll() looping forever while
|
||||
socket is in error state. fixes: https://github.com/MirrorNetworking/Mirror/issues/2733
|
||||
|
||||
V1.29 [2023-01-28]
|
||||
- fix: KcpServer.CreateServerSocket now handles NotSupportedException when setting DualMode
|
||||
https://github.com/MirrorNetworking/Mirror/issues/3358
|
||||
|
||||
@ -24,17 +24,26 @@ public static bool ResolveHostname(string hostname, out IPAddress[] addresses)
|
||||
|
||||
// if connections drop under heavy load, increase to OS limit.
|
||||
// if still not enough, increase the OS limit.
|
||||
public static void MaximizeSocketBuffers(Socket socket)
|
||||
public static void ConfigureSocketBuffers(Socket socket, int recvBufferSize, int sendBufferSize)
|
||||
{
|
||||
// log initial size for comparison.
|
||||
// remember initial size for log comparison
|
||||
int initialReceive = socket.ReceiveBufferSize;
|
||||
int initialSend = socket.SendBufferSize;
|
||||
|
||||
socket.SetReceiveBufferToOSLimit();
|
||||
socket.SetSendBufferToOSLimit();
|
||||
// set to configured size
|
||||
try
|
||||
{
|
||||
socket.ReceiveBufferSize = recvBufferSize;
|
||||
socket.SendBufferSize = sendBufferSize;
|
||||
}
|
||||
catch (SocketException)
|
||||
{
|
||||
Log.Warning($"Kcp: failed to set Socket RecvBufSize = {recvBufferSize} SendBufSize = {sendBufferSize}");
|
||||
}
|
||||
|
||||
Log.Info($"Kcp: RecvBuf = {initialReceive}=>{socket.ReceiveBufferSize} ({socket.ReceiveBufferSize/initialReceive}x) SendBuf = {initialSend}=>{socket.SendBufferSize} ({socket.SendBufferSize/initialSend}x) maximized to OS limits!");
|
||||
|
||||
Log.Info($"Kcp: RecvBuf = {initialReceive}=>{socket.ReceiveBufferSize} ({socket.ReceiveBufferSize/initialReceive}x) SendBuf = {initialSend}=>{socket.SendBufferSize} ({socket.SendBufferSize/initialSend}x)");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,33 +1,6 @@
|
||||
using System.Net.Sockets;
|
||||
|
||||
namespace kcp2k
|
||||
{
|
||||
public static class Extensions
|
||||
{
|
||||
// 100k attempts of 1 KB increases = default + 100 MB max
|
||||
public static void SetReceiveBufferToOSLimit(this Socket socket, int stepSize = 1024, int attempts = 100_000)
|
||||
{
|
||||
// setting a too large size throws a socket exception.
|
||||
// so let's keep increasing until we encounter it.
|
||||
for (int i = 0; i < attempts; ++i)
|
||||
{
|
||||
// increase in 1 KB steps
|
||||
try { socket.ReceiveBufferSize += stepSize; }
|
||||
catch (SocketException) { break; }
|
||||
}
|
||||
}
|
||||
|
||||
// 100k attempts of 1 KB increases = default + 100 MB max
|
||||
public static void SetSendBufferToOSLimit(this Socket socket, int stepSize = 1024, int attempts = 100_000)
|
||||
{
|
||||
// setting a too large size throws a socket exception.
|
||||
// so let's keep increasing until we encounter it.
|
||||
for (int i = 0; i < attempts; ++i)
|
||||
{
|
||||
// increase in 1 KB steps
|
||||
try { socket.SendBufferSize += stepSize; }
|
||||
catch (SocketException) { break; }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -97,15 +97,13 @@ void OnDisconnectedWrap()
|
||||
remoteEndPoint = new IPEndPoint(addresses[0], port);
|
||||
socket = new Socket(remoteEndPoint.AddressFamily, SocketType.Dgram, ProtocolType.Udp);
|
||||
|
||||
// configure buffer sizes:
|
||||
// if connections drop under heavy load, increase to OS limit.
|
||||
// if still not enough, increase the OS limit.
|
||||
if (config.MaximizeSocketBuffers)
|
||||
{
|
||||
Common.MaximizeSocketBuffers(socket);
|
||||
}
|
||||
// otherwise still log the defaults for info.
|
||||
else Log.Info($"KcpClient: RecvBuf = {socket.ReceiveBufferSize} SendBuf = {socket.SendBufferSize}. If connections drop under heavy load, enable {nameof(KcpConfig.MaximizeSocketBuffers)} to increase it to OS limit. If they still drop, increase the OS limit.");
|
||||
// recv & send are called from main thread.
|
||||
// need to ensure this never blocks.
|
||||
// even a 1ms block per connection would stop us from scaling.
|
||||
socket.Blocking = false;
|
||||
|
||||
// configure buffer sizes
|
||||
Common.ConfigureSocketBuffers(socket, config.RecvBufferSize, config.SendBufferSize);
|
||||
|
||||
// bind to endpoint so we can use send/recv instead of sendto/recvfrom.
|
||||
socket.Connect(remoteEndPoint);
|
||||
@ -121,42 +119,56 @@ void OnDisconnectedWrap()
|
||||
protected virtual bool RawReceive(out ArraySegment<byte> segment)
|
||||
{
|
||||
segment = default;
|
||||
if (socket == null) return false;
|
||||
|
||||
try
|
||||
{
|
||||
if (socket != null && socket.Poll(0, SelectMode.SelectRead))
|
||||
{
|
||||
// ReceiveFrom allocates. we used bound Receive.
|
||||
// returns amount of bytes written into buffer.
|
||||
// throws SocketException if datagram was larger than buffer.
|
||||
// https://learn.microsoft.com/en-us/dotnet/api/system.net.sockets.socket.receive?view=net-6.0
|
||||
int msgLength = socket.Receive(rawReceiveBuffer);
|
||||
// ReceiveFrom allocates. we used bound Receive.
|
||||
// returns amount of bytes written into buffer.
|
||||
// throws SocketException if datagram was larger than buffer.
|
||||
// https://learn.microsoft.com/en-us/dotnet/api/system.net.sockets.socket.receive?view=net-6.0
|
||||
int msgLength = socket.Receive(rawReceiveBuffer);
|
||||
|
||||
//Log.Debug($"KCP: client raw recv {msgLength} bytes = {BitConverter.ToString(buffer, 0, msgLength)}");
|
||||
segment = new ArraySegment<byte>(rawReceiveBuffer, 0, msgLength);
|
||||
return true;
|
||||
}
|
||||
//Log.Debug($"KCP: client raw recv {msgLength} bytes = {BitConverter.ToString(buffer, 0, msgLength)}");
|
||||
segment = new ArraySegment<byte>(rawReceiveBuffer, 0, msgLength);
|
||||
return true;
|
||||
}
|
||||
// this is fine, the socket might have been closed in the other end
|
||||
catch (SocketException ex)
|
||||
// for non-blocking sockets, Receive throws WouldBlock if there is
|
||||
// no message to read. that's okay. only log for other errors.
|
||||
catch (SocketException e)
|
||||
{
|
||||
// the other end closing the connection is not an 'error'.
|
||||
// but connections should never just end silently.
|
||||
// at least log a message for easier debugging.
|
||||
// for example, his can happen when connecting without a server.
|
||||
// see test: ConnectWithoutServer().
|
||||
Log.Info($"KcpClient: looks like the other end has closed the connection. This is fine: {ex}");
|
||||
peer.Disconnect();
|
||||
if (e.SocketErrorCode != SocketError.WouldBlock)
|
||||
{
|
||||
// the other end closing the connection is not an 'error'.
|
||||
// but connections should never just end silently.
|
||||
// at least log a message for easier debugging.
|
||||
// for example, his can happen when connecting without a server.
|
||||
// see test: ConnectWithoutServer().
|
||||
Log.Info($"KcpClient: looks like the other end has closed the connection. This is fine: {e}");
|
||||
peer.Disconnect();
|
||||
}
|
||||
// WouldBlock indicates there's no data yet, so return false.
|
||||
return false;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// io - output.
|
||||
// virtual so it may be modified for relays, etc.
|
||||
protected virtual void RawSend(ArraySegment<byte> data)
|
||||
{
|
||||
socket.Send(data.Array, data.Offset, data.Count, SocketFlags.None);
|
||||
try
|
||||
{
|
||||
socket.Send(data.Array, data.Offset, data.Count, SocketFlags.None);
|
||||
}
|
||||
// for non-blocking sockets, SendTo may throw WouldBlock.
|
||||
// in that case, simply drop the message. it's UDP, it's fine.
|
||||
catch (SocketException e)
|
||||
{
|
||||
if (e.SocketErrorCode != SocketError.WouldBlock)
|
||||
{
|
||||
Log.Error($"KcpClient: Send failed: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Send(ArraySegment<byte> segment, KcpChannel channel)
|
||||
|
||||
@ -13,10 +13,15 @@ public class KcpConfig
|
||||
// (Nintendo Switch, etc.)
|
||||
public bool DualMode;
|
||||
|
||||
// attempt to maximize socket send/recv buffers to OS limit.
|
||||
// too small send/receive buffers might cause connection drops under
|
||||
// heavy load. using the OS max size can make a difference already.
|
||||
public bool MaximizeSocketBuffers;
|
||||
// UDP servers use only one socket.
|
||||
// maximize buffer to handle as many connections as possible.
|
||||
//
|
||||
// M1 mac pro:
|
||||
// recv buffer default: 786896 (771 KB)
|
||||
// send buffer default: 9216 (9 KB)
|
||||
// max configurable: ~7 MB
|
||||
public int RecvBufferSize;
|
||||
public int SendBufferSize;
|
||||
|
||||
// kcp configuration ///////////////////////////////////////////////////
|
||||
// NoDelay is recommended to reduce latency. This also scales better
|
||||
@ -59,7 +64,8 @@ public class KcpConfig
|
||||
// makes it easy to define "new KcpConfig(DualMode=false)" etc.
|
||||
public KcpConfig(
|
||||
bool DualMode = true,
|
||||
bool MaximizeSocketBuffers = false,
|
||||
int RecvBufferSize = 1024 * 1024 * 7,
|
||||
int SendBufferSize = 1024 * 1024 * 7,
|
||||
bool NoDelay = true,
|
||||
uint Interval = 10,
|
||||
int FastResend = 0,
|
||||
@ -70,7 +76,8 @@ public KcpConfig(
|
||||
uint MaxRetransmits = Kcp.DEADLINK)
|
||||
{
|
||||
this.DualMode = DualMode;
|
||||
this.MaximizeSocketBuffers = MaximizeSocketBuffers;
|
||||
this.RecvBufferSize = RecvBufferSize;
|
||||
this.SendBufferSize = SendBufferSize;
|
||||
this.NoDelay = NoDelay;
|
||||
this.Interval = Interval;
|
||||
this.FastResend = FastResend;
|
||||
|
||||
@ -103,15 +103,13 @@ public virtual void Start(ushort port)
|
||||
// listen
|
||||
socket = CreateServerSocket(config.DualMode, port);
|
||||
|
||||
// configure buffer sizes:
|
||||
// if connections drop under heavy load, increase to OS limit.
|
||||
// if still not enough, increase the OS limit.
|
||||
if (config.MaximizeSocketBuffers)
|
||||
{
|
||||
Common.MaximizeSocketBuffers(socket);
|
||||
}
|
||||
// otherwise still log the defaults for info.
|
||||
else Log.Info($"KcpServer: RecvBuf = {socket.ReceiveBufferSize} SendBuf = {socket.SendBufferSize}. If connections drop under heavy load, enable {nameof(KcpConfig.MaximizeSocketBuffers)} to increase it to OS limit. If they still drop, increase the OS limit.");
|
||||
// recv & send are called from main thread.
|
||||
// need to ensure this never blocks.
|
||||
// even a 1ms block per connection would stop us from scaling.
|
||||
socket.Blocking = false;
|
||||
|
||||
// configure buffer sizes
|
||||
Common.ConfigureSocketBuffers(socket, config.RecvBufferSize, config.SendBufferSize);
|
||||
}
|
||||
|
||||
public void Send(int connectionId, ArraySegment<byte> segment, KcpChannel channel)
|
||||
@ -149,45 +147,48 @@ protected virtual bool RawReceiveFrom(out ArraySegment<byte> segment, out int co
|
||||
{
|
||||
segment = default;
|
||||
connectionId = 0;
|
||||
if (socket == null) return false;
|
||||
|
||||
try
|
||||
{
|
||||
if (socket != null && socket.Poll(0, SelectMode.SelectRead))
|
||||
{
|
||||
// NOTE: ReceiveFrom allocates.
|
||||
// we pass our IPEndPoint to ReceiveFrom.
|
||||
// receive from calls newClientEP.Create(socketAddr).
|
||||
// IPEndPoint.Create always returns a new IPEndPoint.
|
||||
// https://github.com/mono/mono/blob/f74eed4b09790a0929889ad7fc2cf96c9b6e3757/mcs/class/System/System.Net.Sockets/Socket.cs#L1761
|
||||
//
|
||||
// throws SocketException if datagram was larger than buffer.
|
||||
// https://learn.microsoft.com/en-us/dotnet/api/system.net.sockets.socket.receive?view=net-6.0
|
||||
int size = socket.ReceiveFrom(rawReceiveBuffer, 0, rawReceiveBuffer.Length, SocketFlags.None, ref newClientEP);
|
||||
segment = new ArraySegment<byte>(rawReceiveBuffer, 0, size);
|
||||
// NOTE: ReceiveFrom allocates.
|
||||
// we pass our IPEndPoint to ReceiveFrom.
|
||||
// receive from calls newClientEP.Create(socketAddr).
|
||||
// IPEndPoint.Create always returns a new IPEndPoint.
|
||||
// https://github.com/mono/mono/blob/f74eed4b09790a0929889ad7fc2cf96c9b6e3757/mcs/class/System/System.Net.Sockets/Socket.cs#L1761
|
||||
//
|
||||
// throws SocketException if datagram was larger than buffer.
|
||||
// https://learn.microsoft.com/en-us/dotnet/api/system.net.sockets.socket.receive?view=net-6.0
|
||||
int size = socket.ReceiveFrom(rawReceiveBuffer, 0, rawReceiveBuffer.Length, SocketFlags.None, ref newClientEP);
|
||||
segment = new ArraySegment<byte>(rawReceiveBuffer, 0, size);
|
||||
|
||||
// set connectionId to hash from endpoint
|
||||
// NOTE: IPEndPoint.GetHashCode() allocates.
|
||||
// it calls m_Address.GetHashCode().
|
||||
// m_Address is an IPAddress.
|
||||
// GetHashCode() allocates for IPv6:
|
||||
// https://github.com/mono/mono/blob/bdd772531d379b4e78593587d15113c37edd4a64/mcs/class/referencesource/System/net/System/Net/IPAddress.cs#L699
|
||||
//
|
||||
// => using only newClientEP.Port wouldn't work, because
|
||||
// different connections can have the same port.
|
||||
connectionId = newClientEP.GetHashCode();
|
||||
return true;
|
||||
}
|
||||
// set connectionId to hash from endpoint
|
||||
// NOTE: IPEndPoint.GetHashCode() allocates.
|
||||
// it calls m_Address.GetHashCode().
|
||||
// m_Address is an IPAddress.
|
||||
// GetHashCode() allocates for IPv6:
|
||||
// https://github.com/mono/mono/blob/bdd772531d379b4e78593587d15113c37edd4a64/mcs/class/referencesource/System/net/System/Net/IPAddress.cs#L699
|
||||
//
|
||||
// => using only newClientEP.Port wouldn't work, because
|
||||
// different connections can have the same port.
|
||||
connectionId = newClientEP.GetHashCode();
|
||||
return true;
|
||||
}
|
||||
// this is fine, the socket might have been closed in the other end
|
||||
catch (SocketException ex)
|
||||
// for non-blocking sockets, Receive throws WouldBlock if there is
|
||||
// no message to read. that's okay. only log for other errors.
|
||||
catch (SocketException e)
|
||||
{
|
||||
// the other end closing the connection is not an 'error'.
|
||||
// but connections should never just end silently.
|
||||
// at least log a message for easier debugging.
|
||||
Log.Info($"KcpServer: poll & read failed: {ex}");
|
||||
if (e.SocketErrorCode != SocketError.WouldBlock)
|
||||
{
|
||||
// NOTE: SocketException is not a subclass of IOException.
|
||||
// the other end closing the connection is not an 'error'.
|
||||
// but connections should never just end silently.
|
||||
// at least log a message for easier debugging.
|
||||
Log.Info($"KcpServer: ReceiveFrom failed: {e}");
|
||||
}
|
||||
// WouldBlock indicates there's no data yet, so return false.
|
||||
return false;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// io - out.
|
||||
@ -205,7 +206,19 @@ protected virtual void RawSend(int connectionId, ArraySegment<byte> data)
|
||||
// send to the the endpoint.
|
||||
// do not send to 'newClientEP', as that's always reused.
|
||||
// fixes https://github.com/MirrorNetworking/Mirror/issues/3296
|
||||
socket.SendTo(data.Array, data.Offset, data.Count, SocketFlags.None, connection.remoteEndPoint);
|
||||
try
|
||||
{
|
||||
socket.SendTo(data.Array, data.Offset, data.Count, SocketFlags.None, connection.remoteEndPoint);
|
||||
}
|
||||
// for non-blocking sockets, SendTo may throw WouldBlock.
|
||||
// in that case, simply drop the message. it's UDP, it's fine.
|
||||
catch (SocketException e)
|
||||
{
|
||||
if (e.SocketErrorCode != SocketError.WouldBlock)
|
||||
{
|
||||
Log.Error($"KcpServer: SendTo failed: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected virtual KcpServerConnection CreateConnection(int connectionId)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user