Add uTP over discv5 test and small uTP performance improvements (#663)

- Add a multiple sockets use test for uTP over discv5
- Use assign2 for the biggest consumer of genericAssignAux in uTP
- Avoid calling exists on the growable buffer when there is no
place in the socket window.

eth/utp/growable_buffer.nim

@@ -1,4 +1,4 @@
-# Copyright (c) 2021-2023 Status Research & Development GmbH
+# Copyright (c) 2021-2024 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -9,6 +9,8 @@
 import
   std/[options, math]
 
+from stew/assign2 import assign
+
 export options
 
 # Buffer implementation similar to the one in the reference implementation.
@@ -32,7 +34,7 @@ proc init*[A](T: type GrowableCircularBuffer[A], size: uint32 = 16): T =
   )
 
 proc get*[A](buff: GrowableCircularBuffer[A], i: uint32): Option[A] =
-  buff.items[i and buff.mask]
+  assign(result, buff.items[i and buff.mask])
 
 proc putImpl[A](buff: var GrowableCircularBuffer[A], i: uint32, elem: Option[A]) =
   buff.items[i and buff.mask] = elem

eth/utp/utp_socket.nim

@@ -1,4 +1,4 @@
-# Copyright (c) 2021-2023 Status Research & Development GmbH
+# Copyright (c) 2021-2024 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -481,6 +481,10 @@ proc resetSendTimeout(socket: UtpSocket) =
   socket.rtoTimeout = getMonoTimestamp().moment + socket.retransmitTimeout
 
 proc flushPackets(socket: UtpSocket) =
+  if (socket.freeWindowBytes() == 0):
+    trace "No place in send window, not flushing"
+    return
+
   let oldestOutgoingPacketSeqNr = socket.seqNr - socket.curWindowPackets
   var i: uint16 = oldestOutgoingPacketSeqNr
   while i != socket.seqNr:

tests/utp/test_buffer.nim

@@ -14,7 +14,7 @@ import
 type TestObj = object
   foo: string
 
-suite "Utp ring buffer":
+suite "uTP ring buffer":
   test "Empty buffer":
     let buff = GrowableCircularBuffer[int].init(size = 4)
     check:

tests/utp/test_clock_drift_calculator.nim

@@ -11,8 +11,7 @@ import
   unittest2,
   ../../eth/utp/clock_drift_calculator
 
-suite "Clock drift calculator":
-
+suite "uTP clock drift calculator":
   test "Initial clock drift should be 0":
     let currentTime = Moment.now()
     let calculator = ClockDriftCalculator.init(currentTime)

tests/utp/test_discv5_protocol.nim

@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2021 Status Research & Development GmbH
+# Copyright (c) 2020-2024 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -7,7 +7,7 @@
 {.used.}
 
 import
-  std/options,
+  std/[options, sequtils],
   chronos,
   stew/shims/net, stew/byteutils,
   testutils/unittests,
@@ -19,7 +19,7 @@ import
   ../p2p/discv5_test_helper,
   ../stubloglevel
 
-procSuite "Utp protocol over discovery v5 tests":
+procSuite "uTP over discovery v5 protocol":
   let rng = newRng()
   let utpProtId = "test-utp".toBytes()
 
@@ -230,3 +230,90 @@ procSuite "Utp protocol over discovery v5 tests":
     await serverSocket.destroyWait()
     await node1.closeWait()
     await node2.closeWait()
+
+  asyncTest "Data transfer over multiple sockets":
+    const
+      amountOfTransfers = 25
+      dataToSend: seq[byte] = repeat(byte 0xA0, 1_000_000)
+
+    var readFutures: seq[Future[void]]
+
+    proc readAndCheck(
+        socket: UtpSocket[NodeAddress],
+      ): Future[void] {.async.} =
+      let readData = await socket.read()
+      check:
+        readData == dataToSend
+        socket.atEof()
+
+    proc handleIncomingConnection(
+        server: UtpRouter[NodeAddress],
+        client: UtpSocket[NodeAddress]
+      ): Future[void] =
+      readFutures.add(client.readAndCheck())
+
+      var fut = newFuture[void]("test.AcceptConnectionCallback")
+      fut.complete()
+      return fut
+
+    proc handleIncomingConnectionDummy(
+        server: UtpRouter[NodeAddress],
+        client: UtpSocket[NodeAddress]
+      ): Future[void] =
+        var fut = newFuture[void]("test.AcceptConnectionCallback")
+        fut.complete()
+        return fut
+
+    let
+      address1 = localAddress(20302)
+      address2 = localAddress(20303)
+      node1 = initDiscoveryNode(
+        rng, PrivateKey.random(rng[]), address1)
+      node2 = initDiscoveryNode(
+        rng, PrivateKey.random(rng[]), address2)
+
+      utp1 = UtpDiscv5Protocol.new(
+        node1, utpProtId, handleIncomingConnectionDummy)
+      utp2 {.used.} = UtpDiscv5Protocol.new(
+        node2, utpProtId, handleIncomingConnection)
+
+    # nodes must have session between each other
+    check:
+      (await node1.ping(node2.localNode)).isOk()
+
+    proc connectSendAndCheck(
+        utpProto: UtpDiscv5Protocol,
+        address: NodeAddress
+      ): Future[void] {.async.} =
+      let socketRes = await utpProto.connectTo(address)
+      check:
+        socketRes.isOk()
+      let socket = socketRes.value()
+      let dataSend = await socket.write(dataToSend)
+      check:
+        dataSend.isOk()
+        dataSend.value() == dataToSend.len()
+
+      await socket.closeWait()
+
+    let t0 = Moment.now()
+    for i in 0..<amountOfTransfers:
+      asyncSpawn utp1.connectSendAndCheck(
+        NodeAddress.init(node2.localNode.id, address2))
+
+    while readFutures.len() < amountOfTransfers:
+      await sleepAsync(milliseconds(100))
+
+    await allFutures(readFutures)
+    let elapsed = Moment.now() - t0
+
+    await utp1.shutdownWait()
+    await utp2.shutdownWait()
+
+    let megabitsSent = amountOfTransfers * dataToSend.len() * 8 / 1_000_000
+    let seconds = float(elapsed.nanoseconds) / 1_000_000_000
+    let throughput = megabitsSent / seconds
+
+    echo ""
+    echo "Sent ", amountOfTransfers, " asynchronous uTP transfers in ", seconds,
+      " seconds, payload throughput: ", throughput, " Mbit/s"

tests/utp/test_packets.nim

@@ -12,7 +12,7 @@ import
   ../../eth/utp/packets,
   ../../eth/keys
 
-suite "uTP Packet Encoding":
+suite "uTP packet encoding":
   test "Encode/decode SYN packet":
     let
       synPacket = synPacket(5, 10, 20)

tests/utp/test_protocol.nim

@@ -130,7 +130,7 @@ proc close(s: TwoClientsServerScenario) {.async.} =
   await s.utp2.shutdownWait()
   await s.utp3.shutdownWait()
 
-procSuite "uTP over UDP protocol tests":
+procSuite "uTP over UDP protocol":
   let rng = newRng()
 
   asyncTest "Connect to remote host: test connection callback":

tests/utp/test_protocol_integration.nim

@@ -51,7 +51,7 @@ proc getServerSocket(
   else:
     return some(srvSocket)
 
-procSuite "Utp protocol over udp tests with loss and delays":
+procSuite "uTP over UDP protocol with loss and delays":
   let rng = newRng()
 
   proc sendBuilder(maxDelay: int, packetDropRate: int): SendCallbackBuilder =

tests/utp/test_utp_router.nim

@@ -25,7 +25,7 @@ proc hash*(x: UtpSocketKey[int]): Hash =
 type
   TestError* = object of CatchableError
 
-procSuite "Utp router unit tests":
+procSuite "uTP router unit":
   let rng = newRng()
   let testSender = 1
   let testSender2 = 2

tests/utp/test_utp_socket.nim

@@ -17,7 +17,7 @@ import
   ../../eth/keys,
   ../stubloglevel
 
-procSuite "uTP socket tests":
+procSuite "uTP socket":
   let
     rng = newRng()
     testAddress = initTAddress("127.0.0.1", 9079)

tests/utp/test_utp_socket_sack.nim

@@ -18,7 +18,7 @@ import
   ../../eth/keys,
   ../stubloglevel
 
-procSuite "Utp socket selective acks unit test":
+procSuite "uTP socket selective acks":
   let rng = newRng()
   let testAddress = initTAddress("127.0.0.1", 9079)
   let defaultBufferSize = 1024'u32

tests/utp/utp_packet_test_vectors.nim

@@ -13,7 +13,7 @@ import
   ../../eth/utp/packets,
   ../../eth/keys
 
-suite "Utp packets test vectors":
+suite "uTP packets test vectors":
   test "SYN packet":
     let synPacket = Packet(
       header: PacketHeaderV1(