chore: update libp2p to 2.0.0 (#1443)

Co-authored-by: Arnaud <arnaud@status.im>

.gitmodules

@@ -183,16 +183,16 @@
 [submodule "vendor/nph"]
 	path = vendor/nph
 	url = https://github.com/arnetheduck/nph.git
-[submodule "vendor/nim-quic"]
-	path = vendor/nim-quic
-	url = https://github.com/vacp2p/nim-quic.git
-	ignore = untracked
-	branch = main
-[submodule "vendor/nim-ngtcp2"]
-	path = vendor/nim-ngtcp2
-	url = https://github.com/vacp2p/nim-ngtcp2.git
-	ignore = untracked
-	branch = main
 [submodule "vendor/nim-merkletree"]
 	path = vendor/nim-merkletree
 	url = https://github.com/logos-storage/nim-merkletree
+[submodule "vendor/nim-boringssl"]
+	path = vendor/nim-boringssl
+	url = https://github.com/vacp2p/nim-boringssl.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-lsquic"]
+	path = vendor/nim-lsquic
+	url = https://github.com/vacp2p/nim-lsquic.git
+	ignore = untracked
+	branch = main

library/storage_thread_requests/requests/node_lifecycle_request.nim

@@ -13,7 +13,7 @@ import results
 import confutils
 import confutils/std/net
 import confutils/defs
-import libp2p
+import libp2p except NATConfig
 import json_serialization
 import json_serialization/std/[options, net]
 import ../../alloc

library/storage_thread_requests/storage_thread_request.nim

@@ -58,9 +58,7 @@ proc createShared*(
 # We can improve this by dispatching the callbacks to a thread pool or
 # moving to a MP channel.
 # See: https://github.com/logos-storage/logos-storage-nim/pull/1322#discussion_r2340708316
-proc handleRes[T: string | void | seq[byte]](
-    res: Result[T, string], request: ptr StorageThreadRequest
-) =
+proc handleRes(res: Result[string, string], request: ptr StorageThreadRequest) =
   ## Handles the Result responses, which can either be Result[string, string] or
   ## Result[void, string].
   defer:
@@ -78,12 +76,13 @@ proc handleRes[T: string | void | seq[byte]](
     return
 
   foreignThreadGc:
-    var msg: cstring = ""
-    when T is string:
-      msg = res.get().cstring()
-    request[].callback(
-      RET_OK, unsafeAddr msg[0], cast[csize_t](len(msg)), request[].userData
-    )
+    let msg = res.get()
+    if msg.len == 0:
+      request[].callback(RET_OK, nil, cast[csize_t](0), request[].userData)
+    else:
+      request[].callback(
+        RET_OK, unsafeAddr msg[0], cast[csize_t](msg.len), request[].userData
+      )
   return
 
 proc process*(

storage/blockexchange/network/network.nim

@@ -13,7 +13,7 @@ import std/sequtils
 import pkg/chronos
 
 import pkg/libp2p
-import pkg/libp2p/utils/semaphore
+import pkg/libp2p/protocols/protocol as lp_protocol
 import pkg/questionable
 import pkg/questionable/results
 
@@ -103,17 +103,23 @@ proc send*(
       peerId = id, hasWantList = msg.wantList.entries.len > 0
     return
 
+  var acquired = false
   try:
     let peer = b.peers[id]
 
     await b.inflightSema.acquire()
+    acquired = true
     await peer.send(msg)
   except CancelledError as error:
     raise error
   except CatchableError as err:
     error "Error sending message", peer = id, msg = err.msg
   finally:
-    b.inflightSema.release()
+    if acquired:
+      try:
+        b.inflightSema.release()
+      except AsyncSemaphoreError as err:
+        error "Error releasing inflight semaphore", msg = err.msg
 
 proc handleWantList(
     b: BlockExcNetwork, peer: NetworkPeer, list: WantList
@@ -295,7 +301,6 @@ method init*(self: BlockExcNetwork) {.raises: [].} =
     await blockexcPeer.readLoop(conn) # attach read loop
 
   self.handler = handler
-  self.codec = Codec
 
 proc stop*(self: BlockExcNetwork) {.async: (raises: []).} =
   await self.trackedFutures.cancelTracked()
@@ -309,14 +314,16 @@ proc new*(
   ## Create a new BlockExcNetwork instance
   ##
 
-  let self = BlockExcNetwork(
-    switch: switch,
-    getConn: connProvider,
-    inflightSema: newAsyncSemaphore(maxInflight),
-    maxInflight: maxInflight,
+  let self = lp_protocol.new(
+    BlockExcNetwork, @[Codec], nil, maxIncomingStreamsTotal = maxInflight
   )
+  self.switch = switch
+  self.getConn = connProvider
+  self.inflightSema = newAsyncSemaphore(max(maxInflight, 1))
+  if maxInflight == 0:
+    discard self.inflightSema.tryAcquire()
 
-  self.maxIncomingStreams = self.maxInflight
+  self.maxInflight = maxInflight
 
   proc sendWantList(
       id: PeerId,

storage/blockexchange/peers/peercontext.nim

@@ -7,7 +7,7 @@
 ## This file may not be copied, modified, or distributed except according to
 ## those terms.
 
-import std/math
+import std/[math, options]
 
 import pkg/libp2p
 import pkg/chronos

storage/conf.nim

@@ -30,10 +30,9 @@ import pkg/toml_serialization
 import pkg/metrics
 import pkg/metrics/chronos_httpserver
 import pkg/stew/byteutils
-import pkg/libp2p
+import pkg/libp2p except NATConfig
 import pkg/questionable
 import pkg/questionable/results
-import pkg/stew/base64
 
 import ./storagetypes
 import ./discovery

storage/discovery.nim

@@ -21,7 +21,7 @@ import pkg/contractabi/address as ca
 import pkg/codexdht/discv5/[routing_table, protocol as discv5]
 from pkg/nimcrypto import keccak256
 
-import ./rng
+import ./rng as storage_rng
 import ./errors
 import ./logutils
 
@@ -260,7 +260,7 @@ proc new*(
     bindPort = bindPort,
     record = self.providerRecord.get,
     bootstrapRecords = bootstrapNodes,
-    rng = Rng.instance(),
+    rng = storage_rng.libp2pRng(storage_rng.Rng.instance()),
     providers = ProvidersManager.new(store),
     config = discoveryConfig,
   )

storage/manifest/coders.nim

@@ -14,6 +14,7 @@ import times
 {.push raises: [].}
 
 import std/tables
+import std/options
 
 import pkg/libp2p
 import pkg/questionable

storage/rng.nim

@@ -8,8 +8,8 @@
 ## those terms.
 
 import std/sugar
-import pkg/libp2p/crypto/crypto
 import pkg/bearssl/rand
+import pkg/libp2p/crypto/rng as libp2p_rng
 
 type
   RngSampleError = object of CatchableError
@@ -19,9 +19,12 @@ var rng {.threadvar.}: Rng
 
 proc instance*(t: type Rng): Rng =
   if rng.isNil:
-    rng = newRng()
+    rng = HmacDrbgContext.new()
   rng
 
+proc libp2pRng*(rng: Rng): libp2p_rng.Rng =
+  libp2p_rng.newBearSslRng(rng)
+
 # Random helpers: similar as in stdlib, but with HmacDrbgContext rng
 # TODO: Move these somewhere else?
 const randMax = 18_446_744_073_709_551_615'u64

storage/storage.nim

@@ -174,8 +174,9 @@ proc new*(
   let switch = SwitchBuilder
     .new()
     .withPrivateKey(privateKey)
-    .withAddresses(@[listenMultiAddr])
-    .withRng(random.Rng.instance())
+    .withAddresses(@[listenMultiAddr], enableWildcardResolver = true)
+    .withIdentifyPusher(false)
+    .withRng(random.Rng.instance().libp2pRng)
     .withNoise()
     .withYamux()
     .withMaxConnections(config.maxPeers)

storage/stores/networkstore.nim

@@ -9,6 +9,7 @@
 
 {.push raises: [].}
 
+import std/options
 import pkg/chronos
 import pkg/libp2p
 import pkg/questionable/results

storage/streams/asyncstreamwrapper.nim

@@ -79,7 +79,7 @@ proc completeWrite(
     await fut
 
 method write*(
-    self: AsyncStreamWrapper, msg: seq[byte]
+    self: AsyncStreamWrapper, msg: sink seq[byte]
 ): Future[void] {.async: (raises: [CancelledError, LPStreamError], raw: true).} =
   # Avoid a copy of msg being kept in the closure created by `{.async.}` as this
   # drives up memory usage

storage/utils/keyutils.nim

@@ -27,7 +27,7 @@ proc setupKey*(path: string): ?!PrivateKey =
   if not path.fileAccessible({AccessFlags.Find}):
     info "Creating a private key and saving it"
     let
-      res = ?PrivateKey.random(Rng.instance()[]).mapFailure(StorageKeyError)
+      res = ?PrivateKey.random(Rng.instance().libp2pRng).mapFailure(StorageKeyError)
       bytes = ?res.getBytes().mapFailure(StorageKeyError)
 
     ?path.secureWriteFile(bytes).mapFailure(StorageKeyError)

tests/config.nims

@@ -7,8 +7,8 @@
 
 when not defined(chronicles_log_level):
   --define:
-    "chronicles_log_level:NONE" # compile all log statements
+    "chronicles_log_level:TRACE" # compile trace log statements
   --define:
     "chronicles_sinks:textlines[dynamic]" # allow logs to be filtered at runtime
   --"import":
-    "logging" # ensure that logging is ignored at runtime
+    "logging_config" # ensure that logging is ignored at runtime

tests/logging_config.nim

@@ -1,5 +1,5 @@
 when not defined(nimscript):
-  import pkg/storage/logutils
+  import pkg/chronicles
 
   proc ignoreLogging(level: LogLevel, message: LogOutputStr) =
     discard

tests/storage/blockexchange/discovery/testdiscovery.nim

@@ -1,6 +1,4 @@
-import std/sequtils
-import std/sugar
-import std/tables
+import std/[options, sequtils, sugar, tables]
 
 import pkg/chronos
 

tests/storage/blockexchange/testnetwork.nim

@@ -15,7 +15,7 @@ import ../helpers
 asyncchecksuite "Network - Handlers":
   let
     rng = Rng.instance()
-    seckey = PrivateKey.random(rng[]).tryGet()
+    seckey = PrivateKey.random(rng.libp2pRng).tryGet()
     peerId = PeerId.init(seckey.getPublicKey().tryGet()).tryGet()
     chunker = RandomChunker.new(Rng.instance(), size = 1024, chunkSize = 256)
 

tests/storage/examples.nim

@@ -2,7 +2,7 @@ import std/[random, sequtils]
 
 import pkg/libp2p
 import pkg/stint
-import pkg/storage/rng
+import pkg/storage/rng as storage_rng
 import pkg/storage/stores
 import pkg/storage/blocktype as bt
 import pkg/storage/merkletree
@@ -16,7 +16,7 @@ proc example*(_: type bt.Block, size: int = 4096): bt.Block =
   bt.Block.new(bytes).tryGet()
 
 proc example*(_: type PeerId): PeerId =
-  let key = PrivateKey.random(Rng.instance[]).get
+  let key = PrivateKey.random(storage_rng.Rng.instance().libp2pRng).get
   PeerId.init(key.getPublicKey().get).get
 
 proc example*(_: type PeerContext): PeerContext =

tests/storage/helpers.nim

@@ -19,11 +19,12 @@ import ./helpers/randomchunker
 import ./helpers/mockchunker
 import ./helpers/mockdiscovery
 import ./helpers/always
+import ./helpers/switchutils
 import ../checktest
 
 export
-  randomchunker, nodeutils, datasetutils, mockdiscovery, mockchunker, always, checktest,
-  manifest
+  randomchunker, nodeutils, datasetutils, mockdiscovery, mockchunker, switchutils,
+  always, checktest, manifest
 
 export libp2p except setup, eventually
 

tests/storage/helpers/nodeutils.nim

@@ -2,7 +2,7 @@ import std/[sequtils, sets]
 
 import pkg/chronos
 import pkg/taskpools
-import pkg/libp2p
+import pkg/libp2p except NATConfig
 import pkg/libp2p/errors
 
 import pkg/codexdht/discv5/routing_table
@@ -22,6 +22,7 @@ import ./datasetutils
 import ./mockdiscovery
 import ../examples
 import ../../helpers
+import ./switchutils
 
 proc nextFreePort*(startPort: int): Future[int] {.async.} =
   proc client(server: StreamServer, transp: StreamTransport) {.async: (raises: []).} =

tests/storage/helpers/switchutils.nim

@@ -0,0 +1,19 @@
+import pkg/libp2p
+import pkg/storage/rng as storage_rng
+
+proc newStandardSwitch*(
+    transportFlags: set[ServerFlags] = {},
+    sendSignedPeerRecord = false,
+    addrs: MultiAddress =
+      MultiAddress.init("/ip4/127.0.0.1/tcp/0").expect("invalid multiaddress"),
+): Switch =
+  SwitchBuilder
+    .new()
+    .withAddresses(@[addrs], enableWildcardResolver = true)
+    .withSignedPeerRecord(sendSignedPeerRecord)
+    .withIdentifyPusher(false)
+    .withRng(storage_rng.Rng.instance().libp2pRng)
+    .withNoise()
+    .withMplex()
+    .withTcpTransport(transportFlags)
+    .build()

tests/storage/node/helpers.nim

@@ -7,6 +7,7 @@ import pkg/storage/chunker
 import pkg/storage/stores
 
 import ../../asynctest
+import ../helpers/switchutils
 
 type CountingStore* = ref object of NetworkStore
   lookups*: Table[Cid, int]

vendor/logos-storage-nim-dht

@@ -1 +1 @@
-Subproject commit 1af8dcf50447b5f68d1843e321c71dd871ecf245
+Subproject commit d13c1c1588338d4622221cc47ba88f642a73a94e

vendor/nim-boringssl

@@ -0,0 +1 @@
+Subproject commit f8111056182cf6abd9e35de77a919e873ef94652

vendor/nim-chronos

@@ -1 +1 @@
-Subproject commit 785fcf4ddec1101a3df1f044d6331504d7ab95c6
+Subproject commit 9620a4691ab8bff1dcab33bc13a15c7012f53eb9

vendor/nim-libp2p

@@ -1 +1 @@
-Subproject commit e82080f7b1aa61c6d35fa5311b873f41eff4bb52
+Subproject commit c43199378f46d0aaf61be1cad1ee1d63e8f665d6

vendor/nim-lsquic

@@ -0,0 +1 @@
+Subproject commit 00e4b7dfaa197cd120267aa897b33b0914166b45

vendor/nim-ngtcp2

@@ -1 +0,0 @@
-Subproject commit 791eb859145f9f268eb23eb9cbe777bdd7699c4d

vendor/nim-protobuf-serialization

@@ -1 +1 @@
-Subproject commit 4d74e157cdf1bdcd0ffd41519ebde740c4b80447
+Subproject commit f45476a3c1f4e7bff73845e6450d686be040ddeb

vendor/nim-quic

@@ -1 +0,0 @@
-Subproject commit 6d8678a159bfb902f9725f0081d542134cd93916

vendor/nim-stew

@@ -1 +1 @@
-Subproject commit b66168735d6f3841c5239c3169d3fe5fe98b1257
+Subproject commit 4382b18f04b3c43c8409bfcd6b62063773b2bbaa

vendor/nim-websock

@@ -1 +1 @@
-Subproject commit 35ae76f1559e835c80f9c1a3943bf995d3dd9eb5
+Subproject commit 387a8eb7e961e8fdd3b1a717d36bc53b55e4dc5d