Merge bd712a1c97 into 67facb4b2a

* Rest API: add erasure coding constraints when requesting storage

* clean up

* Make error message for "dataset too small" more informative.

* fix API integration test

---------

Co-authored-by: gmega <giuliano.mega@gmail.com>

.github/workflows/nim-matrix.yml

@@ -6,7 +6,7 @@ on:
 
 env:
   cache_nonce: 0 # Allows for easily busting actions/cache caches
-  nim_version: pinned, v1.6.16, v1.6.18
+  nim_version: pinned
 
 jobs:
   matrix:

Makefile

@@ -15,7 +15,7 @@
 #
 # If NIM_COMMIT is set to "nimbusbuild", this will use the
 # version pinned by nimbus-build-system.
-PINNED_NIM_VERSION := v1.6.14
+PINNED_NIM_VERSION := 38640664088251bbc88917b4bacfd86ec53014b8 # 1.6.21
 
 ifeq ($(NIM_COMMIT),)
 NIM_COMMIT := $(PINNED_NIM_VERSION)

codex/codex.nim

@@ -16,6 +16,10 @@ import std/cpuinfo
 import pkg/chronos
 import pkg/presto
 import pkg/libp2p
+import pkg/libp2p/protocols/connectivity/autonat/client
+import pkg/libp2p/protocols/connectivity/autonat/service
+import pkg/libp2p/protocols/connectivity/relay/client
+import pkg/libp2p/services/[autorelayservice, hpservice]
 import pkg/confutils
 import pkg/confutils/defs
 import pkg/nitro
@@ -196,24 +200,73 @@ proc stop*(s: CodexServer) {.async.} =
     s.repoStore.stop(),
     s.maintenance.stop())
 
+proc getAutonatService*(rng: ref HmacDrbgContext): AutonatService =
+  ## AutonatService request other peers to dial us back
+  ## flagging us as Reachable or NotReachable.
+  ## minConfidence is used as threshold to determine the state.
+  ## If maxQueueSize > numPeersToAsk past samples are considered
+  ## in the calculation.
+  ##
+
+  let
+    autonatService = AutonatService.new(
+      autonatClient = AutonatClient.new(),
+      rng = rng,
+      scheduleInterval = Opt.some(chronos.seconds(120)),
+      askNewConnectedPeers = false,
+      numPeersToAsk = 3,
+      maxQueueSize = 3,
+      minConfidence = 0.7)
+
+  proc statusAndConfidenceHandler(
+    networkReachability: NetworkReachability,
+    confidence: Opt[float]): Future[void] {.gcsafe, async.} =
+    if confidence.isSome():
+      info "Peer reachability status",
+        networkReachability = networkReachability, confidence = confidence.get()
+
+  autonatService.statusAndConfidenceHandler(statusAndConfidenceHandler)
+
+  return autonatService
+
 proc new*(
   T: type CodexServer,
   config: CodexConf,
   privateKey: CodexPrivateKey): CodexServer =
   ## create CodexServer including setting up datastore, repostore, etc
+  ##
+
+  var
+    builder = SwitchBuilder.new()
+      .withPrivateKey(privateKey)
+      .withAddresses(config.listenAddrs)
+      .withRng(Rng.instance())
+      .withNoise()
+      .withYamux()
+      .withMplex(5.minutes, 5.minutes)
+      .withMaxConnections(config.maxPeers)
+      .withAgentVersion(config.agentString)
+      .withSignedPeerRecord(true)
+      .withTcpTransport({ServerFlags.ReuseAddr})
+      .withAutonat()
+      .withRendezVous()
+      # .withObservedAddrManager()
+
+  builder = if config.lpRelay:
+      builder.withCircuitRelay()
+    else:
+      let
+        relayClient = RelayClient.new()
+        autoRelayService = AutoRelayService.new(1, relayClient, nil, Rng.instance())
+        autonatService = getAutonatService(Rng.instance())
+        hpservice = HPService.new(autonatService, autoRelayService)
+
+      builder
+        .withCircuitRelay(relayClient)
+        .withServices(@[Service(hpservice)])
+
   let
-    switch = SwitchBuilder
-    .new()
-    .withPrivateKey(privateKey)
-    .withAddresses(config.listenAddrs)
-    .withRng(Rng.instance())
-    .withNoise()
-    .withMplex(5.minutes, 5.minutes)
-    .withMaxConnections(config.maxPeers)
-    .withAgentVersion(config.agentString)
-    .withSignedPeerRecord(true)
-    .withTcpTransport({ServerFlags.ReuseAddr})
-    .build()
+    switch = builder.build
 
   var
     cache: CacheStore = nil

codex/conf.nim

@@ -167,6 +167,10 @@ type
       abbr: "b"
       name: "bootstrap-node" }: seq[SignedPeerRecord]
 
+    lpRelay* {.
+      desc: "Should this node be a circuit relay"
+      name: "libp2p-relay" }: bool
+
     maxPeers* {.
       desc: "The maximum number of peers to connect to"
       defaultValue: 160

codex/erasure/erasure.nim

@@ -27,6 +27,7 @@ import ../blocktype as bt
 import ../utils
 import ../utils/asynciter
 import ../indexingstrategy
+import ../errors
 
 import pkg/stew/byteutils
 
@@ -82,6 +83,13 @@ type
     blocksCount: Natural
     strategy: StrategyType
 
+  ErasureError* = object of CodexError
+  InsufficientBlocksError* = object of ErasureError
+    # Minimum size, in bytes, that the dataset must have had
+    # for the encoding request to have succeeded with the parameters
+    # provided.
+    minSize*: NBytes
+
 func indexToPos(steps, idx, step: int): int {.inline.} =
   ## Convert an index to a position in the encoded
   ##  dataset
@@ -236,11 +244,13 @@ proc init*(
   ecK: Natural, ecM: Natural,
   strategy: StrategyType): ?!EncodingParams =
   if ecK > manifest.blocksCount:
-    return failure(
-      "Unable to encode manifest, not enough blocks, ecK = " &
+    let exc = (ref InsufficientBlocksError)(
+      msg: "Unable to encode manifest, not enough blocks, ecK = " &
       $ecK &
       ", blocksCount = " &
-      $manifest.blocksCount)
+      $manifest.blocksCount,
+      minSize: ecK.NBytes * manifest.blockSize)
+    return failure(exc)
 
   let
     rounded = roundUp(manifest.blocksCount, ecK)

codex/rest/api.nim

@@ -32,6 +32,7 @@ import ../node
 import ../blocktype
 import ../conf
 import ../contracts
+import ../erasure/erasure
 import ../manifest
 import ../streams/asyncstreamwrapper
 import ../stores
@@ -432,8 +433,16 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
         let nodes = params.nodes |? 1
         let tolerance = params.tolerance |? 0
 
-        if (nodes - tolerance) < 1:
-          return RestApiResponse.error(Http400, "Tolerance cannot be greater or equal than nodes (nodes - tolerance)")
+        # prevent underflow
+        if tolerance > nodes:
+          return RestApiResponse.error(Http400, "Invalid parameters: `tolerance` cannot be greater than `nodes`")
+
+        let ecK = nodes - tolerance
+        let ecM = tolerance # for readability
+
+        # ensure leopard constrainst of 1 < K ≥ M
+        if ecK <= 1 or ecK < ecM:
+          return RestApiResponse.error(Http400, "Invalid parameters: parameters must satify `1 < (nodes - tolerance) ≥ tolerance`")
 
         without expiry =? params.expiry:
           return RestApiResponse.error(Http400, "Expiry required")
@@ -451,6 +460,11 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
           params.collateral,
           expiry), error:
 
+          if error of InsufficientBlocksError:
+            return RestApiResponse.error(Http400,
+            "Dataset too small for erasure parameters, need at least " &
+              $(ref InsufficientBlocksError)(error).minSize.int & " bytes")
+
           return RestApiResponse.error(Http500, error.msg)
 
         return RestApiResponse.response(purchaseId.toHex)

codex/utils/addrutils.nim

@@ -17,10 +17,9 @@ import pkg/libp2p
 import pkg/stew/shims/net
 
 func remapAddr*(
-    address: MultiAddress,
-    ip: Option[ValidIpAddress] = ValidIpAddress.none,
-    port: Option[Port] = Port.none
-): MultiAddress =
+  address: MultiAddress,
+  ip: Option[ValidIpAddress] = ValidIpAddress.none,
+  port: Option[Port] = Port.none): MultiAddress =
   ## Remap addresses to new IP and/or Port
   ##
 

tests/integration/codexclient.nim

@@ -96,7 +96,7 @@ proc requestStorageRaw*(
     proofProbability: UInt256,
     collateral: UInt256,
     expiry: uint = 0,
-    nodes: uint = 1,
+    nodes: uint = 2,
     tolerance: uint = 0
 ): Response =
 
@@ -125,7 +125,7 @@ proc requestStorage*(
     proofProbability: UInt256,
     expiry: uint,
     collateral: UInt256,
-    nodes: uint = 1,
+    nodes: uint = 2,
     tolerance: uint = 0
 ): ?!PurchaseId =
   ## Call request storage REST endpoint

tests/integration/testpurchasing.nim

@@ -8,7 +8,8 @@ import ../examples
 twonodessuite "Purchasing", debug1 = false, debug2 = false:
 
   test "node handles storage request":
-    let cid = client1.upload("some file contents").get
+    let data = await RandomChunker.example(blocks=2)
+    let cid = client1.upload(data).get
     let id1 = client1.requestStorage(cid, duration=100.u256, reward=2.u256, proofProbability=3.u256, expiry=10, collateral=200.u256).get
     let id2 = client1.requestStorage(cid, duration=400.u256, reward=5.u256, proofProbability=6.u256, expiry=10, collateral=201.u256).get
     check id1 != id2
@@ -26,7 +27,7 @@ twonodessuite "Purchasing", debug1 = false, debug2 = false:
       proofProbability=3.u256,
       expiry=30,
       collateral=200.u256,
-      nodes=2,
+      nodes=3,
       tolerance=1).get
 
     let request = client1.getPurchase(id).get.request.get
@@ -35,7 +36,7 @@ twonodessuite "Purchasing", debug1 = false, debug2 = false:
     check request.ask.proofProbability == 3.u256
     check request.expiry == 30
     check request.ask.collateral == 200.u256
-    check request.ask.slots == 2'u64
+    check request.ask.slots == 3'u64
     check request.ask.maxSlotLoss == 1'u64
 
   # TODO: We currently do not support encoding single chunks
@@ -52,7 +53,8 @@ twonodessuite "Purchasing", debug1 = false, debug2 = false:
   #   check request.ask.maxSlotLoss == 1'u64
 
   test "node remembers purchase status after restart":
-    let cid = client1.upload("some file contents").get
+    let data = await RandomChunker.example(blocks=2)
+    let cid = client1.upload(data).get
     let id = client1.requestStorage(cid,
                                     duration=100.u256,
                                     reward=2.u256,
@@ -71,25 +73,12 @@ twonodessuite "Purchasing", debug1 = false, debug2 = false:
     check request.ask.proofProbability == 3.u256
     check request.expiry == 30
     check request.ask.collateral == 200.u256
-    check request.ask.slots == 1'u64
+    check request.ask.slots == 2'u64
     check request.ask.maxSlotLoss == 0'u64
 
-  test "request storage fails if nodes and tolerance aren't correct":
-    let cid = client1.upload("some file contents").get
-    let responseBefore = client1.requestStorageRaw(cid,
-      duration=100.u256,
-      reward=2.u256,
-      proofProbability=3.u256,
-      expiry=30,
-      collateral=200.u256,
-      nodes=1,
-      tolerance=1)
-
-    check responseBefore.status == "400 Bad Request"
-    check responseBefore.body == "Tolerance cannot be greater or equal than nodes (nodes - tolerance)"
-
   test "node requires expiry and its value to be in future":
-    let cid = client1.upload("some file contents").get
+    let data = await RandomChunker.example(blocks=2)
+    let cid = client1.upload(data).get
 
     let responseMissing = client1.requestStorageRaw(cid, duration=1.u256, reward=2.u256, proofProbability=3.u256, collateral=200.u256)
     check responseMissing.status == "400 Bad Request"

tests/integration/testrestapi.nim

@@ -1,7 +1,9 @@
+import std/httpclient
 import std/sequtils
 from pkg/libp2p import `==`
 import pkg/codex/units
 import ./twonodes
+import ../examples
 
 twonodessuite "REST API", debug1 = false, debug2 = false:
 
@@ -36,3 +38,93 @@ twonodessuite "REST API", debug1 = false, debug2 = false:
 
     check:
       [cid1, cid2].allIt(it in list.content.mapIt(it.cid))
+
+  test "request storage fails for datasets that are too small":
+    let cid = client1.upload("some file contents").get
+    let response = client1.requestStorageRaw(cid, duration=10.u256, reward=2.u256, proofProbability=3.u256, nodes=2, collateral=200.u256, expiry=9)
+
+    check:
+      response.status == "400 Bad Request"
+      response.body == "Dataset too small for erasure parameters, need at least "  & $(2*DefaultBlockSize.int) & " bytes"
+
+  test "request storage succeeds for sufficiently sized datasets":
+    let data = await RandomChunker.example(blocks=2)
+    let cid = client1.upload(data).get
+    let response = client1.requestStorageRaw(cid, duration=10.u256, reward=2.u256, proofProbability=3.u256, collateral=200.u256, expiry=9)
+
+    check:
+      response.status == "200 OK"
+
+  test "request storage fails if nodes and tolerance aren't correct":
+    let data = await RandomChunker.example(blocks=2)
+    let cid = client1.upload(data).get
+    let duration = 100.u256
+    let reward = 2.u256
+    let proofProbability = 3.u256
+    let expiry = 30.uint
+    let collateral = 200.u256
+    let ecParams = @[(1, 0), (1, 1), (2, 1), (3, 2), (3, 3)]
+
+    for ecParam in ecParams:
+      let (nodes, tolerance) = ecParam
+
+      var responseBefore = client1.requestStorageRaw(cid,
+        duration,
+        reward,
+        proofProbability,
+        collateral,
+        expiry,
+        nodes.uint,
+        tolerance.uint)
+
+      check responseBefore.status == "400 Bad Request"
+      check responseBefore.body == "Invalid parameters: parameters must satify `1 < (nodes - tolerance) ≥ tolerance`"
+
+  test "request storage fails if tolerance > nodes (underflow protection)":
+    let data = await RandomChunker.example(blocks=2)
+    let cid = client1.upload(data).get
+    let duration = 100.u256
+    let reward = 2.u256
+    let proofProbability = 3.u256
+    let expiry = 30.uint
+    let collateral = 200.u256
+    let ecParams = @[(0, 1), (1, 2), (2, 3)]
+
+    for ecParam in ecParams:
+      let (nodes, tolerance) = ecParam
+
+      var responseBefore = client1.requestStorageRaw(cid,
+        duration,
+        reward,
+        proofProbability,
+        collateral,
+        expiry,
+        nodes.uint,
+        tolerance.uint)
+
+      check responseBefore.status == "400 Bad Request"
+      check responseBefore.body == "Invalid parameters: `tolerance` cannot be greater than `nodes`"
+
+  test "request storage succeeds if nodes and tolerance within range":
+    let data = await RandomChunker.example(blocks=2)
+    let cid = client1.upload(data).get
+    let duration = 100.u256
+    let reward = 2.u256
+    let proofProbability = 3.u256
+    let expiry = 30.uint
+    let collateral = 200.u256
+    let ecParams = @[(2, 0), (3, 1), (5, 2)]
+
+    for ecParam in ecParams:
+      let (nodes, tolerance) = ecParam
+
+      var responseBefore = client1.requestStorageRaw(cid,
+        duration,
+        reward,
+        proofProbability,
+        collateral,
+        expiry,
+        nodes.uint,
+        tolerance.uint)
+
+      check responseBefore.status == "200 OK"