Merge remote-tracking branch 'origin/master' into node-wire-prover

BUILDING.md

@@ -24,7 +24,7 @@ Other approaches may be viable. On macOS, some users may prefer [MacPorts](https
 
 ### Rust
 
-The current implementation of Codex's zero-knowledge proving circuit requires the installation of rust v1.76.0 or greater. Be sure to install it for your OS and add it to your terminal's path such that the command `cargo --version` gives a compatible version.
+The current implementation of Codex's zero-knowledge proving circuit requires the installation of rust v1.74.0 or greater. Be sure to install it for your OS and add it to your terminal's path such that the command `cargo --version` gives a compatible version.
 
 ### Linux
 
@@ -32,35 +32,36 @@ The current implementation of Codex's zero-knowledge proving circuit requires th
 
 On a bare bones installation of Debian (or a distribution derived from Debian, such as Ubuntu), run
 
-```text
-$ apt-get update && apt-get install build-essential cmake curl git
+```shell
+apt-get update && apt-get install build-essential cmake curl git rustc cargo
 ```
 
 Non-Debian distributions have different package managers: `apk`, `dnf`, `pacman`, `rpm`, `yum`, etc.
 
 For example, on a bare bones installation of Fedora, run
 
-```text
-$ dnf install @development-tools cmake gcc-c++ which
+```shell
+dnf install @development-tools cmake gcc-c++ rust cargo
 ```
 
 ### macOS
 
 Install the [Xcode Command Line Tools](https://mac.install.guide/commandlinetools/index.html) by opening a terminal and running
-```text
-$ xcode-select --install
+```shell
+xcode-select --install
 ```
 
 Install [Homebrew (`brew`)](https://brew.sh/) and in a new terminal run
-```text
-$ brew install bash cmake
+```shell
+brew install bash cmake rust
 ```
 
 Check that `PATH` is setup correctly
-```text
-$ which bash cmake
-/usr/local/bin/bash
-/usr/local/bin/cmake
+```shell
+which bash cmake
+
+# /usr/local/bin/bash
+# /usr/local/bin/cmake
 ```
 
 ### Windows + MSYS2
@@ -72,9 +73,9 @@ Download and run the installer from [msys2.org](https://www.msys2.org/).
 Launch an MSYS2 [environment](https://www.msys2.org/docs/environments/). UCRT64 is generally recommended: from the Windows *Start menu* select `MSYS2 MinGW UCRT x64`.
 
 Assuming a UCRT64 environment, in Bash run
-```text
-$ pacman -Suy
-$ pacman -S base-devel git unzip mingw-w64-ucrt-x86_64-toolchain mingw-w64-ucrt-x86_64-cmake mingw-w64-ucrt-x86_64-rust
+```shell
+pacman -Suy
+pacman -S base-devel git unzip mingw-w64-ucrt-x86_64-toolchain mingw-w64-ucrt-x86_64-cmake mingw-w64-ucrt-x86_64-rust
 ```
 
 <!-- #### Headless Windows container -->
@@ -113,27 +114,27 @@ It is possible that nim-codex can be built and run on other platforms supported
 ## Repository
 
 In Bash run
-```text
-$ git clone https://github.com/codex-storage/nim-codex.git repos/nim-codex && cd repos/nim-codex
+```shell
+git clone https://github.com/codex-storage/nim-codex.git repos/nim-codex && cd repos/nim-codex
 ```
 
-nim-codex uses the [nimbus-build-system](https://github.com/status-im/nimbus-build-system#readme), so next run
-```text
-$ make update
+nim-codex uses the [nimbus-build-system](https://github.com/status-im/nimbus-build-system), so next run
+```shell
+make update
 ```
 
 This step can take a while to complete because by default it builds the [Nim compiler](https://nim-lang.org/docs/nimc.html).
 
 To see more output from `make` pass `V=1`. This works for all `make` targets in projects using the nimbus-build-system
-```text
-$ make V=1 update
+```shell
+make V=1 update
 ```
 
 ## Executable
 
 In Bash run
-```text
-$ make
+```shell
+make
 ```
 
 The default `make` target creates the `build/codex` executable.
@@ -145,8 +146,8 @@ See the [instructions](README.md#cli-options) in the main readme.
 ## Tests
 
 In Bash run
-```text
-$ make test
+```shell
+make test
 ```
 
 ### testAll
@@ -156,9 +157,11 @@ $ make test
 To run the integration tests, an Ethereum test node is required. Follow these instructions to set it up.
 
 ##### Windows (do this before 'All platforms')
+
 1. Download and install Visual Studio 2017 or newer. (Not VSCode!) In the Workloads overview, enable `Desktop development with C++`. ( https://visualstudio.microsoft.com )
 
 ##### All platforms
+
 1. Install NodeJS (tested with v18.14.0), consider using NVM as a version manager. [Node Version Manager (`nvm`)](https://github.com/nvm-sh/nvm#readme)
 1. Open a terminal
 1. Go to the vendor/codex-contracts-eth folder: `cd /<git-root>/vendor/codex-contracts-eth/`
@@ -177,6 +180,6 @@ The `testAll` target runs the same tests as `make test` and also runs tests for
 To run `make testAll`.
 
 Use a new terminal to run:
-```text
-$ make testAll
+```shell
+make testAll
 ```

codex.nimble

@@ -12,7 +12,7 @@ requires "bearssl >= 0.1.4"
 requires "chronicles >= 0.7.2"
 requires "chronos >= 2.5.2"
 requires "confutils"
-requires "ethers >= 0.7.1 & < 0.8.0"
+requires "ethers >= 0.7.3 & < 0.8.0"
 requires "libbacktrace"
 requires "libp2p"
 requires "metrics"

codex/blockexchange/engine/engine.nim

@@ -540,13 +540,19 @@ proc taskHandler*(b: BlockExcEngine, task: BlockExcPeerCtx) {.gcsafe, async.} =
 
   var
     wantsBlocks = task.peerWants.filterIt(
-      it.wantType == WantType.WantBlock
+      it.wantType == WantType.WantBlock and not it.inFlight
     )
 
+  proc updateInFlight(addresses: seq[BlockAddress], inFlight: bool) =
+    for peerWant in task.peerWants.mitems:
+      if peerWant.address in addresses:
+        peerWant.inFlight = inFlight
+
   trace "wantsBlocks", peer = task.id, n = wantsBlocks.len
   if wantsBlocks.len > 0:
-    trace "Got peer want blocks list", items = wantsBlocks.len
-
+    # Mark wants as in-flight.
+    let wantAddresses = wantsBlocks.mapIt(it.address)
+    updateInFlight(wantAddresses, true)
     wantsBlocks.sort(SortOrder.Descending)
 
     proc localLookup(e: WantListEntry): Future[?!BlockDelivery] {.async.} =
@@ -563,13 +569,16 @@ proc taskHandler*(b: BlockExcEngine, task: BlockExcPeerCtx) {.gcsafe, async.} =
 
     let
       blocksDeliveryFut = await allFinished(wantsBlocks.map(localLookup))
-
-    # Extract successfully received blocks
-    let
       blocksDelivery = blocksDeliveryFut
         .filterIt(it.completed and it.read.isOk)
         .mapIt(it.read.get)
 
+    # All the wants that failed local lookup must be set to not-in-flight again.
+    let
+      successAddresses = blocksDelivery.mapIt(it.address)
+      failedAddresses = wantAddresses.filterIt(it notin successAddresses)
+    updateInFlight(failedAddresses, false)
+
     if blocksDelivery.len > 0:
       trace "Sending blocks to peer", peer = task.id, blocks = blocksDelivery.len
       await b.network.request.sendBlocksDelivery(
@@ -579,13 +588,8 @@ proc taskHandler*(b: BlockExcEngine, task: BlockExcPeerCtx) {.gcsafe, async.} =
 
       codex_block_exchange_blocks_sent.inc(blocksDelivery.len.int64)
 
-      trace "About to remove entries from peerWants", blocks = blocksDelivery.len, items = task.peerWants.len
-      # Remove successfully sent blocks
-      task.peerWants.keepIf(
-        proc(e: WantListEntry): bool =
-          not blocksDelivery.anyIt( it.address == e.address )
-      )
-      trace "Removed entries from peerWants", items = task.peerWants.len
+      task.peerWants.keepItIf(it.address notin successAddresses)
+      trace "Removed entries from peerWants", peerWants = task.peerWants.len
 
 proc blockexcTaskRunner(b: BlockExcEngine) {.async.} =
   ## process tasks

codex/blockexchange/protobuf/message.nim

@@ -29,6 +29,7 @@ type
     cancel*: bool           # Whether this revokes an entry
     wantType*: WantType     # Note: defaults to enum 0, ie Block
     sendDontHave*: bool     # Note: defaults to false
+    inFlight*: bool         # Whether block sending is in progress. Not serialized.
 
   WantList* = object
     entries*: seq[WantListEntry]    # A list of wantList entries

codex/codex.nim

@@ -58,6 +58,14 @@ type
   CodexPrivateKey* = libp2p.PrivateKey # alias
   EthWallet = ethers.Wallet
 
+proc waitForSync(provider: Provider): Future[void] {.async.} =
+  var sleepTime = 1
+  while await provider.isSyncing:
+    notice "Waiting for Ethereum provider to sync..."
+    await sleepAsync(sleepTime.seconds)
+    if sleepTime < 10:
+      inc sleepTime
+
 proc bootstrapInteractions(
   s: CodexServer): Future[void] {.async.} =
   ## bootstrap interactions and return contracts
@@ -73,6 +81,7 @@ proc bootstrapInteractions(
       quit QuitFailure
 
     let provider = JsonRpcProvider.new(config.ethProvider)
+    await waitForSync(provider)
     var signer: Signer
     if account =? config.ethAccount:
       signer = provider.getSigner(account)

docker/codex.Dockerfile

@@ -1,5 +1,5 @@
 # Variables
-ARG BUILDER=ubuntu:lunar-20230415
+ARG BUILDER=ubuntu:22.04
 ARG IMAGE=${BUILDER}
 ARG BUILD_HOME=/src
 ARG MAKE_PARALLEL=${MAKE_PARALLEL:-4}
@@ -13,9 +13,7 @@ ARG BUILD_HOME
 ARG MAKE_PARALLEL
 ARG NIMFLAGS
 
-RUN apt-get update && apt-get install -y git cmake curl make bash lcov build-essential nim
-RUN curl --proto '=https' --tlsv1.3 https://sh.rustup.rs -sSf | sh -s -- -y
-ENV PATH="/root/.cargo/bin:${PATH}"
+RUN apt-get update && apt-get install -y git cmake curl make bash lcov build-essential rustc cargo
 
 WORKDIR ${BUILD_HOME}
 COPY . .
@@ -32,7 +30,7 @@ ARG NAT_IP_AUTO
 WORKDIR ${APP_HOME}
 COPY --from=builder ${BUILD_HOME}/build/codex /usr/local/bin
 COPY --chmod=0755 docker/docker-entrypoint.sh /
-RUN apt-get update && apt-get install -y libgomp1 bash curl && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y libgomp1 bash curl jq && rm -rf /var/lib/apt/lists/*
 ENV NAT_IP_AUTO=${NAT_IP_AUTO}
 ENTRYPOINT ["/docker-entrypoint.sh"]
 CMD ["codex"]

docker/docker-entrypoint.sh

@@ -2,7 +2,9 @@
 
 # Environment variables from files
 if [[ -n "${ENV_PATH}" ]]; then
-  [[ -f ${ENV_PATH} ]] && source ${ENV_PATH} || for f in ${ENV_PATH}/*; do source $f; done
+  set -a
+  [[ -f "${ENV_PATH}" ]] && source "${ENV_PATH}" || for f in "${ENV_PATH}"/*; do source "$f"; done
+  set +a
 fi
 
 # Parameters

tests/codex/blockexchange/engine/testengine.nim

@@ -490,6 +490,39 @@ asyncchecksuite "Task Handler":
 
     await engine.taskHandler(peersCtx[0])
 
+  test "Should set in-flight for outgoing blocks":
+    proc sendBlocksDelivery(
+      id: PeerId,
+      blocksDelivery: seq[BlockDelivery]) {.gcsafe, async.} =
+      check peersCtx[0].peerWants[0].inFlight
+
+    for blk in blocks:
+      (await engine.localStore.putBlock(blk)).tryGet()
+    engine.network.request.sendBlocksDelivery = sendBlocksDelivery
+
+    peersCtx[0].peerWants.add(WantListEntry(
+      address: blocks[0].address,
+      priority: 50,
+      cancel: false,
+      wantType: WantType.WantBlock,
+      sendDontHave: false,
+      inFlight: false)
+    )
+    await engine.taskHandler(peersCtx[0])
+
+  test "Should clear in-flight when local lookup fails":
+    peersCtx[0].peerWants.add(WantListEntry(
+      address: blocks[0].address,
+      priority: 50,
+      cancel: false,
+      wantType: WantType.WantBlock,
+      sendDontHave: false,
+      inFlight: false)
+    )
+    await engine.taskHandler(peersCtx[0])
+
+    check not peersCtx[0].peerWants[0].inFlight
+
   test "Should send presence":
     let present = blocks
     let missing = @[Block.new("missing".toBytes).tryGet()]