From 5e7ce52fbe650f459eb8923fd07386131aef4067 Mon Sep 17 00:00:00 2001 From: Ben Bierens <39762930+benbierens@users.noreply.github.com> Date: Thu, 29 Feb 2024 08:37:12 +0100 Subject: [PATCH 1/3] Fix block retransmit (#651) * Applies peer-scoped lock to peer task handler. * Replace async lock with delete-first approach. * Cleanup some logging * Adds inFlight flag to WantListEntry * Clears inflight flag when local retrieval fails. * Adds test for setting of in-flight * Adds test for clearing in-flight when lookup fails * Review comments by Tomasz --------- Co-authored-by: gmega --- codex/blockexchange/engine/engine.nim | 30 +++++++++-------- codex/blockexchange/protobuf/message.nim | 1 + .../codex/blockexchange/engine/testengine.nim | 33 +++++++++++++++++++ 3 files changed, 51 insertions(+), 13 deletions(-) diff --git a/codex/blockexchange/engine/engine.nim b/codex/blockexchange/engine/engine.nim index 921e9550..97e7cb99 100644 --- a/codex/blockexchange/engine/engine.nim +++ b/codex/blockexchange/engine/engine.nim @@ -532,13 +532,19 @@ proc taskHandler*(b: BlockExcEngine, task: BlockExcPeerCtx) {.gcsafe, async.} = var wantsBlocks = task.peerWants.filterIt( - it.wantType == WantType.WantBlock + it.wantType == WantType.WantBlock and not it.inFlight ) + proc updateInFlight(addresses: seq[BlockAddress], inFlight: bool) = + for peerWant in task.peerWants.mitems: + if peerWant.address in addresses: + peerWant.inFlight = inFlight + trace "wantsBlocks", peer = task.id, n = wantsBlocks.len if wantsBlocks.len > 0: - trace "Got peer want blocks list", items = wantsBlocks.len - + # Mark wants as in-flight. + let wantAddresses = wantsBlocks.mapIt(it.address) + updateInFlight(wantAddresses, true) wantsBlocks.sort(SortOrder.Descending) proc localLookup(e: WantListEntry): Future[?!BlockDelivery] {.async.} = @@ -555,13 +561,16 @@ proc taskHandler*(b: BlockExcEngine, task: BlockExcPeerCtx) {.gcsafe, async.} = let blocksDeliveryFut = await allFinished(wantsBlocks.map(localLookup)) - - # Extract successfully received blocks - let blocksDelivery = blocksDeliveryFut .filterIt(it.completed and it.read.isOk) .mapIt(it.read.get) + # All the wants that failed local lookup must be set to not-in-flight again. + let + successAddresses = blocksDelivery.mapIt(it.address) + failedAddresses = wantAddresses.filterIt(it notin successAddresses) + updateInFlight(failedAddresses, false) + if blocksDelivery.len > 0: trace "Sending blocks to peer", peer = task.id, blocks = blocksDelivery.len await b.network.request.sendBlocksDelivery( @@ -571,13 +580,8 @@ proc taskHandler*(b: BlockExcEngine, task: BlockExcPeerCtx) {.gcsafe, async.} = codex_block_exchange_blocks_sent.inc(blocksDelivery.len.int64) - trace "About to remove entries from peerWants", blocks = blocksDelivery.len, items = task.peerWants.len - # Remove successfully sent blocks - task.peerWants.keepIf( - proc(e: WantListEntry): bool = - not blocksDelivery.anyIt( it.address == e.address ) - ) - trace "Removed entries from peerWants", items = task.peerWants.len + task.peerWants.keepItIf(it.address notin successAddresses) + trace "Removed entries from peerWants", peerWants = task.peerWants.len proc blockexcTaskRunner(b: BlockExcEngine) {.async.} = ## process tasks diff --git a/codex/blockexchange/protobuf/message.nim b/codex/blockexchange/protobuf/message.nim index 7624bd9c..61488b40 100644 --- a/codex/blockexchange/protobuf/message.nim +++ b/codex/blockexchange/protobuf/message.nim @@ -29,6 +29,7 @@ type cancel*: bool # Whether this revokes an entry wantType*: WantType # Note: defaults to enum 0, ie Block sendDontHave*: bool # Note: defaults to false + inFlight*: bool # Whether block sending is in progress. Not serialized. WantList* = object entries*: seq[WantListEntry] # A list of wantList entries diff --git a/tests/codex/blockexchange/engine/testengine.nim b/tests/codex/blockexchange/engine/testengine.nim index 4cf84724..96571cf3 100644 --- a/tests/codex/blockexchange/engine/testengine.nim +++ b/tests/codex/blockexchange/engine/testengine.nim @@ -490,6 +490,39 @@ asyncchecksuite "Task Handler": await engine.taskHandler(peersCtx[0]) + test "Should set in-flight for outgoing blocks": + proc sendBlocksDelivery( + id: PeerId, + blocksDelivery: seq[BlockDelivery]) {.gcsafe, async.} = + check peersCtx[0].peerWants[0].inFlight + + for blk in blocks: + (await engine.localStore.putBlock(blk)).tryGet() + engine.network.request.sendBlocksDelivery = sendBlocksDelivery + + peersCtx[0].peerWants.add(WantListEntry( + address: blocks[0].address, + priority: 50, + cancel: false, + wantType: WantType.WantBlock, + sendDontHave: false, + inFlight: false) + ) + await engine.taskHandler(peersCtx[0]) + + test "Should clear in-flight when local lookup fails": + peersCtx[0].peerWants.add(WantListEntry( + address: blocks[0].address, + priority: 50, + cancel: false, + wantType: WantType.WantBlock, + sendDontHave: false, + inFlight: false) + ) + await engine.taskHandler(peersCtx[0]) + + check not peersCtx[0].peerWants[0].inFlight + test "Should send presence": let present = blocks let missing = @[Block.new("missing".toBytes).tryGet()] From 53fdc88def1e2ee01456a12e49eaaec87a71e336 Mon Sep 17 00:00:00 2001 From: Ben Bierens <39762930+benbierens@users.noreply.github.com> Date: Fri, 1 Mar 2024 08:50:29 +0100 Subject: [PATCH 2/3] Feature/eth syncing (#721) * Delays starting of codex until eth provider is synced * bumps nim-ethers to 0.7.2 * bumps nim-ethers to 0.7.3 * Adds increasing delay to waitForSync --- codex.nimble | 2 +- codex/codex.nim | 9 +++++++++ vendor/nim-ethers | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/codex.nimble b/codex.nimble index 41da92b7..42347710 100644 --- a/codex.nimble +++ b/codex.nimble @@ -12,7 +12,7 @@ requires "bearssl >= 0.1.4" requires "chronicles >= 0.7.2" requires "chronos >= 2.5.2" requires "confutils" -requires "ethers >= 0.7.1 & < 0.8.0" +requires "ethers >= 0.7.3 & < 0.8.0" requires "libbacktrace" requires "libp2p" requires "metrics" diff --git a/codex/codex.nim b/codex/codex.nim index e5f1bfba..c5bcb7b6 100644 --- a/codex/codex.nim +++ b/codex/codex.nim @@ -56,6 +56,14 @@ type CodexPrivateKey* = libp2p.PrivateKey # alias EthWallet = ethers.Wallet +proc waitForSync(provider: Provider): Future[void] {.async.} = + var sleepTime = 1 + while await provider.isSyncing: + notice "Waiting for Ethereum provider to sync..." + await sleepAsync(sleepTime.seconds) + if sleepTime < 10: + inc sleepTime + proc bootstrapInteractions( s: CodexServer): Future[void] {.async.} = ## bootstrap interactions and return contracts @@ -79,6 +87,7 @@ proc bootstrapInteractions( quit QuitFailure let provider = JsonRpcProvider.new(config.ethProvider) + await waitForSync(provider) var signer: Signer if account =? config.ethAccount: signer = provider.getSigner(account) diff --git a/vendor/nim-ethers b/vendor/nim-ethers index fd16d71e..4c7e351f 160000 --- a/vendor/nim-ethers +++ b/vendor/nim-ethers @@ -1 +1 @@ -Subproject commit fd16d71ea58e8d3e5a7f41e5dfdaa3a9b693a815 +Subproject commit 4c7e351fd95af779786705eac0612b6a52b13380 From 6ee116298929c5419f055abda5eccb7e25caef11 Mon Sep 17 00:00:00 2001 From: Slava <20563034+veaceslavdoina@users.noreply.github.com> Date: Fri, 1 Mar 2024 14:23:05 +0200 Subject: [PATCH 3/3] Update Docker files and building Codex docs (#722) * Make Build Codex copy/paste friendly * Add auto-export for env var from files * Use latest Ubuntu LTS image and install rust via package manager --- BUILDING.md | 61 +++++++++++++++++++------------------ docker/codex.Dockerfile | 8 ++--- docker/docker-entrypoint.sh | 4 ++- 3 files changed, 38 insertions(+), 35 deletions(-) diff --git a/BUILDING.md b/BUILDING.md index 8c72514a..ad66646f 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -24,7 +24,7 @@ Other approaches may be viable. On macOS, some users may prefer [MacPorts](https ### Rust -The current implementation of Codex's zero-knowledge proving circuit requires the installation of rust v1.76.0 or greater. Be sure to install it for your OS and add it to your terminal's path such that the command `cargo --version` gives a compatible version. +The current implementation of Codex's zero-knowledge proving circuit requires the installation of rust v1.74.0 or greater. Be sure to install it for your OS and add it to your terminal's path such that the command `cargo --version` gives a compatible version. ### Linux @@ -32,35 +32,36 @@ The current implementation of Codex's zero-knowledge proving circuit requires th On a bare bones installation of Debian (or a distribution derived from Debian, such as Ubuntu), run -```text -$ apt-get update && apt-get install build-essential cmake curl git +```shell +apt-get update && apt-get install build-essential cmake curl git rustc cargo ``` Non-Debian distributions have different package managers: `apk`, `dnf`, `pacman`, `rpm`, `yum`, etc. For example, on a bare bones installation of Fedora, run -```text -$ dnf install @development-tools cmake gcc-c++ which +```shell +dnf install @development-tools cmake gcc-c++ rust cargo ``` ### macOS Install the [Xcode Command Line Tools](https://mac.install.guide/commandlinetools/index.html) by opening a terminal and running -```text -$ xcode-select --install +```shell +xcode-select --install ``` Install [Homebrew (`brew`)](https://brew.sh/) and in a new terminal run -```text -$ brew install bash cmake +```shell +brew install bash cmake rust ``` Check that `PATH` is setup correctly -```text -$ which bash cmake -/usr/local/bin/bash -/usr/local/bin/cmake +```shell +which bash cmake + +# /usr/local/bin/bash +# /usr/local/bin/cmake ``` ### Windows + MSYS2 @@ -72,9 +73,9 @@ Download and run the installer from [msys2.org](https://www.msys2.org/). Launch an MSYS2 [environment](https://www.msys2.org/docs/environments/). UCRT64 is generally recommended: from the Windows *Start menu* select `MSYS2 MinGW UCRT x64`. Assuming a UCRT64 environment, in Bash run -```text -$ pacman -Suy -$ pacman -S base-devel git unzip mingw-w64-ucrt-x86_64-toolchain mingw-w64-ucrt-x86_64-cmake mingw-w64-ucrt-x86_64-rust +```shell +pacman -Suy +pacman -S base-devel git unzip mingw-w64-ucrt-x86_64-toolchain mingw-w64-ucrt-x86_64-cmake mingw-w64-ucrt-x86_64-rust ``` @@ -113,27 +114,27 @@ It is possible that nim-codex can be built and run on other platforms supported ## Repository In Bash run -```text -$ git clone https://github.com/codex-storage/nim-codex.git repos/nim-codex && cd repos/nim-codex +```shell +git clone https://github.com/codex-storage/nim-codex.git repos/nim-codex && cd repos/nim-codex ``` -nim-codex uses the [nimbus-build-system](https://github.com/status-im/nimbus-build-system#readme), so next run -```text -$ make update +nim-codex uses the [nimbus-build-system](https://github.com/status-im/nimbus-build-system), so next run +```shell +make update ``` This step can take a while to complete because by default it builds the [Nim compiler](https://nim-lang.org/docs/nimc.html). To see more output from `make` pass `V=1`. This works for all `make` targets in projects using the nimbus-build-system -```text -$ make V=1 update +```shell +make V=1 update ``` ## Executable In Bash run -```text -$ make +```shell +make ``` The default `make` target creates the `build/codex` executable. @@ -145,8 +146,8 @@ See the [instructions](README.md#cli-options) in the main readme. ## Tests In Bash run -```text -$ make test +```shell +make test ``` ### testAll @@ -156,9 +157,11 @@ $ make test To run the integration tests, an Ethereum test node is required. Follow these instructions to set it up. ##### Windows (do this before 'All platforms') + 1. Download and install Visual Studio 2017 or newer. (Not VSCode!) In the Workloads overview, enable `Desktop development with C++`. ( https://visualstudio.microsoft.com ) ##### All platforms + 1. Install NodeJS (tested with v18.14.0), consider using NVM as a version manager. [Node Version Manager (`nvm`)](https://github.com/nvm-sh/nvm#readme) 1. Open a terminal 1. Go to the vendor/codex-contracts-eth folder: `cd //vendor/codex-contracts-eth/` @@ -177,6 +180,6 @@ The `testAll` target runs the same tests as `make test` and also runs tests for To run `make testAll`. Use a new terminal to run: -```text -$ make testAll +```shell +make testAll ``` diff --git a/docker/codex.Dockerfile b/docker/codex.Dockerfile index eb312766..673b04a8 100644 --- a/docker/codex.Dockerfile +++ b/docker/codex.Dockerfile @@ -1,5 +1,5 @@ # Variables -ARG BUILDER=ubuntu:lunar-20230415 +ARG BUILDER=ubuntu:22.04 ARG IMAGE=${BUILDER} ARG BUILD_HOME=/src ARG MAKE_PARALLEL=${MAKE_PARALLEL:-4} @@ -13,9 +13,7 @@ ARG BUILD_HOME ARG MAKE_PARALLEL ARG NIMFLAGS -RUN apt-get update && apt-get install -y git cmake curl make bash lcov build-essential nim -RUN curl --proto '=https' --tlsv1.3 https://sh.rustup.rs -sSf | sh -s -- -y -ENV PATH="/root/.cargo/bin:${PATH}" +RUN apt-get update && apt-get install -y git cmake curl make bash lcov build-essential rustc cargo WORKDIR ${BUILD_HOME} COPY . . @@ -32,7 +30,7 @@ ARG NAT_IP_AUTO WORKDIR ${APP_HOME} COPY --from=builder ${BUILD_HOME}/build/codex /usr/local/bin COPY --chmod=0755 docker/docker-entrypoint.sh / -RUN apt-get update && apt-get install -y libgomp1 bash curl && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y libgomp1 bash curl jq && rm -rf /var/lib/apt/lists/* ENV NAT_IP_AUTO=${NAT_IP_AUTO} ENTRYPOINT ["/docker-entrypoint.sh"] CMD ["codex"] diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index f315ddb7..9875d9ae 100644 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -2,7 +2,9 @@ # Environment variables from files if [[ -n "${ENV_PATH}" ]]; then - [[ -f ${ENV_PATH} ]] && source ${ENV_PATH} || for f in ${ENV_PATH}/*; do source $f; done + set -a + [[ -f "${ENV_PATH}" ]] && source "${ENV_PATH}" || for f in "${ENV_PATH}"/*; do source "$f"; done + set +a fi # Parameters