From 8cd10edb69830d3dde79b6cd35468077e3f5072a Mon Sep 17 00:00:00 2001 From: Slava <20563034+veaceslavdoina@users.noreply.github.com> Date: Mon, 28 Jul 2025 13:02:19 +0300 Subject: [PATCH 01/20] ci: auto deploy codex on devnet (#1302) --- .github/workflows/deploy-devnet.yml | 175 ++++++++++++++++++++++++ .github/workflows/docker-dist-tests.yml | 16 +++ .github/workflows/docker-reusable.yml | 50 ++++--- 3 files changed, 220 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/deploy-devnet.yml diff --git a/.github/workflows/deploy-devnet.yml b/.github/workflows/deploy-devnet.yml new file mode 100644 index 00000000..a95a3df0 --- /dev/null +++ b/.github/workflows/deploy-devnet.yml @@ -0,0 +1,175 @@ +name: Deploy - Devnet + + +on: + workflow_dispatch: + inputs: + codex_image: + description: codexstorage/nim-codex:latest-dist-tests + required: false + type: string + workflow_call: + inputs: + codex_image: + description: codexstorage/nim-codex:latest-dist-tests + required: true + type: string + +env: + CODEX_NAMESPACE: codex + TOOLS_NAMESPACE: common + KUBE_CONFIG: ${{ secrets.DEVNET_KUBE_CONFIG }} + KUBE_VERSION: v1.33.1 + CODEX_IMAGE: ${{ inputs.codex_image }} + SSH_HOSTS: ${{ secrets.DEVNET_SSH_HOSTS }} + SSH_PORT: ${{ secrets.DEVNET_SSH_PORT }} + SSH_USERNAME: ${{ secrets.DEVNET_SSH_USERNAME }} + SSH_PRIVATE_KEY: ${{ secrets.DEVNET_SSH_KEY }} + + +jobs: + deploy-contracts: + name: Deploy contracts + runs-on: ubuntu-latest + steps: + - name: Create access token + uses: actions/create-github-app-token@v2 + id: app-token + with: + app-id: ${{ secrets.DEPLOYER_APP_ID }} + private-key: ${{ secrets.DEPLOYER_PRIVATE_KEY }} + repositories: codex-contracts-eth + + - name: Checkout sources + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Get contracts submodule ref + id: contracts + run: echo "ref=$(git rev-parse HEAD:vendor/codex-contracts-eth)" >> $GITHUB_OUTPUT + + - name: Deploy smart contracts + uses: the-actions-org/workflow-dispatch@v4 + with: + repo: codex-storage/codex-contracts-eth + workflow: devnet-contracts.yml + token: ${{ steps.app-token.outputs.token }} + wait-for-completion-timeout: 20m + wait-for-completion-interval: 20s + inputs: '{ "network": "codex_devnet", "contracts_ref": "${{ steps.contracts.outputs.ref }}" }' + + + bootstrap-nodes: + name: Bootstrap nodes + runs-on: ubuntu-latest + needs: deploy-contracts + steps: + - name: Codex Bootstrap - Update + uses: appleboy/ssh-action@v1 + with: + host: ${{ secrets.DEVNET_SSH_HOSTS }} + username: ${{ secrets.DEVNET_SSH_USERNAME }} + key: ${{ secrets.DEVNET_SSH_KEY }} + port: ${{ secrets.DEVNET_SSH_PORT }} + script: /opt/codex/remote-deploy.sh ${{ env.CODEX_IMAGE }} + + cluster-nodes: + name: Cluster nodes + runs-on: ubuntu-latest + needs: bootstrap-nodes + steps: + - name: Kubectl - Install ${{ env.KUBE_VERSION }} + uses: azure/setup-kubectl@v4 + with: + version: ${{ env.KUBE_VERSION }} + + - name: Kubectl - Kubeconfig + run: | + mkdir -p "${HOME}"/.kube + echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config + + - name: Codex Storage - Update + run: | + for node in {1..5}; do + kubectl -n "${{ env.CODEX_NAMESPACE }}" patch statefulset codex-storage-${node} \ + --patch '{"spec": {"template": {"spec":{"containers":[{"name": "codex", "image":"${{ env.CODEX_IMAGE }}"}]}}}}' + done + + - name: Codex Validators - Update + run: | + for node in {1..1}; do + kubectl -n "${{ env.CODEX_NAMESPACE }}" patch statefulset codex-validator-${node} \ + --patch '{"spec": {"template": {"spec":{"containers":[{"name": "codex", "image":"${{ env.CODEX_IMAGE }}"}]}}}}' + done + + - name: Codex Storage - Status + run: | + WAIT=300 + SECONDS=0 + sleep=1 + for instance in {1..5}; do + while (( SECONDS < WAIT )); do + pod=codex-storage-${instance}-1 + phase=$(kubectl get pod "${pod}" -n "${{ env.CODEX_NAMESPACE }}" -o jsonpath='{.status.phase}') + if [[ "${phase}" == "Running" ]]; then + echo "Pod ${pod} is in the ${phase} state" + break + else + echo "Pod ${pod} is in the ${phase} state - Check in ${sleep} second(s) / $((WAIT - SECONDS))" + fi + sleep "${sleep}" + done + done + + - name: Codex Validators - Status + run: | + WAIT=300 + SECONDS=0 + sleep=1 + for instance in {1..1}; do + while (( SECONDS < WAIT )); do + pod=codex-validator-${instance}-1 + phase=$(kubectl get pod "${pod}" -n "${{ env.CODEX_NAMESPACE }}" -o jsonpath='{.status.phase}') + if [[ "${phase}" == "Running" ]]; then + echo "Pod ${pod} is in the ${phase} state" + break + else + echo "Pod ${pod} is in the ${phase} state - Check in ${sleep} second(s) / $((WAIT - SECONDS))" + fi + sleep "${sleep}" + done + done + + - name: Tools - Update + run: | + crawler_pod=$(kubectl get pod -n "${{ env.TOOLS_NAMESPACE }}" -l 'app.kubernetes.io/name=crawler' -ojsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + discordbot_pod=$(kubectl get pod -n "${{ env.TOOLS_NAMESPACE }}" -l 'app=discordbot' -ojsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + + for pod in "${crawler_pod}" "${discordbot_pod}"; do + if [[ -n "${pod}" ]]; then + kubectl delete pod -n "${{ env.TOOLS_NAMESPACE }}" "${pod}" --grace-period=10 + fi + done + + - name: Tools - Status + run: | + WAIT=300 + SECONDS=0 + sleep=1 + crawler_pod=$(kubectl get pod -n "${{ env.TOOLS_NAMESPACE }}" -l 'app.kubernetes.io/name=crawler' -ojsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + discordbot_pod=$(kubectl get pod -n "${{ env.TOOLS_NAMESPACE }}" -l 'app=discordbot' -ojsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + for pod in "${crawler_pod}" "${discordbot_pod}"; do + if [[ -n "${pod}" ]]; then + while (( SECONDS < WAIT )); do + phase=$(kubectl get pod "${pod}" -n "${{ env.TOOLS_NAMESPACE }}" -o jsonpath='{.status.phase}') + if [[ "${phase}" == "Running" ]]; then + echo "Pod ${pod} is in the ${phase} state" + break + else + echo "Pod ${pod} is in the ${phase} state - Check in ${sleep} second(s) / $((WAIT - SECONDS))" + fi + sleep "${sleep}" + done + fi + done diff --git a/.github/workflows/docker-dist-tests.yml b/.github/workflows/docker-dist-tests.yml index 74bd84af..f6bb163c 100644 --- a/.github/workflows/docker-dist-tests.yml +++ b/.github/workflows/docker-dist-tests.yml @@ -13,6 +13,7 @@ on: - '.github/**' - '!.github/workflows/docker-dist-tests.yml' - '!.github/workflows/docker-reusable.yml' + - '!.github/workflows/deploy-devnet.yml' - 'docker/**' - '!docker/codex.Dockerfile' - '!docker/docker-entrypoint.sh' @@ -23,6 +24,11 @@ on: required: false type: boolean default: false + deploy_devnet: + description: Deploy Devnet + required: false + type: boolean + default: false jobs: @@ -40,6 +46,7 @@ jobs: run: | hash=$(git rev-parse --short HEAD:vendor/codex-contracts-eth) echo "hash=$hash" >> $GITHUB_OUTPUT + build-and-push: name: Build and Push uses: ./.github/workflows/docker-reusable.yml @@ -53,3 +60,12 @@ jobs: contract_image: "codexstorage/codex-contracts-eth:sha-${{ needs.get-contracts-hash.outputs.hash }}-dist-tests" run_release_tests: ${{ inputs.run_release_tests }} secrets: inherit + + deploy-devnet: + name: Deploy Devnet + uses: ./.github/workflows/deploy-devnet.yml + needs: build-and-push + if: ${{ inputs.deploy_devnet || github.event_name == 'push' && github.ref_name == github.event.repository.default_branch }} + with: + codex_image: ${{ needs.build-and-push.outputs.codex_image }} + secrets: inherit diff --git a/.github/workflows/docker-reusable.yml b/.github/workflows/docker-reusable.yml index e74927b4..eb614216 100644 --- a/.github/workflows/docker-reusable.yml +++ b/.github/workflows/docker-reusable.yml @@ -68,6 +68,10 @@ on: description: Specifies compatible smart contract image required: false type: string + outputs: + codex_image: + description: Codex Docker image tag + value: ${{ jobs.publish.outputs.codex_image }} env: @@ -91,15 +95,16 @@ env: jobs: + # Compute variables compute: - name: Compute build ID - runs-on: ubuntu-latest - outputs: - build_id: ${{ steps.build_id.outputs.build_id }} - steps: - - name: Generate unique build id - id: build_id - run: echo "build_id=$(openssl rand -hex 5)" >> $GITHUB_OUTPUT + name: Compute build ID + runs-on: ubuntu-latest + outputs: + build_id: ${{ steps.build_id.outputs.build_id }} + steps: + - name: Generate unique build id + id: build_id + run: echo "build_id=$(openssl rand -hex 5)" >> $GITHUB_OUTPUT # Build platform specific image build: @@ -134,7 +139,7 @@ jobs: run: | # Create contract label for compatible contract image if specified if [[ -n "${{ env.CONTRACT_IMAGE }}" ]]; then - echo "CONTRACT_LABEL=storage.codex.nim-codex.blockchain-image=${{ env.CONTRACT_IMAGE }}" >>$GITHUB_ENV + echo "CONTRACT_LABEL=storage.codex.nim-codex.blockchain-image=${{ env.CONTRACT_IMAGE }}" >> $GITHUB_ENV fi - name: Docker - Meta @@ -189,35 +194,35 @@ jobs: runs-on: ubuntu-latest outputs: version: ${{ steps.meta.outputs.version }} + codex_image: ${{ steps.image_tag.outputs.codex_image }} needs: [build, compute] steps: - - name: Docker - Variables run: | - # Adjust custom suffix when set and + # Adjust custom suffix when set if [[ -n "${{ env.TAG_SUFFIX }}" ]]; then - echo "TAG_SUFFIX=-${{ env.TAG_SUFFIX }}" >>$GITHUB_ENV + echo "TAG_SUFFIX=-${{ env.TAG_SUFFIX }}" >> $GITHUB_ENV fi # Disable SHA tags on tagged release if [[ ${{ startsWith(github.ref, 'refs/tags/') }} == "true" ]]; then - echo "TAG_SHA=false" >>$GITHUB_ENV + echo "TAG_SHA=false" >> $GITHUB_ENV fi # Handle latest and latest-custom using raw if [[ ${{ env.TAG_SHA }} == "false" ]]; then - echo "TAG_LATEST=false" >>$GITHUB_ENV - echo "TAG_RAW=true" >>$GITHUB_ENV + echo "TAG_LATEST=false" >> $GITHUB_ENV + echo "TAG_RAW=true" >> $GITHUB_ENV if [[ -z "${{ env.TAG_SUFFIX }}" ]]; then - echo "TAG_RAW_VALUE=latest" >>$GITHUB_ENV + echo "TAG_RAW_VALUE=latest" >> $GITHUB_ENV else - echo "TAG_RAW_VALUE=latest-{{ env.TAG_SUFFIX }}" >>$GITHUB_ENV + echo "TAG_RAW_VALUE=latest-{{ env.TAG_SUFFIX }}" >> $GITHUB_ENV fi else - echo "TAG_RAW=false" >>$GITHUB_ENV + echo "TAG_RAW=false" >> $GITHUB_ENV fi # Create contract label for compatible contract image if specified if [[ -n "${{ env.CONTRACT_IMAGE }}" ]]; then - echo "CONTRACT_LABEL=storage.codex.nim-codex.blockchain-image=${{ env.CONTRACT_IMAGE }}" >>$GITHUB_ENV + echo "CONTRACT_LABEL=storage.codex.nim-codex.blockchain-image=${{ env.CONTRACT_IMAGE }}" >> $GITHUB_ENV fi - name: Docker - Download digests @@ -257,9 +262,12 @@ jobs: docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ $(printf '${{ env.DOCKER_REPO }}@sha256:%s ' *) + - name: Docker - Image tag + id: image_tag + run: echo "codex_image=${{ env.DOCKER_REPO }}:${{ steps.meta.outputs.version }}" >> "$GITHUB_OUTPUT" + - name: Docker - Inspect image - run: | - docker buildx imagetools inspect ${{ env.DOCKER_REPO }}:${{ steps.meta.outputs.version }} + run: docker buildx imagetools inspect ${{ steps.image_tag.outputs.codex_image }} # Compute Tests inputs From d1f2e2399bbcc0ba3b7669fa4ed79e96d28a02de Mon Sep 17 00:00:00 2001 From: Slava <20563034+veaceslavdoina@users.noreply.github.com> Date: Tue, 12 Aug 2025 11:51:41 +0300 Subject: [PATCH 02/20] ci: validate pr title to adhere conventional commits (#1254) --- .github/workflows/ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4cfa6a5..6603aab7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,15 @@ concurrency: cancel-in-progress: true jobs: + pr-title: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - name: PR Conventional Commit Validation + uses: ytanikin/pr-conventional-commits@1.4.1 + with: + task_types: '["feat","fix","docs","test","ci","build","refactor","style","perf","chore","revert"]' + matrix: runs-on: ubuntu-latest outputs: From e3d8d195c37709e49a8f9903cf5d0662eeae5dba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Uhl=C3=AD=C5=99?= Date: Wed, 1 Oct 2025 13:19:15 +0200 Subject: [PATCH 03/20] chore: update nim-libp2p (#1323) --- vendor/nim-libp2p | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p index c08d8073..7eaf79fe 160000 --- a/vendor/nim-libp2p +++ b/vendor/nim-libp2p @@ -1 +1 @@ -Subproject commit c08d80734989b028b3d1705f2188d783a343aac0 +Subproject commit 7eaf79fefe45b03a4281e3505d5fcc97b32df39c From af55a761e622b90b116f20078cf7d7b24cae1555 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 22 Oct 2025 12:22:33 +0400 Subject: [PATCH 04/20] chore: skip marketplace and long integration tests (#1326) --- tests/imports.nim | 5 +++-- tests/testIntegration.nim | 7 ++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/imports.nim b/tests/imports.nim index fbe642fc..6cbc52e5 100644 --- a/tests/imports.nim +++ b/tests/imports.nim @@ -2,13 +2,14 @@ import std/macros import std/os import std/strutils -macro importTests*(dir: static string): untyped = +macro importTests*(dir: static string, excludes: static string = ""): untyped = ## imports all files in the specified directory whose filename ## starts with "test" and ends in ".nim" let imports = newStmtList() + let excludeList = excludes.split(",") for file in walkDirRec(dir): let (_, name, ext) = splitFile(file) - if name.startsWith("test") and ext == ".nim": + if name.startsWith("test") and ext == ".nim" and not (name in excludeList): imports.add( quote do: import `file` diff --git a/tests/testIntegration.nim b/tests/testIntegration.nim index eef2f79e..17a1de7d 100644 --- a/tests/testIntegration.nim +++ b/tests/testIntegration.nim @@ -12,6 +12,11 @@ when includes != "": importAll(includes.split(",")) else: # import all tests in the integration/ directory - importTests(currentSourcePath().parentDir() / "integration") + importTests( + currentSourcePath().parentDir() / "integration" / "1_minute", "testpurchasing" + ) + importTests( + currentSourcePath().parentDir() / "integration" / "5_minutes", "testsales" + ) {.warning[UnusedImport]: off.} From 072bff5cabae3b0ddf9b14d9394953740f7089bd Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Thu, 30 Oct 2025 19:38:11 +1100 Subject: [PATCH 05/20] fix: ci integration tests (#1335) --- .../{testpurchasing.nim => testpurchasing.nim.ignore} | 0 .../{testmarketplace.nim => testmarketplace.nim.ignore} | 0 .../30_minutes/{testproofs.nim => testproofs.nim.ignore} | 0 .../{testslotrepair.nim => testslotrepair.nim.ignore} | 0 .../{testvalidator.nim => testvalidator.nim.ignore} | 0 .../5_minutes/{testsales.nim => testsales.nim.ignore} | 0 tests/testIntegration.nim | 7 +------ tools/scripts/ci-job-matrix.sh | 2 +- 8 files changed, 2 insertions(+), 7 deletions(-) rename tests/integration/1_minute/{testpurchasing.nim => testpurchasing.nim.ignore} (100%) rename tests/integration/30_minutes/{testmarketplace.nim => testmarketplace.nim.ignore} (100%) rename tests/integration/30_minutes/{testproofs.nim => testproofs.nim.ignore} (100%) rename tests/integration/30_minutes/{testslotrepair.nim => testslotrepair.nim.ignore} (100%) rename tests/integration/30_minutes/{testvalidator.nim => testvalidator.nim.ignore} (100%) rename tests/integration/5_minutes/{testsales.nim => testsales.nim.ignore} (100%) diff --git a/tests/integration/1_minute/testpurchasing.nim b/tests/integration/1_minute/testpurchasing.nim.ignore similarity index 100% rename from tests/integration/1_minute/testpurchasing.nim rename to tests/integration/1_minute/testpurchasing.nim.ignore diff --git a/tests/integration/30_minutes/testmarketplace.nim b/tests/integration/30_minutes/testmarketplace.nim.ignore similarity index 100% rename from tests/integration/30_minutes/testmarketplace.nim rename to tests/integration/30_minutes/testmarketplace.nim.ignore diff --git a/tests/integration/30_minutes/testproofs.nim b/tests/integration/30_minutes/testproofs.nim.ignore similarity index 100% rename from tests/integration/30_minutes/testproofs.nim rename to tests/integration/30_minutes/testproofs.nim.ignore diff --git a/tests/integration/30_minutes/testslotrepair.nim b/tests/integration/30_minutes/testslotrepair.nim.ignore similarity index 100% rename from tests/integration/30_minutes/testslotrepair.nim rename to tests/integration/30_minutes/testslotrepair.nim.ignore diff --git a/tests/integration/30_minutes/testvalidator.nim b/tests/integration/30_minutes/testvalidator.nim.ignore similarity index 100% rename from tests/integration/30_minutes/testvalidator.nim rename to tests/integration/30_minutes/testvalidator.nim.ignore diff --git a/tests/integration/5_minutes/testsales.nim b/tests/integration/5_minutes/testsales.nim.ignore similarity index 100% rename from tests/integration/5_minutes/testsales.nim rename to tests/integration/5_minutes/testsales.nim.ignore diff --git a/tests/testIntegration.nim b/tests/testIntegration.nim index 17a1de7d..eef2f79e 100644 --- a/tests/testIntegration.nim +++ b/tests/testIntegration.nim @@ -12,11 +12,6 @@ when includes != "": importAll(includes.split(",")) else: # import all tests in the integration/ directory - importTests( - currentSourcePath().parentDir() / "integration" / "1_minute", "testpurchasing" - ) - importTests( - currentSourcePath().parentDir() / "integration" / "5_minutes", "testsales" - ) + importTests(currentSourcePath().parentDir() / "integration") {.warning[UnusedImport]: off.} diff --git a/tools/scripts/ci-job-matrix.sh b/tools/scripts/ci-job-matrix.sh index 206ac307..15dfa9fb 100755 --- a/tools/scripts/ci-job-matrix.sh +++ b/tools/scripts/ci-job-matrix.sh @@ -82,7 +82,7 @@ find_tests () { batch () { local batch_size=$1 local separator=$2 - xargs -n $batch_size bash -c "IFS=\"$separator\"; echo \"\$*\"" _ + xargs -n "$batch_size" | sed 's/ /'"$separator"'/g' } # outputs a single integration test job From 7aca2f0e61f47ce2b902a8f39eb7df3ba3032437 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Sun, 2 Nov 2025 15:00:55 +1100 Subject: [PATCH 06/20] fix(ci): Move conventional commits job to workflow (#1340) --- .github/workflows/ci.yml | 8 -------- .github/workflows/conventional-commits.yml | 19 +++++++++++++++++++ .github/workflows/nim-matrix.yml | 2 +- 3 files changed, 20 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/conventional-commits.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6603aab7..a09ebce1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,14 +16,6 @@ concurrency: cancel-in-progress: true jobs: - pr-title: - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' - steps: - - name: PR Conventional Commit Validation - uses: ytanikin/pr-conventional-commits@1.4.1 - with: - task_types: '["feat","fix","docs","test","ci","build","refactor","style","perf","chore","revert"]' matrix: runs-on: ubuntu-latest diff --git a/.github/workflows/conventional-commits.yml b/.github/workflows/conventional-commits.yml new file mode 100644 index 00000000..1890a2f8 --- /dev/null +++ b/.github/workflows/conventional-commits.yml @@ -0,0 +1,19 @@ +name: Conventional Commits Linting + +on: + push: + branches: + - master + pull_request: + workflow_dispatch: + merge_group: + +jobs: + pr-title: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - name: PR Conventional Commit Validation + uses: ytanikin/pr-conventional-commits@1.4.1 + with: + task_types: '["feat","fix","docs","test","ci","build","refactor","style","perf","chore","revert"]' diff --git a/.github/workflows/nim-matrix.yml b/.github/workflows/nim-matrix.yml index b5d55415..590805f7 100644 --- a/.github/workflows/nim-matrix.yml +++ b/.github/workflows/nim-matrix.yml @@ -8,7 +8,7 @@ env: cache_nonce: 0 # Allows for easily busting actions/cache caches nim_version: pinned -jobs: +jobs: matrix: runs-on: ubuntu-latest outputs: From db8f866db46d27b91e9a4ae04803efcfd6da74b7 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Sun, 2 Nov 2025 08:32:47 +0400 Subject: [PATCH 07/20] feat: check if CID exists in local store (#1331) --- codex/node.nim | 7 +++++ codex/rest/api.nim | 16 +++++++++++ openapi.yaml | 28 +++++++++++++++++++ tests/codex/node/testnode.nim | 14 ++++++++++ tests/integration/5_minutes/testrestapi.nim | 10 +++++++ .../5_minutes/testrestapivalidation.nim | 7 +++++ tests/integration/codexclient.nim | 21 ++++++++++++++ 7 files changed, 103 insertions(+) diff --git a/codex/node.nim b/codex/node.nim index e010b085..1ca471d5 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -897,3 +897,10 @@ proc new*( contracts: contracts, trackedFutures: TrackedFutures(), ) + +proc hasLocalBlock*( + self: CodexNodeRef, cid: Cid +): Future[bool] {.async: (raises: [CancelledError]).} = + ## Returns true if the given Cid is present in the local store + + return await (cid in self.networkStore.localStore) diff --git a/codex/rest/api.nim b/codex/rest/api.nim index e31a0f59..adab821b 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -365,6 +365,22 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute let json = %formatManifest(cid.get(), manifest) return RestApiResponse.response($json, contentType = "application/json") + router.api(MethodGet, "/api/codex/v1/data/{cid}/exists") do( + cid: Cid, resp: HttpResponseRef + ) -> RestApiResponse: + ## Only test if the give CID is available in the local store + ## + var headers = buildCorsHeaders("GET", allowedOrigin) + + if cid.isErr: + return RestApiResponse.error(Http400, $cid.error(), headers = headers) + + let cid = cid.get() + let hasCid = await node.hasLocalBlock(cid) + + let json = %*{$cid: hasCid} + return RestApiResponse.response($json, contentType = "application/json") + router.api(MethodGet, "/api/codex/v1/space") do() -> RestApiResponse: let json = %RestRepoStore( diff --git a/openapi.yaml b/openapi.yaml index a84842f7..fab3d334 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -727,6 +727,34 @@ paths: "500": description: Well it was bad-bad + "/data/{cid}/exists": + get: + summary: "Check if a block identified by CID exists in the local node." + tags: [Data] + operationId: hasBlock + parameters: + - in: path + name: cid + required: true + schema: + $ref: "#/components/schemas/Cid" + description: "CID of the block to check." + responses: + "200": + description: Block existence information + content: + application/json: + schema: + type: object + properties: + has: + type: boolean + description: Indicates whether the block exists in the local node + "400": + description: Invalid CID is specified + "500": + description: Well it was bad-bad + "/space": get: summary: "Gets a summary of the storage space allocation of the node." diff --git a/tests/codex/node/testnode.nim b/tests/codex/node/testnode.nim index 78298ad7..a3fe7e94 100644 --- a/tests/codex/node/testnode.nim +++ b/tests/codex/node/testnode.nim @@ -229,3 +229,17 @@ asyncchecksuite "Test Node - Basic": check not await manifestCid in localStore for blk in blocks: check not (await blk.cid in localStore) + + test "Should return true when a cid is already in the local store": + let + blocks = await makeRandomBlocks(datasetSize = 1024, blockSize = 256'nb) + manifest = await storeDataGetManifest(localStore, blocks) + manifestBlock = (await store.storeManifest(manifest)).tryGet() + manifestCid = manifestBlock.cid + + check (await node.hasLocalBlock(manifestCid)) == true + + test "Should returns false when a cid is not in the local store": + let randomBlock = bt.Block.new("Random block".toBytes).tryGet() + + check (await node.hasLocalBlock(randomBlock.cid)) == false diff --git a/tests/integration/5_minutes/testrestapi.nim b/tests/integration/5_minutes/testrestapi.nim index 71cf97ef..9a6b9c11 100644 --- a/tests/integration/5_minutes/testrestapi.nim +++ b/tests/integration/5_minutes/testrestapi.nim @@ -220,3 +220,13 @@ twonodessuite "REST API": let response2 = await client1.downloadRaw($cid) check (await response2.body) == contents + + test "should returns true when the block exists", twoNodesConfig: + let cid = (await client2.upload("some file contents")).get + + var response = await client1.hasBlock(cid) + check response.get() == false + + discard (await client1.download(cid)).get + response = await client1.hasBlock(cid) + check response.get() == false diff --git a/tests/integration/5_minutes/testrestapivalidation.nim b/tests/integration/5_minutes/testrestapivalidation.nim index db11da9c..d428402e 100644 --- a/tests/integration/5_minutes/testrestapivalidation.nim +++ b/tests/integration/5_minutes/testrestapivalidation.nim @@ -396,3 +396,10 @@ multinodesuite "Rest API validation": check: response.status == 422 (await response.body) == "totalCollateral must be larger then zero" + + test "has block returns error 400 when the cid is invalid", config: + let response = await client.hasBlockRaw("invalid-cid") + + check: + response.status == 400 + (await response.body) == "Incorrect Cid" diff --git a/tests/integration/codexclient.nim b/tests/integration/codexclient.nim index 17ed6dd4..3fdf564f 100644 --- a/tests/integration/codexclient.nim +++ b/tests/integration/codexclient.nim @@ -16,6 +16,9 @@ type CodexClient* = ref object baseurl: string session: HttpSessionRef +type HasBlockResponse = object + has: bool + proc new*(_: type CodexClient, baseurl: string): CodexClient = CodexClient(session: HttpSessionRef.new(), baseurl: baseurl) @@ -431,3 +434,21 @@ proc getSlots*( let url = client.baseurl & "/sales/slots" let body = await client.getContent(url) seq[Slot].fromJson(body) + +proc hasBlock*( + client: CodexClient, cid: Cid +): Future[?!bool] {.async: (raises: [CancelledError, HttpError]).} = + let url = client.baseurl & "/data/" & $cid & "/exists" + let body = await client.getContent(url) + let response = HasBlockResponse.fromJson(body) + if response.isErr: + return failure "Failed to parse has block response" + return response.get.has.success + +proc hasBlockRaw*( + client: CodexClient, cid: string +): Future[HttpClientResponseRef] {. + async: (raw: true, raises: [CancelledError, HttpError]) +.} = + let url = client.baseurl & "/data/" & cid & "/exists" + return client.get(url) From f791a960f2ea6229685337997abbb40c441f2116 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Mon, 3 Nov 2025 22:45:02 +1100 Subject: [PATCH 08/20] fix(ci): Windows SIGILL in CI (#1339) --- config.nims | 5 +++-- tests/imports.nim | 5 ++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config.nims b/config.nims index 05a31fff..8700f60e 100644 --- a/config.nims +++ b/config.nims @@ -61,11 +61,12 @@ elif defined(macosx) and defined(arm64): switch("passC", "-mcpu=apple-a14") # TODO: newer Clang >=15.0 can: https://github.com/llvm/llvm-project/commit/fcca10c69aaab539962d10fcc59a5f074b73b0de else: - switch("passC", "-march=native") if defined(windows): # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65782 # ("-fno-asynchronous-unwind-tables" breaks Nim's exception raising, sometimes) - switch("passC", "-mno-avx512vl") + switch("passC", "-march=x86-64") + else: switch("passC", "-march=native") + --tlsEmulation: off diff --git a/tests/imports.nim b/tests/imports.nim index 6cbc52e5..fbe642fc 100644 --- a/tests/imports.nim +++ b/tests/imports.nim @@ -2,14 +2,13 @@ import std/macros import std/os import std/strutils -macro importTests*(dir: static string, excludes: static string = ""): untyped = +macro importTests*(dir: static string): untyped = ## imports all files in the specified directory whose filename ## starts with "test" and ends in ".nim" let imports = newStmtList() - let excludeList = excludes.split(",") for file in walkDirRec(dir): let (_, name, ext) = splitFile(file) - if name.startsWith("test") and ext == ".nim" and not (name in excludeList): + if name.startsWith("test") and ext == ".nim": imports.add( quote do: import `file` From ee47ca8760813d9ab81291f241db1ba0d2285b82 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Thu, 13 Nov 2025 15:48:33 +1100 Subject: [PATCH 09/20] feat(libs): Use libp2p multiformats extensions instead of a rolling branch (#1329) --- codex/contentids_exts.nim | 8 ++++ codex/merkletree/codex/codex.nim | 55 ++++++------------------ codex/multicodec_exts.nim | 11 +++++ codex/multihash_exts.nim | 40 +++++++++++++++++ config.nims | 6 +++ tests/codex/merkletree/testcodextree.nim | 6 +-- vendor/nim-libp2p | 2 +- 7 files changed, 83 insertions(+), 45 deletions(-) create mode 100644 codex/contentids_exts.nim create mode 100644 codex/multicodec_exts.nim create mode 100644 codex/multihash_exts.nim diff --git a/codex/contentids_exts.nim b/codex/contentids_exts.nim new file mode 100644 index 00000000..9ef6fbb5 --- /dev/null +++ b/codex/contentids_exts.nim @@ -0,0 +1,8 @@ +const ContentIdsExts = [ + multiCodec("codex-root"), + multiCodec("codex-manifest"), + multiCodec("codex-block"), + multiCodec("codex-slot-root"), + multiCodec("codex-proving-root"), + multiCodec("codex-slot-cell"), +] diff --git a/codex/merkletree/codex/codex.nim b/codex/merkletree/codex/codex.nim index 0eec92e4..bf9f39d6 100644 --- a/codex/merkletree/codex/codex.nim +++ b/codex/merkletree/codex/codex.nim @@ -47,28 +47,6 @@ type CodexProof* = ref object of ByteProof mcodec*: MultiCodec -# CodeHashes is not exported from libp2p -# So we need to recreate it instead of -proc initMultiHashCodeTable(): Table[MultiCodec, MHash] {.compileTime.} = - for item in HashesList: - result[item.mcodec] = item - -const CodeHashes = initMultiHashCodeTable() - -func mhash*(mcodec: MultiCodec): ?!MHash = - let mhash = CodeHashes.getOrDefault(mcodec) - - if isNil(mhash.coder): - return failure "Invalid multihash codec" - - success mhash - -func digestSize*(self: (CodexTree or CodexProof)): int = - ## Number of leaves - ## - - self.mhash.size - func getProof*(self: CodexTree, index: int): ?!CodexProof = var proof = CodexProof(mcodec: self.mcodec) @@ -128,17 +106,12 @@ proc `$`*(self: CodexProof): string = "CodexProof(" & " nleaves: " & $self.nleaves & ", index: " & $self.index & ", path: " & $self.path.mapIt(byteutils.toHex(it)) & ", mcodec: " & $self.mcodec & " )" -func compress*(x, y: openArray[byte], key: ByteTreeKey, mhash: MHash): ?!ByteHash = +func compress*(x, y: openArray[byte], key: ByteTreeKey, codec: MultiCodec): ?!ByteHash = ## Compress two hashes ## - - # Using Constantine's SHA256 instead of mhash for optimal performance on 32-byte merkle node hashing - # See: https://github.com/codex-storage/nim-codex/issues/1162 - let input = @x & @y & @[key.byte] - var digest = hashes.sha256.hash(input) - - success @digest + let digest = ?MultiHash.digest(codec, input).mapFailure + success digest.digestBytes func init*( _: type CodexTree, mcodec: MultiCodec = Sha256HashCodec, leaves: openArray[ByteHash] @@ -147,12 +120,12 @@ func init*( return failure "Empty leaves" let - mhash = ?mcodec.mhash() compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} = - compress(x, y, key, mhash) - Zero: ByteHash = newSeq[byte](mhash.size) + compress(x, y, key, mcodec) + digestSize = ?mcodec.digestSize.mapFailure + Zero: ByteHash = newSeq[byte](digestSize) - if mhash.size != leaves[0].len: + if digestSize != leaves[0].len: return failure "Invalid hash length" var self = CodexTree(mcodec: mcodec, compress: compressor, zero: Zero) @@ -190,12 +163,12 @@ proc fromNodes*( return failure "Empty nodes" let - mhash = ?mcodec.mhash() - Zero = newSeq[byte](mhash.size) + digestSize = ?mcodec.digestSize.mapFailure + Zero = newSeq[byte](digestSize) compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} = - compress(x, y, key, mhash) + compress(x, y, key, mcodec) - if mhash.size != nodes[0].len: + if digestSize != nodes[0].len: return failure "Invalid hash length" var @@ -228,10 +201,10 @@ func init*( return failure "Empty nodes" let - mhash = ?mcodec.mhash() - Zero = newSeq[byte](mhash.size) + digestSize = ?mcodec.digestSize.mapFailure + Zero = newSeq[byte](digestSize) compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!seq[byte] {.noSideEffect.} = - compress(x, y, key, mhash) + compress(x, y, key, mcodec) success CodexProof( compress: compressor, diff --git a/codex/multicodec_exts.nim b/codex/multicodec_exts.nim new file mode 100644 index 00000000..b14fbbe6 --- /dev/null +++ b/codex/multicodec_exts.nim @@ -0,0 +1,11 @@ +const CodecExts = [ + ("poseidon2-alt_bn_128-sponge-r2", 0xCD10), # bn128 rate 2 sponge + ("poseidon2-alt_bn_128-merkle-2kb", 0xCD11), # bn128 2kb compress & merkleize + ("poseidon2-alt_bn_128-keyed-compress", 0xCD12), # bn128 keyed compress] + ("codex-manifest", 0xCD01), + ("codex-block", 0xCD02), + ("codex-root", 0xCD03), + ("codex-slot-root", 0xCD04), + ("codex-proving-root", 0xCD05), + ("codex-slot-cell", 0xCD06), +] diff --git a/codex/multihash_exts.nim b/codex/multihash_exts.nim new file mode 100644 index 00000000..e9477177 --- /dev/null +++ b/codex/multihash_exts.nim @@ -0,0 +1,40 @@ +import blscurve/bls_public_exports +import pkg/constantine/hashes +import poseidon2 + +proc sha2_256hash_constantine(data: openArray[byte], output: var openArray[byte]) = + # Using Constantine's SHA256 instead of mhash for optimal performance on 32-byte merkle node hashing + # See: https://github.com/codex-storage/nim-codex/issues/1162 + if len(output) > 0: + let digest = hashes.sha256.hash(data) + copyMem(addr output[0], addr digest[0], 32) + +proc poseidon2_sponge_rate2(data: openArray[byte], output: var openArray[byte]) = + if len(output) > 0: + var digest = poseidon2.Sponge.digest(data).toBytes() + copyMem(addr output[0], addr digest[0], uint(len(output))) + +proc poseidon2_merkle_2kb_sponge(data: openArray[byte], output: var openArray[byte]) = + if len(output) > 0: + var digest = poseidon2.SpongeMerkle.digest(data, 2048).toBytes() + copyMem(addr output[0], addr digest[0], uint(len(output))) + +const Sha2256MultiHash* = MHash( + mcodec: multiCodec("sha2-256"), + size: sha256.sizeDigest, + coder: sha2_256hash_constantine, +) +const HashExts = [ + # override sha2-256 hash function + Sha2256MultiHash, + MHash( + mcodec: multiCodec("poseidon2-alt_bn_128-sponge-r2"), + size: 32, + coder: poseidon2_sponge_rate2, + ), + MHash( + mcodec: multiCodec("poseidon2-alt_bn_128-merkle-2kb"), + size: 32, + coder: poseidon2_merkle_2kb_sponge, + ), +] diff --git a/config.nims b/config.nims index 8700f60e..b1bc4cbe 100644 --- a/config.nims +++ b/config.nims @@ -92,6 +92,12 @@ else: on --warningAsError: "ProveField:on" +--define: + "libp2p_multicodec_exts:../../../codex/multicodec_exts.nim" +--define: + "libp2p_multihash_exts:../../../codex/multihash_exts.nim" +--define: + "libp2p_contentids_exts:../../../codex/contentids_exts.nim" when (NimMajor, NimMinor) >= (1, 4): --warning: diff --git a/tests/codex/merkletree/testcodextree.nim b/tests/codex/merkletree/testcodextree.nim index 29390c16..91ccd7f5 100644 --- a/tests/codex/merkletree/testcodextree.nim +++ b/tests/codex/merkletree/testcodextree.nim @@ -82,10 +82,10 @@ suite "Test CodexTree": tree == fromNodes let - mhash = sha256.mhash().tryGet - zero: seq[byte] = newSeq[byte](mhash.size) + digestSize = sha256.digestSize.get + zero: seq[byte] = newSeq[byte](digestSize) compress = proc(x, y: seq[byte], key: ByteTreeKey): seq[byte] = - compress(x, y, key, mhash).tryGet + compress(x, y, key, sha256).tryGet makeTree = proc(data: seq[seq[byte]]): CodexTree = CodexTree.init(sha256, leaves = data).tryGet diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p index 7eaf79fe..aa8ce46f 160000 --- a/vendor/nim-libp2p +++ b/vendor/nim-libp2p @@ -1 +1 @@ -Subproject commit 7eaf79fefe45b03a4281e3505d5fcc97b32df39c +Subproject commit aa8ce46f782240cb99a7222c474022b8cfd24e52 From 6147a751f110f3f55580b4c8b1b78a368b56a933 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Thu, 13 Nov 2025 16:37:43 +1100 Subject: [PATCH 10/20] fix(ci): Remove macos amd release build (#1337) --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d4ba1bca..3220cd8c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,7 +32,6 @@ jobs: matrix: | os {linux}, cpu {amd64}, builder {ubuntu-22.04}, nim_version {${{ env.nim_version }}}, rust_version {${{ env.rust_version }}}, shell {bash --noprofile --norc -e -o pipefail} os {linux}, cpu {arm64}, builder {ubuntu-22.04-arm}, nim_version {${{ env.nim_version }}}, rust_version {${{ env.rust_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {macos}, cpu {amd64}, builder {macos-13}, nim_version {${{ env.nim_version }}}, rust_version {${{ env.rust_version }}}, shell {bash --noprofile --norc -e -o pipefail} os {macos}, cpu {arm64}, builder {macos-14}, nim_version {${{ env.nim_version }}}, rust_version {${{ env.rust_version }}}, shell {bash --noprofile --norc -e -o pipefail} os {windows}, cpu {amd64}, builder {windows-latest}, nim_version {${{ env.nim_version }}}, rust_version {${{ env.rust_version }}}, shell {msys2} @@ -189,6 +188,7 @@ jobs: aws s3 cp --recursive "${folder}" s3://${{ env.s3_bucket }}/releases/${branch} --endpoint-url ${{ env.s3_endpoint }} echo "${branch}" > "${folder}"/latest aws s3 cp "${folder}"/latest s3://${{ env.s3_bucket }}/releases/latest --endpoint-url ${{ env.s3_endpoint }} + rm -f "${folder}"/latest # master branch elif [[ "${branch}" == "${{ github.event.repository.default_branch }}" ]]; then From be759baf4de0cac61b8d3bb6f21f9c81335012f7 Mon Sep 17 00:00:00 2001 From: Chrysostomos Nanakos Date: Thu, 13 Nov 2025 07:47:02 +0200 Subject: [PATCH 11/20] feat: Block exchange optimizations (#1325) Signed-off-by: Giuliano Mega Signed-off-by: Chrysostomos Nanakos Co-authored-by: gmega Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Eric <5089238+emizzle@users.noreply.github.com> --- codex/blockexchange/engine/discovery.nim | 35 +- codex/blockexchange/engine/engine.nim | 633 ++++++++++++++---- codex/blockexchange/engine/pendingblocks.nim | 57 +- codex/blockexchange/network/network.nim | 86 ++- codex/blockexchange/network/networkpeer.nim | 11 +- codex/blockexchange/peers/peercontext.nim | 98 ++- codex/blockexchange/peers/peerctxstore.nim | 10 +- codex/blockexchange/protobuf/blockexc.nim | 8 - codex/blockexchange/protobuf/message.nim | 6 +- codex/blocktype.nim | 10 +- codex/codex.nim | 2 +- codex/node.nim | 75 ++- codex/stores/blockstore.nim | 8 + codex/stores/cachestore.nim | 15 + codex/stores/networkstore.nim | 25 + codex/stores/repostore/store.nim | 15 + codex/utils/safeasynciter.nim | 25 + .../blockexchange/discovery/testdiscovery.nim | 11 +- .../discovery/testdiscoveryengine.nim | 2 +- .../blockexchange/engine/testblockexc.nim | 65 +- .../codex/blockexchange/engine/testengine.nim | 364 +++++----- tests/codex/blockexchange/testnetwork.nim | 2 +- .../codex/blockexchange/testpeerctxstore.nim | 5 +- tests/codex/examples.nim | 3 +- tests/codex/helpers.nim | 38 +- tests/codex/helpers/datasetutils.nim | 45 ++ tests/codex/helpers/mockdiscovery.nim | 28 + tests/codex/helpers/nodeutils.nim | 108 ++- tests/codex/node/testnode.nim | 2 +- tests/codex/node/testslotrepair.nim | 1 + tests/codex/stores/commonstoretests.nim | 4 +- tests/codex/stores/testrepostore.nim | 32 +- tests/codex/testblocktype.nim | 44 ++ tests/codex/utils/testsafeasynciter.nim | 19 +- tests/helpers.nim | 6 +- 35 files changed, 1366 insertions(+), 532 deletions(-) create mode 100644 tests/codex/helpers/datasetutils.nim create mode 100644 tests/codex/testblocktype.nim diff --git a/codex/blockexchange/engine/discovery.nim b/codex/blockexchange/engine/discovery.nim index b32b8555..883eea8e 100644 --- a/codex/blockexchange/engine/discovery.nim +++ b/codex/blockexchange/engine/discovery.nim @@ -8,6 +8,7 @@ ## those terms. import std/sequtils +import std/algorithm import pkg/chronos import pkg/libp2p/cid @@ -38,6 +39,7 @@ const DefaultConcurrentDiscRequests = 10 DefaultDiscoveryTimeout = 1.minutes DefaultMinPeersPerBlock = 3 + DefaultMaxPeersPerBlock = 8 DefaultDiscoveryLoopSleep = 3.seconds type DiscoveryEngine* = ref object of RootObj @@ -51,11 +53,32 @@ type DiscoveryEngine* = ref object of RootObj discoveryLoop*: Future[void].Raising([]) # Discovery loop task handle discoveryQueue*: AsyncQueue[Cid] # Discovery queue trackedFutures*: TrackedFutures # Tracked Discovery tasks futures - minPeersPerBlock*: int # Max number of peers with block + minPeersPerBlock*: int # Min number of peers with block + maxPeersPerBlock*: int # Max number of peers with block discoveryLoopSleep: Duration # Discovery loop sleep inFlightDiscReqs*: Table[Cid, Future[seq[SignedPeerRecord]]] # Inflight discovery requests +proc cleanupExcessPeers(b: DiscoveryEngine, cid: Cid) {.gcsafe, raises: [].} = + var haves = b.peers.peersHave(cid) + let count = haves.len - b.maxPeersPerBlock + if count <= 0: + return + + haves.sort( + proc(a, b: BlockExcPeerCtx): int = + cmp(a.lastExchange, b.lastExchange) + ) + + let toRemove = haves[0 ..< count] + for peer in toRemove: + try: + peer.cleanPresence(BlockAddress.init(cid)) + trace "Removed block presence from peer", cid, peer = peer.id + except CatchableError as exc: + error "Failed to clean presence for peer", + cid, peer = peer.id, error = exc.msg, name = exc.name + proc discoveryQueueLoop(b: DiscoveryEngine) {.async: (raises: []).} = try: while b.discEngineRunning: @@ -78,8 +101,16 @@ proc discoveryTaskLoop(b: DiscoveryEngine) {.async: (raises: []).} = trace "Discovery request already in progress", cid continue + trace "Running discovery task for cid", cid + let haves = b.peers.peersHave(cid) + if haves.len > b.maxPeersPerBlock: + trace "Cleaning up excess peers", + cid, peers = haves.len, max = b.maxPeersPerBlock + b.cleanupExcessPeers(cid) + continue + if haves.len < b.minPeersPerBlock: let request = b.discovery.find(cid) b.inFlightDiscReqs[cid] = request @@ -156,6 +187,7 @@ proc new*( concurrentDiscReqs = DefaultConcurrentDiscRequests, discoveryLoopSleep = DefaultDiscoveryLoopSleep, minPeersPerBlock = DefaultMinPeersPerBlock, + maxPeersPerBlock = DefaultMaxPeersPerBlock, ): DiscoveryEngine = ## Create a discovery engine instance for advertising services ## @@ -171,4 +203,5 @@ proc new*( inFlightDiscReqs: initTable[Cid, Future[seq[SignedPeerRecord]]](), discoveryLoopSleep: discoveryLoopSleep, minPeersPerBlock: minPeersPerBlock, + maxPeersPerBlock: maxPeersPerBlock, ) diff --git a/codex/blockexchange/engine/engine.nim b/codex/blockexchange/engine/engine.nim index 0d04fd7f..baacf11c 100644 --- a/codex/blockexchange/engine/engine.nim +++ b/codex/blockexchange/engine/engine.nim @@ -12,12 +12,14 @@ import std/sets import std/options import std/algorithm import std/sugar +import std/random import pkg/chronos import pkg/libp2p/[cid, switch, multihash, multicodec] import pkg/metrics import pkg/stint import pkg/questionable +import pkg/stew/shims/sets import ../../rng import ../../stores/blockstore @@ -63,30 +65,59 @@ declareCounter(codex_block_exchange_blocks_sent, "codex blockexchange blocks sen declareCounter( codex_block_exchange_blocks_received, "codex blockexchange blocks received" ) +declareCounter( + codex_block_exchange_spurious_blocks_received, + "codex blockexchange unrequested/duplicate blocks received", +) +declareCounter( + codex_block_exchange_discovery_requests_total, + "Total number of peer discovery requests sent", +) +declareCounter( + codex_block_exchange_peer_timeouts_total, "Total number of peer activity timeouts" +) +declareCounter( + codex_block_exchange_requests_failed_total, + "Total number of block requests that failed after exhausting retries", +) const - DefaultMaxPeersPerRequest* = 10 + # The default max message length of nim-libp2p is 100 megabytes, meaning we can + # in principle fit up to 1600 64k blocks per message, so 20 is well under + # that number. + DefaultMaxBlocksPerMessage = 20 DefaultTaskQueueSize = 100 DefaultConcurrentTasks = 10 + # Don't do more than one discovery request per `DiscoveryRateLimit` seconds. + DiscoveryRateLimit = 3.seconds + DefaultPeerActivityTimeout = 1.minutes + # Match MaxWantListBatchSize to efficiently respond to incoming WantLists + PresenceBatchSize = MaxWantListBatchSize + CleanupBatchSize = 2048 type TaskHandler* = proc(task: BlockExcPeerCtx): Future[void] {.gcsafe.} TaskScheduler* = proc(task: BlockExcPeerCtx): bool {.gcsafe.} + PeerSelector* = + proc(peers: seq[BlockExcPeerCtx]): BlockExcPeerCtx {.gcsafe, raises: [].} BlockExcEngine* = ref object of RootObj localStore*: BlockStore # Local block store for this instance - network*: BlockExcNetwork # Petwork interface + network*: BlockExcNetwork # Network interface peers*: PeerCtxStore # Peers we're currently actively exchanging with taskQueue*: AsyncHeapQueue[BlockExcPeerCtx] - # Peers we're currently processing tasks for + selectPeer*: PeerSelector # Peers we're currently processing tasks for concurrentTasks: int # Number of concurrent peers we're serving at any given time trackedFutures: TrackedFutures # Tracks futures of blockexc tasks blockexcRunning: bool # Indicates if the blockexc task is running + maxBlocksPerMessage: int + # Maximum number of blocks we can squeeze in a single message pendingBlocks*: PendingBlocksManager # Blocks we're awaiting to be resolved wallet*: WalletRef # Nitro wallet for micropayments pricing*: ?Pricing # Optional bandwidth pricing discovery*: DiscoveryEngine advertiser*: Advertiser + lastDiscRequest: Moment # time of last discovery request Pricing* = object address*: EthAddress @@ -104,7 +135,6 @@ proc blockexcTaskRunner(self: BlockExcEngine) {.async: (raises: []).} proc start*(self: BlockExcEngine) {.async: (raises: []).} = ## Start the blockexc task ## - await self.discovery.start() await self.advertiser.start() @@ -154,8 +184,145 @@ proc sendWantBlock( ) # we want this remote to send us a block codex_block_exchange_want_block_lists_sent.inc() -proc randomPeer(peers: seq[BlockExcPeerCtx]): BlockExcPeerCtx = - Rng.instance.sample(peers) +proc sendBatchedWantList( + self: BlockExcEngine, + peer: BlockExcPeerCtx, + addresses: seq[BlockAddress], + full: bool, +) {.async: (raises: [CancelledError]).} = + var offset = 0 + while offset < addresses.len: + let batchEnd = min(offset + MaxWantListBatchSize, addresses.len) + let batch = addresses[offset ..< batchEnd] + + trace "Sending want list batch", + peer = peer.id, + batchSize = batch.len, + offset = offset, + total = addresses.len, + full = full + + await self.network.request.sendWantList( + peer.id, batch, full = (full and offset == 0) + ) + for address in batch: + peer.lastSentWants.incl(address) + + offset = batchEnd + +proc refreshBlockKnowledge( + self: BlockExcEngine, peer: BlockExcPeerCtx, skipDelta = false, resetBackoff = false +) {.async: (raises: [CancelledError]).} = + if peer.lastSentWants.len > 0: + var toRemove: seq[BlockAddress] + + for address in peer.lastSentWants: + if address notin self.pendingBlocks: + toRemove.add(address) + + if toRemove.len >= CleanupBatchSize: + await idleAsync() + break + + for addr in toRemove: + peer.lastSentWants.excl(addr) + + if self.pendingBlocks.wantListLen == 0: + if peer.lastSentWants.len > 0: + trace "Clearing want list tracking, no pending blocks", peer = peer.id + peer.lastSentWants.clear() + return + + # We send only blocks that the peer hasn't already told us that they already have. + let + peerHave = peer.peerHave + toAsk = toHashSet(self.pendingBlocks.wantList.toSeq.filterIt(it notin peerHave)) + + if toAsk.len == 0: + if peer.lastSentWants.len > 0: + trace "Clearing want list tracking, peer has all blocks", peer = peer.id + peer.lastSentWants.clear() + return + + let newWants = toAsk - peer.lastSentWants + + if peer.lastSentWants.len > 0 and not skipDelta: + if newWants.len > 0: + trace "Sending delta want list update", + peer = peer.id, newWants = newWants.len, totalWants = toAsk.len + + await self.sendBatchedWantList(peer, newWants.toSeq, full = false) + + if resetBackoff: + peer.wantsUpdated + else: + trace "No changes in want list, skipping send", peer = peer.id + peer.lastSentWants = toAsk + else: + trace "Sending full want list", peer = peer.id, length = toAsk.len + + await self.sendBatchedWantList(peer, toAsk.toSeq, full = true) + + if resetBackoff: + peer.wantsUpdated + +proc refreshBlockKnowledge(self: BlockExcEngine) {.async: (raises: [CancelledError]).} = + let runtimeQuota = 10.milliseconds + var lastIdle = Moment.now() + + for peer in self.peers.peers.values.toSeq: + # We refresh block knowledge if: + # 1. the peer hasn't been refreshed in a while; + # 2. the list of blocks we care about has changed. + # + # Note that because of (2), it is important that we update our + # want list in the coarsest way possible instead of over many + # small updates. + # + + # In dynamic swarms, staleness will dominate latency. + let + hasNewBlocks = peer.lastRefresh < self.pendingBlocks.lastInclusion + isKnowledgeStale = peer.isKnowledgeStale + + if isKnowledgeStale or hasNewBlocks: + if not peer.refreshInProgress: + peer.refreshRequested() + await self.refreshBlockKnowledge( + peer, skipDelta = isKnowledgeStale, resetBackoff = hasNewBlocks + ) + else: + trace "Not refreshing: peer is up to date", peer = peer.id + + if (Moment.now() - lastIdle) >= runtimeQuota: + try: + await idleAsync() + except CancelledError: + discard + lastIdle = Moment.now() + +proc searchForNewPeers(self: BlockExcEngine, cid: Cid) = + if self.lastDiscRequest + DiscoveryRateLimit < Moment.now(): + trace "Searching for new peers for", cid = cid + codex_block_exchange_discovery_requests_total.inc() + self.lastDiscRequest = Moment.now() # always refresh before calling await! + self.discovery.queueFindBlocksReq(@[cid]) + else: + trace "Not searching for new peers, rate limit not expired", cid = cid + +proc evictPeer(self: BlockExcEngine, peer: PeerId) = + ## Cleanup disconnected peer + ## + + trace "Evicting disconnected/departed peer", peer + + let peerCtx = self.peers.get(peer) + if not peerCtx.isNil: + for address in peerCtx.blocksRequested: + self.pendingBlocks.clearRequest(address, peer.some) + + # drop the peer from the peers table + self.peers.remove(peer) proc downloadInternal( self: BlockExcEngine, address: BlockAddress @@ -173,41 +340,147 @@ proc downloadInternal( if self.pendingBlocks.retriesExhausted(address): trace "Error retries exhausted" + codex_block_exchange_requests_failed_total.inc() handle.fail(newException(RetriesExhaustedError, "Error retries exhausted")) break - trace "Running retry handle" let peers = self.peers.getPeersForBlock(address) logScope: peersWith = peers.with.len peersWithout = peers.without.len - trace "Peers for block" - if peers.with.len > 0: - self.pendingBlocks.setInFlight(address, true) - await self.sendWantBlock(@[address], peers.with.randomPeer) - else: - self.pendingBlocks.setInFlight(address, false) + if peers.with.len == 0: + # We know of no peers that have the block. if peers.without.len > 0: - await self.sendWantHave(@[address], peers.without) - self.discovery.queueFindBlocksReq(@[address.cidOrTreeCid]) + # If we have peers connected but none of them have the block, this + # could be because our knowledge about what they have has run stale. + # Tries to refresh it. + await self.refreshBlockKnowledge() + # Also tries to look for new peers for good measure. + # TODO: in the future, peer search and knowledge maintenance should + # be completely decoupled from one another. It is very hard to + # control what happens and how many neighbors we get like this. + self.searchForNewPeers(address.cidOrTreeCid) - await (handle or sleepAsync(self.pendingBlocks.retryInterval)) + let nextDiscovery = + if self.lastDiscRequest + DiscoveryRateLimit > Moment.now(): + (self.lastDiscRequest + DiscoveryRateLimit - Moment.now()) + else: + 0.milliseconds + + let retryDelay = + max(secs(rand(self.pendingBlocks.retryInterval.secs)), nextDiscovery) + + # We now wait for a bit and then retry. If the handle gets completed in the + # meantime (cause the presence handler might have requested the block and + # received it in the meantime), we are done. Retry delays are randomized + # so we don't get all block loops spinning at the same time. + await handle or sleepAsync(retryDelay) + if handle.finished: + break + + # Without decrementing the retries count, this would infinitely loop + # trying to find peers. + self.pendingBlocks.decRetries(address) + + # If we still don't have the block, we'll go for another cycle. + trace "No peers for block, will retry shortly" + continue + + # Once again, it might happen that the block was requested to a peer + # in the meantime. If so, we don't need to do anything. Otherwise, + # we'll be the ones placing the request. + let scheduledPeer = + if not self.pendingBlocks.isRequested(address): + let peer = self.selectPeer(peers.with) + discard self.pendingBlocks.markRequested(address, peer.id) + peer.blockRequestScheduled(address) + trace "Request block from block retry loop" + await self.sendWantBlock(@[address], peer) + peer + else: + let peerId = self.pendingBlocks.getRequestPeer(address).get() + self.peers.get(peerId) + + if scheduledPeer.isNil: + trace "Scheduled peer no longer available, clearing stale request", address + self.pendingBlocks.clearRequest(address) + continue + + # Parks until either the block is received, or the peer times out. + let activityTimer = scheduledPeer.activityTimer() + await handle or activityTimer # TODO: or peerDropped + activityTimer.cancel() + + # XXX: we should probably not have this. Blocks should be retried + # to infinity unless cancelled by the client. self.pendingBlocks.decRetries(address) if handle.finished: trace "Handle for block finished", failed = handle.failed break + else: + # If the peer timed out, retries immediately. + trace "Peer timed out during block request", peer = scheduledPeer.id + codex_block_exchange_peer_timeouts_total.inc() + await self.network.dropPeer(scheduledPeer.id) + # Evicts peer immediately or we may end up picking it again in the + # next retry. + self.evictPeer(scheduledPeer.id) except CancelledError as exc: trace "Block download cancelled" if not handle.finished: await handle.cancelAndWait() except RetriesExhaustedError as exc: warn "Retries exhausted for block", address, exc = exc.msg + codex_block_exchange_requests_failed_total.inc() if not handle.finished: handle.fail(exc) finally: - self.pendingBlocks.setInFlight(address, false) + self.pendingBlocks.clearRequest(address) + +proc requestBlocks*( + self: BlockExcEngine, addresses: seq[BlockAddress] +): SafeAsyncIter[Block] = + var handles: seq[BlockHandle] + + # Adds all blocks to pendingBlocks before calling the first downloadInternal. This will + # ensure that we don't send incomplete want lists. + for address in addresses: + if address notin self.pendingBlocks: + handles.add(self.pendingBlocks.getWantHandle(address)) + + for address in addresses: + self.trackedFutures.track(self.downloadInternal(address)) + + let totalHandles = handles.len + var completed = 0 + + proc isFinished(): bool = + completed == totalHandles + + proc genNext(): Future[?!Block] {.async: (raises: [CancelledError]).} = + # Be it success or failure, we're completing this future. + let value = + try: + # FIXME: this is super expensive. We're doing several linear scans, + # not to mention all the copying and callback fumbling in `one`. + let + handle = await one(handles) + i = handles.find(handle) + handles.del(i) + success await handle + except CancelledError as err: + warn "Block request cancelled", addresses, err = err.msg + raise err + except CatchableError as err: + error "Error getting blocks from exchange engine", addresses, err = err.msg + failure err + + inc(completed) + return value + + return SafeAsyncIter[Block].new(genNext, isFinished) proc requestBlock*( self: BlockExcEngine, address: BlockAddress @@ -239,60 +512,64 @@ proc completeBlock*(self: BlockExcEngine, address: BlockAddress, blk: Block) = proc blockPresenceHandler*( self: BlockExcEngine, peer: PeerId, blocks: seq[BlockPresence] ) {.async: (raises: []).} = - trace "Received block presence from peer", peer, blocks = blocks.mapIt($it) + trace "Received block presence from peer", peer, len = blocks.len let peerCtx = self.peers.get(peer) - ourWantList = toSeq(self.pendingBlocks.wantList) + ourWantList = toHashSet(self.pendingBlocks.wantList.toSeq) if peerCtx.isNil: return + peerCtx.refreshReplied() + for blk in blocks: if presence =? Presence.init(blk): peerCtx.setPresence(presence) let peerHave = peerCtx.peerHave - dontWantCids = peerHave.filterIt(it notin ourWantList) + dontWantCids = peerHave - ourWantList if dontWantCids.len > 0: - peerCtx.cleanPresence(dontWantCids) + peerCtx.cleanPresence(dontWantCids.toSeq) let ourWantCids = ourWantList.filterIt( it in peerHave and not self.pendingBlocks.retriesExhausted(it) and - not self.pendingBlocks.isInFlight(it) - ) + self.pendingBlocks.markRequested(it, peer) + ).toSeq for address in ourWantCids: - self.pendingBlocks.setInFlight(address, true) self.pendingBlocks.decRetries(address) + peerCtx.blockRequestScheduled(address) if ourWantCids.len > 0: trace "Peer has blocks in our wantList", peer, wants = ourWantCids + # FIXME: this will result in duplicate requests for blocks if err =? catch(await self.sendWantBlock(ourWantCids, peerCtx)).errorOption: warn "Failed to send wantBlock to peer", peer, err = err.msg + for address in ourWantCids: + self.pendingBlocks.clearRequest(address, peer.some) proc scheduleTasks( self: BlockExcEngine, blocksDelivery: seq[BlockDelivery] ) {.async: (raises: [CancelledError]).} = - let cids = blocksDelivery.mapIt(it.blk.cid) - # schedule any new peers to provide blocks to for p in self.peers: - for c in cids: # for each cid + for blockDelivery in blocksDelivery: # for each cid # schedule a peer if it wants at least one cid # and we have it in our local store - if c in p.peerWantsCids: + if blockDelivery.address in p.wantedBlocks: + let cid = blockDelivery.blk.cid try: - if await (c in self.localStore): + if await (cid in self.localStore): # TODO: the try/except should go away once blockstore tracks exceptions self.scheduleTask(p) break except CancelledError as exc: - warn "Checking local store canceled", cid = c, err = exc.msg + warn "Checking local store canceled", cid = cid, err = exc.msg return except CatchableError as exc: - error "Error checking local store for cid", cid = c, err = exc.msg + error "Error checking local store for cid", cid = cid, err = exc.msg raiseAssert "Unexpected error checking local store for cid" proc cancelBlocks( @@ -301,28 +578,45 @@ proc cancelBlocks( ## Tells neighboring peers that we're no longer interested in a block. ## + let blocksDelivered = toHashSet(addrs) + var scheduledCancellations: Table[PeerId, HashSet[BlockAddress]] + if self.peers.len == 0: return - trace "Sending block request cancellations to peers", - addrs, peers = self.peers.peerIds - - proc processPeer(peerCtx: BlockExcPeerCtx): Future[BlockExcPeerCtx] {.async.} = + proc dispatchCancellations( + entry: tuple[peerId: PeerId, addresses: HashSet[BlockAddress]] + ): Future[PeerId] {.async: (raises: [CancelledError]).} = + trace "Sending block request cancellations to peer", + peer = entry.peerId, addresses = entry.addresses.len await self.network.request.sendWantCancellations( - peer = peerCtx.id, addresses = addrs.filterIt(it in peerCtx) + peer = entry.peerId, addresses = entry.addresses.toSeq ) - return peerCtx + return entry.peerId try: - let (succeededFuts, failedFuts) = await allFinishedFailed[BlockExcPeerCtx]( - toSeq(self.peers.peers.values).filterIt(it.peerHave.anyIt(it in addrs)).map( - processPeer - ) + for peerCtx in self.peers.peers.values: + # Do we have pending requests, towards this peer, for any of the blocks + # that were just delivered? + let intersection = peerCtx.blocksRequested.intersection(blocksDelivered) + if intersection.len > 0: + # If so, schedules a cancellation. + scheduledCancellations[peerCtx.id] = intersection + + if scheduledCancellations.len == 0: + return + + let (succeededFuts, failedFuts) = await allFinishedFailed[PeerId]( + toSeq(scheduledCancellations.pairs).map(dispatchCancellations) ) - (await allFinished(succeededFuts)).mapIt(it.read).apply do(peerCtx: BlockExcPeerCtx): - peerCtx.cleanPresence(addrs) + (await allFinished(succeededFuts)).mapIt(it.read).apply do(peerId: PeerId): + let ctx = self.peers.get(peerId) + if not ctx.isNil: + ctx.cleanPresence(addrs) + for address in scheduledCancellations[peerId]: + ctx.blockRequestCancelled(address) if failedFuts.len > 0: warn "Failed to send block request cancellations to peers", peers = failedFuts.len @@ -392,17 +686,31 @@ proc validateBlockDelivery(self: BlockExcEngine, bd: BlockDelivery): ?!void = return success() proc blocksDeliveryHandler*( - self: BlockExcEngine, peer: PeerId, blocksDelivery: seq[BlockDelivery] + self: BlockExcEngine, + peer: PeerId, + blocksDelivery: seq[BlockDelivery], + allowSpurious: bool = false, ) {.async: (raises: []).} = trace "Received blocks from peer", peer, blocks = (blocksDelivery.mapIt(it.address)) var validatedBlocksDelivery: seq[BlockDelivery] + let peerCtx = self.peers.get(peer) + + let runtimeQuota = 10.milliseconds + var lastIdle = Moment.now() + for bd in blocksDelivery: logScope: peer = peer address = bd.address try: + # Unknown peers and unrequested blocks are dropped with a warning. + if not allowSpurious and (peerCtx == nil or not peerCtx.blockReceived(bd.address)): + warn "Dropping unrequested or duplicate block received from peer" + codex_block_exchange_spurious_blocks_received.inc() + continue + if err =? self.validateBlockDelivery(bd).errorOption: warn "Block validation failed", msg = err.msg continue @@ -422,15 +730,25 @@ proc blocksDeliveryHandler*( ).errorOption: warn "Unable to store proof and cid for a block" continue + except CancelledError: + trace "Block delivery handling cancelled" except CatchableError as exc: warn "Error handling block delivery", error = exc.msg continue validatedBlocksDelivery.add(bd) + if (Moment.now() - lastIdle) >= runtimeQuota: + try: + await idleAsync() + except CancelledError: + discard + except CatchableError: + discard + lastIdle = Moment.now() + codex_block_exchange_blocks_received.inc(validatedBlocksDelivery.len.int64) - let peerCtx = self.peers.get(peer) if peerCtx != nil: if err =? catch(await self.payForBlocks(peerCtx, blocksDelivery)).errorOption: warn "Error paying for blocks", err = err.msg @@ -454,16 +772,17 @@ proc wantListHandler*( presence: seq[BlockPresence] schedulePeer = false + let runtimeQuota = 10.milliseconds + var lastIdle = Moment.now() + try: for e in wantList.entries: - let idx = peerCtx.peerWants.findIt(it.address == e.address) - logScope: peer = peerCtx.id address = e.address wantType = $e.wantType - if idx < 0: # Adding new entry to peer wants + if e.address notin peerCtx.wantedBlocks: # Adding new entry to peer wants let have = try: @@ -474,6 +793,8 @@ proc wantListHandler*( price = @(self.pricing.get(Pricing(price: 0.u256)).price.toBytesBE) if e.cancel: + # This is sort of expected if we sent the block to the peer, as we have removed + # it from the peer's wantlist ourselves. trace "Received cancelation for untracked block, skipping", address = e.address continue @@ -482,12 +803,14 @@ proc wantListHandler*( case e.wantType of WantType.WantHave: if have: + trace "We HAVE the block", address = e.address presence.add( BlockPresence( address: e.address, `type`: BlockPresenceType.Have, price: price ) ) else: + trace "We DON'T HAVE the block", address = e.address if e.sendDontHave: presence.add( BlockPresence( @@ -497,28 +820,35 @@ proc wantListHandler*( codex_block_exchange_want_have_lists_received.inc() of WantType.WantBlock: - peerCtx.peerWants.add(e) + peerCtx.wantedBlocks.incl(e.address) schedulePeer = true codex_block_exchange_want_block_lists_received.inc() else: # Updating existing entry in peer wants # peer doesn't want this block anymore if e.cancel: trace "Canceling want for block", address = e.address - peerCtx.peerWants.del(idx) + peerCtx.wantedBlocks.excl(e.address) trace "Canceled block request", - address = e.address, len = peerCtx.peerWants.len + address = e.address, len = peerCtx.wantedBlocks.len else: + trace "Peer has requested a block more than once", address = e.address if e.wantType == WantType.WantBlock: schedulePeer = true - # peer might want to ask for the same cid with - # different want params - trace "Updating want for block", address = e.address - peerCtx.peerWants[idx] = e # update entry - trace "Updated block request", - address = e.address, len = peerCtx.peerWants.len + if presence.len >= PresenceBatchSize or (Moment.now() - lastIdle) >= runtimeQuota: + if presence.len > 0: + trace "Sending presence batch to remote", items = presence.len + await self.network.request.sendPresence(peer, presence) + presence = @[] + try: + await idleAsync() + except CancelledError: + discard + lastIdle = Moment.now() + + # Send any remaining presence messages if presence.len > 0: - trace "Sending presence to remote", items = presence.mapIt($it).join(",") + trace "Sending final presence to remote", items = presence.len await self.network.request.sendPresence(peer, presence) if schedulePeer: @@ -550,7 +880,7 @@ proc paymentHandler*( else: context.paymentChannel = self.wallet.acceptChannel(payment).option -proc setupPeer*( +proc peerAddedHandler*( self: BlockExcEngine, peer: PeerId ) {.async: (raises: [CancelledError]).} = ## Perform initial setup, such as want @@ -560,88 +890,85 @@ proc setupPeer*( trace "Setting up peer", peer if peer notin self.peers: + let peerCtx = BlockExcPeerCtx(id: peer, activityTimeout: DefaultPeerActivityTimeout) trace "Setting up new peer", peer - self.peers.add(BlockExcPeerCtx(id: peer)) + self.peers.add(peerCtx) trace "Added peer", peers = self.peers.len - - # broadcast our want list, the other peer will do the same - if self.pendingBlocks.wantListLen > 0: - trace "Sending our want list to a peer", peer - let cids = toSeq(self.pendingBlocks.wantList) - await self.network.request.sendWantList(peer, cids, full = true) + await self.refreshBlockKnowledge(peerCtx) if address =? self.pricing .? address: trace "Sending account to peer", peer await self.network.request.sendAccount(peer, Account(address: address)) -proc dropPeer*(self: BlockExcEngine, peer: PeerId) {.raises: [].} = - ## Cleanup disconnected peer - ## +proc localLookup( + self: BlockExcEngine, address: BlockAddress +): Future[?!BlockDelivery] {.async: (raises: [CancelledError]).} = + if address.leaf: + (await self.localStore.getBlockAndProof(address.treeCid, address.index)).map( + (blkAndProof: (Block, CodexProof)) => + BlockDelivery(address: address, blk: blkAndProof[0], proof: blkAndProof[1].some) + ) + else: + (await self.localStore.getBlock(address)).map( + (blk: Block) => BlockDelivery(address: address, blk: blk, proof: CodexProof.none) + ) - trace "Dropping peer", peer +iterator splitBatches[T](sequence: seq[T], batchSize: int): seq[T] = + var batch: seq[T] + for element in sequence: + if batch.len == batchSize: + yield batch + batch = @[] + batch.add(element) - # drop the peer from the peers table - self.peers.remove(peer) + if batch.len > 0: + yield batch proc taskHandler*( - self: BlockExcEngine, task: BlockExcPeerCtx + self: BlockExcEngine, peerCtx: BlockExcPeerCtx ) {.gcsafe, async: (raises: [CancelledError, RetriesExhaustedError]).} = # Send to the peer blocks he wants to get, # if they present in our local store - # TODO: There should be all sorts of accounting of - # bytes sent/received here + # Blocks that have been sent have already been picked up by other tasks and + # should not be re-sent. + var + wantedBlocks = peerCtx.wantedBlocks.filterIt(not peerCtx.isBlockSent(it)) + sent: HashSet[BlockAddress] - var wantsBlocks = - task.peerWants.filterIt(it.wantType == WantType.WantBlock and not it.inFlight) + trace "Running task for peer", peer = peerCtx.id - proc updateInFlight(addresses: seq[BlockAddress], inFlight: bool) = - for peerWant in task.peerWants.mitems: - if peerWant.address in addresses: - peerWant.inFlight = inFlight + for wantedBlock in wantedBlocks: + peerCtx.markBlockAsSent(wantedBlock) - if wantsBlocks.len > 0: - # Mark wants as in-flight. - let wantAddresses = wantsBlocks.mapIt(it.address) - updateInFlight(wantAddresses, true) - wantsBlocks.sort(SortOrder.Descending) + try: + for batch in wantedBlocks.toSeq.splitBatches(self.maxBlocksPerMessage): + var blockDeliveries: seq[BlockDelivery] + for wantedBlock in batch: + # I/O is blocking so looking up blocks sequentially is fine. + without blockDelivery =? await self.localLookup(wantedBlock), err: + error "Error getting block from local store", + err = err.msg, address = wantedBlock + peerCtx.markBlockAsNotSent(wantedBlock) + continue + blockDeliveries.add(blockDelivery) + sent.incl(wantedBlock) - proc localLookup(e: WantListEntry): Future[?!BlockDelivery] {.async.} = - if e.address.leaf: - (await self.localStore.getBlockAndProof(e.address.treeCid, e.address.index)).map( - (blkAndProof: (Block, CodexProof)) => - BlockDelivery( - address: e.address, blk: blkAndProof[0], proof: blkAndProof[1].some - ) - ) - else: - (await self.localStore.getBlock(e.address)).map( - (blk: Block) => - BlockDelivery(address: e.address, blk: blk, proof: CodexProof.none) - ) + if blockDeliveries.len == 0: + continue - let - blocksDeliveryFut = await allFinished(wantsBlocks.map(localLookup)) - blocksDelivery = blocksDeliveryFut.filterIt(it.completed and it.value.isOk).mapIt: - if bd =? it.value: - bd - else: - raiseAssert "Unexpected error in local lookup" - - # All the wants that failed local lookup must be set to not-in-flight again. - let - successAddresses = blocksDelivery.mapIt(it.address) - failedAddresses = wantAddresses.filterIt(it notin successAddresses) - updateInFlight(failedAddresses, false) - - if blocksDelivery.len > 0: - trace "Sending blocks to peer", - peer = task.id, blocks = (blocksDelivery.mapIt(it.address)) - await self.network.request.sendBlocksDelivery(task.id, blocksDelivery) - - codex_block_exchange_blocks_sent.inc(blocksDelivery.len.int64) - - task.peerWants.keepItIf(it.address notin successAddresses) + await self.network.request.sendBlocksDelivery(peerCtx.id, blockDeliveries) + codex_block_exchange_blocks_sent.inc(blockDeliveries.len.int64) + # Drops the batch from the peer's set of wanted blocks; i.e. assumes that after + # we send the blocks, then the peer no longer wants them, so we don't need to + # re-send them. Note that the send might still fail down the line and we will + # have removed those anyway. At that point, we rely on the requester performing + # a retry for the request to succeed. + peerCtx.wantedBlocks.keepItIf(it notin sent) + finally: + # Better safe than sorry: if an exception does happen, we don't want to keep + # those as sent, as it'll effectively prevent the blocks from ever being sent again. + peerCtx.blocksSent.keepItIf(it notin wantedBlocks) proc blockexcTaskRunner(self: BlockExcEngine) {.async: (raises: []).} = ## process tasks @@ -652,11 +979,47 @@ proc blockexcTaskRunner(self: BlockExcEngine) {.async: (raises: []).} = while self.blockexcRunning: let peerCtx = await self.taskQueue.pop() await self.taskHandler(peerCtx) + except CancelledError: + trace "block exchange task runner cancelled" except CatchableError as exc: error "error running block exchange task", error = exc.msg info "Exiting blockexc task runner" +proc selectRandom*( + peers: seq[BlockExcPeerCtx] +): BlockExcPeerCtx {.gcsafe, raises: [].} = + if peers.len == 1: + return peers[0] + + proc evalPeerScore(peer: BlockExcPeerCtx): float = + let + loadPenalty = peer.blocksRequested.len.float * 2.0 + successRate = + if peer.exchanged > 0: + peer.exchanged.float / (peer.exchanged + peer.blocksRequested.len).float + else: + 0.5 + failurePenalty = (1.0 - successRate) * 5.0 + return loadPenalty + failurePenalty + + let + scores = peers.mapIt(evalPeerScore(it)) + maxScore = scores.max() + 1.0 + weights = scores.mapIt(maxScore - it) + + var totalWeight = 0.0 + for w in weights: + totalWeight += w + + var r = rand(totalWeight) + for i, weight in weights: + r -= weight + if r <= 0.0: + return peers[i] + + return peers[^1] + proc new*( T: type BlockExcEngine, localStore: BlockStore, @@ -666,7 +1029,9 @@ proc new*( advertiser: Advertiser, peerStore: PeerCtxStore, pendingBlocks: PendingBlocksManager, + maxBlocksPerMessage = DefaultMaxBlocksPerMessage, concurrentTasks = DefaultConcurrentTasks, + selectPeer: PeerSelector = selectRandom, ): BlockExcEngine = ## Create new block exchange engine instance ## @@ -679,23 +1044,13 @@ proc new*( wallet: wallet, concurrentTasks: concurrentTasks, trackedFutures: TrackedFutures(), + maxBlocksPerMessage: maxBlocksPerMessage, taskQueue: newAsyncHeapQueue[BlockExcPeerCtx](DefaultTaskQueueSize), discovery: discovery, advertiser: advertiser, + selectPeer: selectPeer, ) - proc peerEventHandler( - peerId: PeerId, event: PeerEvent - ): Future[void] {.gcsafe, async: (raises: [CancelledError]).} = - if event.kind == PeerEventKind.Joined: - await self.setupPeer(peerId) - else: - self.dropPeer(peerId) - - if not isNil(network.switch): - network.switch.addPeerEventHandler(peerEventHandler, PeerEventKind.Joined) - network.switch.addPeerEventHandler(peerEventHandler, PeerEventKind.Left) - proc blockWantListHandler( peer: PeerId, wantList: WantList ): Future[void] {.async: (raises: []).} = @@ -721,12 +1076,24 @@ proc new*( ): Future[void] {.async: (raises: []).} = self.paymentHandler(peer, payment) + proc peerAddedHandler( + peer: PeerId + ): Future[void] {.async: (raises: [CancelledError]).} = + await self.peerAddedHandler(peer) + + proc peerDepartedHandler( + peer: PeerId + ): Future[void] {.async: (raises: [CancelledError]).} = + self.evictPeer(peer) + network.handlers = BlockExcHandlers( onWantList: blockWantListHandler, onBlocksDelivery: blocksDeliveryHandler, onPresence: blockPresenceHandler, onAccount: accountHandler, onPayment: paymentHandler, + onPeerJoined: peerAddedHandler, + onPeerDeparted: peerDepartedHandler, ) return self diff --git a/codex/blockexchange/engine/pendingblocks.nim b/codex/blockexchange/engine/pendingblocks.nim index 80c88527..2ff1062c 100644 --- a/codex/blockexchange/engine/pendingblocks.nim +++ b/codex/blockexchange/engine/pendingblocks.nim @@ -34,7 +34,7 @@ declareGauge( const DefaultBlockRetries* = 3000 - DefaultRetryInterval* = 500.millis + DefaultRetryInterval* = 2.seconds type RetriesExhaustedError* = object of CatchableError @@ -42,7 +42,7 @@ type BlockReq* = object handle*: BlockHandle - inFlight*: bool + requested*: ?PeerId blockRetries*: int startTime*: int64 @@ -50,12 +50,13 @@ type blockRetries*: int = DefaultBlockRetries retryInterval*: Duration = DefaultRetryInterval blocks*: Table[BlockAddress, BlockReq] # pending Block requests + lastInclusion*: Moment # time at which we last included a block into our wantlist proc updatePendingBlockGauge(p: PendingBlocksManager) = codex_block_exchange_pending_block_requests.set(p.blocks.len.int64) proc getWantHandle*( - self: PendingBlocksManager, address: BlockAddress, inFlight = false + self: PendingBlocksManager, address: BlockAddress, requested: ?PeerId = PeerId.none ): Future[Block] {.async: (raw: true, raises: [CancelledError, RetriesExhaustedError]).} = ## Add an event for a block ## @@ -65,11 +66,13 @@ proc getWantHandle*( do: let blk = BlockReq( handle: newFuture[Block]("pendingBlocks.getWantHandle"), - inFlight: inFlight, + requested: requested, blockRetries: self.blockRetries, startTime: getMonoTime().ticks, ) self.blocks[address] = blk + self.lastInclusion = Moment.now() + let handle = blk.handle proc cleanUpBlock(data: pointer) {.raises: [].} = @@ -86,9 +89,9 @@ proc getWantHandle*( return handle proc getWantHandle*( - self: PendingBlocksManager, cid: Cid, inFlight = false + self: PendingBlocksManager, cid: Cid, requested: ?PeerId = PeerId.none ): Future[Block] {.async: (raw: true, raises: [CancelledError, RetriesExhaustedError]).} = - self.getWantHandle(BlockAddress.init(cid), inFlight) + self.getWantHandle(BlockAddress.init(cid), requested) proc completeWantHandle*( self: PendingBlocksManager, address: BlockAddress, blk: Block @@ -121,9 +124,6 @@ proc resolve*( blockReq.handle.complete(bd.blk) codex_block_exchange_retrieval_time_us.set(retrievalDurationUs) - - if retrievalDurationUs > 500000: - warn "High block retrieval time", retrievalDurationUs, address = bd.address else: trace "Block handle already finished", address = bd.address @@ -141,19 +141,40 @@ func retriesExhausted*(self: PendingBlocksManager, address: BlockAddress): bool self.blocks.withValue(address, pending): result = pending[].blockRetries <= 0 -func setInFlight*(self: PendingBlocksManager, address: BlockAddress, inFlight = true) = - ## Set inflight status for a block +func isRequested*(self: PendingBlocksManager, address: BlockAddress): bool = + ## Check if a block has been requested to a peer + ## + result = false + self.blocks.withValue(address, pending): + result = pending[].requested.isSome + +func getRequestPeer*(self: PendingBlocksManager, address: BlockAddress): ?PeerId = + ## Returns the peer that requested this block + ## + result = PeerId.none + self.blocks.withValue(address, pending): + result = pending[].requested + +proc markRequested*( + self: PendingBlocksManager, address: BlockAddress, peer: PeerId +): bool = + ## Marks this block as having been requested to a peer ## - self.blocks.withValue(address, pending): - pending[].inFlight = inFlight - -func isInFlight*(self: PendingBlocksManager, address: BlockAddress): bool = - ## Check if a block is in flight - ## + if self.isRequested(address): + return false self.blocks.withValue(address, pending): - result = pending[].inFlight + pending[].requested = peer.some + return true + +proc clearRequest*( + self: PendingBlocksManager, address: BlockAddress, peer: ?PeerId = PeerId.none +) = + self.blocks.withValue(address, pending): + if peer.isSome: + assert peer == pending[].requested + pending[].requested = PeerId.none func contains*(self: PendingBlocksManager, cid: Cid): bool = BlockAddress.init(cid) in self.blocks diff --git a/codex/blockexchange/network/network.nim b/codex/blockexchange/network/network.nim index d4754110..f503a15c 100644 --- a/codex/blockexchange/network/network.nim +++ b/codex/blockexchange/network/network.nim @@ -44,6 +44,7 @@ type AccountHandler* = proc(peer: PeerId, account: Account) {.gcsafe, async: (raises: []).} PaymentHandler* = proc(peer: PeerId, payment: SignedState) {.gcsafe, async: (raises: []).} + PeerEventHandler* = proc(peer: PeerId) {.gcsafe, async: (raises: [CancelledError]).} BlockExcHandlers* = object onWantList*: WantListHandler @@ -51,6 +52,9 @@ type onPresence*: BlockPresenceHandler onAccount*: AccountHandler onPayment*: PaymentHandler + onPeerJoined*: PeerEventHandler + onPeerDeparted*: PeerEventHandler + onPeerDropped*: PeerEventHandler WantListSender* = proc( id: PeerId, @@ -240,86 +244,104 @@ proc handlePayment( await network.handlers.onPayment(peer.id, payment) proc rpcHandler( - b: BlockExcNetwork, peer: NetworkPeer, msg: Message + self: BlockExcNetwork, peer: NetworkPeer, msg: Message ) {.async: (raises: []).} = ## handle rpc messages ## if msg.wantList.entries.len > 0: - b.trackedFutures.track(b.handleWantList(peer, msg.wantList)) + self.trackedFutures.track(self.handleWantList(peer, msg.wantList)) if msg.payload.len > 0: - b.trackedFutures.track(b.handleBlocksDelivery(peer, msg.payload)) + self.trackedFutures.track(self.handleBlocksDelivery(peer, msg.payload)) if msg.blockPresences.len > 0: - b.trackedFutures.track(b.handleBlockPresence(peer, msg.blockPresences)) + self.trackedFutures.track(self.handleBlockPresence(peer, msg.blockPresences)) if account =? Account.init(msg.account): - b.trackedFutures.track(b.handleAccount(peer, account)) + self.trackedFutures.track(self.handleAccount(peer, account)) if payment =? SignedState.init(msg.payment): - b.trackedFutures.track(b.handlePayment(peer, payment)) + self.trackedFutures.track(self.handlePayment(peer, payment)) -proc getOrCreatePeer(b: BlockExcNetwork, peer: PeerId): NetworkPeer = +proc getOrCreatePeer(self: BlockExcNetwork, peer: PeerId): NetworkPeer = ## Creates or retrieves a BlockExcNetwork Peer ## - if peer in b.peers: - return b.peers.getOrDefault(peer, nil) + if peer in self.peers: + return self.peers.getOrDefault(peer, nil) var getConn: ConnProvider = proc(): Future[Connection] {. async: (raises: [CancelledError]) .} = try: trace "Getting new connection stream", peer - return await b.switch.dial(peer, Codec) + return await self.switch.dial(peer, Codec) except CancelledError as error: raise error except CatchableError as exc: trace "Unable to connect to blockexc peer", exc = exc.msg - if not isNil(b.getConn): - getConn = b.getConn + if not isNil(self.getConn): + getConn = self.getConn let rpcHandler = proc(p: NetworkPeer, msg: Message) {.async: (raises: []).} = - await b.rpcHandler(p, msg) + await self.rpcHandler(p, msg) # create new pubsub peer let blockExcPeer = NetworkPeer.new(peer, getConn, rpcHandler) debug "Created new blockexc peer", peer - b.peers[peer] = blockExcPeer + self.peers[peer] = blockExcPeer return blockExcPeer -proc setupPeer*(b: BlockExcNetwork, peer: PeerId) = - ## Perform initial setup, such as want - ## list exchange - ## - - discard b.getOrCreatePeer(peer) - -proc dialPeer*(b: BlockExcNetwork, peer: PeerRecord) {.async.} = +proc dialPeer*(self: BlockExcNetwork, peer: PeerRecord) {.async.} = ## Dial a peer ## - if b.isSelf(peer.peerId): + if self.isSelf(peer.peerId): trace "Skipping dialing self", peer = peer.peerId return - if peer.peerId in b.peers: + if peer.peerId in self.peers: trace "Already connected to peer", peer = peer.peerId return - await b.switch.connect(peer.peerId, peer.addresses.mapIt(it.address)) + await self.switch.connect(peer.peerId, peer.addresses.mapIt(it.address)) -proc dropPeer*(b: BlockExcNetwork, peer: PeerId) = +proc dropPeer*( + self: BlockExcNetwork, peer: PeerId +) {.async: (raises: [CancelledError]).} = + trace "Dropping peer", peer + + try: + if not self.switch.isNil: + await self.switch.disconnect(peer) + except CatchableError as error: + warn "Error attempting to disconnect from peer", peer = peer, error = error.msg + + if not self.handlers.onPeerDropped.isNil: + await self.handlers.onPeerDropped(peer) + +proc handlePeerJoined*( + self: BlockExcNetwork, peer: PeerId +) {.async: (raises: [CancelledError]).} = + discard self.getOrCreatePeer(peer) + if not self.handlers.onPeerJoined.isNil: + await self.handlers.onPeerJoined(peer) + +proc handlePeerDeparted*( + self: BlockExcNetwork, peer: PeerId +) {.async: (raises: [CancelledError]).} = ## Cleanup disconnected peer ## - trace "Dropping peer", peer - b.peers.del(peer) + trace "Cleaning up departed peer", peer + self.peers.del(peer) + if not self.handlers.onPeerDeparted.isNil: + await self.handlers.onPeerDeparted(peer) -method init*(self: BlockExcNetwork) = +method init*(self: BlockExcNetwork) {.raises: [].} = ## Perform protocol initialization ## @@ -327,9 +349,11 @@ method init*(self: BlockExcNetwork) = peerId: PeerId, event: PeerEvent ): Future[void] {.gcsafe, async: (raises: [CancelledError]).} = if event.kind == PeerEventKind.Joined: - self.setupPeer(peerId) + await self.handlePeerJoined(peerId) + elif event.kind == PeerEventKind.Left: + await self.handlePeerDeparted(peerId) else: - self.dropPeer(peerId) + warn "Unknown peer event", event self.switch.addPeerEventHandler(peerEventHandler, PeerEventKind.Joined) self.switch.addPeerEventHandler(peerEventHandler, PeerEventKind.Left) diff --git a/codex/blockexchange/network/networkpeer.nim b/codex/blockexchange/network/networkpeer.nim index 66c39294..1964a307 100644 --- a/codex/blockexchange/network/networkpeer.nim +++ b/codex/blockexchange/network/networkpeer.nim @@ -65,7 +65,9 @@ proc readLoop*(self: NetworkPeer, conn: Connection) {.async: (raises: []).} = except CatchableError as err: warn "Exception in blockexc read loop", msg = err.msg finally: - trace "Detaching read loop", peer = self.id, connId = conn.oid + warn "Detaching read loop", peer = self.id, connId = conn.oid + if self.sendConn == conn: + self.sendConn = nil await conn.close() proc connect*( @@ -89,7 +91,12 @@ proc send*( return trace "Sending message", peer = self.id, connId = conn.oid - await conn.writeLp(protobufEncode(msg)) + try: + await conn.writeLp(protobufEncode(msg)) + except CatchableError as err: + if self.sendConn == conn: + self.sendConn = nil + raise newException(LPStreamError, "Failed to send message: " & err.msg) func new*( T: type NetworkPeer, diff --git a/codex/blockexchange/peers/peercontext.nim b/codex/blockexchange/peers/peercontext.nim index 7a299b6b..f9b3d586 100644 --- a/codex/blockexchange/peers/peercontext.nim +++ b/codex/blockexchange/peers/peercontext.nim @@ -25,28 +25,77 @@ import ../../logutils export payments, nitro +const + MinRefreshInterval = 1.seconds + MaxRefreshBackoff = 36 # 36 seconds + MaxWantListBatchSize* = 1024 # Maximum blocks to send per WantList message + type BlockExcPeerCtx* = ref object of RootObj id*: PeerId blocks*: Table[BlockAddress, Presence] # remote peer have list including price - peerWants*: seq[WantListEntry] # remote peers want lists + wantedBlocks*: HashSet[BlockAddress] # blocks that the peer wants exchanged*: int # times peer has exchanged with us - lastExchange*: Moment # last time peer has exchanged with us + refreshInProgress*: bool # indicates if a refresh is in progress + lastRefresh*: Moment # last time we refreshed our knowledge of the blocks this peer has + refreshBackoff*: int = 1 # backoff factor for refresh requests account*: ?Account # ethereum account of this peer paymentChannel*: ?ChannelId # payment channel id + blocksSent*: HashSet[BlockAddress] # blocks sent to peer + blocksRequested*: HashSet[BlockAddress] # pending block requests to this peer + lastExchange*: Moment # last time peer has sent us a block + activityTimeout*: Duration + lastSentWants*: HashSet[BlockAddress] + # track what wantList we last sent for delta updates -proc peerHave*(self: BlockExcPeerCtx): seq[BlockAddress] = - toSeq(self.blocks.keys) +proc isKnowledgeStale*(self: BlockExcPeerCtx): bool = + let staleness = + self.lastRefresh + self.refreshBackoff * MinRefreshInterval < Moment.now() -proc peerHaveCids*(self: BlockExcPeerCtx): HashSet[Cid] = - self.blocks.keys.toSeq.mapIt(it.cidOrTreeCid).toHashSet + if staleness and self.refreshInProgress: + trace "Cleaning up refresh state", peer = self.id + self.refreshInProgress = false + self.refreshBackoff = 1 -proc peerWantsCids*(self: BlockExcPeerCtx): HashSet[Cid] = - self.peerWants.mapIt(it.address.cidOrTreeCid).toHashSet + staleness + +proc isBlockSent*(self: BlockExcPeerCtx, address: BlockAddress): bool = + address in self.blocksSent + +proc markBlockAsSent*(self: BlockExcPeerCtx, address: BlockAddress) = + self.blocksSent.incl(address) + +proc markBlockAsNotSent*(self: BlockExcPeerCtx, address: BlockAddress) = + self.blocksSent.excl(address) + +proc refreshRequested*(self: BlockExcPeerCtx) = + trace "Refresh requested for peer", peer = self.id, backoff = self.refreshBackoff + self.refreshInProgress = true + self.lastRefresh = Moment.now() + +proc refreshReplied*(self: BlockExcPeerCtx) = + self.refreshInProgress = false + self.lastRefresh = Moment.now() + self.refreshBackoff = min(self.refreshBackoff * 2, MaxRefreshBackoff) + +proc havesUpdated(self: BlockExcPeerCtx) = + self.refreshBackoff = 1 + +proc wantsUpdated*(self: BlockExcPeerCtx) = + self.refreshBackoff = 1 + +proc peerHave*(self: BlockExcPeerCtx): HashSet[BlockAddress] = + # XXX: this is ugly an inefficient, but since those will typically + # be used in "joins", it's better to pay the price here and have + # a linear join than to not do it and have a quadratic join. + toHashSet(self.blocks.keys.toSeq) proc contains*(self: BlockExcPeerCtx, address: BlockAddress): bool = address in self.blocks func setPresence*(self: BlockExcPeerCtx, presence: Presence) = + if presence.address notin self.blocks: + self.havesUpdated() + self.blocks[presence.address] = presence func cleanPresence*(self: BlockExcPeerCtx, addresses: seq[BlockAddress]) = @@ -63,3 +112,36 @@ func price*(self: BlockExcPeerCtx, addresses: seq[BlockAddress]): UInt256 = price += precense[].price price + +proc blockRequestScheduled*(self: BlockExcPeerCtx, address: BlockAddress) = + ## Adds a block the set of blocks that have been requested to this peer + ## (its request schedule). + if self.blocksRequested.len == 0: + self.lastExchange = Moment.now() + self.blocksRequested.incl(address) + +proc blockRequestCancelled*(self: BlockExcPeerCtx, address: BlockAddress) = + ## Removes a block from the set of blocks that have been requested to this peer + ## (its request schedule). + self.blocksRequested.excl(address) + +proc blockReceived*(self: BlockExcPeerCtx, address: BlockAddress): bool = + let wasRequested = address in self.blocksRequested + self.blocksRequested.excl(address) + self.lastExchange = Moment.now() + wasRequested + +proc activityTimer*( + self: BlockExcPeerCtx +): Future[void] {.async: (raises: [CancelledError]).} = + ## This is called by the block exchange when a block is scheduled for this peer. + ## If the peer sends no blocks for a while, it is considered inactive/uncooperative + ## and the peer is dropped. Note that ANY block that the peer sends will reset this + ## timer for all blocks. + ## + while true: + let idleTime = Moment.now() - self.lastExchange + if idleTime > self.activityTimeout: + return + + await sleepAsync(self.activityTimeout - idleTime) diff --git a/codex/blockexchange/peers/peerctxstore.nim b/codex/blockexchange/peers/peerctxstore.nim index ce2506a8..d2762fc8 100644 --- a/codex/blockexchange/peers/peerctxstore.nim +++ b/codex/blockexchange/peers/peerctxstore.nim @@ -62,21 +62,23 @@ func len*(self: PeerCtxStore): int = self.peers.len func peersHave*(self: PeerCtxStore, address: BlockAddress): seq[BlockExcPeerCtx] = - toSeq(self.peers.values).filterIt(it.peerHave.anyIt(it == address)) + toSeq(self.peers.values).filterIt(address in it.peerHave) func peersHave*(self: PeerCtxStore, cid: Cid): seq[BlockExcPeerCtx] = + # FIXME: this is way slower and can end up leading to unexpected performance loss. toSeq(self.peers.values).filterIt(it.peerHave.anyIt(it.cidOrTreeCid == cid)) func peersWant*(self: PeerCtxStore, address: BlockAddress): seq[BlockExcPeerCtx] = - toSeq(self.peers.values).filterIt(it.peerWants.anyIt(it == address)) + toSeq(self.peers.values).filterIt(address in it.wantedBlocks) func peersWant*(self: PeerCtxStore, cid: Cid): seq[BlockExcPeerCtx] = - toSeq(self.peers.values).filterIt(it.peerWants.anyIt(it.address.cidOrTreeCid == cid)) + # FIXME: this is way slower and can end up leading to unexpected performance loss. + toSeq(self.peers.values).filterIt(it.wantedBlocks.anyIt(it.cidOrTreeCid == cid)) proc getPeersForBlock*(self: PeerCtxStore, address: BlockAddress): PeersForBlock = var res: PeersForBlock = (@[], @[]) for peer in self: - if peer.peerHave.anyIt(it == address): + if address in peer: res.with.add(peer) else: res.without.add(peer) diff --git a/codex/blockexchange/protobuf/blockexc.nim b/codex/blockexchange/protobuf/blockexc.nim index 69868681..a0512cd3 100644 --- a/codex/blockexchange/protobuf/blockexc.nim +++ b/codex/blockexchange/protobuf/blockexc.nim @@ -9,7 +9,6 @@ import std/hashes import std/sequtils -import pkg/stew/endians2 import message @@ -20,13 +19,6 @@ export Wantlist, WantType, WantListEntry export BlockDelivery, BlockPresenceType, BlockPresence export AccountMessage, StateChannelUpdate -proc hash*(a: BlockAddress): Hash = - if a.leaf: - let data = a.treeCid.data.buffer & @(a.index.uint64.toBytesBE) - hash(data) - else: - hash(a.cid.data.buffer) - proc hash*(e: WantListEntry): Hash = hash(e.address) diff --git a/codex/blockexchange/protobuf/message.nim b/codex/blockexchange/protobuf/message.nim index 4db89729..00dbc57b 100644 --- a/codex/blockexchange/protobuf/message.nim +++ b/codex/blockexchange/protobuf/message.nim @@ -25,11 +25,15 @@ type WantListEntry* = object address*: BlockAddress + # XXX: I think explicit priority is pointless as the peer will request + # the blocks in the order it wants to receive them, and all we have to + # do is process those in the same order as we send them back. It also + # complicates things for no reason at the moment, as the priority is + # always set to 0. priority*: int32 # The priority (normalized). default to 1 cancel*: bool # Whether this revokes an entry wantType*: WantType # Note: defaults to enum 0, ie Block sendDontHave*: bool # Note: defaults to false - inFlight*: bool # Whether block sending is in progress. Not serialized. WantList* = object entries*: seq[WantListEntry] # A list of wantList entries diff --git a/codex/blocktype.nim b/codex/blocktype.nim index 7e13493d..233d4e8f 100644 --- a/codex/blocktype.nim +++ b/codex/blocktype.nim @@ -9,6 +9,7 @@ import std/tables import std/sugar +import std/hashes export tables @@ -18,7 +19,7 @@ push: {.upraises: [].} import pkg/libp2p/[cid, multicodec, multihash] -import pkg/stew/byteutils +import pkg/stew/[byteutils, endians2] import pkg/questionable import pkg/questionable/results @@ -67,6 +68,13 @@ proc `$`*(a: BlockAddress): string = else: "cid: " & $a.cid +proc hash*(a: BlockAddress): Hash = + if a.leaf: + let data = a.treeCid.data.buffer & @(a.index.uint64.toBytesBE) + hash(data) + else: + hash(a.cid.data.buffer) + proc cidOrTreeCid*(a: BlockAddress): Cid = if a.leaf: a.treeCid else: a.cid diff --git a/codex/codex.nim b/codex/codex.nim index 81357464..4cb6cfb8 100644 --- a/codex/codex.nim +++ b/codex/codex.nim @@ -211,7 +211,7 @@ proc new*( .withMaxConnections(config.maxPeers) .withAgentVersion(config.agentString) .withSignedPeerRecord(true) - .withTcpTransport({ServerFlags.ReuseAddr}) + .withTcpTransport({ServerFlags.ReuseAddr, ServerFlags.TcpNoDelay}) .build() var diff --git a/codex/node.nim b/codex/node.nim index 1ca471d5..67145d22 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -44,7 +44,7 @@ import ./indexingstrategy import ./utils import ./errors import ./logutils -import ./utils/asynciter +import ./utils/safeasynciter import ./utils/trackedfutures export logutils @@ -52,7 +52,10 @@ export logutils logScope: topics = "codex node" -const DefaultFetchBatch = 10 +const + DefaultFetchBatch = 1024 + MaxOnBatchBlocks = 128 + BatchRefillThreshold = 0.75 # Refill when 75% of window completes type Contracts* = @@ -186,34 +189,62 @@ proc fetchBatched*( # (i: int) => self.networkStore.getBlock(BlockAddress.init(cid, i)) # ) - while not iter.finished: - let blockFutures = collect: - for i in 0 ..< batchSize: - if not iter.finished: - let address = BlockAddress.init(cid, iter.next()) - if not (await address in self.networkStore) or fetchLocal: - self.networkStore.getBlock(address) + # Sliding window: maintain batchSize blocks in-flight + let + refillThreshold = int(float(batchSize) * BatchRefillThreshold) + refillSize = max(refillThreshold, 1) + maxCallbackBlocks = min(batchSize, MaxOnBatchBlocks) - if blockFutures.len == 0: + var + blockData: seq[bt.Block] + failedBlocks = 0 + successfulBlocks = 0 + completedInWindow = 0 + + var addresses = newSeqOfCap[BlockAddress](batchSize) + for i in 0 ..< batchSize: + if not iter.finished: + let address = BlockAddress.init(cid, iter.next()) + if fetchLocal or not (await address in self.networkStore): + addresses.add(address) + + var blockResults = await self.networkStore.getBlocks(addresses) + + while not blockResults.finished: + without blk =? await blockResults.next(), err: + inc(failedBlocks) continue - without blockResults =? await allFinishedValues[?!bt.Block](blockFutures), err: - trace "Some blocks failed to fetch", err = err.msg - return failure(err) + inc(successfulBlocks) + inc(completedInWindow) - let blocks = blockResults.filterIt(it.isSuccess()).mapIt(it.value) + if not onBatch.isNil: + blockData.add(blk) + if blockData.len >= maxCallbackBlocks: + if batchErr =? (await onBatch(blockData)).errorOption: + return failure(batchErr) + blockData = @[] - let numOfFailedBlocks = blockResults.len - blocks.len - if numOfFailedBlocks > 0: - return - failure("Some blocks failed (Result) to fetch (" & $numOfFailedBlocks & ")") + if completedInWindow >= refillThreshold and not iter.finished: + var refillAddresses = newSeqOfCap[BlockAddress](refillSize) + for i in 0 ..< refillSize: + if not iter.finished: + let address = BlockAddress.init(cid, iter.next()) + if fetchLocal or not (await address in self.networkStore): + refillAddresses.add(address) - if not onBatch.isNil and batchErr =? (await onBatch(blocks)).errorOption: + if refillAddresses.len > 0: + blockResults = + chain(blockResults, await self.networkStore.getBlocks(refillAddresses)) + completedInWindow = 0 + + if failedBlocks > 0: + return failure("Some blocks failed (Result) to fetch (" & $failedBlocks & ")") + + if not onBatch.isNil and blockData.len > 0: + if batchErr =? (await onBatch(blockData)).errorOption: return failure(batchErr) - if not iter.finished: - await sleepAsync(1.millis) - success() proc fetchBatched*( diff --git a/codex/stores/blockstore.nim b/codex/stores/blockstore.nim index e436577c..78809191 100644 --- a/codex/stores/blockstore.nim +++ b/codex/stores/blockstore.nim @@ -70,6 +70,14 @@ method completeBlock*( ) {.base, gcsafe.} = discard +method getBlocks*( + self: BlockStore, addresses: seq[BlockAddress] +): Future[SafeAsyncIter[Block]] {.async: (raises: [CancelledError]).} = + ## Gets a set of blocks from the blockstore. Blocks might + ## be returned in any order. + + raiseAssert("getBlocks not implemented!") + method getBlockAndProof*( self: BlockStore, treeCid: Cid, index: Natural ): Future[?!(Block, CodexProof)] {.base, async: (raises: [CancelledError]), gcsafe.} = diff --git a/codex/stores/cachestore.nim b/codex/stores/cachestore.nim index ff3fd6df..a86e6b10 100644 --- a/codex/stores/cachestore.nim +++ b/codex/stores/cachestore.nim @@ -66,6 +66,21 @@ method getBlock*( trace "Error requesting block from cache", cid, error = exc.msg return failure exc +method getBlocks*( + self: CacheStore, addresses: seq[BlockAddress] +): Future[SafeAsyncIter[Block]] {.async: (raises: [CancelledError]).} = + var i = 0 + + proc isFinished(): bool = + i == addresses.len + + proc genNext(): Future[?!Block] {.async: (raises: [CancelledError]).} = + let value = await self.getBlock(addresses[i]) + inc(i) + return value + + return SafeAsyncIter[Block].new(genNext, isFinished) + method getCidAndProof*( self: CacheStore, treeCid: Cid, index: Natural ): Future[?!(Cid, CodexProof)] {.async: (raises: [CancelledError]).} = diff --git a/codex/stores/networkstore.nim b/codex/stores/networkstore.nim index 06b96b77..d047dc51 100644 --- a/codex/stores/networkstore.nim +++ b/codex/stores/networkstore.nim @@ -31,6 +31,31 @@ type NetworkStore* = ref object of BlockStore engine*: BlockExcEngine # blockexc decision engine localStore*: BlockStore # local block store +method getBlocks*( + self: NetworkStore, addresses: seq[BlockAddress] +): Future[SafeAsyncIter[Block]] {.async: (raises: [CancelledError]).} = + var + localAddresses: seq[BlockAddress] + remoteAddresses: seq[BlockAddress] + + let runtimeQuota = 10.milliseconds + var lastIdle = Moment.now() + + for address in addresses: + if not (await address in self.localStore): + remoteAddresses.add(address) + else: + localAddresses.add(address) + + if (Moment.now() - lastIdle) >= runtimeQuota: + await idleAsync() + lastIdle = Moment.now() + + return chain( + await self.localStore.getBlocks(localAddresses), + self.engine.requestBlocks(remoteAddresses), + ) + method getBlock*( self: NetworkStore, address: BlockAddress ): Future[?!Block] {.async: (raises: [CancelledError]).} = diff --git a/codex/stores/repostore/store.nim b/codex/stores/repostore/store.nim index bea2971c..ad6f03fc 100644 --- a/codex/stores/repostore/store.nim +++ b/codex/stores/repostore/store.nim @@ -38,6 +38,21 @@ logScope: # BlockStore API ########################################################### +method getBlocks*( + self: RepoStore, addresses: seq[BlockAddress] +): Future[SafeAsyncIter[Block]] {.async: (raises: [CancelledError]).} = + var i = 0 + + proc isFinished(): bool = + i == addresses.len + + proc genNext(): Future[?!Block] {.async: (raises: [CancelledError]).} = + let value = await self.getBlock(addresses[i]) + inc(i) + return value + + return SafeAsyncIter[Block].new(genNext, isFinished) + method getBlock*( self: RepoStore, cid: Cid ): Future[?!Block] {.async: (raises: [CancelledError]).} = diff --git a/codex/utils/safeasynciter.nim b/codex/utils/safeasynciter.nim index d582fec3..be46cc0f 100644 --- a/codex/utils/safeasynciter.nim +++ b/codex/utils/safeasynciter.nim @@ -232,3 +232,28 @@ proc empty*[T](_: type SafeAsyncIter[T]): SafeAsyncIter[T] = true SafeAsyncIter[T].new(genNext, isFinished) + +proc chain*[T](iters: seq[SafeAsyncIter[T]]): SafeAsyncIter[T] = + if iters.len == 0: + return SafeAsyncIter[T].empty + + var curIdx = 0 + + proc ensureNext(): void = + while curIdx < iters.len and iters[curIdx].finished: + inc(curIdx) + + proc isFinished(): bool = + curIdx == iters.len + + proc genNext(): Future[?!T] {.async: (raises: [CancelledError]).} = + let item = await iters[curIdx].next() + ensureNext() + return item + + ensureNext() + + return SafeAsyncIter[T].new(genNext, isFinished) + +proc chain*[T](iters: varargs[SafeAsyncIter[T]]): SafeAsyncIter[T] = + chain(iters.toSeq) diff --git a/tests/codex/blockexchange/discovery/testdiscovery.nim b/tests/codex/blockexchange/discovery/testdiscovery.nim index c54a1fff..6b36a630 100644 --- a/tests/codex/blockexchange/discovery/testdiscovery.nim +++ b/tests/codex/blockexchange/discovery/testdiscovery.nim @@ -54,7 +54,7 @@ asyncchecksuite "Block Advertising and Discovery": peerStore = PeerCtxStore.new() pendingBlocks = PendingBlocksManager.new() - (manifest, tree) = makeManifestAndTree(blocks).tryGet() + (_, tree, manifest) = makeDataset(blocks).tryGet() manifestBlock = bt.Block.new(manifest.encode().tryGet(), codec = ManifestCodec).tryGet() @@ -172,7 +172,7 @@ asyncchecksuite "E2E - Multiple Nodes Discovery": break blocks.add(bt.Block.new(chunk).tryGet()) - let (manifest, tree) = makeManifestAndTree(blocks).tryGet() + let (_, tree, manifest) = makeDataset(blocks).tryGet() manifests.add(manifest) mBlocks.add(manifest.asBlock()) trees.add(tree) @@ -216,7 +216,6 @@ asyncchecksuite "E2E - Multiple Nodes Discovery": test "E2E - Should advertise and discover blocks": # Distribute the manifests and trees amongst 1..3 # Ask 0 to download everything without connecting him beforehand - var advertised: Table[Cid, SignedPeerRecord] MockDiscovery(blockexc[1].engine.discovery.discovery).publishBlockProvideHandler = proc( @@ -242,6 +241,7 @@ asyncchecksuite "E2E - Multiple Nodes Discovery": blk: mBlocks[0], address: BlockAddress(leaf: false, cid: mBlocks[0].cid) ) ], + allowSpurious = true, ) discard blockexc[2].engine.pendingBlocks.getWantHandle(mBlocks[1].cid) @@ -252,6 +252,7 @@ asyncchecksuite "E2E - Multiple Nodes Discovery": blk: mBlocks[1], address: BlockAddress(leaf: false, cid: mBlocks[1].cid) ) ], + allowSpurious = true, ) discard blockexc[3].engine.pendingBlocks.getWantHandle(mBlocks[2].cid) @@ -262,6 +263,7 @@ asyncchecksuite "E2E - Multiple Nodes Discovery": blk: mBlocks[2], address: BlockAddress(leaf: false, cid: mBlocks[2].cid) ) ], + allowSpurious = true, ) MockDiscovery(blockexc[0].engine.discovery.discovery).findBlockProvidersHandler = proc( @@ -311,6 +313,7 @@ asyncchecksuite "E2E - Multiple Nodes Discovery": blk: mBlocks[0], address: BlockAddress(leaf: false, cid: mBlocks[0].cid) ) ], + allowSpurious = true, ) discard blockexc[2].engine.pendingBlocks.getWantHandle(mBlocks[1].cid) @@ -321,6 +324,7 @@ asyncchecksuite "E2E - Multiple Nodes Discovery": blk: mBlocks[1], address: BlockAddress(leaf: false, cid: mBlocks[1].cid) ) ], + allowSpurious = true, ) discard blockexc[3].engine.pendingBlocks.getWantHandle(mBlocks[2].cid) @@ -331,6 +335,7 @@ asyncchecksuite "E2E - Multiple Nodes Discovery": blk: mBlocks[2], address: BlockAddress(leaf: false, cid: mBlocks[2].cid) ) ], + allowSpurious = true, ) MockDiscovery(blockexc[0].engine.discovery.discovery).findBlockProvidersHandler = proc( diff --git a/tests/codex/blockexchange/discovery/testdiscoveryengine.nim b/tests/codex/blockexchange/discovery/testdiscoveryengine.nim index 9efab1a6..df3f8c80 100644 --- a/tests/codex/blockexchange/discovery/testdiscoveryengine.nim +++ b/tests/codex/blockexchange/discovery/testdiscoveryengine.nim @@ -43,7 +43,7 @@ asyncchecksuite "Test Discovery Engine": blocks.add(bt.Block.new(chunk).tryGet()) - (manifest, tree) = makeManifestAndTree(blocks).tryGet() + (_, tree, manifest) = makeDataset(blocks).tryGet() manifestBlock = manifest.asBlock() blocks.add(manifestBlock) diff --git a/tests/codex/blockexchange/engine/testblockexc.nim b/tests/codex/blockexchange/engine/testblockexc.nim index 6ab345d1..7c2a9ed8 100644 --- a/tests/codex/blockexchange/engine/testblockexc.nim +++ b/tests/codex/blockexchange/engine/testblockexc.nim @@ -29,14 +29,7 @@ asyncchecksuite "NetworkStore engine - 2 nodes": nodeCmps1 = generateNodes(1, blocks1).components[0] nodeCmps2 = generateNodes(1, blocks2).components[0] - await allFuturesThrowing( - nodeCmps1.switch.start(), - nodeCmps1.blockDiscovery.start(), - nodeCmps1.engine.start(), - nodeCmps2.switch.start(), - nodeCmps2.blockDiscovery.start(), - nodeCmps2.engine.start(), - ) + await allFuturesThrowing(nodeCmps1.start(), nodeCmps2.start()) # initialize our want lists pendingBlocks1 = @@ -65,14 +58,7 @@ asyncchecksuite "NetworkStore engine - 2 nodes": check isNil(peerCtx2).not teardown: - await allFuturesThrowing( - nodeCmps1.blockDiscovery.stop(), - nodeCmps1.engine.stop(), - nodeCmps1.switch.stop(), - nodeCmps2.blockDiscovery.stop(), - nodeCmps2.engine.stop(), - nodeCmps2.switch.stop(), - ) + await allFuturesThrowing(nodeCmps1.stop(), nodeCmps2.stop()) test "Should exchange blocks on connect": await allFuturesThrowing(allFinished(pendingBlocks1)).wait(10.seconds) @@ -96,17 +82,11 @@ asyncchecksuite "NetworkStore engine - 2 nodes": test "Should send want-have for block": let blk = bt.Block.new("Block 1".toBytes).tryGet() let blkFut = nodeCmps1.pendingBlocks.getWantHandle(blk.cid) + peerCtx2.blockRequestScheduled(blk.address) + (await nodeCmps2.localStore.putBlock(blk)).tryGet() - let entry = WantListEntry( - address: blk.address, - priority: 1, - cancel: false, - wantType: WantType.WantBlock, - sendDontHave: false, - ) - - peerCtx1.peerWants.add(entry) + peerCtx1.wantedBlocks.incl(blk.address) check nodeCmps2.engine.taskQueue.pushOrUpdateNoWait(peerCtx1).isOk check eventually (await nodeCmps1.localStore.hasBlock(blk.cid)).tryGet() @@ -209,3 +189,38 @@ asyncchecksuite "NetworkStore - multiple nodes": check pendingBlocks1.mapIt(it.read) == blocks[0 .. 3] check pendingBlocks2.mapIt(it.read) == blocks[12 .. 15] + +asyncchecksuite "NetworkStore - dissemination": + var nodes: seq[NodesComponents] + + teardown: + if nodes.len > 0: + await nodes.stop() + + test "Should disseminate blocks across large diameter swarm": + let dataset = makeDataset(await makeRandomBlocks(60 * 256, 256'nb)).tryGet() + + nodes = generateNodes( + 6, + config = NodeConfig( + useRepoStore: false, + findFreePorts: false, + basePort: 8080, + createFullNode: false, + enableBootstrap: false, + enableDiscovery: true, + ), + ) + + await assignBlocks(nodes[0], dataset, 0 .. 9) + await assignBlocks(nodes[1], dataset, 10 .. 19) + await assignBlocks(nodes[2], dataset, 20 .. 29) + await assignBlocks(nodes[3], dataset, 30 .. 39) + await assignBlocks(nodes[4], dataset, 40 .. 49) + await assignBlocks(nodes[5], dataset, 50 .. 59) + + await nodes.start() + await nodes.linearTopology() + + let downloads = nodes.mapIt(downloadDataset(it, dataset)) + await allFuturesThrowing(downloads).wait(30.seconds) diff --git a/tests/codex/blockexchange/engine/testengine.nim b/tests/codex/blockexchange/engine/testengine.nim index 0541c119..1afe2147 100644 --- a/tests/codex/blockexchange/engine/testengine.nim +++ b/tests/codex/blockexchange/engine/testengine.nim @@ -27,8 +27,6 @@ const NopSendWantCancellationsProc = proc( asyncchecksuite "NetworkStore engine basic": var - rng: Rng - seckey: PrivateKey peerId: PeerId chunker: Chunker wallet: WalletRef @@ -39,9 +37,7 @@ asyncchecksuite "NetworkStore engine basic": done: Future[void] setup: - rng = Rng.instance() - seckey = PrivateKey.random(rng[]).tryGet() - peerId = PeerId.init(seckey.getPublicKey().tryGet()).tryGet() + peerId = PeerId.example chunker = RandomChunker.new(Rng.instance(), size = 1024'nb, chunkSize = 256'nb) wallet = WalletRef.example blockDiscovery = Discovery.new() @@ -83,7 +79,7 @@ asyncchecksuite "NetworkStore engine basic": for b in blocks: discard engine.pendingBlocks.getWantHandle(b.cid) - await engine.setupPeer(peerId) + await engine.peerAddedHandler(peerId) await done.wait(100.millis) @@ -111,14 +107,12 @@ asyncchecksuite "NetworkStore engine basic": ) engine.pricing = pricing.some - await engine.setupPeer(peerId) + await engine.peerAddedHandler(peerId) await done.wait(100.millis) asyncchecksuite "NetworkStore engine handlers": var - rng: Rng - seckey: PrivateKey peerId: PeerId chunker: Chunker wallet: WalletRef @@ -134,8 +128,7 @@ asyncchecksuite "NetworkStore engine handlers": blocks: seq[Block] setup: - rng = Rng.instance() - chunker = RandomChunker.new(rng, size = 1024'nb, chunkSize = 256'nb) + chunker = RandomChunker.new(Rng.instance(), size = 1024'nb, chunkSize = 256'nb) while true: let chunk = await chunker.getBytes() @@ -144,8 +137,7 @@ asyncchecksuite "NetworkStore engine handlers": blocks.add(Block.new(chunk).tryGet()) - seckey = PrivateKey.random(rng[]).tryGet() - peerId = PeerId.init(seckey.getPublicKey().tryGet()).tryGet() + peerId = PeerId.example wallet = WalletRef.example blockDiscovery = Discovery.new() peerStore = PeerCtxStore.new() @@ -174,7 +166,7 @@ asyncchecksuite "NetworkStore engine handlers": let ctx = await engine.taskQueue.pop() check ctx.id == peerId # only `wantBlock` scheduled - check ctx.peerWants.mapIt(it.address.cidOrTreeCid) == blocks.mapIt(it.cid) + check ctx.wantedBlocks == blocks.mapIt(it.address).toHashSet let done = handler() await engine.wantListHandler(peerId, wantList) @@ -249,6 +241,9 @@ asyncchecksuite "NetworkStore engine handlers": test "Should store blocks in local store": let pending = blocks.mapIt(engine.pendingBlocks.getWantHandle(it.cid)) + for blk in blocks: + peerCtx.blockRequestScheduled(blk.address) + let blocksDelivery = blocks.mapIt(BlockDelivery(blk: it, address: it.address)) # Install NOP for want list cancellations so they don't cause a crash @@ -274,6 +269,9 @@ asyncchecksuite "NetworkStore engine handlers": (it.address, Presence(address: it.address, price: rand(uint16).u256, have: true)) ).toTable + for blk in blocks: + peerContext.blockRequestScheduled(blk.address) + engine.network = BlockExcNetwork( request: BlockExcRequest( sendPayment: proc( @@ -337,33 +335,44 @@ asyncchecksuite "NetworkStore engine handlers": check a in peerCtx.peerHave check peerCtx.blocks[a].price == price - test "Should send cancellations for received blocks": + test "Should send cancellations for requested blocks only": let - pending = blocks.mapIt(engine.pendingBlocks.getWantHandle(it.cid)) - blocksDelivery = blocks.mapIt(BlockDelivery(blk: it, address: it.address)) - cancellations = newTable(blocks.mapIt((it.address, newFuture[void]())).toSeq) + pendingPeer = peerId # peer towards which we have pending block requests + pendingPeerCtx = peerCtx + senderPeer = PeerId.example # peer that will actually send the blocks + senderPeerCtx = BlockExcPeerCtx(id: senderPeer) + reqBlocks = @[blocks[0], blocks[4]] # blocks that we requested to pendingPeer + reqBlockAddrs = reqBlocks.mapIt(it.address) + blockHandles = blocks.mapIt(engine.pendingBlocks.getWantHandle(it.cid)) - peerCtx.blocks = blocks.mapIt( - (it.address, Presence(address: it.address, have: true, price: UInt256.example)) - ).toTable + var cancelled: HashSet[BlockAddress] + + engine.peers.add(senderPeerCtx) + for address in reqBlockAddrs: + pendingPeerCtx.blockRequestScheduled(address) + + for address in blocks.mapIt(it.address): + senderPeerCtx.blockRequestScheduled(address) proc sendWantCancellations( id: PeerId, addresses: seq[BlockAddress] ) {.async: (raises: [CancelledError]).} = + assert id == pendingPeer for address in addresses: - cancellations[address].catch.expect("address should exist").complete() + cancelled.incl(address) engine.network = BlockExcNetwork( request: BlockExcRequest(sendWantCancellations: sendWantCancellations) ) - await engine.blocksDeliveryHandler(peerId, blocksDelivery) - discard await allFinished(pending).wait(100.millis) - await allFuturesThrowing(cancellations.values().toSeq) + let blocksDelivery = blocks.mapIt(BlockDelivery(blk: it, address: it.address)) + await engine.blocksDeliveryHandler(senderPeer, blocksDelivery) + discard await allFinished(blockHandles).wait(100.millis) + + check cancelled == reqBlockAddrs.toHashSet() asyncchecksuite "Block Download": var - rng: Rng seckey: PrivateKey peerId: PeerId chunker: Chunker @@ -380,8 +389,7 @@ asyncchecksuite "Block Download": blocks: seq[Block] setup: - rng = Rng.instance() - chunker = RandomChunker.new(rng, size = 1024'nb, chunkSize = 256'nb) + chunker = RandomChunker.new(Rng.instance(), size = 1024'nb, chunkSize = 256'nb) while true: let chunk = await chunker.getBytes() @@ -390,8 +398,7 @@ asyncchecksuite "Block Download": blocks.add(Block.new(chunk).tryGet()) - seckey = PrivateKey.random(rng[]).tryGet() - peerId = PeerId.init(seckey.getPublicKey().tryGet()).tryGet() + peerId = PeerId.example wallet = WalletRef.example blockDiscovery = Discovery.new() peerStore = PeerCtxStore.new() @@ -409,13 +416,27 @@ asyncchecksuite "Block Download": localStore, wallet, network, discovery, advertiser, peerStore, pendingBlocks ) - peerCtx = BlockExcPeerCtx(id: peerId) + peerCtx = BlockExcPeerCtx(id: peerId, activityTimeout: 100.milliseconds) engine.peers.add(peerCtx) - test "Should exhaust retries": + test "Should reschedule blocks on peer timeout": + let + slowPeer = peerId + fastPeer = PeerId.example + slowPeerCtx = peerCtx + # "Fast" peer has in fact a generous timeout. This should avoid timing issues + # in the test. + fastPeerCtx = BlockExcPeerCtx(id: fastPeer, activityTimeout: 60.seconds) + requestedBlock = blocks[0] + var - retries = 2 - address = BlockAddress.init(blocks[0].cid) + slowPeerWantList = newFuture[void]("slowPeerWantList") + fastPeerWantList = newFuture[void]("fastPeerWantList") + slowPeerDropped = newFuture[void]("slowPeerDropped") + slowPeerBlockRequest = newFuture[void]("slowPeerBlockRequest") + fastPeerBlockRequest = newFuture[void]("fastPeerBlockRequest") + + engine.peers.add(fastPeerCtx) proc sendWantList( id: PeerId, @@ -426,68 +447,63 @@ asyncchecksuite "Block Download": full: bool = false, sendDontHave: bool = false, ) {.async: (raises: [CancelledError]).} = - check wantType == WantHave - check not engine.pendingBlocks.isInFlight(address) - check engine.pendingBlocks.retries(address) == retries - retries -= 1 + check addresses == @[requestedBlock.address] - engine.pendingBlocks.blockRetries = 2 - engine.pendingBlocks.retryInterval = 10.millis + if wantType == WantBlock: + if id == slowPeer: + slowPeerBlockRequest.complete() + else: + fastPeerBlockRequest.complete() + + if wantType == WantHave: + if id == slowPeer: + slowPeerWantList.complete() + else: + fastPeerWantList.complete() + + proc onPeerDropped( + peer: PeerId + ): Future[void] {.async: (raises: [CancelledError]).} = + assert peer == slowPeer + slowPeerDropped.complete() + + proc selectPeer(peers: seq[BlockExcPeerCtx]): BlockExcPeerCtx = + # Looks for the slow peer. + for peer in peers: + if peer.id == slowPeer: + return peer + + return peers[0] + + engine.selectPeer = selectPeer + engine.pendingBlocks.retryInterval = 200.milliseconds engine.network = BlockExcNetwork(request: BlockExcRequest(sendWantList: sendWantList)) + engine.network.handlers.onPeerDropped = onPeerDropped - let pending = engine.requestBlock(address) + let blockHandle = engine.requestBlock(requestedBlock.address) - expect RetriesExhaustedError: - discard (await pending).tryGet() + # Waits for the peer to send its want list to both peers. + await slowPeerWantList.wait(5.seconds) + await fastPeerWantList.wait(5.seconds) - test "Should retry block request": - var - address = BlockAddress.init(blocks[0].cid) - steps = newAsyncEvent() + let blockPresence = + @[BlockPresence(address: requestedBlock.address, type: BlockPresenceType.Have)] - proc sendWantList( - id: PeerId, - addresses: seq[BlockAddress], - priority: int32 = 0, - cancel: bool = false, - wantType: WantType = WantType.WantHave, - full: bool = false, - sendDontHave: bool = false, - ) {.async: (raises: [CancelledError]).} = - case wantType - of WantHave: - check engine.pendingBlocks.isInFlight(address) == false - check engine.pendingBlocks.retriesExhausted(address) == false - steps.fire() - of WantBlock: - check engine.pendingBlocks.isInFlight(address) == true - check engine.pendingBlocks.retriesExhausted(address) == false - steps.fire() - - engine.pendingBlocks.blockRetries = 10 - engine.pendingBlocks.retryInterval = 10.millis - engine.network = BlockExcNetwork( - request: BlockExcRequest( - sendWantList: sendWantList, sendWantCancellations: NopSendWantCancellationsProc - ) - ) - - let pending = engine.requestBlock(address) - await steps.wait() - - # add blocks precense - peerCtx.blocks = blocks.mapIt( - (it.address, Presence(address: it.address, have: true, price: UInt256.example)) - ).toTable - - steps.clear() - await steps.wait() + await engine.blockPresenceHandler(slowPeer, blockPresence) + await engine.blockPresenceHandler(fastPeer, blockPresence) + # Waits for the peer to ask for the block. + await slowPeerBlockRequest.wait(5.seconds) + # Don't reply and wait for the peer to be dropped by timeout. + await slowPeerDropped.wait(5.seconds) + # The engine should retry and ask the fast peer for the block. + await fastPeerBlockRequest.wait(5.seconds) await engine.blocksDeliveryHandler( - peerId, @[BlockDelivery(blk: blocks[0], address: address)] + fastPeer, @[BlockDelivery(blk: requestedBlock, address: requestedBlock.address)] ) - check (await pending).tryGet() == blocks[0] + + discard await blockHandle.wait(5.seconds) test "Should cancel block request": var @@ -522,8 +538,6 @@ asyncchecksuite "Block Download": asyncchecksuite "Task Handler": var - rng: Rng - seckey: PrivateKey peerId: PeerId chunker: Chunker wallet: WalletRef @@ -541,8 +555,7 @@ asyncchecksuite "Task Handler": blocks: seq[Block] setup: - rng = Rng.instance() - chunker = RandomChunker.new(rng, size = 1024, chunkSize = 256'nb) + chunker = RandomChunker.new(Rng.instance(), size = 1024, chunkSize = 256'nb) while true: let chunk = await chunker.getBytes() if chunk.len <= 0: @@ -550,8 +563,7 @@ asyncchecksuite "Task Handler": blocks.add(Block.new(chunk).tryGet()) - seckey = PrivateKey.random(rng[]).tryGet() - peerId = PeerId.init(seckey.getPublicKey().tryGet()).tryGet() + peerId = PeerId.example wallet = WalletRef.example blockDiscovery = Discovery.new() peerStore = PeerCtxStore.new() @@ -571,138 +583,72 @@ asyncchecksuite "Task Handler": peersCtx = @[] for i in 0 .. 3: - let seckey = PrivateKey.random(rng[]).tryGet() - peers.add(PeerId.init(seckey.getPublicKey().tryGet()).tryGet()) - + peers.add(PeerId.example) peersCtx.add(BlockExcPeerCtx(id: peers[i])) peerStore.add(peersCtx[i]) engine.pricing = Pricing.example.some - test "Should send want-blocks in priority order": + # FIXME: this is disabled for now: I've dropped block priorities to make + # my life easier as I try to optimize the protocol, and also because + # they were not being used anywhere. + # + # test "Should send want-blocks in priority order": + # proc sendBlocksDelivery( + # id: PeerId, blocksDelivery: seq[BlockDelivery] + # ) {.async: (raises: [CancelledError]).} = + # check blocksDelivery.len == 2 + # check: + # blocksDelivery[1].address == blocks[0].address + # blocksDelivery[0].address == blocks[1].address + + # for blk in blocks: + # (await engine.localStore.putBlock(blk)).tryGet() + # engine.network.request.sendBlocksDelivery = sendBlocksDelivery + + # # second block to send by priority + # peersCtx[0].peerWants.add( + # WantListEntry( + # address: blocks[0].address, + # priority: 49, + # cancel: false, + # wantType: WantType.WantBlock, + # sendDontHave: false, + # ) + # ) + + # # first block to send by priority + # peersCtx[0].peerWants.add( + # WantListEntry( + # address: blocks[1].address, + # priority: 50, + # cancel: false, + # wantType: WantType.WantBlock, + # sendDontHave: false, + # ) + # ) + + # await engine.taskHandler(peersCtx[0]) + + test "Should mark outgoing blocks as sent": proc sendBlocksDelivery( id: PeerId, blocksDelivery: seq[BlockDelivery] ) {.async: (raises: [CancelledError]).} = - check blocksDelivery.len == 2 - check: - blocksDelivery[1].address == blocks[0].address - blocksDelivery[0].address == blocks[1].address + let blockAddress = peersCtx[0].wantedBlocks.toSeq[0] + check peersCtx[0].isBlockSent(blockAddress) for blk in blocks: (await engine.localStore.putBlock(blk)).tryGet() engine.network.request.sendBlocksDelivery = sendBlocksDelivery - # second block to send by priority - peersCtx[0].peerWants.add( - WantListEntry( - address: blocks[0].address, - priority: 49, - cancel: false, - wantType: WantType.WantBlock, - sendDontHave: false, - ) - ) - - # first block to send by priority - peersCtx[0].peerWants.add( - WantListEntry( - address: blocks[1].address, - priority: 50, - cancel: false, - wantType: WantType.WantBlock, - sendDontHave: false, - ) - ) + peersCtx[0].wantedBlocks.incl(blocks[0].address) await engine.taskHandler(peersCtx[0]) - test "Should set in-flight for outgoing blocks": - proc sendBlocksDelivery( - id: PeerId, blocksDelivery: seq[BlockDelivery] - ) {.async: (raises: [CancelledError]).} = - check peersCtx[0].peerWants[0].inFlight - - for blk in blocks: - (await engine.localStore.putBlock(blk)).tryGet() - engine.network.request.sendBlocksDelivery = sendBlocksDelivery - - peersCtx[0].peerWants.add( - WantListEntry( - address: blocks[0].address, - priority: 50, - cancel: false, - wantType: WantType.WantBlock, - sendDontHave: false, - inFlight: false, - ) - ) - await engine.taskHandler(peersCtx[0]) - - test "Should clear in-flight when local lookup fails": - peersCtx[0].peerWants.add( - WantListEntry( - address: blocks[0].address, - priority: 50, - cancel: false, - wantType: WantType.WantBlock, - sendDontHave: false, - inFlight: false, - ) - ) - await engine.taskHandler(peersCtx[0]) - - check not peersCtx[0].peerWants[0].inFlight - - test "Should send presence": - let present = blocks - let missing = @[Block.new("missing".toBytes).tryGet()] - let price = (!engine.pricing).price - - proc sendPresence( - id: PeerId, presence: seq[BlockPresence] - ) {.async: (raises: [CancelledError]).} = - check presence.mapIt(!Presence.init(it)) == - @[ - Presence(address: present[0].address, have: true, price: price), - Presence(address: present[1].address, have: true, price: price), - Presence(address: missing[0].address, have: false), - ] - - for blk in blocks: - (await engine.localStore.putBlock(blk)).tryGet() - engine.network.request.sendPresence = sendPresence - - # have block - peersCtx[0].peerWants.add( - WantListEntry( - address: present[0].address, - priority: 1, - cancel: false, - wantType: WantType.WantHave, - sendDontHave: false, - ) - ) - - # have block - peersCtx[0].peerWants.add( - WantListEntry( - address: present[1].address, - priority: 1, - cancel: false, - wantType: WantType.WantHave, - sendDontHave: false, - ) - ) - - # don't have block - peersCtx[0].peerWants.add( - WantListEntry( - address: missing[0].address, - priority: 1, - cancel: false, - wantType: WantType.WantHave, - sendDontHave: false, - ) - ) + test "Should not mark blocks for which local look fails as sent": + peersCtx[0].wantedBlocks.incl(blocks[0].address) await engine.taskHandler(peersCtx[0]) + + let blockAddress = peersCtx[0].wantedBlocks.toSeq[0] + check not peersCtx[0].isBlockSent(blockAddress) diff --git a/tests/codex/blockexchange/testnetwork.nim b/tests/codex/blockexchange/testnetwork.nim index b9a51c9d..ab19a6ae 100644 --- a/tests/codex/blockexchange/testnetwork.nim +++ b/tests/codex/blockexchange/testnetwork.nim @@ -40,7 +40,7 @@ asyncchecksuite "Network - Handlers": done = newFuture[void]() buffer = BufferStream.new() network = BlockExcNetwork.new(switch = newStandardSwitch(), connProvider = getConn) - network.setupPeer(peerId) + await network.handlePeerJoined(peerId) networkPeer = network.peers[peerId] discard await networkPeer.connect() diff --git a/tests/codex/blockexchange/testpeerctxstore.nim b/tests/codex/blockexchange/testpeerctxstore.nim index e2983d10..f348c1d5 100644 --- a/tests/codex/blockexchange/testpeerctxstore.nim +++ b/tests/codex/blockexchange/testpeerctxstore.nim @@ -81,8 +81,9 @@ suite "Peer Context Store Peer Selection": ) ) - peerCtxs[0].peerWants = entries - peerCtxs[5].peerWants = entries + for address in addresses: + peerCtxs[0].wantedBlocks.incl(address) + peerCtxs[5].wantedBlocks.incl(address) let peers = store.peersWant(addresses[4]) diff --git a/tests/codex/examples.nim b/tests/codex/examples.nim index 52b8a0b8..260adbfc 100644 --- a/tests/codex/examples.nim +++ b/tests/codex/examples.nim @@ -38,8 +38,7 @@ proc example*(_: type Pricing): Pricing = Pricing(address: EthAddress.example, price: uint32.rand.u256) proc example*(_: type bt.Block, size: int = 4096): bt.Block = - let length = rand(size) - let bytes = newSeqWith(length, rand(uint8)) + let bytes = newSeqWith(size, rand(uint8)) bt.Block.new(bytes).tryGet() proc example*(_: type PeerId): PeerId = diff --git a/tests/codex/helpers.nim b/tests/codex/helpers.nim index 898dd16e..b855f412 100644 --- a/tests/codex/helpers.nim +++ b/tests/codex/helpers.nim @@ -12,13 +12,16 @@ import pkg/codex/rng import pkg/codex/utils import ./helpers/nodeutils +import ./helpers/datasetutils import ./helpers/randomchunker import ./helpers/mockchunker import ./helpers/mockdiscovery import ./helpers/always import ../checktest -export randomchunker, nodeutils, mockdiscovery, mockchunker, always, checktest, manifest +export + randomchunker, nodeutils, datasetutils, mockdiscovery, mockchunker, always, checktest, + manifest export libp2p except setup, eventually @@ -46,23 +49,6 @@ proc lenPrefix*(msg: openArray[byte]): seq[byte] = return buf -proc makeManifestAndTree*(blocks: seq[Block]): ?!(Manifest, CodexTree) = - if blocks.len == 0: - return failure("Blocks list was empty") - - let - datasetSize = blocks.mapIt(it.data.len).foldl(a + b) - blockSize = blocks.mapIt(it.data.len).foldl(max(a, b)) - tree = ?CodexTree.init(blocks.mapIt(it.cid)) - treeCid = ?tree.rootCid - manifest = Manifest.new( - treeCid = treeCid, - blockSize = NBytes(blockSize), - datasetSize = NBytes(datasetSize), - ) - - return success((manifest, tree)) - proc makeWantList*( cids: seq[Cid], priority: int = 0, @@ -91,7 +77,7 @@ proc storeDataGetManifest*( (await store.putBlock(blk)).tryGet() let - (manifest, tree) = makeManifestAndTree(blocks).tryGet() + (_, tree, manifest) = makeDataset(blocks).tryGet() treeCid = tree.rootCid.tryGet() for i in 0 ..< tree.leavesCount: @@ -110,19 +96,6 @@ proc storeDataGetManifest*( return await storeDataGetManifest(store, blocks) -proc makeRandomBlocks*( - datasetSize: int, blockSize: NBytes -): Future[seq[Block]] {.async.} = - var chunker = - RandomChunker.new(Rng.instance(), size = datasetSize, chunkSize = blockSize) - - while true: - let chunk = await chunker.getBytes() - if chunk.len <= 0: - break - - result.add(Block.new(chunk).tryGet()) - proc corruptBlocks*( store: BlockStore, manifest: Manifest, blks, bytes: int ): Future[seq[int]] {.async.} = @@ -147,4 +120,5 @@ proc corruptBlocks*( bytePos.add(ii) blk.data[ii] = byte 0 + return pos diff --git a/tests/codex/helpers/datasetutils.nim b/tests/codex/helpers/datasetutils.nim new file mode 100644 index 00000000..56f26e34 --- /dev/null +++ b/tests/codex/helpers/datasetutils.nim @@ -0,0 +1,45 @@ +import std/random + +import pkg/chronos +import pkg/codex/blocktype as bt +import pkg/codex/merkletree +import pkg/codex/manifest +import pkg/codex/rng + +import ./randomchunker + +type TestDataset* = tuple[blocks: seq[Block], tree: CodexTree, manifest: Manifest] + +proc makeRandomBlock*(size: NBytes): Block = + let bytes = newSeqWith(size.int, rand(uint8)) + Block.new(bytes).tryGet() + +proc makeRandomBlocks*( + datasetSize: int, blockSize: NBytes +): Future[seq[Block]] {.async.} = + var chunker = + RandomChunker.new(Rng.instance(), size = datasetSize, chunkSize = blockSize) + + while true: + let chunk = await chunker.getBytes() + if chunk.len <= 0: + break + + result.add(Block.new(chunk).tryGet()) + +proc makeDataset*(blocks: seq[Block]): ?!TestDataset = + if blocks.len == 0: + return failure("Blocks list was empty") + + let + datasetSize = blocks.mapIt(it.data.len).foldl(a + b) + blockSize = blocks.mapIt(it.data.len).foldl(max(a, b)) + tree = ?CodexTree.init(blocks.mapIt(it.cid)) + treeCid = ?tree.rootCid + manifest = Manifest.new( + treeCid = treeCid, + blockSize = NBytes(blockSize), + datasetSize = NBytes(datasetSize), + ) + + return success((blocks, tree, manifest)) diff --git a/tests/codex/helpers/mockdiscovery.nim b/tests/codex/helpers/mockdiscovery.nim index 4110c577..69b61d49 100644 --- a/tests/codex/helpers/mockdiscovery.nim +++ b/tests/codex/helpers/mockdiscovery.nim @@ -70,3 +70,31 @@ method provide*( return await d.publishHostProvideHandler(d, host) + +proc nullDiscovery*(): MockDiscovery = + proc findBlockProvidersHandler( + d: MockDiscovery, cid: Cid + ): Future[seq[SignedPeerRecord]] {.async: (raises: [CancelledError]).} = + return @[] + + proc publishBlockProvideHandler( + d: MockDiscovery, cid: Cid + ): Future[void] {.async: (raises: [CancelledError]).} = + return + + proc findHostProvidersHandler( + d: MockDiscovery, host: ca.Address + ): Future[seq[SignedPeerRecord]] {.async: (raises: [CancelledError]).} = + return @[] + + proc publishHostProvideHandler( + d: MockDiscovery, host: ca.Address + ): Future[void] {.async: (raises: [CancelledError]).} = + return + + return MockDiscovery( + findBlockProvidersHandler: findBlockProvidersHandler, + publishBlockProvideHandler: publishBlockProvideHandler, + findHostProvidersHandler: findHostProvidersHandler, + publishHostProvideHandler: publishHostProvideHandler, + ) diff --git a/tests/codex/helpers/nodeutils.nim b/tests/codex/helpers/nodeutils.nim index 6d2edd46..12c38350 100644 --- a/tests/codex/helpers/nodeutils.nim +++ b/tests/codex/helpers/nodeutils.nim @@ -1,4 +1,5 @@ import std/sequtils +import std/sets import pkg/chronos import pkg/taskpools @@ -12,10 +13,15 @@ import pkg/codex/blockexchange import pkg/codex/systemclock import pkg/codex/nat import pkg/codex/utils/natutils +import pkg/codex/utils/safeasynciter import pkg/codex/slots +import pkg/codex/merkletree +import pkg/codex/manifest import pkg/codex/node +import ./datasetutils +import ./mockdiscovery import ../examples import ../../helpers @@ -58,6 +64,7 @@ type basePort*: int = 8080 createFullNode*: bool = false enableBootstrap*: bool = false + enableDiscovery*: bool = true converter toTuple*( nc: NodesComponents @@ -90,6 +97,36 @@ proc localStores*(cluster: NodesCluster): seq[BlockStore] = proc switches*(cluster: NodesCluster): seq[Switch] = cluster.components.mapIt(it.switch) +proc assignBlocks*( + node: NodesComponents, + dataset: TestDataset, + indices: seq[int], + putMerkleProofs = true, +): Future[void] {.async: (raises: [CatchableError]).} = + let rootCid = dataset.tree.rootCid.tryGet() + + for i in indices: + assert (await node.networkStore.putBlock(dataset.blocks[i])).isOk + if putMerkleProofs: + assert ( + await node.networkStore.putCidAndProof( + rootCid, i, dataset.blocks[i].cid, dataset.tree.getProof(i).tryGet() + ) + ).isOk + +proc assignBlocks*( + node: NodesComponents, + dataset: TestDataset, + indices: HSlice[int, int], + putMerkleProofs = true, +): Future[void] {.async: (raises: [CatchableError]).} = + await assignBlocks(node, dataset, indices.toSeq, putMerkleProofs) + +proc assignBlocks*( + node: NodesComponents, dataset: TestDataset, putMerkleProofs = true +): Future[void] {.async: (raises: [CatchableError]).} = + await assignBlocks(node, dataset, 0 ..< dataset.blocks.len, putMerkleProofs) + proc generateNodes*( num: Natural, blocks: openArray[bt.Block] = [], config: NodeConfig = NodeConfig() ): NodesCluster = @@ -145,13 +182,18 @@ proc generateNodes*( store = RepoStore.new(repoStore.newDb(), mdStore.newDb(), clock = SystemClock.new()) blockDiscoveryStore = bdStore.newDb() - discovery = Discovery.new( - switch.peerInfo.privateKey, - announceAddrs = @[listenAddr], - bindPort = bindPort.Port, - store = blockDiscoveryStore, - bootstrapNodes = bootstrapNodes, - ) + discovery = + if config.enableDiscovery: + Discovery.new( + switch.peerInfo.privateKey, + announceAddrs = @[listenAddr], + bindPort = bindPort.Port, + store = blockDiscoveryStore, + bootstrapNodes = bootstrapNodes, + ) + else: + nullDiscovery() + waitFor store.start() (store.BlockStore, @[bdStore, repoStore, mdStore], discovery) else: @@ -225,6 +267,26 @@ proc generateNodes*( return NodesCluster(components: components, taskpool: taskpool) +proc start*(nodes: NodesComponents) {.async: (raises: [CatchableError]).} = + await allFuturesThrowing( + nodes.switch.start(), + #nodes.blockDiscovery.start(), + nodes.engine.start(), + ) + +proc stop*(nodes: NodesComponents) {.async: (raises: [CatchableError]).} = + await allFuturesThrowing( + nodes.switch.stop(), + # nodes.blockDiscovery.stop(), + nodes.engine.stop(), + ) + +proc start*(nodes: seq[NodesComponents]) {.async: (raises: [CatchableError]).} = + await allFuturesThrowing(nodes.mapIt(it.start()).toSeq) + +proc stop*(nodes: seq[NodesComponents]) {.async: (raises: [CatchableError]).} = + await allFuturesThrowing(nodes.mapIt(it.stop()).toSeq) + proc connectNodes*(nodes: seq[Switch]) {.async.} = for dialer in nodes: for node in nodes: @@ -234,6 +296,15 @@ proc connectNodes*(nodes: seq[Switch]) {.async.} = proc connectNodes*(nodes: seq[NodesComponents]) {.async.} = await connectNodes(nodes.mapIt(it.switch)) +proc connectNodes*(nodes: varargs[NodesComponents]): Future[void] = + # varargs can't be captured on closures, and async procs are closures, + # so we have to do this mess + let copy = nodes.toSeq + ( + proc() {.async.} = + await connectNodes(copy.mapIt(it.switch)) + )() + proc connectNodes*(cluster: NodesCluster) {.async.} = await connectNodes(cluster.components) @@ -252,3 +323,26 @@ proc cleanup*(cluster: NodesCluster) {.async.} = await RepoStore(component.localStore).stop() cluster.taskpool.shutdown() + +proc linearTopology*(nodes: seq[NodesComponents]) {.async.} = + for i in 0 .. nodes.len - 2: + await connectNodes(nodes[i], nodes[i + 1]) + +proc downloadDataset*( + node: NodesComponents, dataset: TestDataset +): Future[void] {.async.} = + # This is the same as fetchBatched, but we don't construct CodexNodes so I can't use + # it here. + let requestAddresses = collect: + for i in 0 ..< dataset.manifest.blocksCount: + BlockAddress.init(dataset.manifest.treeCid, i) + + let blockCids = dataset.blocks.mapIt(it.cid).toHashSet() + + var count = 0 + for blockFut in (await node.networkStore.getBlocks(requestAddresses)): + let blk = (await blockFut).tryGet() + assert blk.cid in blockCids, "Unknown block CID: " & $blk.cid + count += 1 + + assert count == dataset.blocks.len, "Incorrect number of blocks downloaded" diff --git a/tests/codex/node/testnode.nim b/tests/codex/node/testnode.nim index a3fe7e94..dc3f7a8f 100644 --- a/tests/codex/node/testnode.nim +++ b/tests/codex/node/testnode.nim @@ -82,7 +82,7 @@ asyncchecksuite "Test Node - Basic": ).tryGet() test "Block Batching with corrupted blocks": - let blocks = await makeRandomBlocks(datasetSize = 64.KiBs.int, blockSize = 64.KiBs) + let blocks = await makeRandomBlocks(datasetSize = 65536, blockSize = 64.KiBs) assert blocks.len == 1 let blk = blocks[0] diff --git a/tests/codex/node/testslotrepair.nim b/tests/codex/node/testslotrepair.nim index d96078d2..2c778f44 100644 --- a/tests/codex/node/testslotrepair.nim +++ b/tests/codex/node/testslotrepair.nim @@ -48,6 +48,7 @@ asyncchecksuite "Test Node - Slot Repair": findFreePorts: true, createFullNode: true, enableBootstrap: true, + enableDiscovery: true, ) var manifest: Manifest diff --git a/tests/codex/stores/commonstoretests.nim b/tests/codex/stores/commonstoretests.nim index e4287dd2..d3132773 100644 --- a/tests/codex/stores/commonstoretests.nim +++ b/tests/codex/stores/commonstoretests.nim @@ -38,8 +38,8 @@ proc commonBlockStoreTests*( newBlock2 = Block.new("2".repeat(100).toBytes()).tryGet() newBlock3 = Block.new("3".repeat(100).toBytes()).tryGet() - (manifest, tree) = - makeManifestAndTree(@[newBlock, newBlock1, newBlock2, newBlock3]).tryGet() + (_, tree, manifest) = + makeDataset(@[newBlock, newBlock1, newBlock2, newBlock3]).tryGet() if not isNil(before): await before() diff --git a/tests/codex/stores/testrepostore.nim b/tests/codex/stores/testrepostore.nim index 69f38711..7eb9fd0d 100644 --- a/tests/codex/stores/testrepostore.nim +++ b/tests/codex/stores/testrepostore.nim @@ -364,9 +364,11 @@ asyncchecksuite "RepoStore": let repo = RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 1000'nb) - dataset = await makeRandomBlocks(datasetSize = 512, blockSize = 256'nb) - blk = dataset[0] - (manifest, tree) = makeManifestAndTree(dataset).tryGet() + (blocks, tree, manifest) = makeDataset( + await makeRandomBlocks(datasetSize = 2 * 256, blockSize = 256'nb) + ) + .tryGet() + blk = blocks[0] treeCid = tree.rootCid.tryGet() proof = tree.getProof(0).tryGet() @@ -381,9 +383,11 @@ asyncchecksuite "RepoStore": let repo = RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 1000'nb) - dataset = await makeRandomBlocks(datasetSize = 512, blockSize = 256'nb) - blk = dataset[0] - (manifest, tree) = makeManifestAndTree(dataset).tryGet() + (blocks, tree, manifest) = makeDataset( + await makeRandomBlocks(datasetSize = 2 * 256, blockSize = 256'nb) + ) + .tryGet() + blk = blocks[0] treeCid = tree.rootCid.tryGet() proof = tree.getProof(0).tryGet() @@ -406,9 +410,9 @@ asyncchecksuite "RepoStore": let sharedBlock = blockPool[1] let - (manifest1, tree1) = makeManifestAndTree(dataset1).tryGet() + (_, tree1, manifest1) = makeDataset(dataset1).tryGet() treeCid1 = tree1.rootCid.tryGet() - (manifest2, tree2) = makeManifestAndTree(dataset2).tryGet() + (_, tree2, manifest2) = makeDataset(dataset2).tryGet() treeCid2 = tree2.rootCid.tryGet() (await repo.putBlock(sharedBlock)).tryGet() @@ -435,9 +439,9 @@ asyncchecksuite "RepoStore": let repo = RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 1000'nb) - dataset = await makeRandomBlocks(datasetSize = 512, blockSize = 256'nb) - blk = dataset[0] - (manifest, tree) = makeManifestAndTree(dataset).tryGet() + blocks = await makeRandomBlocks(datasetSize = 512, blockSize = 256'nb) + blk = blocks[0] + (_, tree, manifest) = makeDataset(blocks).tryGet() treeCid = tree.rootCid.tryGet() proof = tree.getProof(1).tryGet() @@ -455,9 +459,9 @@ asyncchecksuite "RepoStore": let repo = RepoStore.new(repoDs, metaDs, clock = mockClock, quotaMaxBytes = 1000'nb) - dataset = await makeRandomBlocks(datasetSize = 512, blockSize = 256'nb) - blk = dataset[0] - (manifest, tree) = makeManifestAndTree(dataset).tryGet() + blocks = await makeRandomBlocks(datasetSize = 512, blockSize = 256'nb) + blk = blocks[0] + (_, tree, manifest) = makeDataset(blocks).tryGet() treeCid = tree.rootCid.tryGet() proof = tree.getProof(1).tryGet() diff --git a/tests/codex/testblocktype.nim b/tests/codex/testblocktype.nim new file mode 100644 index 00000000..b0ea2732 --- /dev/null +++ b/tests/codex/testblocktype.nim @@ -0,0 +1,44 @@ +import pkg/unittest2 +import pkg/libp2p/cid + +import pkg/codex/blocktype + +import ./examples + +suite "blocktype": + test "should hash equal non-leaf block addresses onto the same hash": + let + cid1 = Cid.example + nonLeaf1 = BlockAddress.init(cid1) + nonLeaf2 = BlockAddress.init(cid1) + + check nonLeaf1 == nonLeaf2 + check nonLeaf1.hash == nonLeaf2.hash + + test "should hash equal leaf block addresses onto the same hash": + let + cid1 = Cid.example + leaf1 = BlockAddress.init(cid1, 0) + leaf2 = BlockAddress.init(cid1, 0) + + check leaf1 == leaf2 + check leaf1.hash == leaf2.hash + + test "should hash different non-leaf block addresses onto different hashes": + let + cid1 = Cid.example + cid2 = Cid.example + nonLeaf1 = BlockAddress.init(cid1) + nonLeaf2 = BlockAddress.init(cid2) + + check nonLeaf1 != nonLeaf2 + check nonLeaf1.hash != nonLeaf2.hash + + test "should hash different leaf block addresses onto different hashes": + let + cid1 = Cid.example + leaf1 = BlockAddress.init(cid1, 0) + leaf2 = BlockAddress.init(cid1, 1) + + check leaf1 != leaf2 + check leaf1.hash != leaf2.hash diff --git a/tests/codex/utils/testsafeasynciter.nim b/tests/codex/utils/testsafeasynciter.nim index 1aeba4d2..87b0d84a 100644 --- a/tests/codex/utils/testsafeasynciter.nim +++ b/tests/codex/utils/testsafeasynciter.nim @@ -373,7 +373,7 @@ asyncchecksuite "Test SafeAsyncIter": # Now, to make sure that this mechanism works, and to document its # cancellation semantics, this test shows that when the async predicate # function is cancelled, this cancellation has immediate effect, which means - # that `next()` (or more precisely `getNext()` in `mapFilter` function), is + # that `next()` (or more precisely `getNext()` in `mapFilter` function), is # interrupted immediately. If this is the case, the the iterator be interrupted # before `next()` returns this locally captured value from the previous # iteration and this is exactly the reason why at the end of the test @@ -415,3 +415,20 @@ asyncchecksuite "Test SafeAsyncIter": # will not be returned because of the cancellation. collected == @["0", "1"] iter2.finished + + test "should allow chaining": + let + iter1 = SafeAsyncIter[int].new(0 ..< 5) + iter2 = SafeAsyncIter[int].new(5 ..< 10) + iter3 = chain[int](iter1, SafeAsyncIter[int].empty, iter2) + + var collected: seq[int] + + for fut in iter3: + without i =? (await fut), err: + fail() + collected.add(i) + + check: + iter3.finished + collected == @[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] diff --git a/tests/helpers.nim b/tests/helpers.nim index b48b787e..742bc10d 100644 --- a/tests/helpers.nim +++ b/tests/helpers.nim @@ -7,12 +7,11 @@ import std/sequtils, chronos export multisetup, trackers, templeveldb ### taken from libp2p errorhelpers.nim -proc allFuturesThrowing*(args: varargs[FutureBase]): Future[void] = +proc allFuturesThrowing(futs: seq[FutureBase]): Future[void] = # This proc is only meant for use in tests / not suitable for general use. # - Swallowing errors arbitrarily instead of aggregating them is bad design # - It raises `CatchableError` instead of the union of the `futs` errors, # inflating the caller's `raises` list unnecessarily. `macro` could fix it - let futs = @args ( proc() {.async: (raises: [CatchableError]).} = await allFutures(futs) @@ -28,6 +27,9 @@ proc allFuturesThrowing*(args: varargs[FutureBase]): Future[void] = raise firstErr )() +proc allFuturesThrowing*(args: varargs[FutureBase]): Future[void] = + allFuturesThrowing(@args) + proc allFuturesThrowing*[T](futs: varargs[Future[T]]): Future[void] = allFuturesThrowing(futs.mapIt(FutureBase(it))) From bd360322515addd487c8f97fc4447bf25eda6c99 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Thu, 13 Nov 2025 11:34:09 +0400 Subject: [PATCH 12/20] feat: add c binding (#1322) Co-authored-by: Eric <5089238+emizzle@users.noreply.github.com> --- Makefile | 29 + README.md | 50 + build.nims | 44 + codex.nim | 12 +- codex/clock.nim | 6 +- codex/codex.nim | 61 +- codex/conf.nim | 122 ++- codex/contracts/clock.nim | 6 +- codex/discovery.nim | 6 + codex/node.nim | 6 + codex/stores/repostore/store.nim | 2 - codex/validation.nim | 10 +- examples/golang/README.md | 24 + examples/golang/codex.go | 885 ++++++++++++++++++ examples/golang/hello.txt | 1 + library/README.md | 37 + library/alloc.nim | 42 + library/codex_context.nim | 225 +++++ .../codex_thread_request.nim | 126 +++ .../requests/node_debug_request.nim | 126 +++ .../requests/node_download_request.nim | 336 +++++++ .../requests/node_info_request.nim | 76 ++ .../requests/node_lifecycle_request.nim | 188 ++++ .../requests/node_p2p_request.nim | 95 ++ .../requests/node_storage_request.nim | 180 ++++ .../requests/node_upload_request.nim | 372 ++++++++ library/events/json_base_event.nim | 14 + library/ffi_types.nim | 62 ++ library/libcodex.h | 206 ++++ library/libcodex.nim | 565 +++++++++++ tests/codex/helpers/mockclock.nim | 17 +- tests/integration/codexprocess.nim | 3 +- 32 files changed, 3862 insertions(+), 72 deletions(-) create mode 100644 examples/golang/README.md create mode 100644 examples/golang/codex.go create mode 100644 examples/golang/hello.txt create mode 100644 library/README.md create mode 100644 library/alloc.nim create mode 100644 library/codex_context.nim create mode 100644 library/codex_thread_requests/codex_thread_request.nim create mode 100644 library/codex_thread_requests/requests/node_debug_request.nim create mode 100644 library/codex_thread_requests/requests/node_download_request.nim create mode 100644 library/codex_thread_requests/requests/node_info_request.nim create mode 100644 library/codex_thread_requests/requests/node_lifecycle_request.nim create mode 100644 library/codex_thread_requests/requests/node_p2p_request.nim create mode 100644 library/codex_thread_requests/requests/node_storage_request.nim create mode 100644 library/codex_thread_requests/requests/node_upload_request.nim create mode 100644 library/events/json_base_event.nim create mode 100644 library/ffi_types.nim create mode 100644 library/libcodex.h create mode 100644 library/libcodex.nim diff --git a/Makefile b/Makefile index f39a3394..4ec12fb5 100644 --- a/Makefile +++ b/Makefile @@ -232,6 +232,7 @@ format: $(NPH) *.nim $(NPH) codex/ $(NPH) tests/ + $(NPH) library/ clean-nph: rm -f $(NPH) @@ -242,4 +243,32 @@ print-nph-path: clean: | clean-nph +################ +## C Bindings ## +################ +.PHONY: libcodex + +STATIC ?= 0 + +ifneq ($(strip $(CODEX_LIB_PARAMS)),) +NIM_PARAMS := $(NIM_PARAMS) $(CODEX_LIB_PARAMS) +endif + +libcodex: + $(MAKE) deps + rm -f build/libcodex* + +ifeq ($(STATIC), 1) + echo -e $(BUILD_MSG) "build/$@.a" && \ + $(ENV_SCRIPT) nim libcodexStatic $(NIM_PARAMS) -d:LeopardCmakeFlags="\"-DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=Release\"" codex.nims +else ifeq ($(detected_OS),Windows) + echo -e $(BUILD_MSG) "build/$@.dll" && \ + $(ENV_SCRIPT) nim libcodexDynamic $(NIM_PARAMS) -d:LeopardCmakeFlags="\"-G \\\"MSYS Makefiles\\\" -DCMAKE_BUILD_TYPE=Release\"" codex.nims +else ifeq ($(detected_OS),macOS) + echo -e $(BUILD_MSG) "build/$@.dylib" && \ + $(ENV_SCRIPT) nim libcodexDynamic $(NIM_PARAMS) -d:LeopardCmakeFlags="\"-DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=Release\"" codex.nims +else + echo -e $(BUILD_MSG) "build/$@.so" && \ + $(ENV_SCRIPT) nim libcodexDynamic $(NIM_PARAMS) -d:LeopardCmakeFlags="\"-DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=Release\"" codex.nims +endif endif # "variables.mk" was not included diff --git a/README.md b/README.md index 2a15051f..78b4e6f5 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,56 @@ To get acquainted with Codex, consider: The client exposes a REST API that can be used to interact with the clients. Overview of the API can be found on [api.codex.storage](https://api.codex.storage). +## Bindings + +Codex provides a C API that can be wrapped by other languages. The bindings is located in the `library` folder. +Currently, only a Go binding is included. + +### Build the C library + +```bash +make libcodex +``` + +This produces the shared library under `build/`. + +### Run the Go example + +Build the Go example: + +```bash +go build -o codex-go examples/golang/codex.go +``` + +Export the library path: + +```bash +export LD_LIBRARY_PATH=build +``` + +Run the example: + +```bash +./codex-go +``` + +### Static vs Dynamic build + +By default, Codex builds a dynamic library (`libcodex.so`), which you can load at runtime. +If you prefer a static library (`libcodex.a`), set the `STATIC` flag: + +```bash +# Build dynamic (default) +make libcodex + +# Build static +make STATIC=1 libcodex +``` + +### Limitation + +Callbacks must be fast and non-blocking; otherwise, the working thread will hang and prevent other requests from being processed. + ## Contributing and development Feel free to dive in, contributions are welcomed! Open an issue or submit PRs. diff --git a/build.nims b/build.nims index 88660321..dcfe5c13 100644 --- a/build.nims +++ b/build.nims @@ -25,6 +25,30 @@ proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") = exec(cmd) +proc buildLibrary(name: string, srcDir = "./", params = "", `type` = "dynamic") = + if not dirExists "build": + mkDir "build" + + if `type` == "dynamic": + let lib_name = ( + when defined(windows): name & ".dll" + elif defined(macosx): name & ".dylib" + else: name & ".so" + ) + exec "nim c" & " --out:build/" & lib_name & + " --threads:on --app:lib --opt:size --noMain --mm:refc --header --d:metrics " & + "--nimMainPrefix:libcodex -d:noSignalHandler " & + "-d:LeopardExtraCompilerFlags=-fPIC " & "-d:chronicles_runtime_filtering " & + "-d:chronicles_log_level=TRACE " & params & " " & srcDir & name & ".nim" + else: + exec "nim c" & " --out:build/" & name & + ".a --threads:on --app:staticlib --opt:size --noMain --mm:refc --header --d:metrics " & + "--nimMainPrefix:libcodex -d:noSignalHandler " & + "-d:LeopardExtraCompilerFlags=-fPIC " & + "-d:chronicles_runtime_filtering " & + "-d:chronicles_log_level=TRACE " & + params & " " & srcDir & name & ".nim" + proc test(name: string, srcDir = "tests/", params = "", lang = "c") = buildBinary name, srcDir, params exec "build/" & name @@ -121,3 +145,23 @@ task showCoverage, "open coverage html": echo " ======== Opening HTML coverage report in browser... ======== " if findExe("open") != "": exec("open coverage/report/index.html") + +task libcodexDynamic, "Generate bindings": + var params = "" + when compiles(commandLineParams): + for param in commandLineParams(): + if param.len > 0 and param.startsWith("-"): + params.add " " & param + + let name = "libcodex" + buildLibrary name, "library/", params, "dynamic" + +task libcodexStatic, "Generate bindings": + var params = "" + when compiles(commandLineParams): + for param in commandLineParams(): + if param.len > 0 and param.startsWith("-"): + params.add " " & param + + let name = "libcodex" + buildLibrary name, "library/", params, "static" diff --git a/codex.nim b/codex.nim index 7749bdee..b3e40608 100644 --- a/codex.nim +++ b/codex.nim @@ -54,6 +54,16 @@ when isMainModule: , ) config.setupLogging() + + try: + updateLogLevel(config.logLevel) + except ValueError as err: + try: + stderr.write "Invalid value for --log-level. " & err.msg & "\n" + except IOError: + echo "Invalid value for --log-level. " & err.msg + quit QuitFailure + config.setupMetrics() if not (checkAndCreateDataDir((config.dataDir).string)): @@ -94,7 +104,7 @@ when isMainModule: ## Ctrl+C handling proc doShutdown() = - shutdown = server.stop() + shutdown = server.shutdown() state = CodexStatus.Stopping notice "Stopping Codex" diff --git a/codex/clock.nim b/codex/clock.nim index c02e04aa..5c63d15e 100644 --- a/codex/clock.nim +++ b/codex/clock.nim @@ -1,3 +1,5 @@ +{.push raises: [].} + import pkg/chronos import pkg/stew/endians2 import pkg/upraises @@ -11,7 +13,9 @@ type method now*(clock: Clock): SecondsSince1970 {.base, gcsafe, upraises: [].} = raiseAssert "not implemented" -method waitUntil*(clock: Clock, time: SecondsSince1970) {.base, async.} = +method waitUntil*( + clock: Clock, time: SecondsSince1970 +) {.base, async: (raises: [CancelledError]).} = raiseAssert "not implemented" method start*(clock: Clock) {.base, async.} = diff --git a/codex/codex.nim b/codex/codex.nim index 4cb6cfb8..682fe9b5 100644 --- a/codex/codex.nim +++ b/codex/codex.nim @@ -57,10 +57,20 @@ type repoStore: RepoStore maintenance: BlockMaintainer taskpool: Taskpool + isStarted: bool CodexPrivateKey* = libp2p.PrivateKey # alias EthWallet = ethers.Wallet +func config*(self: CodexServer): CodexConf = + return self.config + +func node*(self: CodexServer): CodexNodeRef = + return self.codexNode + +func repoStore*(self: CodexServer): RepoStore = + return self.repoStore + proc waitForSync(provider: Provider): Future[void] {.async.} = var sleepTime = 1 trace "Checking sync state of Ethereum provider..." @@ -159,9 +169,13 @@ proc bootstrapInteractions(s: CodexServer): Future[void] {.async.} = s.codexNode.contracts = (client, host, validator) proc start*(s: CodexServer) {.async.} = - trace "Starting codex node", config = $s.config + if s.isStarted: + warn "Codex server already started, skipping" + return + trace "Starting codex node", config = $s.config await s.repoStore.start() + s.maintenance.start() await s.codexNode.switch.start() @@ -175,27 +189,55 @@ proc start*(s: CodexServer) {.async.} = await s.bootstrapInteractions() await s.codexNode.start() - s.restServer.start() + + if s.restServer != nil: + s.restServer.start() + + s.isStarted = true proc stop*(s: CodexServer) {.async.} = + if not s.isStarted: + warn "Codex is not started" + return + notice "Stopping codex node" - let res = await noCancel allFinishedFailed[void]( + var futures = @[ - s.restServer.stop(), s.codexNode.switch.stop(), s.codexNode.stop(), s.repoStore.stop(), s.maintenance.stop(), ] - ) + + if s.restServer != nil: + futures.add(s.restServer.stop()) + + let res = await noCancel allFinishedFailed[void](futures) if res.failure.len > 0: error "Failed to stop codex node", failures = res.failure.len raiseAssert "Failed to stop codex node" +proc close*(s: CodexServer) {.async.} = + var futures = @[s.codexNode.close(), s.repoStore.close()] + + let res = await noCancel allFinishedFailed[void](futures) + if not s.taskpool.isNil: - s.taskpool.shutdown() + try: + s.taskpool.shutdown() + except Exception as exc: + error "Failed to stop the taskpool", failures = res.failure.len + raiseAssert("Failure in taskpool shutdown:" & exc.msg) + + if res.failure.len > 0: + error "Failed to close codex node", failures = res.failure.len + raiseAssert "Failed to close codex node" + +proc shutdown*(server: CodexServer) {.async.} = + await server.stop() + await server.close() proc new*( T: type CodexServer, config: CodexConf, privateKey: CodexPrivateKey @@ -295,7 +337,7 @@ proc new*( ) peerStore = PeerCtxStore.new() - pendingBlocks = PendingBlocksManager.new() + pendingBlocks = PendingBlocksManager.new(retries = config.blockRetries) advertiser = Advertiser.new(repoStore, discovery) blockDiscovery = DiscoveryEngine.new(repoStore, peerStore, network, discovery, pendingBlocks) @@ -320,10 +362,13 @@ proc new*( taskPool = taskpool, ) + var restServer: RestServerRef = nil + + if config.apiBindAddress.isSome: restServer = RestServerRef .new( codexNode.initRestApi(config, repoStore, config.apiCorsAllowedOrigin), - initTAddress(config.apiBindAddress, config.apiPort), + initTAddress(config.apiBindAddress.get(), config.apiPort), bufferSize = (1024 * 64), maxRequestBodySize = int.high, ) diff --git a/codex/conf.nim b/codex/conf.nim index 77ef96ca..206a3e04 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -34,6 +34,7 @@ import pkg/libp2p import pkg/ethers import pkg/questionable import pkg/questionable/results +import pkg/stew/base64 import ./codextypes import ./discovery @@ -46,13 +47,14 @@ import ./utils/natutils from ./contracts/config import DefaultRequestCacheSize, DefaultMaxPriorityFeePerGas from ./validationconfig import MaxSlots, ValidationGroups +from ./blockexchange/engine/pendingblocks import DefaultBlockRetries export units, net, codextypes, logutils, completeCmdArg, parseCmdArg, NatConfig export ValidationGroups, MaxSlots export DefaultQuotaBytes, DefaultBlockTtl, DefaultBlockInterval, DefaultNumBlocksPerInterval, - DefaultRequestCacheSize, DefaultMaxPriorityFeePerGas + DefaultRequestCacheSize, DefaultMaxPriorityFeePerGas, DefaultBlockRetries type ThreadCount* = distinct Natural @@ -202,8 +204,10 @@ type .}: string apiBindAddress* {. - desc: "The REST API bind address", defaultValue: "127.0.0.1", name: "api-bindaddr" - .}: string + desc: "The REST API bind address", + defaultValue: "127.0.0.1".some, + name: "api-bindaddr" + .}: Option[string] apiPort* {. desc: "The REST Api port", @@ -261,6 +265,13 @@ type name: "block-mn" .}: int + blockRetries* {. + desc: "Number of times to retry fetching a block before giving up", + defaultValue: DefaultBlockRetries, + defaultValueDesc: $DefaultBlockRetries, + name: "block-retries" + .}: int + cacheSize* {. desc: "The size of the block cache, 0 disables the cache - " & @@ -474,7 +485,7 @@ func prover*(self: CodexConf): bool = self.persistence and self.persistenceCmd == PersistenceCmd.prover proc getCodexVersion(): string = - let tag = strip(staticExec("git tag")) + let tag = strip(staticExec("git describe --tags --abbrev=0")) if tag.isEmptyOrWhitespace: return "untagged build" return tag @@ -510,55 +521,73 @@ proc parseCmdArg*( if res.isOk: ma = res.get() else: - warn "Invalid MultiAddress", input = input, error = res.error() + fatal "Invalid MultiAddress", input = input, error = res.error() quit QuitFailure except LPError as exc: - warn "Invalid MultiAddress uri", uri = input, error = exc.msg + fatal "Invalid MultiAddress uri", uri = input, error = exc.msg quit QuitFailure ma -proc parseCmdArg*(T: type ThreadCount, input: string): T {.upraises: [ValueError].} = - let count = parseInt(input) - if count != 0 and count < 2: - warn "Invalid number of threads", input = input - quit QuitFailure - ThreadCount(count) +proc parse*(T: type ThreadCount, p: string): Result[ThreadCount, string] = + try: + let count = parseInt(p) + if count != 0 and count < 2: + return err("Invalid number of threads: " & p) + return ok(ThreadCount(count)) + except ValueError as e: + return err("Invalid number of threads: " & p & ", error=" & e.msg) -proc parseCmdArg*(T: type SignedPeerRecord, uri: string): T = +proc parseCmdArg*(T: type ThreadCount, input: string): T = + let val = ThreadCount.parse(input) + if val.isErr: + fatal "Cannot parse the thread count.", input = input, error = val.error() + quit QuitFailure + return val.get() + +proc parse*(T: type SignedPeerRecord, p: string): Result[SignedPeerRecord, string] = var res: SignedPeerRecord try: - if not res.fromURI(uri): - warn "Invalid SignedPeerRecord uri", uri = uri - quit QuitFailure - except LPError as exc: - warn "Invalid SignedPeerRecord uri", uri = uri, error = exc.msg - quit QuitFailure - except CatchableError as exc: - warn "Invalid SignedPeerRecord uri", uri = uri, error = exc.msg - quit QuitFailure - res + if not res.fromURI(p): + return err("The uri is not a valid SignedPeerRecord: " & p) + return ok(res) + except LPError, Base64Error: + let e = getCurrentException() + return err(e.msg) -func parseCmdArg*(T: type NatConfig, p: string): T {.raises: [ValueError].} = +proc parseCmdArg*(T: type SignedPeerRecord, uri: string): T = + let res = SignedPeerRecord.parse(uri) + if res.isErr: + fatal "Cannot parse the signed peer.", error = res.error(), input = uri + quit QuitFailure + return res.get() + +func parse*(T: type NatConfig, p: string): Result[NatConfig, string] = case p.toLowerAscii of "any": - NatConfig(hasExtIp: false, nat: NatStrategy.NatAny) + return ok(NatConfig(hasExtIp: false, nat: NatStrategy.NatAny)) of "none": - NatConfig(hasExtIp: false, nat: NatStrategy.NatNone) + return ok(NatConfig(hasExtIp: false, nat: NatStrategy.NatNone)) of "upnp": - NatConfig(hasExtIp: false, nat: NatStrategy.NatUpnp) + return ok(NatConfig(hasExtIp: false, nat: NatStrategy.NatUpnp)) of "pmp": - NatConfig(hasExtIp: false, nat: NatStrategy.NatPmp) + return ok(NatConfig(hasExtIp: false, nat: NatStrategy.NatPmp)) else: if p.startsWith("extip:"): try: let ip = parseIpAddress(p[6 ..^ 1]) - NatConfig(hasExtIp: true, extIp: ip) + return ok(NatConfig(hasExtIp: true, extIp: ip)) except ValueError: let error = "Not a valid IP address: " & p[6 ..^ 1] - raise newException(ValueError, error) + return err(error) else: - let error = "Not a valid NAT option: " & p - raise newException(ValueError, error) + return err("Not a valid NAT option: " & p) + +proc parseCmdArg*(T: type NatConfig, p: string): T = + let res = NatConfig.parse(p) + if res.isErr: + fatal "Cannot parse the NAT config.", error = res.error(), input = p + quit QuitFailure + return res.get() proc completeCmdArg*(T: type NatConfig, val: string): seq[string] = return @[] @@ -566,19 +595,25 @@ proc completeCmdArg*(T: type NatConfig, val: string): seq[string] = proc parseCmdArg*(T: type EthAddress, address: string): T = EthAddress.init($address).get() -proc parseCmdArg*(T: type NBytes, val: string): T = +func parse*(T: type NBytes, p: string): Result[NBytes, string] = var num = 0'i64 - let count = parseSize(val, num, alwaysBin = true) + let count = parseSize(p, num, alwaysBin = true) if count == 0: - warn "Invalid number of bytes", nbytes = val + return err("Invalid number of bytes: " & p) + return ok(NBytes(num)) + +proc parseCmdArg*(T: type NBytes, val: string): T = + let res = NBytes.parse(val) + if res.isErr: + fatal "Cannot parse NBytes.", error = res.error(), input = val quit QuitFailure - NBytes(num) + return res.get() proc parseCmdArg*(T: type Duration, val: string): T = var dur: Duration let count = parseDuration(val, dur) if count == 0: - warn "Cannot parse duration", dur = dur + fatal "Cannot parse duration", dur = dur quit QuitFailure dur @@ -595,7 +630,7 @@ proc readValue*(r: var TomlReader, val: var SignedPeerRecord) = try: val = SignedPeerRecord.parseCmdArg(uri) except LPError as err: - warn "Invalid SignedPeerRecord uri", uri = uri, error = err.msg + fatal "Invalid SignedPeerRecord uri", uri = uri, error = err.msg quit QuitFailure proc readValue*(r: var TomlReader, val: var MultiAddress) = @@ -607,7 +642,7 @@ proc readValue*(r: var TomlReader, val: var MultiAddress) = if res.isOk: val = res.get() else: - warn "Invalid MultiAddress", input = input, error = res.error() + fatal "Invalid MultiAddress", input = input, error = res.error() quit QuitFailure proc readValue*( @@ -779,15 +814,6 @@ proc setupLogging*(conf: CodexConf) = else: defaultChroniclesStream.outputs[0].writer = writer - try: - updateLogLevel(conf.logLevel) - except ValueError as err: - try: - stderr.write "Invalid value for --log-level. " & err.msg & "\n" - except IOError: - echo "Invalid value for --log-level. " & err.msg - quit QuitFailure - proc setupMetrics*(config: CodexConf) = if config.metricsEnabled: let metricsAddress = config.metricsAddress diff --git a/codex/contracts/clock.nim b/codex/contracts/clock.nim index b7863539..1d4f57ba 100644 --- a/codex/contracts/clock.nim +++ b/codex/contracts/clock.nim @@ -1,3 +1,5 @@ +{.push raises: [].} + import std/times import pkg/ethers import pkg/questionable @@ -72,7 +74,9 @@ method now*(clock: OnChainClock): SecondsSince1970 = doAssert clock.started, "clock should be started before calling now()" return toUnix(getTime() + clock.offset) -method waitUntil*(clock: OnChainClock, time: SecondsSince1970) {.async.} = +method waitUntil*( + clock: OnChainClock, time: SecondsSince1970 +) {.async: (raises: [CancelledError]).} = while (let difference = time - clock.now(); difference > 0): clock.newBlock.clear() discard await clock.newBlock.wait().withTimeout(chronos.seconds(difference)) diff --git a/codex/discovery.nim b/codex/discovery.nim index 4a211c20..9ff89caf 100644 --- a/codex/discovery.nim +++ b/codex/discovery.nim @@ -43,6 +43,7 @@ type Discovery* = ref object of RootObj # record to advertice node connection information, this carry any # address that the node can be connected on dhtRecord*: ?SignedPeerRecord # record to advertice DHT connection information + isStarted: bool proc toNodeId*(cid: Cid): NodeId = ## Cid to discovery id @@ -203,10 +204,15 @@ proc start*(d: Discovery) {.async: (raises: []).} = try: d.protocol.open() await d.protocol.start() + d.isStarted = true except CatchableError as exc: error "Error starting discovery", exc = exc.msg proc stop*(d: Discovery) {.async: (raises: []).} = + if not d.isStarted: + warn "Discovery not started, skipping stop" + return + try: await noCancel d.protocol.closeWait() except CatchableError as exc: diff --git a/codex/node.nim b/codex/node.nim index 67145d22..ccae080c 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -84,6 +84,7 @@ type BatchProc* = proc(blocks: seq[bt.Block]): Future[?!void] {. gcsafe, async: (raises: [CancelledError]) .} + OnBlockStoredProc = proc(chunk: seq[byte]): void {.gcsafe, raises: [].} func switch*(self: CodexNodeRef): Switch = return self.switch @@ -434,6 +435,7 @@ proc store*( filename: ?string = string.none, mimetype: ?string = string.none, blockSize = DefaultBlockSize, + onBlockStored: OnBlockStoredProc = nil, ): Future[?!Cid] {.async.} = ## Save stream contents as dataset with given blockSize ## to nodes's BlockStore, and return Cid of its manifest @@ -463,6 +465,9 @@ proc store*( if err =? (await self.networkStore.putBlock(blk)).errorOption: error "Unable to store block", cid = blk.cid, err = err.msg return failure(&"Unable to store block {blk.cid}") + + if not onBlockStored.isNil: + onBlockStored(chunk) except CancelledError as exc: raise exc except CatchableError as exc: @@ -902,6 +907,7 @@ proc stop*(self: CodexNodeRef) {.async.} = if not self.clock.isNil: await self.clock.stop() +proc close*(self: CodexNodeRef) {.async.} = if not self.networkStore.isNil: await self.networkStore.close diff --git a/codex/stores/repostore/store.nim b/codex/stores/repostore/store.nim index ad6f03fc..16813a16 100644 --- a/codex/stores/repostore/store.nim +++ b/codex/stores/repostore/store.nim @@ -443,7 +443,6 @@ proc start*( ): Future[void] {.async: (raises: [CancelledError, CodexError]).} = ## Start repo ## - if self.started: trace "Repo already started" return @@ -465,6 +464,5 @@ proc stop*(self: RepoStore): Future[void] {.async: (raises: []).} = return trace "Stopping repo" - await self.close() self.started = false diff --git a/codex/validation.nim b/codex/validation.nim index 58a0e6b7..d9f8fb5e 100644 --- a/codex/validation.nim +++ b/codex/validation.nim @@ -80,7 +80,7 @@ proc removeSlotsThatHaveEnded(validation: Validation) {.async.} = proc markProofAsMissing( validation: Validation, slotId: SlotId, period: Period -) {.async.} = +) {.async: (raises: [CancelledError]).} = logScope: currentPeriod = validation.getCurrentPeriod() @@ -91,18 +91,18 @@ proc markProofAsMissing( else: let inDowntime {.used.} = await validation.market.inDowntime(slotId) trace "Proof not missing", checkedPeriod = period, inDowntime - except CancelledError: - raise + except CancelledError as e: + raise e except CatchableError as e: error "Marking proof as missing failed", msg = e.msg -proc markProofsAsMissing(validation: Validation) {.async.} = +proc markProofsAsMissing(validation: Validation) {.async: (raises: [CancelledError]).} = let slots = validation.slots for slotId in slots: let previousPeriod = validation.getCurrentPeriod() - 1 await validation.markProofAsMissing(slotId, previousPeriod) -proc run(validation: Validation) {.async: (raises: []).} = +proc run(validation: Validation) {.async: (raises: [CancelledError]).} = trace "Validation started" try: while true: diff --git a/examples/golang/README.md b/examples/golang/README.md new file mode 100644 index 00000000..30a12932 --- /dev/null +++ b/examples/golang/README.md @@ -0,0 +1,24 @@ + +## Pre-requisite + +libcodex.so is needed to be compiled and present in build folder. + +## Compilation + +From the codex root folder: + +```code +go build -o codex-go examples/golang/codex.go +``` + +## Run +From the codex root folder: + + +```code +export LD_LIBRARY_PATH=build +``` + +```code +./codex-go +``` diff --git a/examples/golang/codex.go b/examples/golang/codex.go new file mode 100644 index 00000000..cf6f9aa3 --- /dev/null +++ b/examples/golang/codex.go @@ -0,0 +1,885 @@ +package main + +/* + #cgo LDFLAGS: -L../../build/ -lcodex + #cgo LDFLAGS: -L../../ -Wl,-rpath,../../ + + #include + #include + #include "../../library/libcodex.h" + + typedef struct { + int ret; + char* msg; + size_t len; + uintptr_t h; + } Resp; + + static void* allocResp(uintptr_t h) { + Resp* r = (Resp*)calloc(1, sizeof(Resp)); + r->h = h; + return r; + } + + static void freeResp(void* resp) { + if (resp != NULL) { + free(resp); + } + } + + static int getRet(void* resp) { + if (resp == NULL) { + return 0; + } + Resp* m = (Resp*) resp; + return m->ret; + } + + void libcodexNimMain(void); + + static void codex_host_init_once(void){ + static int done; + if (!__atomic_exchange_n(&done, 1, __ATOMIC_SEQ_CST)) libcodexNimMain(); + } + + // resp must be set != NULL in case interest on retrieving data from the callback + void callback(int ret, char* msg, size_t len, void* resp); + + static void* cGoCodexNew(const char* configJson, void* resp) { + void* ret = codex_new(configJson, (CodexCallback) callback, resp); + return ret; + } + + static int cGoCodexStart(void* codexCtx, void* resp) { + return codex_start(codexCtx, (CodexCallback) callback, resp); + } + + static int cGoCodexStop(void* codexCtx, void* resp) { + return codex_stop(codexCtx, (CodexCallback) callback, resp); + } + + static int cGoCodexClose(void* codexCtx, void* resp) { + return codex_close(codexCtx, (CodexCallback) callback, resp); + } + + static int cGoCodexDestroy(void* codexCtx, void* resp) { + return codex_destroy(codexCtx, (CodexCallback) callback, resp); + } + + static int cGoCodexVersion(void* codexCtx, void* resp) { + return codex_version(codexCtx, (CodexCallback) callback, resp); + } + + static int cGoCodexRevision(void* codexCtx, void* resp) { + return codex_revision(codexCtx, (CodexCallback) callback, resp); + } + + static int cGoCodexRepo(void* codexCtx, void* resp) { + return codex_repo(codexCtx, (CodexCallback) callback, resp); + } + + static int cGoCodexSpr(void* codexCtx, void* resp) { + return codex_spr(codexCtx, (CodexCallback) callback, resp); + } + + static int cGoCodexPeerId(void* codexCtx, void* resp) { + return codex_peer_id(codexCtx, (CodexCallback) callback, resp); + } + + static int cGoCodexUploadInit(void* codexCtx, char* filepath, size_t chunkSize, void* resp) { + return codex_upload_init(codexCtx, filepath, chunkSize, (CodexCallback) callback, resp); + } + + static int cGoCodexUploadChunk(void* codexCtx, char* sessionId, const uint8_t* chunk, size_t len, void* resp) { + return codex_upload_chunk(codexCtx, sessionId, chunk, len, (CodexCallback) callback, resp); + } + + static int cGoCodexUploadFinalize(void* codexCtx, char* sessionId, void* resp) { + return codex_upload_finalize(codexCtx, sessionId, (CodexCallback) callback, resp); + } + + static int cGoCodexUploadCancel(void* codexCtx, char* sessionId, void* resp) { + return codex_upload_cancel(codexCtx, sessionId, (CodexCallback) callback, resp); + } + + static int cGoCodexUploadFile(void* codexCtx, char* sessionId, void* resp) { + return codex_upload_file(codexCtx, sessionId, (CodexCallback) callback, resp); + } + + static int cGoCodexLogLevel(void* codexCtx, char* logLevel, void* resp) { + return codex_log_level(codexCtx, logLevel, (CodexCallback) callback, resp); + } + + static int cGoCodexExists(void* codexCtx, char* cid, void* resp) { + return codex_storage_exists(codexCtx, cid, (CodexCallback) callback, resp); + } +*/ +import "C" +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "os" + "os/signal" + "runtime/cgo" + "sync" + "syscall" + "unsafe" +) + +type LogFormat string + +const ( + LogFormatAuto LogFormat = "auto" + LogFormatColors LogFormat = "colors" + LogFormatNoColors LogFormat = "nocolors" + LogFormatJSON LogFormat = "json" +) + +type RepoKind string + +const ( + FS RepoKind = "fs" + SQLite RepoKind = "sqlite" + LevelDb RepoKind = "leveldb" +) + +const defaultBlockSize = 1024 * 64 + +type Config struct { + // Default: INFO + LogLevel string `json:"log-level,omitempty"` + + // Specifies what kind of logs should be written to stdout + // Default: auto + LogFormat LogFormat `json:"log-format,omitempty"` + + // Enable the metrics server + // Default: false + MetricsEnabled bool `json:"metrics,omitempty"` + + // Listening address of the metrics server + // Default: 127.0.0.1 + MetricsAddress string `json:"metrics-address,omitempty"` + + // Listening HTTP port of the metrics server + // Default: 8008 + MetricsPort int `json:"metrics-port,omitempty"` + + // The directory where codex will store configuration and data + // Default: + // $HOME\AppData\Roaming\Codex on Windows + // $HOME/Library/Application Support/Codex on macOS + // $HOME/.cache/codex on Linux + DataDir string `json:"data-dir,omitempty"` + + // Multi Addresses to listen on + // Default: ["/ip4/0.0.0.0/tcp/0"] + ListenAddrs []string `json:"listen-addrs,omitempty"` + + // Specify method to use for determining public address. + // Must be one of: any, none, upnp, pmp, extip: + // Default: any + Nat string `json:"nat,omitempty"` + + // Discovery (UDP) port + // Default: 8090 + DiscoveryPort int `json:"disc-port,omitempty"` + + // Source of network (secp256k1) private key file path or name + // Default: "key" + NetPrivKeyFile string `json:"net-privkey,omitempty"` + + // Specifies one or more bootstrap nodes to use when connecting to the network. + BootstrapNodes []string `json:"bootstrap-node,omitempty"` + + // The maximum number of peers to connect to. + // Default: 160 + MaxPeers int `json:"max-peers,omitempty"` + + // Number of worker threads (\"0\" = use as many threads as there are CPU cores available) + // Default: 0 + NumThreads int `json:"num-threads,omitempty"` + + // Node agent string which is used as identifier in network + // Default: "Codex" + AgentString string `json:"agent-string,omitempty"` + + // Backend for main repo store (fs, sqlite, leveldb) + // Default: fs + RepoKind RepoKind `json:"repo-kind,omitempty"` + + // The size of the total storage quota dedicated to the node + // Default: 20 GiBs + StorageQuota int `json:"storage-quota,omitempty"` + + // Default block timeout in seconds - 0 disables the ttl + // Default: 30 days + BlockTtl int `json:"block-ttl,omitempty"` + + // Time interval in seconds - determines frequency of block + // maintenance cycle: how often blocks are checked for expiration and cleanup + // Default: 10 minutes + BlockMaintenanceInterval int `json:"block-mi,omitempty"` + + // Number of blocks to check every maintenance cycle + // Default: 1000 + BlockMaintenanceNumberOfBlocks int `json:"block-mn,omitempty"` + + // Number of times to retry fetching a block before giving up + // Default: 3000 + BlockRetries int `json:"block-retries,omitempty"` + + // The size of the block cache, 0 disables the cache - + // might help on slow hardrives + // Default: 0 + CacheSize int `json:"cache-size,omitempty"` + + // Default: "" (no log file) + LogFile string `json:"log-file,omitempty"` +} + +type CodexNode struct { + ctx unsafe.Pointer +} + +type ChunkSize int + +func (c ChunkSize) valOrDefault() int { + if c == 0 { + return defaultBlockSize + } + + return int(c) +} + +func (c ChunkSize) toSizeT() C.size_t { + return C.size_t(c.valOrDefault()) +} + +// bridgeCtx is used for managing the C-Go bridge calls. +// It contains a wait group for synchronizing the calls, +// a cgo.Handle for passing context to the C code, +// a response pointer for receiving data from the C code, +// and fields for storing the result and error of the call. +type bridgeCtx struct { + wg *sync.WaitGroup + h cgo.Handle + resp unsafe.Pointer + result string + err error + + // Callback used for receiving progress updates during upload/download. + // + // For the upload, the bytes parameter indicates the number of bytes uploaded. + // If the chunk size is superior or equal to the blocksize (passed in init function), + // the callback will be called when a block is put in the store. + // Otherwise, it will be called when a chunk is pushed into the stream. + // + // For the download, the bytes is the size of the chunk received, and the chunk + // is the actual chunk of data received. + onProgress func(bytes int, chunk []byte) +} + +// newBridgeCtx creates a new bridge context for managing C-Go calls. +// The bridge context is initialized with a wait group and a cgo.Handle. +func newBridgeCtx() *bridgeCtx { + bridge := &bridgeCtx{} + bridge.wg = &sync.WaitGroup{} + bridge.wg.Add(1) + bridge.h = cgo.NewHandle(bridge) + bridge.resp = C.allocResp(C.uintptr_t(uintptr(bridge.h))) + return bridge +} + +// callError creates an error message for a failed C-Go call. +func (b *bridgeCtx) callError(name string) error { + return fmt.Errorf("failed the call to %s returned code %d", name, C.getRet(b.resp)) +} + +// free releases the resources associated with the bridge context, +// including the cgo.Handle and the response pointer. +func (b *bridgeCtx) free() { + if b.h > 0 { + b.h.Delete() + b.h = 0 + } + + if b.resp != nil { + C.freeResp(b.resp) + b.resp = nil + } +} + +// callback is the function called by the C code to communicate back to Go. +// It handles progress updates, successful completions, and errors. +// The function uses the response pointer to retrieve the bridge context +// and update its state accordingly. +// +//export callback +func callback(ret C.int, msg *C.char, len C.size_t, resp unsafe.Pointer) { + if resp == nil { + return + } + + m := (*C.Resp)(resp) + m.ret = ret + m.msg = msg + m.len = len + + if m.h == 0 { + return + } + + h := cgo.Handle(m.h) + if h == 0 { + return + } + + if v, ok := h.Value().(*bridgeCtx); ok { + switch ret { + case C.RET_PROGRESS: + if v.onProgress == nil { + return + } + if msg != nil { + chunk := C.GoBytes(unsafe.Pointer(msg), C.int(len)) + v.onProgress(int(C.int(len)), chunk) + } else { + v.onProgress(int(C.int(len)), nil) + } + case C.RET_OK: + retMsg := C.GoStringN(msg, C.int(len)) + v.result = retMsg + v.err = nil + if v.wg != nil { + v.wg.Done() + } + case C.RET_ERR: + retMsg := C.GoStringN(msg, C.int(len)) + v.err = errors.New(retMsg) + if v.wg != nil { + v.wg.Done() + } + } + } +} + +// wait waits for the bridge context to complete its operation. +// It returns the result and error of the operation. +func (b *bridgeCtx) wait() (string, error) { + b.wg.Wait() + return b.result, b.err +} + +type OnUploadProgressFunc func(read, total int, percent float64, err error) + +type UploadOptions struct { + // Filepath can be the full path when using UploadFile + // otherwise the file name. + // It is used to detect the mimetype. + Filepath string + + // ChunkSize is the size of each upload chunk, passed as `blockSize` to the Codex node + // store. Default is to 64 KB. + ChunkSize ChunkSize + + // OnProgress is a callback function that is called after each chunk is uploaded with: + // - read: the number of bytes read in the last chunk. + // - total: the total number of bytes read so far. + // - percent: the percentage of the total file size that has been uploaded. It is + // determined from a `stat` call if it is a file and from the length of the buffer + // if it is a buffer. Otherwise, it is 0. + // - err: an error, if one occurred. + // + // If the chunk size is more than the `chunkSize` parameter, the callback is called + // after the block is actually stored in the block store. Otherwise, it is called + // after the chunk is sent to the stream. + OnProgress OnUploadProgressFunc +} + +func getReaderSize(r io.Reader) int64 { + switch v := r.(type) { + case *os.File: + stat, err := v.Stat() + if err != nil { + return 0 + } + return stat.Size() + case *bytes.Buffer: + return int64(v.Len()) + default: + return 0 + } +} + +// New creates a new Codex node with the provided configuration. +// The node is not started automatically; you need to call CodexStart +// to start it. +// It returns a Codex node that can be used to interact +// with the Codex network. +func New(config Config) (*CodexNode, error) { + bridge := newBridgeCtx() + defer bridge.free() + + jsonConfig, err := json.Marshal(config) + if err != nil { + return nil, err + } + + cJsonConfig := C.CString(string(jsonConfig)) + defer C.free(unsafe.Pointer(cJsonConfig)) + + ctx := C.cGoCodexNew(cJsonConfig, bridge.resp) + + if _, err := bridge.wait(); err != nil { + return nil, bridge.err + } + + return &CodexNode{ctx: ctx}, bridge.err +} + +// Start starts the Codex node. +func (node CodexNode) Start() error { + bridge := newBridgeCtx() + defer bridge.free() + + if C.cGoCodexStart(node.ctx, bridge.resp) != C.RET_OK { + return bridge.callError("cGoCodexStart") + } + + _, err := bridge.wait() + return err +} + +// StartAsync is the asynchronous version of Start. +func (node CodexNode) StartAsync(onDone func(error)) { + go func() { + err := node.Start() + onDone(err) + }() +} + +// Stop stops the Codex node. +func (node CodexNode) Stop() error { + bridge := newBridgeCtx() + defer bridge.free() + + if C.cGoCodexStop(node.ctx, bridge.resp) != C.RET_OK { + return bridge.callError("cGoCodexStop") + } + + _, err := bridge.wait() + return err +} + +// Destroy destroys the Codex node, freeing all resources. +// The node must be stopped before calling this method. +func (node CodexNode) Destroy() error { + bridge := newBridgeCtx() + defer bridge.free() + + if C.cGoCodexClose(node.ctx, bridge.resp) != C.RET_OK { + return bridge.callError("cGoCodexClose") + } + + _, err := bridge.wait() + if err != nil { + return err + } + + if C.cGoCodexDestroy(node.ctx, bridge.resp) != C.RET_OK { + return errors.New("Failed to destroy the codex node.") + } + + return err +} + +// Version returns the version of the Codex node. +func (node CodexNode) Version() (string, error) { + bridge := newBridgeCtx() + defer bridge.free() + + if C.cGoCodexVersion(node.ctx, bridge.resp) != C.RET_OK { + return "", bridge.callError("cGoCodexVersion") + } + + return bridge.wait() +} + +func (node CodexNode) Revision() (string, error) { + bridge := newBridgeCtx() + defer bridge.free() + + if C.cGoCodexRevision(node.ctx, bridge.resp) != C.RET_OK { + return "", bridge.callError("cGoCodexRevision") + } + + return bridge.wait() +} + +// Repo returns the path of the data dir folder. +func (node CodexNode) Repo() (string, error) { + bridge := newBridgeCtx() + defer bridge.free() + + if C.cGoCodexRepo(node.ctx, bridge.resp) != C.RET_OK { + return "", bridge.callError("cGoCodexRepo") + } + + return bridge.wait() +} + +func (node CodexNode) Spr() (string, error) { + bridge := newBridgeCtx() + defer bridge.free() + + if C.cGoCodexSpr(node.ctx, bridge.resp) != C.RET_OK { + return "", bridge.callError("cGoCodexSpr") + } + + return bridge.wait() +} + +func (node CodexNode) PeerId() (string, error) { + bridge := newBridgeCtx() + defer bridge.free() + + if C.cGoCodexPeerId(node.ctx, bridge.resp) != C.RET_OK { + return "", bridge.callError("cGoCodexPeerId") + } + + return bridge.wait() +} + +// UploadInit initializes a new upload session. +// It returns a session ID that can be used for subsequent upload operations. +// This function is called by UploadReader and UploadFile internally. +// You should use this function only if you need to manage the upload session manually. +func (node CodexNode) UploadInit(options *UploadOptions) (string, error) { + bridge := newBridgeCtx() + defer bridge.free() + + var cFilename = C.CString(options.Filepath) + defer C.free(unsafe.Pointer(cFilename)) + + if C.cGoCodexUploadInit(node.ctx, cFilename, options.ChunkSize.toSizeT(), bridge.resp) != C.RET_OK { + return "", bridge.callError("cGoCodexUploadInit") + } + + return bridge.wait() +} + +// UploadChunk uploads a chunk of data to the Codex node. +// It takes the session ID returned by UploadInit +// and a byte slice containing the chunk data. +// This function is called by UploadReader internally. +// You should use this function only if you need to manage the upload session manually. +func (node CodexNode) UploadChunk(sessionId string, chunk []byte) error { + bridge := newBridgeCtx() + defer bridge.free() + + var cSessionId = C.CString(sessionId) + defer C.free(unsafe.Pointer(cSessionId)) + + var cChunkPtr *C.uint8_t + if len(chunk) > 0 { + cChunkPtr = (*C.uint8_t)(unsafe.Pointer(&chunk[0])) + } + + if C.cGoCodexUploadChunk(node.ctx, cSessionId, cChunkPtr, C.size_t(len(chunk)), bridge.resp) != C.RET_OK { + return bridge.callError("cGoCodexUploadChunk") + } + + _, err := bridge.wait() + return err +} + +// UploadFinalize finalizes the upload session and returns the CID of the uploaded file. +// It takes the session ID returned by UploadInit. +// This function is called by UploadReader and UploadFile internally. +// You should use this function only if you need to manage the upload session manually. +func (node CodexNode) UploadFinalize(sessionId string) (string, error) { + bridge := newBridgeCtx() + defer bridge.free() + + var cSessionId = C.CString(sessionId) + defer C.free(unsafe.Pointer(cSessionId)) + + if C.cGoCodexUploadFinalize(node.ctx, cSessionId, bridge.resp) != C.RET_OK { + return "", bridge.callError("cGoCodexUploadFinalize") + } + + return bridge.wait() +} + +// UploadCancel cancels an ongoing upload session. +// It can be only if the upload session is managed manually. +// It doesn't work with UploadFile. +func (node CodexNode) UploadCancel(sessionId string) error { + bridge := newBridgeCtx() + defer bridge.free() + + var cSessionId = C.CString(sessionId) + defer C.free(unsafe.Pointer(cSessionId)) + + if C.cGoCodexUploadCancel(node.ctx, cSessionId, bridge.resp) != C.RET_OK { + return bridge.callError("cGoCodexUploadCancel") + } + + _, err := bridge.wait() + return err +} + +// UploadReader uploads data from an io.Reader to the Codex node. +// It takes the upload options and the reader as parameters. +// It returns the CID of the uploaded file or an error. +// +// Internally, it calls: +// - UploadInit to create the upload session. +// - UploadChunk to upload a chunk to codex. +// - UploadFinalize to finalize the upload session. +// - UploadCancel if an error occurs. +func (node CodexNode) UploadReader(options UploadOptions, r io.Reader) (string, error) { + sessionId, err := node.UploadInit(&options) + if err != nil { + return "", err + } + + buf := make([]byte, options.ChunkSize.valOrDefault()) + total := 0 + + var size int64 + if options.OnProgress != nil { + size = getReaderSize(r) + } + + for { + n, err := r.Read(buf) + if err == io.EOF { + break + } + + if err != nil { + if cancelErr := node.UploadCancel(sessionId); cancelErr != nil { + return "", fmt.Errorf("failed to upload chunk %v and failed to cancel upload session %v", err, cancelErr) + } + + return "", err + } + + if n == 0 { + break + } + + if err := node.UploadChunk(sessionId, buf[:n]); err != nil { + if cancelErr := node.UploadCancel(sessionId); cancelErr != nil { + return "", fmt.Errorf("failed to upload chunk %v and failed to cancel upload session %v", err, cancelErr) + } + + return "", err + } + + total += n + if options.OnProgress != nil && size > 0 { + percent := float64(total) / float64(size) * 100.0 + // The last block could be a bit over the size due to padding + // on the chunk size. + if percent > 100.0 { + percent = 100.0 + } + options.OnProgress(n, total, percent, nil) + } else if options.OnProgress != nil { + options.OnProgress(n, total, 0, nil) + } + } + + return node.UploadFinalize(sessionId) +} + +// UploadReaderAsync is the asynchronous version of UploadReader using a goroutine. +func (node CodexNode) UploadReaderAsync(options UploadOptions, r io.Reader, onDone func(cid string, err error)) { + go func() { + cid, err := node.UploadReader(options, r) + onDone(cid, err) + }() +} + +// UploadFile uploads a file to the Codex node. +// It takes the upload options as parameter. +// It returns the CID of the uploaded file or an error. +// +// The options parameter contains the following fields: +// - filepath: the full path of the file to upload. +// - chunkSize: the size of each upload chunk, passed as `blockSize` to the Codex node +// store. Default is to 64 KB. +// - onProgress: a callback function that is called after each chunk is uploaded with: +// - read: the number of bytes read in the last chunk. +// - total: the total number of bytes read so far. +// - percent: the percentage of the total file size that has been uploaded. It is +// determined from a `stat` call. +// - err: an error, if one occurred. +// +// If the chunk size is more than the `chunkSize` parameter, the callback is called after +// the block is actually stored in the block store. Otherwise, it is called after the chunk +// is sent to the stream. +// +// Internally, it calls UploadInit to create the upload session. +func (node CodexNode) UploadFile(options UploadOptions) (string, error) { + bridge := newBridgeCtx() + defer bridge.free() + + if options.OnProgress != nil { + stat, err := os.Stat(options.Filepath) + if err != nil { + return "", err + } + + size := stat.Size() + total := 0 + + if size > 0 { + bridge.onProgress = func(read int, _ []byte) { + if read == 0 { + return + } + + total += read + percent := float64(total) / float64(size) * 100.0 + // The last block could be a bit over the size due to padding + // on the chunk size. + if percent > 100.0 { + percent = 100.0 + } + + options.OnProgress(read, int(size), percent, nil) + } + } + } + + sessionId, err := node.UploadInit(&options) + if err != nil { + return "", err + } + + var cSessionId = C.CString(sessionId) + defer C.free(unsafe.Pointer(cSessionId)) + + if C.cGoCodexUploadFile(node.ctx, cSessionId, bridge.resp) != C.RET_OK { + return "", bridge.callError("cGoCodexUploadFile") + } + + return bridge.wait() +} + +// UploadFileAsync is the asynchronous version of UploadFile using a goroutine. +func (node CodexNode) UploadFileAsync(options UploadOptions, onDone func(cid string, err error)) { + go func() { + cid, err := node.UploadFile(options) + onDone(cid, err) + }() +} + +func (node CodexNode) UpdateLogLevel(logLevel string) error { + bridge := newBridgeCtx() + defer bridge.free() + + var cLogLevel = C.CString(string(logLevel)) + defer C.free(unsafe.Pointer(cLogLevel)) + + if C.cGoCodexLogLevel(node.ctx, cLogLevel, bridge.resp) != C.RET_OK { + return bridge.callError("cGoCodexLogLevel") + } + + _, err := bridge.wait() + return err +} + +func (node CodexNode) Exists(cid string) (bool, error) { + bridge := newBridgeCtx() + defer bridge.free() + + var cCid = C.CString(cid) + defer C.free(unsafe.Pointer(cCid)) + + if C.cGoCodexExists(node.ctx, cCid, bridge.resp) != C.RET_OK { + return false, bridge.callError("cGoCodexUploadCancel") + } + + result, err := bridge.wait() + return result == "true", err +} + +func main() { + dataDir := os.TempDir() + "/data-dir" + + node, err := New(Config{ + BlockRetries: 5, + LogLevel: "WARN", + DataDir: dataDir, + }) + if err != nil { + log.Fatalf("Failed to create Codex node: %v", err) + } + defer os.RemoveAll(dataDir) + + if err := node.Start(); err != nil { + log.Fatalf("Failed to start Codex node: %v", err) + } + log.Println("Codex node started") + + version, err := node.Version() + if err != nil { + log.Fatalf("Failed to get Codex version: %v", err) + } + log.Printf("Codex version: %s", version) + + err = node.UpdateLogLevel("ERROR") + if err != nil { + log.Fatalf("Failed to update log level: %v", err) + } + + cid := "zDvZRwzmAkhzDRPH5EW242gJBNZ2T7aoH2v1fVH66FxXL4kSbvyM" + exists, err := node.Exists(cid) + if err != nil { + log.Fatalf("Failed to check data existence: %v", err) + } + + if exists { + log.Fatalf("The data should not exist") + } + + buf := bytes.NewBuffer([]byte("Hello World!")) + len := buf.Len() + cid, err = node.UploadReader(UploadOptions{Filepath: "hello.txt"}, buf) + if err != nil { + log.Fatalf("Failed to upload data: %v", err) + } + log.Printf("Uploaded data with CID: %s (size: %d bytes)", cid, len) + + exists, err = node.Exists(cid) + if err != nil { + log.Fatalf("Failed to check data existence: %v", err) + } + + if !exists { + log.Fatalf("The data should exist") + } + + // Wait for a SIGINT or SIGTERM signal + ch := make(chan os.Signal, 1) + signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM) + <-ch + + if err := node.Stop(); err != nil { + log.Fatalf("Failed to stop Codex node: %v", err) + } + log.Println("Codex node stopped") + + if err := node.Destroy(); err != nil { + log.Fatalf("Failed to destroy Codex node: %v", err) + } +} diff --git a/examples/golang/hello.txt b/examples/golang/hello.txt new file mode 100644 index 00000000..c57eff55 --- /dev/null +++ b/examples/golang/hello.txt @@ -0,0 +1 @@ +Hello World! \ No newline at end of file diff --git a/library/README.md b/library/README.md new file mode 100644 index 00000000..db6423fe --- /dev/null +++ b/library/README.md @@ -0,0 +1,37 @@ +# Codex Library + +Codex exposes a C binding that serves as a stable contract, making it straightforward to integrate Codex into other languages such as Go. + +The implementation was inspired by [nim-library-template](https://github.com/logos-co/nim-library-template) +and by the [nwaku](https://github.com/waku-org/nwaku/tree/master/library) library. + +The source code contains detailed comments to explain the threading and callback flow. +The diagram below summarizes the lifecycle: context creation, request execution, and shutdown. + +```mermaid +sequenceDiagram + autonumber + actor App as App/User + participant Go as Go Wrapper + participant C as C API (libcodex.h) + participant Ctx as CodexContext + participant Thr as Worker Thread + participant Eng as CodexServer + + App->>Go: Start + Go->>C: codex_start_node + C->>Ctx: enqueue request + C->>Ctx: fire signal + Ctx->>Thr: wake worker + Thr->>Ctx: dequeue request + Thr-->>Ctx: ACK + Ctx-->>C: forward ACK + C-->>Go: RET OK + Go->>App: Unblock + Thr->>Eng: execute (async) + Eng-->>Thr: result ready + Thr-->>Ctx: callback + Ctx-->>C: forward callback + C-->>Go: forward callback + Go-->>App: done +``` \ No newline at end of file diff --git a/library/alloc.nim b/library/alloc.nim new file mode 100644 index 00000000..1a6f118b --- /dev/null +++ b/library/alloc.nim @@ -0,0 +1,42 @@ +## Can be shared safely between threads +type SharedSeq*[T] = tuple[data: ptr UncheckedArray[T], len: int] + +proc alloc*(str: cstring): cstring = + # Byte allocation from the given address. + # There should be the corresponding manual deallocation with deallocShared ! + if str.isNil(): + var ret = cast[cstring](allocShared(1)) # Allocate memory for the null terminator + ret[0] = '\0' # Set the null terminator + return ret + + let ret = cast[cstring](allocShared(len(str) + 1)) + copyMem(ret, str, len(str) + 1) + return ret + +proc alloc*(str: string): cstring = + ## Byte allocation from the given address. + ## There should be the corresponding manual deallocation with deallocShared ! + var ret = cast[cstring](allocShared(str.len + 1)) + let s = cast[seq[char]](str) + for i in 0 ..< str.len: + ret[i] = s[i] + ret[str.len] = '\0' + return ret + +proc allocSharedSeq*[T](s: seq[T]): SharedSeq[T] = + let data = allocShared(sizeof(T) * s.len) + if s.len != 0: + copyMem(data, unsafeAddr s[0], s.len) + return (cast[ptr UncheckedArray[T]](data), s.len) + +proc deallocSharedSeq*[T](s: var SharedSeq[T]) = + deallocShared(s.data) + s.len = 0 + +proc toSeq*[T](s: SharedSeq[T]): seq[T] = + ## Creates a seq[T] from a SharedSeq[T]. No explicit dealloc is required + ## as req[T] is a GC managed type. + var ret = newSeq[T]() + for i in 0 ..< s.len: + ret.add(s.data[i]) + return ret diff --git a/library/codex_context.nim b/library/codex_context.nim new file mode 100644 index 00000000..f1b6b1a5 --- /dev/null +++ b/library/codex_context.nim @@ -0,0 +1,225 @@ +## This file defines the Codex context and its thread flow: +## 1. Client enqueues a request and signals the Codex thread. +## 2. The Codex thread dequeues the request and sends an ack (reqReceivedSignal). +## 3. The Codex thread executes the request asynchronously. +## 4. On completion, the Codex thread invokes the client callback with the result and userData. + +{.pragma: exported, exportc, cdecl, raises: [].} +{.pragma: callback, cdecl, raises: [], gcsafe.} +{.passc: "-fPIC".} + +import std/[options, locks, atomics] +import chronicles +import chronos +import chronos/threadsync +import taskpools/channels_spsc_single +import ./ffi_types +import ./codex_thread_requests/[codex_thread_request] + +from ../codex/codex import CodexServer + +logScope: + topics = "codexlib" + +type CodexContext* = object + thread: Thread[(ptr CodexContext)] + + # This lock is only necessary while we use a SP Channel and while the signalling + # between threads assumes that there aren't concurrent requests. + # Rearchitecting the signaling + migrating to a MP Channel will allow us to receive + # requests concurrently and spare us the need of locks + lock: Lock + + # Channel to send requests to the Codex thread. + # Requests will be popped from this channel. + reqChannel: ChannelSPSCSingle[ptr CodexThreadRequest] + + # To notify the Codex thread that a request is ready + reqSignal: ThreadSignalPtr + + # To notify the client thread that the request was received. + # It is acknowledgment signal (handshake). + reqReceivedSignal: ThreadSignalPtr + + # Custom state attached by the client to a request, + # returned when its callback is invoked + userData*: pointer + + # Function called by the library to notify the client of global events + eventCallback*: pointer + + # Custom state attached by the client to the context, + # returned with every event callback + eventUserData*: pointer + + # Set to false to stop the Codex thread (during codex_destroy) + running: Atomic[bool] + +template callEventCallback(ctx: ptr CodexContext, eventName: string, body: untyped) = + ## Template used to notify the client of global events + ## Example: onConnectionChanged, onProofMissing, etc. + if isNil(ctx[].eventCallback): + error eventName & " - eventCallback is nil" + return + + foreignThreadGc: + try: + let event = body + cast[CodexCallback](ctx[].eventCallback)( + RET_OK, unsafeAddr event[0], cast[csize_t](len(event)), ctx[].eventUserData + ) + except CatchableError: + let msg = + "Exception " & eventName & " when calling 'eventCallBack': " & + getCurrentExceptionMsg() + cast[CodexCallback](ctx[].eventCallback)( + RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), ctx[].eventUserData + ) + +proc sendRequestToCodexThread*( + ctx: ptr CodexContext, + reqType: RequestType, + reqContent: pointer, + callback: CodexCallback, + userData: pointer, + timeout = InfiniteDuration, +): Result[void, string] = + ctx.lock.acquire() + + defer: + ctx.lock.release() + + let req = CodexThreadRequest.createShared(reqType, reqContent, callback, userData) + + # Send the request to the Codex thread + let sentOk = ctx.reqChannel.trySend(req) + if not sentOk: + deallocShared(req) + return err("Failed to send request to the codex thread: " & $req[]) + + # Notify the Codex thread that a request is available + let fireSyncRes = ctx.reqSignal.fireSync() + if fireSyncRes.isErr(): + deallocShared(req) + return err( + "Failed to send request to the codex thread: unable to fireSync: " & + $fireSyncRes.error + ) + + if fireSyncRes.get() == false: + deallocShared(req) + return err("Failed to send request to the codex thread: fireSync timed out.") + + # Wait until the Codex Thread properly received the request + let res = ctx.reqReceivedSignal.waitSync(timeout) + if res.isErr(): + deallocShared(req) + return err( + "Failed to send request to the codex thread: unable to receive reqReceivedSignal signal." + ) + + ## Notice that in case of "ok", the deallocShared(req) is performed by the Codex Thread in the + ## process proc. See the 'codex_thread_request.nim' module for more details. + ok() + +proc runCodex(ctx: ptr CodexContext) {.async: (raises: []).} = + var codex: CodexServer + + while true: + try: + # Wait until a request is available + await ctx.reqSignal.wait() + except Exception as e: + error "Failure in run codex thread while waiting for reqSignal.", error = e.msg + continue + + # If codex_destroy was called, exit the loop + if ctx.running.load == false: + break + + var request: ptr CodexThreadRequest + + # Pop a request from the channel + let recvOk = ctx.reqChannel.tryRecv(request) + if not recvOk: + error "Failure in run codex: unable to receive request in codex thread." + continue + + # yield immediately to the event loop + # with asyncSpawn only, the code will be executed + # synchronously until the first await + asyncSpawn ( + proc() {.async.} = + await sleepAsync(0) + await CodexThreadRequest.process(request, addr codex) + )() + + # Notify the main thread that we picked up the request + let fireRes = ctx.reqReceivedSignal.fireSync() + if fireRes.isErr(): + error "Failure in run codex: unable to fire back to requester thread.", + error = fireRes.error + +proc run(ctx: ptr CodexContext) {.thread.} = + waitFor runCodex(ctx) + +proc createCodexContext*(): Result[ptr CodexContext, string] = + ## This proc is called from the main thread and it creates + ## the Codex working thread. + + # Allocates a CodexContext in shared memory (for the main thread) + var ctx = createShared(CodexContext, 1) + + # This signal is used by the main side to wake the Codex thread + # when a new request is enqueued. + ctx.reqSignal = ThreadSignalPtr.new().valueOr: + return + err("Failed to create a context: unable to create reqSignal ThreadSignalPtr.") + + # Used to let the caller know that the Codex thread has + # acknowledged / picked up a request (like a handshake). + ctx.reqReceivedSignal = ThreadSignalPtr.new().valueOr: + return err( + "Failed to create codex context: unable to create reqReceivedSignal ThreadSignalPtr." + ) + + # Protects shared state inside CodexContext + ctx.lock.initLock() + + # Codex thread will loop until codex_destroy is called + ctx.running.store(true) + + try: + createThread(ctx.thread, run, ctx) + except ValueError, ResourceExhaustedError: + freeShared(ctx) + return err( + "Failed to create codex context: unable to create thread: " & + getCurrentExceptionMsg() + ) + + return ok(ctx) + +proc destroyCodexContext*(ctx: ptr CodexContext): Result[void, string] = + # Signal the Codex thread to stop + ctx.running.store(false) + + # Wake the worker up if it's waiting + let signaledOnTime = ctx.reqSignal.fireSync().valueOr: + return err("Failed to destroy codex context: " & $error) + + if not signaledOnTime: + return err( + "Failed to destroy codex context: unable to get signal reqSignal on time in destroyCodexContext." + ) + + # Wait for the thread to finish + joinThread(ctx.thread) + + # Clean up + ctx.lock.deinitLock() + ?ctx.reqSignal.close() + ?ctx.reqReceivedSignal.close() + freeShared(ctx) + + return ok() diff --git a/library/codex_thread_requests/codex_thread_request.nim b/library/codex_thread_requests/codex_thread_request.nim new file mode 100644 index 00000000..2229cddf --- /dev/null +++ b/library/codex_thread_requests/codex_thread_request.nim @@ -0,0 +1,126 @@ +## This file contains the base message request type that will be handled. +## The requests are created by the main thread and processed by +## the Codex Thread. + +import std/json +import results +import chronos +import ../ffi_types +import ./requests/node_lifecycle_request +import ./requests/node_info_request +import ./requests/node_debug_request +import ./requests/node_p2p_request +import ./requests/node_upload_request +import ./requests/node_download_request +import ./requests/node_storage_request + +from ../../codex/codex import CodexServer + +type RequestType* {.pure.} = enum + LIFECYCLE + INFO + DEBUG + P2P + UPLOAD + DOWNLOAD + STORAGE + +type CodexThreadRequest* = object + reqType: RequestType + + # Request payloed + reqContent: pointer + + # Callback to notify the client thread of the result + callback: CodexCallback + + # Custom state attached by the client to the request, + # returned when its callback is invoked. + userData: pointer + +proc createShared*( + T: type CodexThreadRequest, + reqType: RequestType, + reqContent: pointer, + callback: CodexCallback, + userData: pointer, +): ptr type T = + var ret = createShared(T) + ret[].reqType = reqType + ret[].reqContent = reqContent + ret[].callback = callback + ret[].userData = userData + return ret + +# NOTE: User callbacks are executed on the working thread. +# They must be fast and non-blocking; otherwise this thread will be blocked +# and no further requests can be processed. +# We can improve this by dispatching the callbacks to a thread pool or +# moving to a MP channel. +# See: https://github.com/codex-storage/nim-codex/pull/1322#discussion_r2340708316 +proc handleRes[T: string | void | seq[byte]]( + res: Result[T, string], request: ptr CodexThreadRequest +) = + ## Handles the Result responses, which can either be Result[string, string] or + ## Result[void, string]. + defer: + deallocShared(request) + + if res.isErr(): + foreignThreadGc: + let msg = $res.error + if msg == "": + request[].callback(RET_ERR, nil, cast[csize_t](0), request[].userData) + else: + request[].callback( + RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), request[].userData + ) + return + + foreignThreadGc: + var msg: cstring = "" + when T is string: + msg = res.get().cstring() + request[].callback( + RET_OK, unsafeAddr msg[0], cast[csize_t](len(msg)), request[].userData + ) + return + +proc process*( + T: type CodexThreadRequest, request: ptr CodexThreadRequest, codex: ptr CodexServer +) {.async: (raises: []).} = + ## Processes the request in the Codex thread. + ## Dispatch to the appropriate request handler based on reqType. + let retFut = + case request[].reqType + of LIFECYCLE: + cast[ptr NodeLifecycleRequest](request[].reqContent).process(codex) + of INFO: + cast[ptr NodeInfoRequest](request[].reqContent).process(codex) + of RequestType.DEBUG: + cast[ptr NodeDebugRequest](request[].reqContent).process(codex) + of P2P: + cast[ptr NodeP2PRequest](request[].reqContent).process(codex) + of STORAGE: + cast[ptr NodeStorageRequest](request[].reqContent).process(codex) + of DOWNLOAD: + let onChunk = proc(bytes: seq[byte]) = + if bytes.len > 0: + request[].callback( + RET_PROGRESS, + cast[ptr cchar](unsafeAddr bytes[0]), + cast[csize_t](bytes.len), + request[].userData, + ) + + cast[ptr NodeDownloadRequest](request[].reqContent).process(codex, onChunk) + of UPLOAD: + let onBlockReceived = proc(bytes: int) = + request[].callback(RET_PROGRESS, nil, cast[csize_t](bytes), request[].userData) + + cast[ptr NodeUploadRequest](request[].reqContent).process(codex, onBlockReceived) + + handleRes(await retFut, request) + +proc `$`*(self: CodexThreadRequest): string = + return $self.reqType diff --git a/library/codex_thread_requests/requests/node_debug_request.nim b/library/codex_thread_requests/requests/node_debug_request.nim new file mode 100644 index 00000000..d051ea16 --- /dev/null +++ b/library/codex_thread_requests/requests/node_debug_request.nim @@ -0,0 +1,126 @@ +{.push raises: [].} + +## This file contains the debug info available with Codex. +## The DEBUG type will return info about the P2P node. +## The PEER type is available only with codex_enable_api_debug_peers flag. +## It will return info about a specific peer if available. + +import std/[options] +import chronos +import chronicles +import codexdht/discv5/spr +import ../../alloc +import ../../../codex/conf +import ../../../codex/rest/json +import ../../../codex/node + +from ../../../codex/codex import CodexServer, node + +logScope: + topics = "codexlib codexlibdebug" + +type NodeDebugMsgType* = enum + DEBUG + PEER + LOG_LEVEL + +type NodeDebugRequest* = object + operation: NodeDebugMsgType + peerId: cstring + logLevel: cstring + +proc createShared*( + T: type NodeDebugRequest, + op: NodeDebugMsgType, + peerId: cstring = "", + logLevel: cstring = "", +): ptr type T = + var ret = createShared(T) + ret[].operation = op + ret[].peerId = peerId.alloc() + ret[].logLevel = logLevel.alloc() + return ret + +proc destroyShared(self: ptr NodeDebugRequest) = + deallocShared(self[].peerId) + deallocShared(self[].logLevel) + deallocShared(self) + +proc getDebug( + codex: ptr CodexServer +): Future[Result[string, string]] {.async: (raises: []).} = + let node = codex[].node + let table = RestRoutingTable.init(node.discovery.protocol.routingTable) + + let json = + %*{ + "id": $node.switch.peerInfo.peerId, + "addrs": node.switch.peerInfo.addrs.mapIt($it), + "spr": + if node.discovery.dhtRecord.isSome: node.discovery.dhtRecord.get.toURI else: "", + "announceAddresses": node.discovery.announceAddrs, + "table": table, + } + + return ok($json) + +proc getPeer( + codex: ptr CodexServer, peerId: cstring +): Future[Result[string, string]] {.async: (raises: []).} = + when codex_enable_api_debug_peers: + let node = codex[].node + let res = PeerId.init($peerId) + if res.isErr: + return err("Failed to get peer: invalid peer ID " & $peerId & ": " & $res.error()) + + let id = res.get() + + try: + let peerRecord = await node.findPeer(id) + if peerRecord.isNone: + return err("Failed to get peer: peer not found") + + return ok($ %RestPeerRecord.init(peerRecord.get())) + except CancelledError: + return err("Failed to get peer: operation cancelled") + except CatchableError as e: + return err("Failed to get peer: " & e.msg) + else: + return err("Failed to get peer: peer debug API is disabled") + +proc updateLogLevel( + codex: ptr CodexServer, logLevel: cstring +): Future[Result[string, string]] {.async: (raises: []).} = + try: + {.gcsafe.}: + updateLogLevel($logLevel) + except ValueError as err: + return err("Failed to update log level: invalid value for log level: " & err.msg) + + return ok("") + +proc process*( + self: ptr NodeDebugRequest, codex: ptr CodexServer +): Future[Result[string, string]] {.async: (raises: []).} = + defer: + destroyShared(self) + + case self.operation + of NodeDebugMsgType.DEBUG: + let res = (await getDebug(codex)) + if res.isErr: + error "Failed to get DEBUG.", error = res.error + return err($res.error) + return res + of NodeDebugMsgType.PEER: + let res = (await getPeer(codex, self.peerId)) + if res.isErr: + error "Failed to get PEER.", error = res.error + return err($res.error) + return res + of NodeDebugMsgType.LOG_LEVEL: + let res = (await updateLogLevel(codex, self.logLevel)) + if res.isErr: + error "Failed to update LOG_LEVEL.", error = res.error + return err($res.error) + return res diff --git a/library/codex_thread_requests/requests/node_download_request.nim b/library/codex_thread_requests/requests/node_download_request.nim new file mode 100644 index 00000000..394b667e --- /dev/null +++ b/library/codex_thread_requests/requests/node_download_request.nim @@ -0,0 +1,336 @@ +{.push raises: [].} + +## This file contains the download request. +## A session is created for each download identified by the CID, +## allowing to resume, pause and cancel the download (using chunks). +## +## There are two ways to download a file: +## 1. Via chunks: the cid parameter is the CID of the file to download. Steps are: +## - INIT: initializes the download session +## - CHUNK: downloads the next chunk of the file +## - CANCEL: cancels the download session +## 2. Via stream. +## - INIT: initializes the download session +## - STREAM: downloads the file in a streaming manner, calling +## the onChunk handler for each chunk and / or writing to a file if filepath is set. +## - CANCEL: cancels the download session + +import std/[options, streams] +import chronos +import chronicles +import libp2p/stream/[lpstream] +import serde/json as serde +import ../../alloc +import ../../../codex/units +import ../../../codex/codextypes + +from ../../../codex/codex import CodexServer, node +from ../../../codex/node import retrieve, fetchManifest +from ../../../codex/rest/json import `%`, RestContent +from libp2p import Cid, init, `$` + +logScope: + topics = "codexlib codexlibdownload" + +type NodeDownloadMsgType* = enum + INIT + CHUNK + STREAM + CANCEL + MANIFEST + +type OnChunkHandler = proc(bytes: seq[byte]): void {.gcsafe, raises: [].} + +type NodeDownloadRequest* = object + operation: NodeDownloadMsgType + cid: cstring + chunkSize: csize_t + local: bool + filepath: cstring + +type + DownloadSessionId* = string + DownloadSessionCount* = int + DownloadSession* = object + stream: LPStream + chunkSize: int + +var downloadSessions {.threadvar.}: Table[DownloadSessionId, DownloadSession] + +proc createShared*( + T: type NodeDownloadRequest, + op: NodeDownloadMsgType, + cid: cstring = "", + chunkSize: csize_t = 0, + local: bool = false, + filepath: cstring = "", +): ptr type T = + var ret = createShared(T) + ret[].operation = op + ret[].cid = cid.alloc() + ret[].chunkSize = chunkSize + ret[].local = local + ret[].filepath = filepath.alloc() + + return ret + +proc destroyShared(self: ptr NodeDownloadRequest) = + deallocShared(self[].cid) + deallocShared(self[].filepath) + deallocShared(self) + +proc init( + codex: ptr CodexServer, cCid: cstring = "", chunkSize: csize_t = 0, local: bool +): Future[Result[string, string]] {.async: (raises: []).} = + ## Init a new session to download the file identified by cid. + ## + ## If the session already exists, do nothing and return ok. + ## Meaning that a cid can only have one active download session. + ## If the chunkSize is 0, the default block size will be used. + ## If local is true, the file will be retrived from the local store. + + let cid = Cid.init($cCid) + if cid.isErr: + return err("Failed to download locally: cannot parse cid: " & $cCid) + + if downloadSessions.contains($cid): + return ok("Download session already exists.") + + let node = codex[].node + var stream: LPStream + + try: + let res = await node.retrieve(cid.get(), local) + if res.isErr(): + return err("Failed to init the download: " & res.error.msg) + stream = res.get() + except CancelledError: + downloadSessions.del($cid) + return err("Failed to init the download: download cancelled.") + + let blockSize = if chunkSize.int > 0: chunkSize.int else: DefaultBlockSize.int + downloadSessions[$cid] = DownloadSession(stream: stream, chunkSize: blockSize) + + return ok("") + +proc chunk( + codex: ptr CodexServer, cCid: cstring = "", onChunk: OnChunkHandler +): Future[Result[string, string]] {.async: (raises: []).} = + ## Download the next chunk of the file identified by cid. + ## The chunk is passed to the onChunk handler. + ## + ## If the stream is at EOF, return ok with empty string. + ## + ## If an error is raised while reading the stream, the session is deleted + ## and an error is returned. + + let cid = Cid.init($cCid) + if cid.isErr: + return err("Failed to download locally: cannot parse cid: " & $cCid) + + if not downloadSessions.contains($cid): + return err("Failed to download chunk: no session for cid " & $cid) + + var session: DownloadSession + try: + session = downloadSessions[$cid] + except KeyError: + return err("Failed to download chunk: no session for cid " & $cid) + + let stream = session.stream + if stream.atEof: + return ok("") + + let chunkSize = session.chunkSize + var buf = newSeq[byte](chunkSize) + + try: + let read = await stream.readOnce(addr buf[0], buf.len) + buf.setLen(read) + except LPStreamError as e: + await stream.close() + downloadSessions.del($cid) + return err("Failed to download chunk: " & $e.msg) + except CancelledError: + await stream.close() + downloadSessions.del($cid) + return err("Failed to download chunk: download cancelled.") + + if buf.len <= 0: + return err("Failed to download chunk: no data") + + onChunk(buf) + + return ok("") + +proc streamData( + codex: ptr CodexServer, + stream: LPStream, + onChunk: OnChunkHandler, + chunkSize: csize_t, + filepath: cstring, +): Future[Result[string, string]] {. + async: (raises: [CancelledError, LPStreamError, IOError]) +.} = + let blockSize = if chunkSize.int > 0: chunkSize.int else: DefaultBlockSize.int + var buf = newSeq[byte](blockSize) + var read = 0 + var outputStream: OutputStreamHandle + var filedest: string = $filepath + + try: + if filepath != "": + outputStream = filedest.fileOutput() + + while not stream.atEof: + ## Yield immediately to the event loop + ## It gives a chance to cancel request to be processed + await sleepAsync(0) + + let read = await stream.readOnce(addr buf[0], buf.len) + buf.setLen(read) + + if buf.len <= 0: + break + + onChunk(buf) + + if outputStream != nil: + outputStream.write(buf) + + if outputStream != nil: + outputStream.close() + finally: + if outputStream != nil: + outputStream.close() + + return ok("") + +proc stream( + codex: ptr CodexServer, + cCid: cstring, + chunkSize: csize_t, + local: bool, + filepath: cstring, + onChunk: OnChunkHandler, +): Future[Result[string, string]] {.raises: [], async: (raises: []).} = + ## Stream the file identified by cid, calling the onChunk handler for each chunk + ## and / or writing to a file if filepath is set. + ## + ## If local is true, the file will be retrieved from the local store. + + let cid = Cid.init($cCid) + if cid.isErr: + return err("Failed to stream: cannot parse cid: " & $cCid) + + if not downloadSessions.contains($cid): + return err("Failed to stream: no session for cid " & $cid) + + var session: DownloadSession + try: + session = downloadSessions[$cid] + except KeyError: + return err("Failed to stream: no session for cid " & $cid) + + let node = codex[].node + + try: + let res = + await noCancel codex.streamData(session.stream, onChunk, chunkSize, filepath) + if res.isErr: + return err($res.error) + except LPStreamError as e: + return err("Failed to stream file: " & $e.msg) + except IOError as e: + return err("Failed to stream file: " & $e.msg) + finally: + if session.stream != nil: + await session.stream.close() + downloadSessions.del($cid) + + return ok("") + +proc cancel( + codex: ptr CodexServer, cCid: cstring +): Future[Result[string, string]] {.raises: [], async: (raises: []).} = + ## Cancel the download session identified by cid. + ## This operation is not supported when using the stream mode, + ## because the worker will be busy downloading the file. + + let cid = Cid.init($cCid) + if cid.isErr: + return err("Failed to cancel : cannot parse cid: " & $cCid) + + if not downloadSessions.contains($cid): + # The session is already cancelled + return ok("") + + var session: DownloadSession + try: + session = downloadSessions[$cid] + except KeyError: + # The session is already cancelled + return ok("") + + let stream = session.stream + await stream.close() + downloadSessions.del($cCid) + + return ok("") + +proc manifest( + codex: ptr CodexServer, cCid: cstring +): Future[Result[string, string]] {.raises: [], async: (raises: []).} = + let cid = Cid.init($cCid) + if cid.isErr: + return err("Failed to fetch manifest: cannot parse cid: " & $cCid) + + try: + let node = codex[].node + let manifest = await node.fetchManifest(cid.get()) + if manifest.isErr: + return err("Failed to fetch manifest: " & manifest.error.msg) + + return ok(serde.toJson(manifest.get())) + except CancelledError: + return err("Failed to fetch manifest: download cancelled.") + +proc process*( + self: ptr NodeDownloadRequest, codex: ptr CodexServer, onChunk: OnChunkHandler +): Future[Result[string, string]] {.async: (raises: []).} = + defer: + destroyShared(self) + + case self.operation + of NodeDownloadMsgType.INIT: + let res = (await init(codex, self.cid, self.chunkSize, self.local)) + if res.isErr: + error "Failed to INIT.", error = res.error + return err($res.error) + return res + of NodeDownloadMsgType.CHUNK: + let res = (await chunk(codex, self.cid, onChunk)) + if res.isErr: + error "Failed to CHUNK.", error = res.error + return err($res.error) + return res + of NodeDownloadMsgType.STREAM: + let res = ( + await stream(codex, self.cid, self.chunkSize, self.local, self.filepath, onChunk) + ) + if res.isErr: + error "Failed to STREAM.", error = res.error + return err($res.error) + return res + of NodeDownloadMsgType.CANCEL: + let res = (await cancel(codex, self.cid)) + if res.isErr: + error "Failed to CANCEL.", error = res.error + return err($res.error) + return res + of NodeDownloadMsgType.MANIFEST: + let res = (await manifest(codex, self.cid)) + if res.isErr: + error "Failed to MANIFEST.", error = res.error + return err($res.error) + return res diff --git a/library/codex_thread_requests/requests/node_info_request.nim b/library/codex_thread_requests/requests/node_info_request.nim new file mode 100644 index 00000000..2e397fcf --- /dev/null +++ b/library/codex_thread_requests/requests/node_info_request.nim @@ -0,0 +1,76 @@ +## This file contains the lifecycle request type that will be handled. + +import std/[options] +import chronos +import chronicles +import confutils +import codexdht/discv5/spr +import ../../../codex/conf +import ../../../codex/rest/json +import ../../../codex/node + +from ../../../codex/codex import CodexServer, config, node + +logScope: + topics = "codexlib codexlibinfo" + +type NodeInfoMsgType* = enum + REPO + SPR + PEERID + +type NodeInfoRequest* = object + operation: NodeInfoMsgType + +proc createShared*(T: type NodeInfoRequest, op: NodeInfoMsgType): ptr type T = + var ret = createShared(T) + ret[].operation = op + return ret + +proc destroyShared(self: ptr NodeInfoRequest) = + deallocShared(self) + +proc getRepo( + codex: ptr CodexServer +): Future[Result[string, string]] {.async: (raises: []).} = + return ok($(codex[].config.dataDir)) + +proc getSpr( + codex: ptr CodexServer +): Future[Result[string, string]] {.async: (raises: []).} = + let spr = codex[].node.discovery.dhtRecord + if spr.isNone: + return err("Failed to get SPR: no SPR record found.") + + return ok(spr.get.toURI) + +proc getPeerId( + codex: ptr CodexServer +): Future[Result[string, string]] {.async: (raises: []).} = + return ok($codex[].node.switch.peerInfo.peerId) + +proc process*( + self: ptr NodeInfoRequest, codex: ptr CodexServer +): Future[Result[string, string]] {.async: (raises: []).} = + defer: + destroyShared(self) + + case self.operation + of REPO: + let res = (await getRepo(codex)) + if res.isErr: + error "Failed to get REPO.", error = res.error + return err($res.error) + return res + of SPR: + let res = (await getSpr(codex)) + if res.isErr: + error "Failed to get SPR.", error = res.error + return err($res.error) + return res + of PEERID: + let res = (await getPeerId(codex)) + if res.isErr: + error "Failed to get PEERID.", error = res.error + return err($res.error) + return res diff --git a/library/codex_thread_requests/requests/node_lifecycle_request.nim b/library/codex_thread_requests/requests/node_lifecycle_request.nim new file mode 100644 index 00000000..532facf3 --- /dev/null +++ b/library/codex_thread_requests/requests/node_lifecycle_request.nim @@ -0,0 +1,188 @@ +## This file contains the lifecycle request type that will be handled. +## CREATE_NODE: create a new Codex node with the provided config.json. +## START_NODE: start the provided Codex node. +## STOP_NODE: stop the provided Codex node. + +import std/[options, json, strutils, net, os] +import codexdht/discv5/spr +import stew/shims/parseutils +import contractabi/address +import chronos +import chronicles +import results +import confutils +import confutils/std/net +import confutils/defs +import libp2p +import json_serialization +import json_serialization/std/[options, net] +import ../../alloc +import ../../../codex/conf +import ../../../codex/utils +import ../../../codex/utils/[keyutils, fileutils] +import ../../../codex/units + +from ../../../codex/codex import CodexServer, new, start, stop, close + +logScope: + topics = "codexlib codexliblifecycle" + +type NodeLifecycleMsgType* = enum + CREATE_NODE + START_NODE + STOP_NODE + CLOSE_NODE + +proc readValue*[T: InputFile | InputDir | OutPath | OutDir | OutFile]( + r: var JsonReader, val: var T +) = + val = T(r.readValue(string)) + +proc readValue*(r: var JsonReader, val: var MultiAddress) = + val = MultiAddress.init(r.readValue(string)).get() + +proc readValue*(r: var JsonReader, val: var NatConfig) = + let res = NatConfig.parse(r.readValue(string)) + if res.isErr: + raise + newException(SerializationError, "Cannot parse the NAT config: " & res.error()) + val = res.get() + +proc readValue*(r: var JsonReader, val: var SignedPeerRecord) = + let res = SignedPeerRecord.parse(r.readValue(string)) + if res.isErr: + raise + newException(SerializationError, "Cannot parse the signed peer: " & res.error()) + val = res.get() + +proc readValue*(r: var JsonReader, val: var ThreadCount) = + val = ThreadCount(r.readValue(int)) + +proc readValue*(r: var JsonReader, val: var NBytes) = + val = NBytes(r.readValue(int)) + +proc readValue*(r: var JsonReader, val: var Duration) = + var dur: Duration + let input = r.readValue(string) + let count = parseDuration(input, dur) + if count == 0: + raise newException(SerializationError, "Cannot parse the duration: " & input) + val = dur + +proc readValue*(r: var JsonReader, val: var EthAddress) = + val = EthAddress.init(r.readValue(string)).get() + +type NodeLifecycleRequest* = object + operation: NodeLifecycleMsgType + configJson: cstring + +proc createShared*( + T: type NodeLifecycleRequest, op: NodeLifecycleMsgType, configJson: cstring = "" +): ptr type T = + var ret = createShared(T) + ret[].operation = op + ret[].configJson = configJson.alloc() + return ret + +proc destroyShared(self: ptr NodeLifecycleRequest) = + deallocShared(self[].configJson) + deallocShared(self) + +proc createCodex( + configJson: cstring +): Future[Result[CodexServer, string]] {.async: (raises: []).} = + var conf: CodexConf + + try: + conf = CodexConf.load( + version = codexFullVersion, + envVarsPrefix = "codex", + cmdLine = @[], + secondarySources = proc( + config: CodexConf, sources: auto + ) {.gcsafe, raises: [ConfigurationError].} = + if configJson.len > 0: + sources.addConfigFileContent(Json, $(configJson)) + , + ) + except ConfigurationError as e: + return err("Failed to create codex: unable to load configuration: " & e.msg) + + conf.setupLogging() + + try: + {.gcsafe.}: + updateLogLevel(conf.logLevel) + except ValueError as err: + return err("Failed to create codex: invalid value for log level: " & err.msg) + + conf.setupMetrics() + + if not (checkAndCreateDataDir((conf.dataDir).string)): + # We are unable to access/create data folder or data folder's + # permissions are insecure. + return err( + "Failed to create codex: unable to access/create data folder or data folder's permissions are insecure." + ) + + if not (checkAndCreateDataDir((conf.dataDir / "repo"))): + # We are unable to access/create data folder or data folder's + # permissions are insecure. + return err( + "Failed to create codex: unable to access/create data folder or data folder's permissions are insecure." + ) + + let keyPath = + if isAbsolute(conf.netPrivKeyFile): + conf.netPrivKeyFile + else: + conf.dataDir / conf.netPrivKeyFile + let privateKey = setupKey(keyPath) + if privateKey.isErr: + return err("Failed to create codex: unable to get the private key.") + let pk = privateKey.get() + + conf.apiBindAddress = string.none + + let server = + try: + CodexServer.new(conf, pk) + except Exception as exc: + return err("Failed to create codex: " & exc.msg) + + return ok(server) + +proc process*( + self: ptr NodeLifecycleRequest, codex: ptr CodexServer +): Future[Result[string, string]] {.async: (raises: []).} = + defer: + destroyShared(self) + + case self.operation + of CREATE_NODE: + codex[] = ( + await createCodex( + self.configJson # , self.appCallbacks + ) + ).valueOr: + error "Failed to CREATE_NODE.", error = error + return err($error) + of START_NODE: + try: + await codex[].start() + except Exception as e: + error "Failed to START_NODE.", error = e.msg + return err(e.msg) + of STOP_NODE: + try: + await codex[].stop() + except Exception as e: + error "Failed to STOP_NODE.", error = e.msg + return err(e.msg) + of CLOSE_NODE: + try: + await codex[].close() + except Exception as e: + error "Failed to STOP_NODE.", error = e.msg + return err(e.msg) + return ok("") diff --git a/library/codex_thread_requests/requests/node_p2p_request.nim b/library/codex_thread_requests/requests/node_p2p_request.nim new file mode 100644 index 00000000..3bdbbf97 --- /dev/null +++ b/library/codex_thread_requests/requests/node_p2p_request.nim @@ -0,0 +1,95 @@ +{.push raises: [].} + +## This file contains the P2p request type that will be handled. +## CONNECT: connect to a peer with the provided peer ID and optional addresses. + +import std/[options] +import chronos +import chronicles +import libp2p +import ../../alloc +import ../../../codex/node + +from ../../../codex/codex import CodexServer, node + +logScope: + topics = "codexlib codexlibp2p" + +type NodeP2PMsgType* = enum + CONNECT + +type NodeP2PRequest* = object + operation: NodeP2PMsgType + peerId: cstring + peerAddresses: seq[cstring] + +proc createShared*( + T: type NodeP2PRequest, + op: NodeP2PMsgType, + peerId: cstring = "", + peerAddresses: seq[cstring] = @[], +): ptr type T = + var ret = createShared(T) + ret[].operation = op + ret[].peerId = peerId.alloc() + ret[].peerAddresses = peerAddresses + return ret + +proc destroyShared(self: ptr NodeP2PRequest) = + deallocShared(self[].peerId) + deallocShared(self) + +proc connect( + codex: ptr CodexServer, peerId: cstring, peerAddresses: seq[cstring] = @[] +): Future[Result[string, string]] {.async: (raises: []).} = + let node = codex[].node + let res = PeerId.init($peerId) + if res.isErr: + return err("Failed to connect to peer: invalid peer ID: " & $res.error()) + + let id = res.get() + + let addresses = + if peerAddresses.len > 0: + var addrs: seq[MultiAddress] + for addrStr in peerAddresses: + let res = MultiAddress.init($addrStr) + if res.isOk: + addrs.add(res[]) + else: + return err("Failed to connect to peer: invalid address: " & $addrStr) + addrs + else: + try: + let peerRecord = await node.findPeer(id) + if peerRecord.isNone: + return err("Failed to connect to peer: peer not found.") + + peerRecord.get().addresses.mapIt(it.address) + except CancelledError: + return err("Failed to connect to peer: operation cancelled.") + except CatchableError as e: + return err("Failed to connect to peer: " & $e.msg) + + try: + await node.connect(id, addresses) + except CancelledError: + return err("Failed to connect to peer: operation cancelled.") + except CatchableError as e: + return err("Failed to connect to peer: " & $e.msg) + + return ok("") + +proc process*( + self: ptr NodeP2PRequest, codex: ptr CodexServer +): Future[Result[string, string]] {.async: (raises: []).} = + defer: + destroyShared(self) + + case self.operation + of NodeP2PMsgType.CONNECT: + let res = (await connect(codex, self.peerId, self.peerAddresses)) + if res.isErr: + error "Failed to CONNECT.", error = res.error + return err($res.error) + return res diff --git a/library/codex_thread_requests/requests/node_storage_request.nim b/library/codex_thread_requests/requests/node_storage_request.nim new file mode 100644 index 00000000..21eceb17 --- /dev/null +++ b/library/codex_thread_requests/requests/node_storage_request.nim @@ -0,0 +1,180 @@ +{.push raises: [].} + +## This file contains the node storage request. +## 4 operations are available: +## - LIST: list all manifests stored in the node. +## - DELETE: Deletes either a single block or an entire dataset from the local node. +## - FETCH: download a file from the network to the local node. +## - SPACE: get the amount of space used by the local node. +## - EXISTS: check the existence of a cid in a node (local store). + +import std/[options] +import chronos +import chronicles +import libp2p/stream/[lpstream] +import serde/json as serde +import ../../alloc +import ../../../codex/units +import ../../../codex/manifest +import ../../../codex/stores/repostore + +from ../../../codex/codex import CodexServer, node, repoStore +from ../../../codex/node import + iterateManifests, fetchManifest, fetchDatasetAsyncTask, delete, hasLocalBlock +from libp2p import Cid, init, `$` + +logScope: + topics = "codexlib codexlibstorage" + +type NodeStorageMsgType* = enum + LIST + DELETE + FETCH + SPACE + EXISTS + +type NodeStorageRequest* = object + operation: NodeStorageMsgType + cid: cstring + +type StorageSpace = object + totalBlocks* {.serialize.}: Natural + quotaMaxBytes* {.serialize.}: NBytes + quotaUsedBytes* {.serialize.}: NBytes + quotaReservedBytes* {.serialize.}: NBytes + +proc createShared*( + T: type NodeStorageRequest, op: NodeStorageMsgType, cid: cstring = "" +): ptr type T = + var ret = createShared(T) + ret[].operation = op + ret[].cid = cid.alloc() + + return ret + +proc destroyShared(self: ptr NodeStorageRequest) = + deallocShared(self[].cid) + deallocShared(self) + +type ManifestWithCid = object + cid {.serialize.}: string + manifest {.serialize.}: Manifest + +proc list( + codex: ptr CodexServer +): Future[Result[string, string]] {.async: (raises: []).} = + var manifests = newSeq[ManifestWithCid]() + proc onManifest(cid: Cid, manifest: Manifest) {.raises: [], gcsafe.} = + manifests.add(ManifestWithCid(cid: $cid, manifest: manifest)) + + try: + let node = codex[].node + await node.iterateManifests(onManifest) + except CancelledError: + return err("Failed to list manifests: cancelled operation.") + except CatchableError as err: + return err("Failed to list manifest: : " & err.msg) + + return ok(serde.toJson(manifests)) + +proc delete( + codex: ptr CodexServer, cCid: cstring +): Future[Result[string, string]] {.async: (raises: []).} = + let cid = Cid.init($cCid) + if cid.isErr: + return err("Failed to delete the data: cannot parse cid: " & $cCid) + + let node = codex[].node + try: + let res = await node.delete(cid.get()) + if res.isErr: + return err("Failed to delete the data: " & res.error.msg) + except CancelledError: + return err("Failed to delete the data: cancelled operation.") + except CatchableError as err: + return err("Failed to delete the data: " & err.msg) + + return ok("") + +proc fetch( + codex: ptr CodexServer, cCid: cstring +): Future[Result[string, string]] {.async: (raises: []).} = + let cid = Cid.init($cCid) + if cid.isErr: + return err("Failed to fetch the data: cannot parse cid: " & $cCid) + + try: + let node = codex[].node + let manifest = await node.fetchManifest(cid.get()) + if manifest.isErr: + return err("Failed to fetch the data: " & manifest.error.msg) + + node.fetchDatasetAsyncTask(manifest.get()) + + return ok(serde.toJson(manifest.get())) + except CancelledError: + return err("Failed to fetch the data: download cancelled.") + +proc space( + codex: ptr CodexServer +): Future[Result[string, string]] {.async: (raises: []).} = + let repoStore = codex[].repoStore + let space = StorageSpace( + totalBlocks: repoStore.totalBlocks, + quotaMaxBytes: repoStore.quotaMaxBytes, + quotaUsedBytes: repoStore.quotaUsedBytes, + quotaReservedBytes: repoStore.quotaReservedBytes, + ) + return ok(serde.toJson(space)) + +proc exists( + codex: ptr CodexServer, cCid: cstring +): Future[Result[string, string]] {.async: (raises: []).} = + let cid = Cid.init($cCid) + if cid.isErr: + return err("Failed to check the data existence: cannot parse cid: " & $cCid) + + try: + let node = codex[].node + let exists = await node.hasLocalBlock(cid.get()) + return ok($exists) + except CancelledError: + return err("Failed to check the data existence: operation cancelled.") + +proc process*( + self: ptr NodeStorageRequest, codex: ptr CodexServer +): Future[Result[string, string]] {.async: (raises: []).} = + defer: + destroyShared(self) + + case self.operation + of NodeStorageMsgType.LIST: + let res = (await list(codex)) + if res.isErr: + error "Failed to LIST.", error = res.error + return err($res.error) + return res + of NodeStorageMsgType.DELETE: + let res = (await delete(codex, self.cid)) + if res.isErr: + error "Failed to DELETE.", error = res.error + return err($res.error) + return res + of NodeStorageMsgType.FETCH: + let res = (await fetch(codex, self.cid)) + if res.isErr: + error "Failed to FETCH.", error = res.error + return err($res.error) + return res + of NodeStorageMsgType.SPACE: + let res = (await space(codex)) + if res.isErr: + error "Failed to SPACE.", error = res.error + return err($res.error) + return res + of NodeStorageMsgType.EXISTS: + let res = (await exists(codex, self.cid)) + if res.isErr: + error "Failed to EXISTS.", error = res.error + return err($res.error) + return res diff --git a/library/codex_thread_requests/requests/node_upload_request.nim b/library/codex_thread_requests/requests/node_upload_request.nim new file mode 100644 index 00000000..a850abb7 --- /dev/null +++ b/library/codex_thread_requests/requests/node_upload_request.nim @@ -0,0 +1,372 @@ +{.push raises: [].} + +## This file contains the upload request. +## A session is created for each upload allowing to resume, +## pause and cancel uploads (using chunks). +## +## There are two ways to upload a file: +## 1. Via chunks: the filepath parameter is the data filename. Steps are: +## - INIT: creates a new upload session and returns its ID. +## - CHUNK: sends a chunk of data to the upload session. +## - FINALIZE: finalizes the upload and returns the CID of the uploaded file. +## - CANCEL: cancels the upload session. +## +## 2. Directly from a file path: the filepath has to be absolute. +## - INIT: creates a new upload session and returns its ID +## - FILE: starts the upload and returns the CID of the uploaded file +## - CANCEL: cancels the upload session. + +import std/[options, os, mimetypes] +import chronos +import chronicles +import questionable +import questionable/results +import faststreams/inputs +import libp2p/stream/[bufferstream, lpstream] +import ../../alloc +import ../../../codex/units +import ../../../codex/codextypes + +from ../../../codex/codex import CodexServer, node +from ../../../codex/node import store +from libp2p import Cid, `$` + +logScope: + topics = "codexlib codexlibupload" + +type NodeUploadMsgType* = enum + INIT + CHUNK + FINALIZE + CANCEL + FILE + +type OnProgressHandler = proc(bytes: int): void {.gcsafe, raises: [].} + +type NodeUploadRequest* = object + operation: NodeUploadMsgType + sessionId: cstring + filepath: cstring + chunk: seq[byte] + chunkSize: csize_t + +type + UploadSessionId* = string + UploadSessionCount* = int + UploadSession* = object + stream: BufferStream + fut: Future[?!Cid] + filepath: string + chunkSize: int + onProgress: OnProgressHandler + +var uploadSessions {.threadvar.}: Table[UploadSessionId, UploadSession] +var nexUploadSessionCount {.threadvar.}: UploadSessionCount + +proc createShared*( + T: type NodeUploadRequest, + op: NodeUploadMsgType, + sessionId: cstring = "", + filepath: cstring = "", + chunk: seq[byte] = @[], + chunkSize: csize_t = 0, +): ptr type T = + var ret = createShared(T) + ret[].operation = op + ret[].sessionId = sessionId.alloc() + ret[].filepath = filepath.alloc() + ret[].chunk = chunk + ret[].chunkSize = chunkSize + + return ret + +proc destroyShared(self: ptr NodeUploadRequest) = + deallocShared(self[].filepath) + deallocShared(self[].sessionId) + deallocShared(self) + +proc init( + codex: ptr CodexServer, filepath: cstring = "", chunkSize: csize_t = 0 +): Future[Result[string, string]] {.async: (raises: []).} = + ## Init a new session upload and return its ID. + ## The session contains the future corresponding to the + ## `node.store` call. + ## The filepath can be: + ## - the filename when uploading via chunks + ## - the absolute path to a file when uploading directly. + ## The mimetype is deduced from the filename extension. + ## + ## The chunkSize matches by default the block size used to store the file. + ## + ## A callback `onBlockStore` is provided to `node.store` to + ## report the progress of the upload. This callback will check + ## that an `onProgress` handler is set in the session + ## and call it with the number of bytes stored each time a block + ## is stored. + + var filenameOpt, mimetypeOpt = string.none + + if isAbsolute($filepath): + if not fileExists($filepath): + return err( + "Failed to create an upload session, the filepath does not exist: " & $filepath + ) + + if filepath != "": + let (_, name, ext) = splitFile($filepath) + + filenameOpt = (name & ext).some + + if ext != "": + let extNoDot = + if ext.len > 0: + ext[1 ..^ 1] + else: + "" + let mime = newMimetypes() + let mimetypeStr = mime.getMimetype(extNoDot, "") + + mimetypeOpt = if mimetypeStr == "": string.none else: mimetypeStr.some + + let sessionId = $nexUploadSessionCount + nexUploadSessionCount.inc() + + let stream = BufferStream.new() + let lpStream = LPStream(stream) + let node = codex[].node + + let onBlockStored = proc(chunk: seq[byte]): void {.gcsafe, raises: [].} = + try: + if uploadSessions.contains($sessionId): + let session = uploadSessions[$sessionId] + if session.onProgress != nil: + session.onProgress(chunk.len) + except KeyError: + error "Failed to push progress update, session is not found: ", + sessionId = $sessionId + + let blockSize = + if chunkSize.NBytes > 0.NBytes: chunkSize.NBytes else: DefaultBlockSize + let fut = node.store(lpStream, filenameOpt, mimetypeOpt, blockSize, onBlockStored) + + uploadSessions[sessionId] = UploadSession( + stream: stream, fut: fut, filepath: $filepath, chunkSize: blockSize.int + ) + + return ok(sessionId) + +proc chunk( + codex: ptr CodexServer, sessionId: cstring, chunk: seq[byte] +): Future[Result[string, string]] {.async: (raises: []).} = + ## Upload a chunk of data to the session identified by sessionId. + ## The chunk is pushed to the BufferStream of the session. + ## If the chunk size is equal or greater than the session chunkSize, + ## the `onProgress` callback is temporarily set to receive the progress + ## from `onBlockStored` callback. This provide a way to report progress + ## precisely when a block is stored. + ## If the chunk size is smaller than the session chunkSize, + ## the `onProgress` callback is not set because the LPStream will + ## wait until enough data is received to form a block before storing it. + ## The wrapper may then report the progress because the data is in the stream + ## but not yet stored. + + if not uploadSessions.contains($sessionId): + return err("Failed to upload the chunk, the session is not found: " & $sessionId) + + var fut = newFuture[void]() + + try: + let session = uploadSessions[$sessionId] + + if chunk.len >= session.chunkSize: + uploadSessions[$sessionId].onProgress = proc( + bytes: int + ): void {.gcsafe, raises: [].} = + fut.complete() + await session.stream.pushData(chunk) + else: + fut = session.stream.pushData(chunk) + + await fut + + uploadSessions[$sessionId].onProgress = nil + except KeyError: + return err("Failed to upload the chunk, the session is not found: " & $sessionId) + except LPError as e: + return err("Failed to upload the chunk, stream error: " & $e.msg) + except CancelledError: + return err("Failed to upload the chunk, operation cancelled.") + except CatchableError as e: + return err("Failed to upload the chunk: " & $e.msg) + finally: + if not fut.finished(): + fut.cancelSoon() + + return ok("") + +proc finalize( + codex: ptr CodexServer, sessionId: cstring +): Future[Result[string, string]] {.async: (raises: []).} = + ## Finalize the upload session identified by sessionId. + ## This closes the BufferStream and waits for the `node.store` future + ## to complete. It returns the CID of the uploaded file. + + if not uploadSessions.contains($sessionId): + return + err("Failed to finalize the upload session, session not found: " & $sessionId) + + var session: UploadSession + try: + session = uploadSessions[$sessionId] + await session.stream.pushEof() + + let res = await session.fut + if res.isErr: + return err("Failed to finalize the upload session: " & res.error().msg) + + return ok($res.get()) + except KeyError: + return + err("Failed to finalize the upload session, invalid session ID: " & $sessionId) + except LPStreamError as e: + return err("Failed to finalize the upload session, stream error: " & $e.msg) + except CancelledError: + return err("Failed to finalize the upload session, operation cancelled") + except CatchableError as e: + return err("Failed to finalize the upload session: " & $e.msg) + finally: + if uploadSessions.contains($sessionId): + uploadSessions.del($sessionId) + + if session.fut != nil and not session.fut.finished(): + session.fut.cancelSoon() + +proc cancel( + codex: ptr CodexServer, sessionId: cstring +): Future[Result[string, string]] {.async: (raises: []).} = + ## Cancel the upload session identified by sessionId. + ## This cancels the `node.store` future and removes the session + ## from the table. + + if not uploadSessions.contains($sessionId): + # Session not found, nothing to cancel + return ok("") + + try: + let session = uploadSessions[$sessionId] + session.fut.cancelSoon() + except KeyError: + # Session not found, nothing to cancel + return ok("") + + uploadSessions.del($sessionId) + + return ok("") + +proc streamFile( + filepath: string, stream: BufferStream, chunkSize: int +): Future[Result[void, string]] {.async: (raises: [CancelledError]).} = + ## Streams a file from the given filepath using faststream. + ## fsMultiSync cannot be used with chronos because of this warning: + ## Warning: chronos backend uses nested calls to `waitFor` which + ## is not supported by chronos - it is not recommended to use it until + ## this has been resolved. + ## + ## Ideally when it is solved, we should use fsMultiSync or find a way to use async + ## file I/O with chronos, see https://github.com/status-im/nim-chronos/issues/501. + + try: + let inputStreamHandle = filepath.fileInput() + let inputStream = inputStreamHandle.implicitDeref + + var buf = newSeq[byte](chunkSize) + while inputStream.readable: + let read = inputStream.readIntoEx(buf) + if read == 0: + break + await stream.pushData(buf[0 ..< read]) + # let byt = inputStream.read + # await stream.pushData(@[byt]) + return ok() + except IOError, OSError, LPStreamError: + let e = getCurrentException() + return err("Failed to stream the file: " & $e.msg) + +proc file( + codex: ptr CodexServer, sessionId: cstring, onProgress: OnProgressHandler +): Future[Result[string, string]] {.async: (raises: []).} = + ## Starts the file upload for the session identified by sessionId. + ## Will call finalize when done and return the CID of the uploaded file. + ## + ## The onProgress callback is called with the number of bytes + ## to report the progress of the upload. + + if not uploadSessions.contains($sessionId): + return err("Failed to upload the file, invalid session ID: " & $sessionId) + + var session: UploadSession + + try: + uploadSessions[$sessionId].onProgress = onProgress + session = uploadSessions[$sessionId] + + let res = await streamFile(session.filepath, session.stream, session.chunkSize) + if res.isErr: + return err("Failed to upload the file: " & res.error) + + return await codex.finalize(sessionId) + except KeyError: + return err("Failed to upload the file, the session is not found: " & $sessionId) + except LPStreamError, IOError: + let e = getCurrentException() + return err("Failed to upload the file: " & $e.msg) + except CancelledError: + return err("Failed to upload the file, the operation is cancelled.") + except CatchableError as e: + return err("Failed to upload the file: " & $e.msg) + finally: + if uploadSessions.contains($sessionId): + uploadSessions.del($sessionId) + + if session.fut != nil and not session.fut.finished(): + session.fut.cancelSoon() + +proc process*( + self: ptr NodeUploadRequest, + codex: ptr CodexServer, + onUploadProgress: OnProgressHandler = nil, +): Future[Result[string, string]] {.async: (raises: []).} = + defer: + destroyShared(self) + + case self.operation + of NodeUploadMsgType.INIT: + let res = (await init(codex, self.filepath, self.chunkSize)) + if res.isErr: + error "Failed to INIT.", error = res.error + return err($res.error) + return res + of NodeUploadMsgType.CHUNK: + let res = (await chunk(codex, self.sessionId, self.chunk)) + if res.isErr: + error "Failed to CHUNK.", error = res.error + return err($res.error) + return res + of NodeUploadMsgType.FINALIZE: + let res = (await finalize(codex, self.sessionId)) + if res.isErr: + error "Failed to FINALIZE.", error = res.error + return err($res.error) + return res + of NodeUploadMsgType.CANCEL: + let res = (await cancel(codex, self.sessionId)) + if res.isErr: + error "Failed to CANCEL.", error = res.error + return err($res.error) + return res + of NodeUploadMsgType.FILE: + let res = (await file(codex, self.sessionId, onUploadProgress)) + if res.isErr: + error "Failed to FILE.", error = res.error + return err($res.error) + return res diff --git a/library/events/json_base_event.nim b/library/events/json_base_event.nim new file mode 100644 index 00000000..743444ed --- /dev/null +++ b/library/events/json_base_event.nim @@ -0,0 +1,14 @@ +# JSON Event definition +# +# This file defines de JsonEvent type, which serves as the base +# for all event types in the library +# +# Reference specification: +# https://github.com/vacp2p/rfc/blob/master/content/docs/rfcs/36/README.md#jsonsignal-type + +type JsonEvent* = ref object of RootObj + eventType* {.requiresInit.}: string + +method `$`*(jsonEvent: JsonEvent): string {.base.} = + discard + # All events should implement this diff --git a/library/ffi_types.nim b/library/ffi_types.nim new file mode 100644 index 00000000..1a865eaf --- /dev/null +++ b/library/ffi_types.nim @@ -0,0 +1,62 @@ +# FFI Types and Utilities +# +# This file defines the core types and utilities for the library's foreign +# function interface (FFI), enabling interoperability with external code. + +################################################################################ +### Exported types +import results + +type CodexCallback* = proc( + callerRet: cint, msg: ptr cchar, len: csize_t, userData: pointer +) {.cdecl, gcsafe, raises: [].} + +const RET_OK*: cint = 0 +const RET_ERR*: cint = 1 +const RET_MISSING_CALLBACK*: cint = 2 +const RET_PROGRESS*: cint = 3 + +## Returns RET_OK as acknowledgment and call the callback +## with RET_OK code and the provided message. +proc success*(callback: CodexCallback, msg: string, userData: pointer): cint = + callback(RET_OK, cast[ptr cchar](msg), cast[csize_t](len(msg)), userData) + + return RET_OK + +## Returns RET_ERR as acknowledgment and call the callback +## with RET_ERR code and the provided message. +proc error*(callback: CodexCallback, msg: string, userData: pointer): cint = + let msg = "libcodex error: " & msg + callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData) + + return RET_ERR + +## Returns RET_OK as acknowledgment if the result is ok. +## If not, return RET_ERR and call the callback with the error message. +proc okOrError*[T]( + callback: CodexCallback, res: Result[T, string], userData: pointer +): cint = + if res.isOk: + return RET_OK + + return callback.error($res.error, userData) + +### End of exported types +################################################################################ + +################################################################################ +### FFI utils + +template foreignThreadGc*(body: untyped) = + when declared(setupForeignThreadGc): + setupForeignThreadGc() + + body + + when declared(tearDownForeignThreadGc): + tearDownForeignThreadGc() + +type onDone* = proc() + +### End of FFI utils +################################################################################ diff --git a/library/libcodex.h b/library/libcodex.h new file mode 100644 index 00000000..bade4c95 --- /dev/null +++ b/library/libcodex.h @@ -0,0 +1,206 @@ +/** +* libcodex.h - C Interface for Example Library +* +* This header provides the public API for libcodex +* +* To see the auto-generated header by Nim, run `make libcodex` from the +* repository root. The generated file will be created at: +* nimcache/release/libcodex/libcodex.h +*/ + +#ifndef __libcodex__ +#define __libcodex__ + +#include +#include + +// The possible returned values for the functions that return int +#define RET_OK 0 +#define RET_ERR 1 +#define RET_MISSING_CALLBACK 2 +#define RET_PROGRESS 3 + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void (*CodexCallback) (int callerRet, const char* msg, size_t len, void* userData); + +void* codex_new( + const char* configJson, + CodexCallback callback, + void* userData); + +int codex_version( + void* ctx, + CodexCallback callback, + void* userData); + +int codex_revision( + void* ctx, + CodexCallback callback, + void* userData); + +int codex_repo( + void* ctx, + CodexCallback callback, + void* userData); + +int codex_debug( + void* ctx, + CodexCallback callback, + void* userData); + +int codex_spr( + void* ctx, + CodexCallback callback, + void* userData); + +int codex_peer_id( + void* ctx, + CodexCallback callback, + void* userData); + +int codex_log_level( + void* ctx, + const char* logLevel, + CodexCallback callback, + void* userData); + +int codex_connect( + void* ctx, + const char* peerId, + const char** peerAddresses, + size_t peerAddressesSize, + CodexCallback callback, + void* userData); + +int codex_peer_debug( + void* ctx, + const char* peerId, + CodexCallback callback, + void* userData); + + +int codex_upload_init( + void* ctx, + const char* filepath, + size_t chunkSize, + CodexCallback callback, + void* userData); + +int codex_upload_chunk( + void* ctx, + const char* sessionId, + const uint8_t* chunk, + size_t len, + CodexCallback callback, + void* userData); + +int codex_upload_finalize( + void* ctx, + const char* sessionId, + CodexCallback callback, + void* userData); + +int codex_upload_cancel( + void* ctx, + const char* sessionId, + CodexCallback callback, + void* userData); + +int codex_upload_file( + void* ctx, + const char* sessionId, + CodexCallback callback, + void* userData); + +int codex_download_stream( + void* ctx, + const char* cid, + size_t chunkSize, + bool local, + const char* filepath, + CodexCallback callback, + void* userData); + +int codex_download_init( + void* ctx, + const char* cid, + size_t chunkSize, + bool local, + CodexCallback callback, + void* userData); + +int codex_download_chunk( + void* ctx, + const char* cid, + CodexCallback callback, + void* userData); + +int codex_download_cancel( + void* ctx, + const char* cid, + CodexCallback callback, + void* userData); + +int codex_download_manifest( + void* ctx, + const char* cid, + CodexCallback callback, + void* userData); + +int codex_storage_list( + void* ctx, + CodexCallback callback, + void* userData); + +int codex_storage_space( + void* ctx, + CodexCallback callback, + void* userData); + +int codex_storage_delete( + void* ctx, + const char* cid, + CodexCallback callback, + void* userData); + +int codex_storage_fetch( + void* ctx, + const char* cid, + CodexCallback callback, + void* userData); + +int codex_storage_exists( + void* ctx, + const char* cid, + CodexCallback callback, + void* userData); + +int codex_start(void* ctx, + CodexCallback callback, + void* userData); + +int codex_stop(void* ctx, + CodexCallback callback, + void* userData); + +int codex_close(void* ctx, + CodexCallback callback, + void* userData); + +// Destroys an instance of a codex node created with codex_new +int codex_destroy(void* ctx, + CodexCallback callback, + void* userData); + +void codex_set_event_callback(void* ctx, + CodexCallback callback, + void* userData); + +#ifdef __cplusplus +} +#endif + +#endif /* __libcodex__ */ \ No newline at end of file diff --git a/library/libcodex.nim b/library/libcodex.nim new file mode 100644 index 00000000..d1c7e052 --- /dev/null +++ b/library/libcodex.nim @@ -0,0 +1,565 @@ +# libcodex.nim - C-exported interface for the Codex shared library +# +# This file implements the public C API for libcodex. +# It acts as the bridge between C programs and the internal Nim implementation. +# +# This file defines: +# - Initialization logic for the Nim runtime (once per process) +# - Thread-safe exported procs callable from C +# - Callback registration and invocation for asynchronous communication + +# cdecl is C declaration calling convention. +# It’s the standard way C compilers expect functions to behave: +# 1- Caller cleans up the stack after the call +# 2- Symbol names are exported in a predictable way +# In other termes, it is a glue that makes Nim functions callable as normal C functions. +{.pragma: exported, exportc, cdecl, raises: [].} +{.pragma: callback, cdecl, raises: [], gcsafe.} + +# Ensure code is position-independent so it can be built into a shared library (.so). +# In other terms, the code that can run no matter where it’s placed in memory. +{.passc: "-fPIC".} + +when defined(linux): + # Define the canonical name for this library + {.passl: "-Wl,-soname,libcodex.so".} + +import std/[atomics] +import chronicles +import chronos +import chronos/threadsync +import ./codex_context +import ./codex_thread_requests/codex_thread_request +import ./codex_thread_requests/requests/node_lifecycle_request +import ./codex_thread_requests/requests/node_info_request +import ./codex_thread_requests/requests/node_debug_request +import ./codex_thread_requests/requests/node_p2p_request +import ./codex_thread_requests/requests/node_upload_request +import ./codex_thread_requests/requests/node_download_request +import ./codex_thread_requests/requests/node_storage_request +import ./ffi_types + +from ../codex/conf import codexVersion + +logScope: + topics = "codexlib" + +template checkLibcodexParams*( + ctx: ptr CodexContext, callback: CodexCallback, userData: pointer +) = + if not isNil(ctx): + ctx[].userData = userData + + if isNil(callback): + return RET_MISSING_CALLBACK + +# From Nim doc: +# "the C targets require you to initialize Nim's internals, which is done calling a NimMain function." +# "The name NimMain can be influenced via the --nimMainPrefix:prefix switch." +# "Use --nimMainPrefix:MyLib and the function to call is named MyLibNimMain." +proc libcodexNimMain() {.importc.} + +# Atomic flag to prevent multiple initializations +var initialized: Atomic[bool] + +if defined(android): + # Redirect chronicles to Android System logs + when compiles(defaultChroniclesStream.outputs[0].writer): + defaultChroniclesStream.outputs[0].writer = proc( + logLevel: LogLevel, msg: LogOutputStr + ) {.raises: [].} = + echo logLevel, msg + +# Initializes the Nim runtime and foreign-thread GC +proc initializeLibrary() {.exported.} = + if not initialized.exchange(true): + ## Every Nim library must call `NimMain()` once + libcodexNimMain() + when declared(setupForeignThreadGc): + setupForeignThreadGc() + when declared(nimGC_setStackBottom): + var locals {.volatile, noinit.}: pointer + locals = addr(locals) + nimGC_setStackBottom(locals) + +proc codex_new( + configJson: cstring, callback: CodexCallback, userData: pointer +): pointer {.dynlib, exported.} = + initializeLibrary() + + if isNil(callback): + error "Failed to create codex instance: the callback is missing." + return nil + + var ctx = codex_context.createCodexContext().valueOr: + let msg = $error + callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData) + return nil + + ctx.userData = userData + + let reqContent = + NodeLifecycleRequest.createShared(NodeLifecycleMsgType.CREATE_NODE, configJson) + + codex_context.sendRequestToCodexThread( + ctx, RequestType.LIFECYCLE, reqContent, callback, userData + ).isOkOr: + let msg = $error + callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData) + return nil + + return ctx + +proc codex_version( + ctx: ptr CodexContext, callback: CodexCallback, userData: pointer +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + callback( + RET_OK, + cast[ptr cchar](conf.codexVersion), + cast[csize_t](len(conf.codexVersion)), + userData, + ) + + return RET_OK + +proc codex_revision( + ctx: ptr CodexContext, callback: CodexCallback, userData: pointer +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + callback( + RET_OK, + cast[ptr cchar](conf.codexRevision), + cast[csize_t](len(conf.codexRevision)), + userData, + ) + + return RET_OK + +proc codex_repo( + ctx: ptr CodexContext, callback: CodexCallback, userData: pointer +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let reqContent = NodeInfoRequest.createShared(NodeInfoMsgType.REPO) + let res = codex_context.sendRequestToCodexThread( + ctx, RequestType.INFO, reqContent, callback, userData + ) + + return callback.okOrError(res, userData) + +proc codex_debug( + ctx: ptr CodexContext, callback: CodexCallback, userData: pointer +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let reqContent = NodeDebugRequest.createShared(NodeDebugMsgType.DEBUG) + let res = codex_context.sendRequestToCodexThread( + ctx, RequestType.DEBUG, reqContent, callback, userData + ) + + return callback.okOrError(res, userData) + +proc codex_spr( + ctx: ptr CodexContext, callback: CodexCallback, userData: pointer +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let reqContent = NodeInfoRequest.createShared(NodeInfoMsgType.SPR) + let res = codex_context.sendRequestToCodexThread( + ctx, RequestType.INFO, reqContent, callback, userData + ) + + return callback.okOrError(res, userData) + +proc codex_peer_id( + ctx: ptr CodexContext, callback: CodexCallback, userData: pointer +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let reqContent = NodeInfoRequest.createShared(NodeInfoMsgType.PEERID) + let res = codex_context.sendRequestToCodexThread( + ctx, RequestType.INFO, reqContent, callback, userData + ) + + return callback.okOrError(res, userData) + +## Set the log level of the library at runtime. +## It uses updateLogLevel which is a synchronous proc and +## cannot be used inside an async context because of gcsafe issue. +proc codex_log_level( + ctx: ptr CodexContext, logLevel: cstring, callback: CodexCallback, userData: pointer +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let reqContent = + NodeDebugRequest.createShared(NodeDebugMsgType.LOG_LEVEL, logLevel = logLevel) + let res = codex_context.sendRequestToCodexThread( + ctx, RequestType.DEBUG, reqContent, callback, userData + ) + + return callback.okOrError(res, userData) + +proc codex_connect( + ctx: ptr CodexContext, + peerId: cstring, + peerAddressesPtr: ptr cstring, + peerAddressesLength: csize_t, + callback: CodexCallback, + userData: pointer, +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + var peerAddresses = newSeq[cstring](peerAddressesLength) + let peers = cast[ptr UncheckedArray[cstring]](peerAddressesPtr) + for i in 0 ..< peerAddressesLength: + peerAddresses[i] = peers[i] + + let reqContent = NodeP2PRequest.createShared( + NodeP2PMsgType.CONNECT, peerId = peerId, peerAddresses = peerAddresses + ) + let res = codex_context.sendRequestToCodexThread( + ctx, RequestType.P2P, reqContent, callback, userData + ) + + return callback.okOrError(res, userData) + +proc codex_peer_debug( + ctx: ptr CodexContext, peerId: cstring, callback: CodexCallback, userData: pointer +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let reqContent = NodeDebugRequest.createShared(NodeDebugMsgType.PEER, peerId = peerId) + let res = codex_context.sendRequestToCodexThread( + ctx, RequestType.DEBUG, reqContent, callback, userData + ) + + return callback.okOrError(res, userData) + +proc codex_close( + ctx: ptr CodexContext, callback: CodexCallback, userData: pointer +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let reqContent = NodeLifecycleRequest.createShared(NodeLifecycleMsgType.CLOSE_NODE) + var res = codex_context.sendRequestToCodexThread( + ctx, RequestType.LIFECYCLE, reqContent, callback, userData + ) + if res.isErr: + return callback.error(res.error, userData) + + return callback.okOrError(res, userData) + +proc codex_destroy( + ctx: ptr CodexContext, callback: CodexCallback, userData: pointer +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let res = codex_context.destroyCodexContext(ctx) + if res.isErr: + return RET_ERR + + return RET_OK + +proc codex_upload_init( + ctx: ptr CodexContext, + filepath: cstring, + chunkSize: csize_t, + callback: CodexCallback, + userData: pointer, +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let reqContent = NodeUploadRequest.createShared( + NodeUploadMsgType.INIT, filepath = filepath, chunkSize = chunkSize + ) + + let res = codex_context.sendRequestToCodexThread( + ctx, RequestType.UPLOAD, reqContent, callback, userData + ) + + return callback.okOrError(res, userData) + +proc codex_upload_chunk( + ctx: ptr CodexContext, + sessionId: cstring, + data: ptr byte, + len: csize_t, + callback: CodexCallback, + userData: pointer, +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let chunk = newSeq[byte](len) + copyMem(addr chunk[0], data, len) + + let reqContent = NodeUploadRequest.createShared( + NodeUploadMsgType.CHUNK, sessionId = sessionId, chunk = chunk + ) + let res = codex_context.sendRequestToCodexThread( + ctx, RequestType.UPLOAD, reqContent, callback, userData + ) + + return callback.okOrError(res, userData) + +proc codex_upload_finalize( + ctx: ptr CodexContext, + sessionId: cstring, + callback: CodexCallback, + userData: pointer, +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let reqContent = + NodeUploadRequest.createShared(NodeUploadMsgType.FINALIZE, sessionId = sessionId) + let res = codex_context.sendRequestToCodexThread( + ctx, RequestType.UPLOAD, reqContent, callback, userData + ) + + return callback.okOrError(res, userData) + +proc codex_upload_cancel( + ctx: ptr CodexContext, + sessionId: cstring, + callback: CodexCallback, + userData: pointer, +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let reqContent = + NodeUploadRequest.createShared(NodeUploadMsgType.CANCEL, sessionId = sessionId) + + let res = codex_context.sendRequestToCodexThread( + ctx, RequestType.UPLOAD, reqContent, callback, userData + ) + + return callback.okOrError(res, userData) + +proc codex_upload_file( + ctx: ptr CodexContext, + sessionId: cstring, + callback: CodexCallback, + userData: pointer, +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let reqContent = + NodeUploadRequest.createShared(NodeUploadMsgType.FILE, sessionId = sessionId) + + let res = codex_context.sendRequestToCodexThread( + ctx, RequestType.UPLOAD, reqContent, callback, userData + ) + + return callback.okOrError(res, userData) + +proc codex_download_init( + ctx: ptr CodexContext, + cid: cstring, + chunkSize: csize_t, + local: bool, + callback: CodexCallback, + userData: pointer, +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let req = NodeDownloadRequest.createShared( + NodeDownloadMsgType.INIT, cid = cid, chunkSize = chunkSize, local = local + ) + + let res = codex_context.sendRequestToCodexThread( + ctx, RequestType.DOWNLOAD, req, callback, userData + ) + + return callback.okOrError(res, userData) + +proc codex_download_chunk( + ctx: ptr CodexContext, cid: cstring, callback: CodexCallback, userData: pointer +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let req = NodeDownloadRequest.createShared(NodeDownloadMsgType.CHUNK, cid = cid) + + let res = codex_context.sendRequestToCodexThread( + ctx, RequestType.DOWNLOAD, req, callback, userData + ) + + return callback.okOrError(res, userData) + +proc codex_download_stream( + ctx: ptr CodexContext, + cid: cstring, + chunkSize: csize_t, + local: bool, + filepath: cstring, + callback: CodexCallback, + userData: pointer, +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let req = NodeDownloadRequest.createShared( + NodeDownloadMsgType.STREAM, + cid = cid, + chunkSize = chunkSize, + local = local, + filepath = filepath, + ) + + let res = codex_context.sendRequestToCodexThread( + ctx, RequestType.DOWNLOAD, req, callback, userData + ) + + return callback.okOrError(res, userData) + +proc codex_download_cancel( + ctx: ptr CodexContext, cid: cstring, callback: CodexCallback, userData: pointer +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let req = NodeDownloadRequest.createShared(NodeDownloadMsgType.CANCEL, cid = cid) + + let res = codex_context.sendRequestToCodexThread( + ctx, RequestType.DOWNLOAD, req, callback, userData + ) + + return callback.okOrError(res, userData) + +proc codex_download_manifest( + ctx: ptr CodexContext, cid: cstring, callback: CodexCallback, userData: pointer +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let req = NodeDownloadRequest.createShared(NodeDownloadMsgType.MANIFEST, cid = cid) + + let res = codex_context.sendRequestToCodexThread( + ctx, RequestType.DOWNLOAD, req, callback, userData + ) + + return callback.okOrError(res, userData) + +proc codex_storage_list( + ctx: ptr CodexContext, callback: CodexCallback, userData: pointer +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let req = NodeStorageRequest.createShared(NodeStorageMsgType.LIST) + + let res = codex_context.sendRequestToCodexThread( + ctx, RequestType.STORAGE, req, callback, userData + ) + + return callback.okOrError(res, userData) + +proc codex_storage_space( + ctx: ptr CodexContext, callback: CodexCallback, userData: pointer +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let req = NodeStorageRequest.createShared(NodeStorageMsgType.SPACE) + + let res = codex_context.sendRequestToCodexThread( + ctx, RequestType.STORAGE, req, callback, userData + ) + + return callback.okOrError(res, userData) + +proc codex_storage_delete( + ctx: ptr CodexContext, cid: cstring, callback: CodexCallback, userData: pointer +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let req = NodeStorageRequest.createShared(NodeStorageMsgType.DELETE, cid = cid) + + let res = codex_context.sendRequestToCodexThread( + ctx, RequestType.STORAGE, req, callback, userData + ) + + return callback.okOrError(res, userData) + +proc codex_storage_fetch( + ctx: ptr CodexContext, cid: cstring, callback: CodexCallback, userData: pointer +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let req = NodeStorageRequest.createShared(NodeStorageMsgType.FETCH, cid = cid) + + let res = codex_context.sendRequestToCodexThread( + ctx, RequestType.STORAGE, req, callback, userData + ) + + return callback.okOrError(res, userData) + +proc codex_storage_exists( + ctx: ptr CodexContext, cid: cstring, callback: CodexCallback, userData: pointer +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let req = NodeStorageRequest.createShared(NodeStorageMsgType.EXISTS, cid = cid) + + let res = codex_context.sendRequestToCodexThread( + ctx, RequestType.STORAGE, req, callback, userData + ) + + return callback.okOrError(res, userData) + +proc codex_start( + ctx: ptr CodexContext, callback: CodexCallback, userData: pointer +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let reqContent: ptr NodeLifecycleRequest = + NodeLifecycleRequest.createShared(NodeLifecycleMsgType.START_NODE) + let res = codex_context.sendRequestToCodexThread( + ctx, RequestType.LIFECYCLE, reqContent, callback, userData + ) + + return callback.okOrError(res, userData) + +proc codex_stop( + ctx: ptr CodexContext, callback: CodexCallback, userData: pointer +): cint {.dynlib, exportc.} = + initializeLibrary() + checkLibcodexParams(ctx, callback, userData) + + let reqContent: ptr NodeLifecycleRequest = + NodeLifecycleRequest.createShared(NodeLifecycleMsgType.STOP_NODE) + let res = codex_context.sendRequestToCodexThread( + ctx, RequestType.LIFECYCLE, reqContent, callback, userData + ) + + return callback.okOrError(res, userData) + +proc codex_set_event_callback( + ctx: ptr CodexContext, callback: CodexCallback, userData: pointer +) {.dynlib, exportc.} = + initializeLibrary() + ctx[].eventCallback = cast[pointer](callback) + ctx[].eventUserData = userData diff --git a/tests/codex/helpers/mockclock.nim b/tests/codex/helpers/mockclock.nim index be1eb4d2..76446041 100644 --- a/tests/codex/helpers/mockclock.nim +++ b/tests/codex/helpers/mockclock.nim @@ -33,11 +33,18 @@ proc advance*(clock: MockClock, seconds: int64) = method now*(clock: MockClock): SecondsSince1970 = clock.time -method waitUntil*(clock: MockClock, time: SecondsSince1970) {.async.} = - if time > clock.now(): - let future = newFuture[void]() - clock.waiting.add(Waiting(until: time, future: future)) - await future +method waitUntil*( + clock: MockClock, time: SecondsSince1970 +) {.async: (raises: [CancelledError]).} = + try: + if time > clock.now(): + let future = newFuture[void]() + clock.waiting.add(Waiting(until: time, future: future)) + await future + except CancelledError as e: + raise e + except Exception as e: + discard proc isWaiting*(clock: MockClock): bool = clock.waiting.len > 0 diff --git a/tests/integration/codexprocess.nim b/tests/integration/codexprocess.nim index 3eca5b04..351a78e2 100644 --- a/tests/integration/codexprocess.nim +++ b/tests/integration/codexprocess.nim @@ -51,7 +51,8 @@ proc ethAccount*(node: CodexProcess): Address = proc apiUrl*(node: CodexProcess): string = let config = CodexConf.load(cmdLine = node.arguments, quitOnFailure = false) - return "http://" & config.apiBindAddress & ":" & $config.apiPort & "/api/codex/v1" + return + "http://" & config.apiBindAddress.get() & ":" & $config.apiPort & "/api/codex/v1" proc client*(node: CodexProcess): CodexClient = if client =? node.client: From 9ac9f6ff3c62dfa2dd93f0738711c428dbe27a9f Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 11 Dec 2025 10:03:43 +0100 Subject: [PATCH 13/20] chore: drop usage of upraises (#1348) --- codex/blockexchange/engine/engine.nim | 2 +- codex/blockexchange/network/network.nim | 16 +++++++------- codex/blockexchange/network/networkpeer.nim | 5 ++--- codex/blocktype.nim | 5 +---- codex/chunker.nim | 11 ++++------ codex/clock.nim | 3 +-- codex/conf.nim | 12 +++++------ codex/contracts/market.nim | 23 ++++++++++----------- codex/discovery.nim | 2 +- codex/erasure/backend.nim | 5 +---- codex/erasure/erasure.nim | 5 +---- codex/manifest/coders.nim | 4 +--- codex/manifest/manifest.nim | 5 +---- codex/market.nim | 19 ++++++++--------- codex/merkletree/codex/coders.nim | 5 +---- codex/node.nim | 5 ++--- codex/rest/api.nim | 5 +---- codex/rng.nim | 5 +---- codex/sales/reservations.nim | 16 +++++--------- codex/sales/salesagent.nim | 9 +++----- codex/sales/salescontext.nim | 12 +++++------ codex/sales/slotqueue.nim | 3 +-- codex/sales/statemachine.nim | 7 +++---- codex/sales/states/errored.nim | 1 - codex/stores/blockstore.nim | 2 +- codex/stores/keyutils.nim | 4 +--- codex/stores/treehelper.nim | 5 +---- codex/streams/asyncstreamwrapper.nim | 4 +--- codex/streams/storestream.nim | 5 +---- codex/systemclock.nim | 3 +-- codex/utils/addrutils.nim | 4 +--- codex/utils/fileutils.nim | 5 +---- codex/utils/keyutils.nim | 4 +--- codex/utils/timer.nim | 2 +- tests/codex/helpers/mockchunker.nim | 2 +- tests/codex/node/testnode.nim | 4 ++-- tests/codex/sales/testreservations.nim | 16 ++++---------- tests/codex/testchunking.nim | 4 ++-- tests/codex/utils/testasyncstatemachine.nim | 3 +-- 39 files changed, 89 insertions(+), 163 deletions(-) diff --git a/codex/blockexchange/engine/engine.nim b/codex/blockexchange/engine/engine.nim index baacf11c..90fd513d 100644 --- a/codex/blockexchange/engine/engine.nim +++ b/codex/blockexchange/engine/engine.nim @@ -926,7 +926,7 @@ iterator splitBatches[T](sequence: seq[T], batchSize: int): seq[T] = proc taskHandler*( self: BlockExcEngine, peerCtx: BlockExcPeerCtx -) {.gcsafe, async: (raises: [CancelledError, RetriesExhaustedError]).} = +) {.async: (raises: [CancelledError, RetriesExhaustedError]).} = # Send to the peer blocks he wants to get, # if they present in our local store diff --git a/codex/blockexchange/network/network.nim b/codex/blockexchange/network/network.nim index f503a15c..904b60d8 100644 --- a/codex/blockexchange/network/network.nim +++ b/codex/blockexchange/network/network.nim @@ -35,16 +35,14 @@ const DefaultMaxInflight* = 100 type - WantListHandler* = - proc(peer: PeerId, wantList: WantList) {.gcsafe, async: (raises: []).} + WantListHandler* = proc(peer: PeerId, wantList: WantList) {.async: (raises: []).} BlocksDeliveryHandler* = - proc(peer: PeerId, blocks: seq[BlockDelivery]) {.gcsafe, async: (raises: []).} + proc(peer: PeerId, blocks: seq[BlockDelivery]) {.async: (raises: []).} BlockPresenceHandler* = - proc(peer: PeerId, precense: seq[BlockPresence]) {.gcsafe, async: (raises: []).} - AccountHandler* = proc(peer: PeerId, account: Account) {.gcsafe, async: (raises: []).} - PaymentHandler* = - proc(peer: PeerId, payment: SignedState) {.gcsafe, async: (raises: []).} - PeerEventHandler* = proc(peer: PeerId) {.gcsafe, async: (raises: [CancelledError]).} + proc(peer: PeerId, precense: seq[BlockPresence]) {.async: (raises: []).} + AccountHandler* = proc(peer: PeerId, account: Account) {.async: (raises: []).} + PaymentHandler* = proc(peer: PeerId, payment: SignedState) {.async: (raises: []).} + PeerEventHandler* = proc(peer: PeerId) {.async: (raises: [CancelledError]).} BlockExcHandlers* = object onWantList*: WantListHandler @@ -347,7 +345,7 @@ method init*(self: BlockExcNetwork) {.raises: [].} = proc peerEventHandler( peerId: PeerId, event: PeerEvent - ): Future[void] {.gcsafe, async: (raises: [CancelledError]).} = + ): Future[void] {.async: (raises: [CancelledError]).} = if event.kind == PeerEventKind.Joined: await self.handlePeerJoined(peerId) elif event.kind == PeerEventKind.Left: diff --git a/codex/blockexchange/network/networkpeer.nim b/codex/blockexchange/network/networkpeer.nim index 1964a307..8363fdc2 100644 --- a/codex/blockexchange/network/networkpeer.nim +++ b/codex/blockexchange/network/networkpeer.nim @@ -24,10 +24,9 @@ logScope: const DefaultYieldInterval = 50.millis type - ConnProvider* = - proc(): Future[Connection] {.gcsafe, async: (raises: [CancelledError]).} + ConnProvider* = proc(): Future[Connection] {.async: (raises: [CancelledError]).} - RPCHandler* = proc(peer: NetworkPeer, msg: Message) {.gcsafe, async: (raises: []).} + RPCHandler* = proc(peer: NetworkPeer, msg: Message) {.async: (raises: []).} NetworkPeer* = ref object of RootObj id*: PeerId diff --git a/codex/blocktype.nim b/codex/blocktype.nim index 233d4e8f..edacb927 100644 --- a/codex/blocktype.nim +++ b/codex/blocktype.nim @@ -13,10 +13,7 @@ import std/hashes export tables -import pkg/upraises - -push: - {.upraises: [].} +{.push raises: [], gcsafe.} import pkg/libp2p/[cid, multicodec, multihash] import pkg/stew/[byteutils, endians2] diff --git a/codex/chunker.nim b/codex/chunker.nim index 908dd0c0..f0022859 100644 --- a/codex/chunker.nim +++ b/codex/chunker.nim @@ -9,10 +9,7 @@ # TODO: This is super inneficient and needs a rewrite, but it'll do for now -import pkg/upraises - -push: - {.upraises: [].} +{.push raises: [], gcsafe.} import pkg/questionable import pkg/questionable/results @@ -31,7 +28,7 @@ type ChunkerError* = object of CatchableError ChunkBuffer* = ptr UncheckedArray[byte] Reader* = proc(data: ChunkBuffer, len: int): Future[int] {. - gcsafe, async: (raises: [ChunkerError, CancelledError]) + async: (raises: [ChunkerError, CancelledError]) .} # Reader that splits input data into fixed-size chunks @@ -77,7 +74,7 @@ proc new*( proc reader( data: ChunkBuffer, len: int - ): Future[int] {.gcsafe, async: (raises: [ChunkerError, CancelledError]).} = + ): Future[int] {.async: (raises: [ChunkerError, CancelledError]).} = var res = 0 try: while res < len: @@ -105,7 +102,7 @@ proc new*( proc reader( data: ChunkBuffer, len: int - ): Future[int] {.gcsafe, async: (raises: [ChunkerError, CancelledError]).} = + ): Future[int] {.async: (raises: [ChunkerError, CancelledError]).} = var total = 0 try: while total < len: diff --git a/codex/clock.nim b/codex/clock.nim index 5c63d15e..c0867afa 100644 --- a/codex/clock.nim +++ b/codex/clock.nim @@ -2,7 +2,6 @@ import pkg/chronos import pkg/stew/endians2 -import pkg/upraises import pkg/stint type @@ -10,7 +9,7 @@ type SecondsSince1970* = int64 Timeout* = object of CatchableError -method now*(clock: Clock): SecondsSince1970 {.base, gcsafe, upraises: [].} = +method now*(clock: Clock): SecondsSince1970 {.base, gcsafe, raises: [].} = raiseAssert "not implemented" method waitUntil*( diff --git a/codex/conf.nim b/codex/conf.nim index 206a3e04..cec9c11a 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -514,7 +514,7 @@ const proc parseCmdArg*( T: typedesc[MultiAddress], input: string -): MultiAddress {.upraises: [ValueError].} = +): MultiAddress {.raises: [ValueError].} = var ma: MultiAddress try: let res = MultiAddress.init(input) @@ -619,7 +619,7 @@ proc parseCmdArg*(T: type Duration, val: string): T = proc readValue*( r: var TomlReader, val: var EthAddress -) {.upraises: [SerializationError, IOError].} = +) {.raises: [SerializationError, IOError].} = val = EthAddress.init(r.readValue(string)).get() proc readValue*(r: var TomlReader, val: var SignedPeerRecord) = @@ -647,7 +647,7 @@ proc readValue*(r: var TomlReader, val: var MultiAddress) = proc readValue*( r: var TomlReader, val: var NBytes -) {.upraises: [SerializationError, IOError].} = +) {.raises: [SerializationError, IOError].} = var value = 0'i64 var str = r.readValue(string) let count = parseSize(str, value, alwaysBin = true) @@ -658,7 +658,7 @@ proc readValue*( proc readValue*( r: var TomlReader, val: var ThreadCount -) {.upraises: [SerializationError, IOError].} = +) {.raises: [SerializationError, IOError].} = var str = r.readValue(string) try: val = parseCmdArg(ThreadCount, str) @@ -667,7 +667,7 @@ proc readValue*( proc readValue*( r: var TomlReader, val: var Duration -) {.upraises: [SerializationError, IOError].} = +) {.raises: [SerializationError, IOError].} = var str = r.readValue(string) var dur: Duration let count = parseDuration(str, dur) @@ -734,7 +734,7 @@ proc stripAnsi*(v: string): string = res -proc updateLogLevel*(logLevel: string) {.upraises: [ValueError].} = +proc updateLogLevel*(logLevel: string) {.raises: [ValueError].} = # Updates log levels (without clearing old ones) let directives = logLevel.split(";") try: diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index f676012b..884441d4 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -1,7 +1,6 @@ import std/strformat import std/strutils import pkg/ethers -import pkg/upraises import pkg/questionable import pkg/lrucache import ../utils/exceptions @@ -436,7 +435,7 @@ method canReserveSlot*( method subscribeRequests*( market: OnChainMarket, callback: OnRequest ): Future[MarketSubscription] {.async.} = - proc onEvent(eventResult: ?!StorageRequested) {.upraises: [].} = + proc onEvent(eventResult: ?!StorageRequested) {.raises: [].} = without event =? eventResult, eventErr: error "There was an error in Request subscription", msg = eventErr.msg return @@ -450,7 +449,7 @@ method subscribeRequests*( method subscribeSlotFilled*( market: OnChainMarket, callback: OnSlotFilled ): Future[MarketSubscription] {.async.} = - proc onEvent(eventResult: ?!SlotFilled) {.upraises: [].} = + proc onEvent(eventResult: ?!SlotFilled) {.raises: [].} = without event =? eventResult, eventErr: error "There was an error in SlotFilled subscription", msg = eventErr.msg return @@ -477,7 +476,7 @@ method subscribeSlotFilled*( method subscribeSlotFreed*( market: OnChainMarket, callback: OnSlotFreed ): Future[MarketSubscription] {.async.} = - proc onEvent(eventResult: ?!SlotFreed) {.upraises: [].} = + proc onEvent(eventResult: ?!SlotFreed) {.raises: [].} = without event =? eventResult, eventErr: error "There was an error in SlotFreed subscription", msg = eventErr.msg return @@ -491,7 +490,7 @@ method subscribeSlotFreed*( method subscribeSlotReservationsFull*( market: OnChainMarket, callback: OnSlotReservationsFull ): Future[MarketSubscription] {.async.} = - proc onEvent(eventResult: ?!SlotReservationsFull) {.upraises: [].} = + proc onEvent(eventResult: ?!SlotReservationsFull) {.raises: [].} = without event =? eventResult, eventErr: error "There was an error in SlotReservationsFull subscription", msg = eventErr.msg @@ -506,7 +505,7 @@ method subscribeSlotReservationsFull*( method subscribeFulfillment( market: OnChainMarket, callback: OnFulfillment ): Future[MarketSubscription] {.async.} = - proc onEvent(eventResult: ?!RequestFulfilled) {.upraises: [].} = + proc onEvent(eventResult: ?!RequestFulfilled) {.raises: [].} = without event =? eventResult, eventErr: error "There was an error in RequestFulfillment subscription", msg = eventErr.msg return @@ -520,7 +519,7 @@ method subscribeFulfillment( method subscribeFulfillment( market: OnChainMarket, requestId: RequestId, callback: OnFulfillment ): Future[MarketSubscription] {.async.} = - proc onEvent(eventResult: ?!RequestFulfilled) {.upraises: [].} = + proc onEvent(eventResult: ?!RequestFulfilled) {.raises: [].} = without event =? eventResult, eventErr: error "There was an error in RequestFulfillment subscription", msg = eventErr.msg return @@ -535,7 +534,7 @@ method subscribeFulfillment( method subscribeRequestCancelled*( market: OnChainMarket, callback: OnRequestCancelled ): Future[MarketSubscription] {.async.} = - proc onEvent(eventResult: ?!RequestCancelled) {.upraises: [].} = + proc onEvent(eventResult: ?!RequestCancelled) {.raises: [].} = without event =? eventResult, eventErr: error "There was an error in RequestCancelled subscription", msg = eventErr.msg return @@ -549,7 +548,7 @@ method subscribeRequestCancelled*( method subscribeRequestCancelled*( market: OnChainMarket, requestId: RequestId, callback: OnRequestCancelled ): Future[MarketSubscription] {.async.} = - proc onEvent(eventResult: ?!RequestCancelled) {.upraises: [].} = + proc onEvent(eventResult: ?!RequestCancelled) {.raises: [].} = without event =? eventResult, eventErr: error "There was an error in RequestCancelled subscription", msg = eventErr.msg return @@ -564,7 +563,7 @@ method subscribeRequestCancelled*( method subscribeRequestFailed*( market: OnChainMarket, callback: OnRequestFailed ): Future[MarketSubscription] {.async.} = - proc onEvent(eventResult: ?!RequestFailed) {.upraises: [].} = + proc onEvent(eventResult: ?!RequestFailed) {.raises: [].} = without event =? eventResult, eventErr: error "There was an error in RequestFailed subscription", msg = eventErr.msg return @@ -578,7 +577,7 @@ method subscribeRequestFailed*( method subscribeRequestFailed*( market: OnChainMarket, requestId: RequestId, callback: OnRequestFailed ): Future[MarketSubscription] {.async.} = - proc onEvent(eventResult: ?!RequestFailed) {.upraises: [].} = + proc onEvent(eventResult: ?!RequestFailed) {.raises: [].} = without event =? eventResult, eventErr: error "There was an error in RequestFailed subscription", msg = eventErr.msg return @@ -593,7 +592,7 @@ method subscribeRequestFailed*( method subscribeProofSubmission*( market: OnChainMarket, callback: OnProofSubmitted ): Future[MarketSubscription] {.async.} = - proc onEvent(eventResult: ?!ProofSubmitted) {.upraises: [].} = + proc onEvent(eventResult: ?!ProofSubmitted) {.raises: [].} = without event =? eventResult, eventErr: error "There was an error in ProofSubmitted subscription", msg = eventErr.msg return diff --git a/codex/discovery.nim b/codex/discovery.nim index 9ff89caf..905f0902 100644 --- a/codex/discovery.nim +++ b/codex/discovery.nim @@ -158,7 +158,7 @@ method provide*( method removeProvider*( d: Discovery, peerId: PeerId -): Future[void] {.base, gcsafe, async: (raises: [CancelledError]).} = +): Future[void] {.base, async: (raises: [CancelledError]).} = ## Remove provider from providers table ## diff --git a/codex/erasure/backend.nim b/codex/erasure/backend.nim index 32009829..14bb0ddb 100644 --- a/codex/erasure/backend.nim +++ b/codex/erasure/backend.nim @@ -7,10 +7,7 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. -import pkg/upraises - -push: - {.upraises: [].} +{.push raises: [], gcsafe.} import ../stores diff --git a/codex/erasure/erasure.nim b/codex/erasure/erasure.nim index 95516500..bec2f135 100644 --- a/codex/erasure/erasure.nim +++ b/codex/erasure/erasure.nim @@ -7,10 +7,7 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. -import pkg/upraises - -push: - {.upraises: [].} +{.push raises: [], gcsafe.} import std/[sugar, atomics, sequtils] diff --git a/codex/manifest/coders.nim b/codex/manifest/coders.nim index 30e0c7ca..207b6c99 100644 --- a/codex/manifest/coders.nim +++ b/codex/manifest/coders.nim @@ -9,11 +9,9 @@ # This module implements serialization and deserialization of Manifest -import pkg/upraises import times -push: - {.upraises: [].} +{.push raises: [].} import std/tables import std/sequtils diff --git a/codex/manifest/manifest.nim b/codex/manifest/manifest.nim index cbb0bace..101fbb89 100644 --- a/codex/manifest/manifest.nim +++ b/codex/manifest/manifest.nim @@ -9,10 +9,7 @@ # This module defines all operations on Manifest -import pkg/upraises - -push: - {.upraises: [].} +{.push raises: [], gcsafe.} import pkg/libp2p/protobuf/minprotobuf import pkg/libp2p/[cid, multihash, multicodec] diff --git a/codex/market.nim b/codex/market.nim index 0fe69347..968f204e 100644 --- a/codex/market.nim +++ b/codex/market.nim @@ -1,5 +1,4 @@ import pkg/chronos -import pkg/upraises import pkg/questionable import pkg/ethers/erc20 import ./contracts/requests @@ -23,15 +22,15 @@ type ProofInvalidError* = object of MarketError Subscription* = ref object of RootObj OnRequest* = - proc(id: RequestId, ask: StorageAsk, expiry: uint64) {.gcsafe, upraises: [].} - OnFulfillment* = proc(requestId: RequestId) {.gcsafe, upraises: [].} - OnSlotFilled* = proc(requestId: RequestId, slotIndex: uint64) {.gcsafe, upraises: [].} - OnSlotFreed* = proc(requestId: RequestId, slotIndex: uint64) {.gcsafe, upraises: [].} + proc(id: RequestId, ask: StorageAsk, expiry: uint64) {.gcsafe, raises: [].} + OnFulfillment* = proc(requestId: RequestId) {.gcsafe, raises: [].} + OnSlotFilled* = proc(requestId: RequestId, slotIndex: uint64) {.gcsafe, raises: [].} + OnSlotFreed* = proc(requestId: RequestId, slotIndex: uint64) {.gcsafe, raises: [].} OnSlotReservationsFull* = - proc(requestId: RequestId, slotIndex: uint64) {.gcsafe, upraises: [].} - OnRequestCancelled* = proc(requestId: RequestId) {.gcsafe, upraises: [].} - OnRequestFailed* = proc(requestId: RequestId) {.gcsafe, upraises: [].} - OnProofSubmitted* = proc(id: SlotId) {.gcsafe, upraises: [].} + proc(requestId: RequestId, slotIndex: uint64) {.gcsafe, raises: [].} + OnRequestCancelled* = proc(requestId: RequestId) {.gcsafe, raises: [].} + OnRequestFailed* = proc(requestId: RequestId) {.gcsafe, raises: [].} + OnProofSubmitted* = proc(id: SlotId) {.gcsafe, raises: [].} ProofChallenge* = array[32, byte] # Marketplace events -- located here due to the Market abstraction @@ -275,7 +274,7 @@ method subscribeProofSubmission*( ): Future[Subscription] {.base, async.} = raiseAssert("not implemented") -method unsubscribe*(subscription: Subscription) {.base, async, upraises: [].} = +method unsubscribe*(subscription: Subscription) {.base, async.} = raiseAssert("not implemented") method queryPastSlotFilledEvents*( diff --git a/codex/merkletree/codex/coders.nim b/codex/merkletree/codex/coders.nim index 1d50707c..bfcccbd6 100644 --- a/codex/merkletree/codex/coders.nim +++ b/codex/merkletree/codex/coders.nim @@ -7,10 +7,7 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. -import pkg/upraises - -push: - {.upraises: [].} +{.push raises: [], gcsafe.} import pkg/libp2p import pkg/questionable diff --git a/codex/node.nim b/codex/node.nim index ccae080c..4c259bfc 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -81,9 +81,8 @@ type CodexNodeRef* = ref CodexNode OnManifest* = proc(cid: Cid, manifest: Manifest): void {.gcsafe, raises: [].} - BatchProc* = proc(blocks: seq[bt.Block]): Future[?!void] {. - gcsafe, async: (raises: [CancelledError]) - .} + BatchProc* = + proc(blocks: seq[bt.Block]): Future[?!void] {.async: (raises: [CancelledError]).} OnBlockStoredProc = proc(chunk: seq[byte]): void {.gcsafe, raises: [].} func switch*(self: CodexNodeRef): Switch = diff --git a/codex/rest/api.nim b/codex/rest/api.nim index adab821b..938ede12 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -7,10 +7,7 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. -import pkg/upraises - -push: - {.upraises: [].} +{.push raises: [], gcsafe.} import std/sequtils import std/mimetypes diff --git a/codex/rng.nim b/codex/rng.nim index 866d65f8..0d789305 100644 --- a/codex/rng.nim +++ b/codex/rng.nim @@ -7,10 +7,7 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. -import pkg/upraises - -push: - {.upraises: [].} +{.push raises: [], gcsafe.} import pkg/libp2p/crypto/crypto import pkg/bearssl/rand diff --git a/codex/sales/reservations.nim b/codex/sales/reservations.nim index f27a66fe..627ef6bb 100644 --- a/codex/sales/reservations.nim +++ b/codex/sales/reservations.nim @@ -27,9 +27,7 @@ ## | UInt256 | totalRemainingCollateral | | ## +---------------------------------------------------+ -import pkg/upraises -push: - {.upraises: [].} +{.push raises: [], gcsafe.} import std/sequtils import std/sugar @@ -92,14 +90,10 @@ type repo: RepoStore OnAvailabilitySaved: ?OnAvailabilitySaved - GetNext* = proc(): Future[?seq[byte]] {. - upraises: [], gcsafe, async: (raises: [CancelledError]), closure - .} - IterDispose* = - proc(): Future[?!void] {.gcsafe, async: (raises: [CancelledError]), closure.} - OnAvailabilitySaved* = proc(availability: Availability): Future[void] {. - upraises: [], gcsafe, async: (raises: []) - .} + GetNext* = proc(): Future[?seq[byte]] {.async: (raises: [CancelledError]), closure.} + IterDispose* = proc(): Future[?!void] {.async: (raises: [CancelledError]), closure.} + OnAvailabilitySaved* = + proc(availability: Availability): Future[void] {.async: (raises: []).} StorableIter* = ref object finished*: bool next*: GetNext diff --git a/codex/sales/salesagent.nim b/codex/sales/salesagent.nim index 96137fe0..6584353c 100644 --- a/codex/sales/salesagent.nim +++ b/codex/sales/salesagent.nim @@ -2,7 +2,6 @@ import pkg/chronos import pkg/questionable import pkg/questionable/results import pkg/stint -import pkg/upraises import ../contracts/requests import ../errors import ../logutils @@ -113,14 +112,12 @@ proc subscribeCancellation(agent: SalesAgent) {.async.} = method onFulfilled*( agent: SalesAgent, requestId: RequestId -) {.base, gcsafe, upraises: [].} = +) {.base, gcsafe, raises: [].} = let cancelled = agent.data.cancelled if agent.data.requestId == requestId and not cancelled.isNil and not cancelled.finished: cancelled.cancelSoon() -method onFailed*( - agent: SalesAgent, requestId: RequestId -) {.base, gcsafe, upraises: [].} = +method onFailed*(agent: SalesAgent, requestId: RequestId) {.base, gcsafe, raises: [].} = without request =? agent.data.request: return if agent.data.requestId == requestId: @@ -128,7 +125,7 @@ method onFailed*( method onSlotFilled*( agent: SalesAgent, requestId: RequestId, slotIndex: uint64 -) {.base, gcsafe, upraises: [].} = +) {.base, gcsafe, raises: [].} = if agent.data.requestId == requestId and agent.data.slotIndex == slotIndex: agent.schedule(slotFilledEvent(requestId, slotIndex)) diff --git a/codex/sales/salescontext.nim b/codex/sales/salescontext.nim index ac0908df..5fd7099c 100644 --- a/codex/sales/salescontext.nim +++ b/codex/sales/salescontext.nim @@ -1,6 +1,5 @@ import pkg/questionable import pkg/questionable/results -import pkg/upraises import pkg/libp2p/cid import ../market @@ -24,21 +23,20 @@ type slotQueue*: SlotQueue simulateProofFailures*: int - BlocksCb* = proc(blocks: seq[bt.Block]): Future[?!void] {. - gcsafe, async: (raises: [CancelledError]) - .} + BlocksCb* = + proc(blocks: seq[bt.Block]): Future[?!void] {.async: (raises: [CancelledError]).} OnStore* = proc( request: StorageRequest, expiry: SecondsSince1970, slot: uint64, blocksCb: BlocksCb, isRepairing: bool, - ): Future[?!void] {.gcsafe, async: (raises: [CancelledError]).} + ): Future[?!void] {.async: (raises: [CancelledError]).} OnProve* = proc(slot: Slot, challenge: ProofChallenge): Future[?!Groth16Proof] {. - gcsafe, async: (raises: [CancelledError]) + async: (raises: [CancelledError]) .} OnExpiryUpdate* = proc(rootCid: Cid, expiry: SecondsSince1970): Future[?!void] {. - gcsafe, async: (raises: [CancelledError]) + async: (raises: [CancelledError]) .} OnClear* = proc(request: StorageRequest, slotIndex: uint64) {.gcsafe, raises: [].} OnSale* = proc(request: StorageRequest, slotIndex: uint64) {.gcsafe, raises: [].} diff --git a/codex/sales/slotqueue.nim b/codex/sales/slotqueue.nim index b6e77395..ad9a07db 100644 --- a/codex/sales/slotqueue.nim +++ b/codex/sales/slotqueue.nim @@ -15,8 +15,7 @@ logScope: topics = "marketplace slotqueue" type - OnProcessSlot* = - proc(item: SlotQueueItem): Future[void] {.gcsafe, async: (raises: []).} + OnProcessSlot* = proc(item: SlotQueueItem): Future[void] {.async: (raises: []).} # Non-ref obj copies value when assigned, preventing accidental modification # of values which could cause an incorrect order (eg diff --git a/codex/sales/statemachine.nim b/codex/sales/statemachine.nim index d1732549..dc199ade 100644 --- a/codex/sales/statemachine.nim +++ b/codex/sales/statemachine.nim @@ -1,5 +1,4 @@ import pkg/questionable -import pkg/upraises import ../errors import ../utils/asyncstatemachine import ../market @@ -16,17 +15,17 @@ type method onCancelled*( state: SaleState, request: StorageRequest -): ?State {.base, upraises: [].} = +): ?State {.base, raises: [].} = discard method onFailed*( state: SaleState, request: StorageRequest -): ?State {.base, upraises: [].} = +): ?State {.base, raises: [].} = discard method onSlotFilled*( state: SaleState, requestId: RequestId, slotIndex: uint64 -): ?State {.base, upraises: [].} = +): ?State {.base, raises: [].} = discard proc cancelledEvent*(request: StorageRequest): Event = diff --git a/codex/sales/states/errored.nim b/codex/sales/states/errored.nim index 95848fd3..3887c652 100644 --- a/codex/sales/states/errored.nim +++ b/codex/sales/states/errored.nim @@ -1,6 +1,5 @@ import pkg/questionable import pkg/questionable/results -import pkg/upraises import ../statemachine import ../salesagent diff --git a/codex/stores/blockstore.nim b/codex/stores/blockstore.nim index 78809191..253346a8 100644 --- a/codex/stores/blockstore.nim +++ b/codex/stores/blockstore.nim @@ -29,7 +29,7 @@ type Block Both - CidCallback* = proc(cid: Cid): Future[void] {.gcsafe, async: (raises: []).} + CidCallback* = proc(cid: Cid): Future[void] {.async: (raises: []).} BlockStore* = ref object of RootObj onBlockStored*: ?CidCallback diff --git a/codex/stores/keyutils.nim b/codex/stores/keyutils.nim index 0634b6a2..0d14c2e0 100644 --- a/codex/stores/keyutils.nim +++ b/codex/stores/keyutils.nim @@ -7,9 +7,7 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. -import pkg/upraises -push: - {.upraises: [].} +{.push raises: [], gcsafe.} import std/sugar import pkg/questionable/results diff --git a/codex/stores/treehelper.nim b/codex/stores/treehelper.nim index e1f5d48d..3c214856 100644 --- a/codex/stores/treehelper.nim +++ b/codex/stores/treehelper.nim @@ -7,10 +7,7 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. -import pkg/upraises - -push: - {.upraises: [].} +{.push raises: [], gcsafe.} import std/sugar import pkg/chronos diff --git a/codex/streams/asyncstreamwrapper.nim b/codex/streams/asyncstreamwrapper.nim index 6708816d..f80c0696 100644 --- a/codex/streams/asyncstreamwrapper.nim +++ b/codex/streams/asyncstreamwrapper.nim @@ -7,9 +7,7 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. -import pkg/upraises -push: - {.upraises: [].} +{.push raises: [], gcsafe.} import pkg/chronos import pkg/libp2p diff --git a/codex/streams/storestream.nim b/codex/streams/storestream.nim index 2e06d39d..42eb5148 100644 --- a/codex/streams/storestream.nim +++ b/codex/streams/storestream.nim @@ -9,10 +9,7 @@ import std/options -import pkg/upraises - -push: - {.upraises: [].} +{.push raises: [], gcsafe.} import pkg/chronos import pkg/stew/ptrops diff --git a/codex/systemclock.nim b/codex/systemclock.nim index 6226f627..93c70d89 100644 --- a/codex/systemclock.nim +++ b/codex/systemclock.nim @@ -1,9 +1,8 @@ import std/times -import pkg/upraises import ./clock type SystemClock* = ref object of Clock -method now*(clock: SystemClock): SecondsSince1970 {.upraises: [].} = +method now*(clock: SystemClock): SecondsSince1970 {.raises: [].} = let now = times.now().utc now.toTime().toUnix() diff --git a/codex/utils/addrutils.nim b/codex/utils/addrutils.nim index a9ec54f5..cd9dbd0e 100644 --- a/codex/utils/addrutils.nim +++ b/codex/utils/addrutils.nim @@ -7,9 +7,7 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. -import pkg/upraises -push: - {.upraises: [].} +{.push raises: [], gcsafe.} import std/strutils import std/options diff --git a/codex/utils/fileutils.nim b/codex/utils/fileutils.nim index 6f12dd76..28b82b84 100644 --- a/codex/utils/fileutils.nim +++ b/codex/utils/fileutils.nim @@ -9,10 +9,7 @@ ## Partially taken from nim beacon chain -import pkg/upraises - -push: - {.upraises: [].} +{.push raises: [], gcsafe.} import std/strutils import pkg/stew/io2 diff --git a/codex/utils/keyutils.nim b/codex/utils/keyutils.nim index 664396d3..e0881df6 100644 --- a/codex/utils/keyutils.nim +++ b/codex/utils/keyutils.nim @@ -7,9 +7,7 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. -import pkg/upraises -push: - {.upraises: [].} +{.push raises: [], gcsafe.} import pkg/questionable/results import pkg/libp2p/crypto/crypto diff --git a/codex/utils/timer.nim b/codex/utils/timer.nim index c4b6c4a6..e5097e21 100644 --- a/codex/utils/timer.nim +++ b/codex/utils/timer.nim @@ -17,7 +17,7 @@ import pkg/chronos import ../logutils type - TimerCallback* = proc(): Future[void] {.gcsafe, async: (raises: []).} + TimerCallback* = proc(): Future[void] {.async: (raises: []).} Timer* = ref object of RootObj callback: TimerCallback interval: Duration diff --git a/tests/codex/helpers/mockchunker.nim b/tests/codex/helpers/mockchunker.nim index eb51f7ca..1ecd8a21 100644 --- a/tests/codex/helpers/mockchunker.nim +++ b/tests/codex/helpers/mockchunker.nim @@ -21,7 +21,7 @@ proc new*( var consumed = 0 proc reader( data: ChunkBuffer, len: int - ): Future[int] {.gcsafe, async: (raises: [ChunkerError, CancelledError]).} = + ): Future[int] {.async: (raises: [ChunkerError, CancelledError]).} = if consumed >= dataset.len: return 0 diff --git a/tests/codex/node/testnode.nim b/tests/codex/node/testnode.nim index dc3f7a8f..09ed7ca7 100644 --- a/tests/codex/node/testnode.nim +++ b/tests/codex/node/testnode.nim @@ -75,7 +75,7 @@ asyncchecksuite "Test Node - Basic": batchSize = batchSize, proc( blocks: seq[bt.Block] - ): Future[?!void] {.gcsafe, async: (raises: [CancelledError]).} = + ): Future[?!void] {.async: (raises: [CancelledError]).} = check blocks.len > 0 and blocks.len <= batchSize return success(), ) @@ -100,7 +100,7 @@ asyncchecksuite "Test Node - Basic": batchSize = batchSize, proc( blocks: seq[bt.Block] - ): Future[?!void] {.gcsafe, async: (raises: [CancelledError]).} = + ): Future[?!void] {.async: (raises: [CancelledError]).} = return failure("Should not be called"), ) ) diff --git a/tests/codex/sales/testreservations.nim b/tests/codex/sales/testreservations.nim index 48bdee9c..7d958ee3 100644 --- a/tests/codex/sales/testreservations.nim +++ b/tests/codex/sales/testreservations.nim @@ -342,9 +342,7 @@ asyncchecksuite "Reservations module": test "OnAvailabilitySaved called when availability is created": var added: Availability - reservations.OnAvailabilitySaved = proc( - a: Availability - ) {.gcsafe, async: (raises: []).} = + reservations.OnAvailabilitySaved = proc(a: Availability) {.async: (raises: []).} = added = a let availability = createAvailability() @@ -354,9 +352,7 @@ asyncchecksuite "Reservations module": test "OnAvailabilitySaved called when availability size is increased": var availability = createAvailability() var added: Availability - reservations.OnAvailabilitySaved = proc( - a: Availability - ) {.gcsafe, async: (raises: []).} = + reservations.OnAvailabilitySaved = proc(a: Availability) {.async: (raises: []).} = added = a availability.freeSize += 1 discard await reservations.update(availability) @@ -366,9 +362,7 @@ asyncchecksuite "Reservations module": test "OnAvailabilitySaved is not called when availability size is decreased": var availability = createAvailability() var called = false - reservations.OnAvailabilitySaved = proc( - a: Availability - ) {.gcsafe, async: (raises: []).} = + reservations.OnAvailabilitySaved = proc(a: Availability) {.async: (raises: []).} = called = true availability.freeSize -= 1.uint64 discard await reservations.update(availability) @@ -378,9 +372,7 @@ asyncchecksuite "Reservations module": test "OnAvailabilitySaved is not called when availability is disabled": var availability = createAvailability(enabled = false) var called = false - reservations.OnAvailabilitySaved = proc( - a: Availability - ) {.gcsafe, async: (raises: []).} = + reservations.OnAvailabilitySaved = proc(a: Availability) {.async: (raises: []).} = called = true availability.freeSize -= 1 discard await reservations.update(availability) diff --git a/tests/codex/testchunking.nim b/tests/codex/testchunking.nim index 44202c40..531d92bf 100644 --- a/tests/codex/testchunking.nim +++ b/tests/codex/testchunking.nim @@ -18,7 +18,7 @@ type CrashingStreamWrapper* = ref object of LPStream method readOnce*( self: CrashingStreamWrapper, pbytes: pointer, nbytes: int -): Future[int] {.gcsafe, async: (raises: [CancelledError, LPStreamError]).} = +): Future[int] {.async: (raises: [CancelledError, LPStreamError]).} = self.toRaise() asyncchecksuite "Chunking": @@ -27,7 +27,7 @@ asyncchecksuite "Chunking": let contents = [1.byte, 2, 3, 4, 5, 6, 7, 8, 9, 0] proc reader( data: ChunkBuffer, len: int - ): Future[int] {.gcsafe, async: (raises: [ChunkerError, CancelledError]).} = + ): Future[int] {.async: (raises: [ChunkerError, CancelledError]).} = let read = min(contents.len - offset, len) if read == 0: return 0 diff --git a/tests/codex/utils/testasyncstatemachine.nim b/tests/codex/utils/testasyncstatemachine.nim index ed3ea747..c46bb605 100644 --- a/tests/codex/utils/testasyncstatemachine.nim +++ b/tests/codex/utils/testasyncstatemachine.nim @@ -1,6 +1,5 @@ import pkg/questionable import pkg/chronos -import pkg/upraises import codex/utils/asyncstatemachine import ../../asynctest @@ -36,7 +35,7 @@ method run(state: State2, machine: Machine): Future[?State] {.async: (raises: [] method run(state: State3, machine: Machine): Future[?State] {.async: (raises: []).} = inc runs[2] -method onMoveToNextStateEvent*(state: State): ?State {.base, upraises: [].} = +method onMoveToNextStateEvent*(state: State): ?State {.base, raises: [].} = discard method onMoveToNextStateEvent(state: State2): ?State = From 45fec4b524e77af5f364675242d40dc7215514de Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 11 Dec 2025 21:42:53 +0100 Subject: [PATCH 14/20] chore: bump libbacktrace (#1349) --- vendor/nim-libbacktrace | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/nim-libbacktrace b/vendor/nim-libbacktrace index 6da0cda8..99bc2ba1 160000 --- a/vendor/nim-libbacktrace +++ b/vendor/nim-libbacktrace @@ -1 +1 @@ -Subproject commit 6da0cda88ab7780bd5fd342327adb91ab84692aa +Subproject commit 99bc2ba16bc2d44f9a97e706304f64744d913d7f From 6765beee2caf138c1a584499b890ce759c5ebf0e Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 11 Dec 2025 22:03:36 +0100 Subject: [PATCH 15/20] chore: assorted bumps (#1351) --- .gitmodules | 19 +++++++------------ benchmarks/utils.nim | 5 ++--- codex/codex.nim | 2 +- codex/conf.nim | 4 ++-- codex/contracts/requests.nim | 2 +- codex/discovery.nim | 2 +- codex/nat.nim | 6 +++--- codex/sales/reservations.nim | 3 ++- codex/utils/addrutils.nim | 2 +- codex/utils/natutils.nim | 3 +-- tests/codex/testnat.nim | 2 +- tests/examples.nim | 2 +- vendor/lrucache.nim | 2 +- vendor/nim-bearssl | 2 +- vendor/nim-blscurve | 2 +- vendor/nim-codex-dht | 2 +- vendor/nim-contract-abi | 2 +- vendor/nim-http-utils | 2 +- vendor/nim-leopard | 2 +- vendor/nim-leveldbstatic | 2 +- vendor/nim-metrics | 2 +- vendor/nim-nat-traversal | 2 +- vendor/nim-nitro | 2 +- vendor/nim-presto | 2 +- vendor/nim-sqlite3-abi | 2 +- vendor/nim-stew | 2 +- vendor/nim-taskpools | 2 +- vendor/nim-testutils | 2 +- vendor/nim-websock | 2 +- vendor/nim-zlib | 2 +- vendor/stint | 2 +- vendor/upraises | 1 - 32 files changed, 42 insertions(+), 49 deletions(-) delete mode 160000 vendor/upraises diff --git a/.gitmodules b/.gitmodules index 5cc2bfab..0260b1c7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -37,22 +37,17 @@ path = vendor/nim-nitro url = https://github.com/status-im/nim-nitro.git ignore = untracked - branch = master + branch = main [submodule "vendor/questionable"] path = vendor/questionable url = https://github.com/status-im/questionable.git ignore = untracked - branch = master -[submodule "vendor/upraises"] - path = vendor/upraises - url = https://github.com/markspanbroek/upraises.git - ignore = untracked - branch = master + branch = main [submodule "vendor/asynctest"] path = vendor/asynctest url = https://github.com/status-im/asynctest.git ignore = untracked - branch = master + branch = main [submodule "vendor/nim-presto"] path = vendor/nim-presto url = https://github.com/status-im/nim-presto.git @@ -132,7 +127,7 @@ path = vendor/nim-websock url = https://github.com/status-im/nim-websock.git ignore = untracked - branch = master + branch = main [submodule "vendor/nim-contract-abi"] path = vendor/nim-contract-abi url = https://github.com/status-im/nim-contract-abi @@ -160,7 +155,7 @@ path = vendor/nim-taskpools url = https://github.com/status-im/nim-taskpools.git ignore = untracked - branch = master + branch = stable [submodule "vendor/nim-leopard"] path = vendor/nim-leopard url = https://github.com/status-im/nim-leopard.git @@ -225,9 +220,9 @@ path = vendor/nim-quic url = https://github.com/vacp2p/nim-quic.git ignore = untracked - branch = master + branch = main [submodule "vendor/nim-ngtcp2"] path = vendor/nim-ngtcp2 url = https://github.com/vacp2p/nim-ngtcp2.git ignore = untracked - branch = master + branch = main diff --git a/benchmarks/utils.nim b/benchmarks/utils.nim index af5cdc25..7bd0f2e0 100644 --- a/benchmarks/utils.nim +++ b/benchmarks/utils.nim @@ -41,19 +41,18 @@ template benchmark*(name: untyped, count: int, blk: untyped) = ) benchRuns[benchmarkName] = (runs.avg(), count) -template printBenchMarkSummaries*(printRegular=true, printTsv=true) = +template printBenchMarkSummaries*(printRegular = true, printTsv = true) = if printRegular: echo "" for k, v in benchRuns: echo "Benchmark average run ", v.avgTimeSec, " for ", v.count, " runs ", "for ", k - + if printTsv: echo "" echo "name", "\t", "avgTimeSec", "\t", "count" for k, v in benchRuns: echo k, "\t", v.avgTimeSec, "\t", v.count - import std/math func floorLog2*(x: int): int = diff --git a/codex/codex.nim b/codex/codex.nim index 682fe9b5..63f56c6b 100644 --- a/codex/codex.nim +++ b/codex/codex.nim @@ -12,6 +12,7 @@ import std/strutils import std/os import std/tables import std/cpuinfo +import std/net import pkg/chronos import pkg/taskpools @@ -21,7 +22,6 @@ import pkg/confutils import pkg/confutils/defs import pkg/nitro import pkg/stew/io2 -import pkg/stew/shims/net as stewnet import pkg/datastore import pkg/ethers except Rng import pkg/stew/io2 diff --git a/codex/conf.nim b/codex/conf.nim index cec9c11a..0e71b562 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -16,8 +16,10 @@ import std/terminal # Is not used in tests {.pop.} import std/options +import std/parseutils import std/strutils import std/typetraits +import std/net import pkg/chronos import pkg/chronicles/helpers @@ -27,8 +29,6 @@ import pkg/confutils/std/net import pkg/toml_serialization import pkg/metrics import pkg/metrics/chronos_httpserver -import pkg/stew/shims/net as stewnet -import pkg/stew/shims/parseutils import pkg/stew/byteutils import pkg/libp2p import pkg/ethers diff --git a/codex/contracts/requests.nim b/codex/contracts/requests.nim index 035e9648..f6f630e2 100644 --- a/codex/contracts/requests.nim +++ b/codex/contracts/requests.nim @@ -2,7 +2,7 @@ import std/hashes import std/sequtils import std/typetraits import pkg/contractabi -import pkg/nimcrypto +import pkg/nimcrypto/keccak import pkg/ethers/contracts/fields import pkg/questionable/results import pkg/stew/byteutils diff --git a/codex/discovery.nim b/codex/discovery.nim index 905f0902..4b555db1 100644 --- a/codex/discovery.nim +++ b/codex/discovery.nim @@ -10,13 +10,13 @@ {.push raises: [].} import std/algorithm +import std/net import std/sequtils import pkg/chronos import pkg/libp2p/[cid, multicodec, routing_record, signed_envelope] import pkg/questionable import pkg/questionable/results -import pkg/stew/shims/net import pkg/contractabi/address as ca import pkg/codexdht/discv5/[routing_table, protocol as discv5] from pkg/nimcrypto import keccak256 diff --git a/codex/nat.nim b/codex/nat.nim index d022dad6..2038ca3d 100644 --- a/codex/nat.nim +++ b/codex/nat.nim @@ -10,10 +10,10 @@ import std/[options, os, strutils, times, net, atomics], - stew/shims/net as stewNet, - stew/[objects, results], + stew/[objects], nat_traversal/[miniupnpc, natpmp], - json_serialization/std/net + json_serialization/std/net, + results import pkg/chronos import pkg/chronicles diff --git a/codex/sales/reservations.nim b/codex/sales/reservations.nim index 627ef6bb..8ab16105 100644 --- a/codex/sales/reservations.nim +++ b/codex/sales/reservations.nim @@ -36,7 +36,6 @@ import std/sequtils import std/times import pkg/chronos import pkg/datastore -import pkg/nimcrypto import pkg/questionable import pkg/questionable/results import pkg/stint @@ -53,6 +52,8 @@ import ../units export requests export logutils +from nimcrypto import randomBytes + logScope: topics = "marketplace sales reservations" diff --git a/codex/utils/addrutils.nim b/codex/utils/addrutils.nim index cd9dbd0e..641ad9d4 100644 --- a/codex/utils/addrutils.nim +++ b/codex/utils/addrutils.nim @@ -9,11 +9,11 @@ {.push raises: [], gcsafe.} +import std/net import std/strutils import std/options import pkg/libp2p -import pkg/stew/shims/net import pkg/stew/endians2 func remapAddr*( diff --git a/codex/utils/natutils.nim b/codex/utils/natutils.nim index 996d8dd0..45ad7589 100644 --- a/codex/utils/natutils.nim +++ b/codex/utils/natutils.nim @@ -1,7 +1,6 @@ {.push raises: [].} -import - std/[tables, hashes], pkg/results, pkg/stew/shims/net as stewNet, chronos, chronicles +import std/[net, tables, hashes], pkg/results, chronos, chronicles import pkg/libp2p diff --git a/tests/codex/testnat.nim b/tests/codex/testnat.nim index 3981b2e6..a4161c6f 100644 --- a/tests/codex/testnat.nim +++ b/tests/codex/testnat.nim @@ -1,4 +1,4 @@ -import std/[unittest, options, net], stew/shims/net as stewNet +import std/[unittest, options, net] import pkg/chronos import pkg/libp2p/[multiaddress, multihash, multicodec] import pkg/results diff --git a/tests/examples.nim b/tests/examples.nim index 9ef4e292..2b7e96d6 100644 --- a/tests/examples.nim +++ b/tests/examples.nim @@ -38,7 +38,7 @@ proc example*[T](_: type seq[T]): seq[T] = newSeqWith(length, T.example) proc example*(_: type UInt256): UInt256 = - UInt256.fromBytes(array[32, byte].example) + UInt256.fromBytesBE(array[32, byte].example) proc example*[T: distinct](_: type T): T = type baseType = T.distinctBase diff --git a/vendor/lrucache.nim b/vendor/lrucache.nim index 8767ade0..ba577369 160000 --- a/vendor/lrucache.nim +++ b/vendor/lrucache.nim @@ -1 +1 @@ -Subproject commit 8767ade0b76ea5b5d4ce24a52d0c58a6ebeb66cd +Subproject commit ba57736921b2972163b673fc706e7659e7c5cbd6 diff --git a/vendor/nim-bearssl b/vendor/nim-bearssl index 667b4044..ff43a1da 160000 --- a/vendor/nim-bearssl +++ b/vendor/nim-bearssl @@ -1 +1 @@ -Subproject commit 667b40440a53a58e9f922e29e20818720c62d9ac +Subproject commit ff43a1dad1af1cc4bd928c243a1c0470a315aa2c diff --git a/vendor/nim-blscurve b/vendor/nim-blscurve index de2d3c79..f4d0de2e 160000 --- a/vendor/nim-blscurve +++ b/vendor/nim-blscurve @@ -1 +1 @@ -Subproject commit de2d3c79264bba18dbea469c8c5c4b3bb3c8bc55 +Subproject commit f4d0de2eece20380541fbf73d4b8bf57dc214b3b diff --git a/vendor/nim-codex-dht b/vendor/nim-codex-dht index f6eef1ac..6c7de036 160000 --- a/vendor/nim-codex-dht +++ b/vendor/nim-codex-dht @@ -1 +1 @@ -Subproject commit f6eef1ac95c70053b2518f1e3909c909ed8701a6 +Subproject commit 6c7de036224724b064dcaa6b1d898be1c6d03242 diff --git a/vendor/nim-contract-abi b/vendor/nim-contract-abi index 842f4891..0a7b4cec 160000 --- a/vendor/nim-contract-abi +++ b/vendor/nim-contract-abi @@ -1 +1 @@ -Subproject commit 842f48910be4f388bcbf8abf1f02aba1d5e2ee64 +Subproject commit 0a7b4cecce725bcb11ad8648035a92704a8854d3 diff --git a/vendor/nim-http-utils b/vendor/nim-http-utils index 8bb1acba..c53852d9 160000 --- a/vendor/nim-http-utils +++ b/vendor/nim-http-utils @@ -1 +1 @@ -Subproject commit 8bb1acbaa4b86eb866145b0d468eff64a57d1897 +Subproject commit c53852d9e24205b6363bba517fa8ee7bde823691 diff --git a/vendor/nim-leopard b/vendor/nim-leopard index 7506b90f..0478b12d 160000 --- a/vendor/nim-leopard +++ b/vendor/nim-leopard @@ -1 +1 @@ -Subproject commit 7506b90f9c650c02b96bf525d4fd1bd4942a495f +Subproject commit 0478b12df90cbbe531efa69422cff67b5a3a5d93 diff --git a/vendor/nim-leveldbstatic b/vendor/nim-leveldbstatic index 378ef63e..5a0cd8de 160000 --- a/vendor/nim-leveldbstatic +++ b/vendor/nim-leveldbstatic @@ -1 +1 @@ -Subproject commit 378ef63e261e3b5834a3567404edc3ce838498b3 +Subproject commit 5a0cd8de6b2363827c43cafd3ed346ecee427e1e diff --git a/vendor/nim-metrics b/vendor/nim-metrics index cacfdc12..9b9afee9 160000 --- a/vendor/nim-metrics +++ b/vendor/nim-metrics @@ -1 +1 @@ -Subproject commit cacfdc12454a0804c65112b9f4f50d1375208dcd +Subproject commit 9b9afee96357ad82dabf4563cf292f89b50423df diff --git a/vendor/nim-nat-traversal b/vendor/nim-nat-traversal index 6508ce75..860e18c3 160000 --- a/vendor/nim-nat-traversal +++ b/vendor/nim-nat-traversal @@ -1 +1 @@ -Subproject commit 6508ce75060878dfcdfa21f94721672c69a1823b +Subproject commit 860e18c37667b5dd005b94c63264560c35d88004 diff --git a/vendor/nim-nitro b/vendor/nim-nitro index e3719433..5ccdeb46 160000 --- a/vendor/nim-nitro +++ b/vendor/nim-nitro @@ -1 +1 @@ -Subproject commit e3719433d5ace25947c468787c805969642b3913 +Subproject commit 5ccdeb46e06dcf5cef80d0acbb80ee8a17d596e7 diff --git a/vendor/nim-presto b/vendor/nim-presto index 92b1c7ff..62225bfa 160000 --- a/vendor/nim-presto +++ b/vendor/nim-presto @@ -1 +1 @@ -Subproject commit 92b1c7ff141e6920e1f8a98a14c35c1fa098e3be +Subproject commit 62225bfa7ce703a99e04680bfc3498e69b52897f diff --git a/vendor/nim-sqlite3-abi b/vendor/nim-sqlite3-abi index 05bbff1a..6797c318 160000 --- a/vendor/nim-sqlite3-abi +++ b/vendor/nim-sqlite3-abi @@ -1 +1 @@ -Subproject commit 05bbff1af4e8fe2d972ba4b0667b89ca94d3ebba +Subproject commit 6797c31836bff377bf50f1ac7bf8122449bf99ba diff --git a/vendor/nim-stew b/vendor/nim-stew index a6e19813..b6616873 160000 --- a/vendor/nim-stew +++ b/vendor/nim-stew @@ -1 +1 @@ -Subproject commit a6e198132097fb544d04959aeb3b839e1408f942 +Subproject commit b66168735d6f3841c5239c3169d3fe5fe98b1257 diff --git a/vendor/nim-taskpools b/vendor/nim-taskpools index 66585e2e..4acdc6ef 160000 --- a/vendor/nim-taskpools +++ b/vendor/nim-taskpools @@ -1 +1 @@ -Subproject commit 66585e2e960b7695e48ea60377fb3aeac96406e8 +Subproject commit 4acdc6ef005a93dba09f902ed75197548cf7b451 diff --git a/vendor/nim-testutils b/vendor/nim-testutils index 4d37244f..e4d37dc1 160000 --- a/vendor/nim-testutils +++ b/vendor/nim-testutils @@ -1 +1 @@ -Subproject commit 4d37244f9f5e1acd8592a4ceb5c3fc47bc160181 +Subproject commit e4d37dc1652d5c63afb89907efb5a5e812261797 diff --git a/vendor/nim-websock b/vendor/nim-websock index ebe308a7..35ae76f1 160000 --- a/vendor/nim-websock +++ b/vendor/nim-websock @@ -1 +1 @@ -Subproject commit ebe308a79a7b440a11dfbe74f352be86a3883508 +Subproject commit 35ae76f1559e835c80f9c1a3943bf995d3dd9eb5 diff --git a/vendor/nim-zlib b/vendor/nim-zlib index 91cf360b..c71efff5 160000 --- a/vendor/nim-zlib +++ b/vendor/nim-zlib @@ -1 +1 @@ -Subproject commit 91cf360b1aeb2e0c753ff8bac6de22a41c5ed8cd +Subproject commit c71efff5fd1721362b3363dc7d0e2a4c0dbc6453 diff --git a/vendor/stint b/vendor/stint index 5c5e01ce..470b7892 160000 --- a/vendor/stint +++ b/vendor/stint @@ -1 +1 @@ -Subproject commit 5c5e01cef089a261474b7abfe246b37447aaa8ed +Subproject commit 470b7892561b5179ab20bd389a69217d6213fe58 diff --git a/vendor/upraises b/vendor/upraises deleted file mode 160000 index bc262898..00000000 --- a/vendor/upraises +++ /dev/null @@ -1 +0,0 @@ -Subproject commit bc2628989b63854d980e92dadbd58f83e34b6f25 From bd49591fffb642158fba6d53d6a187a5efb6f63a Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Fri, 12 Dec 2025 09:03:56 +0100 Subject: [PATCH 16/20] chore: bump *-serialization (#1352) --- vendor/nim-faststreams | 2 +- vendor/nim-json-serialization | 2 +- vendor/nim-protobuf-serialization | 2 +- vendor/nim-serialization | 2 +- vendor/nim-toml-serialization | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/vendor/nim-faststreams b/vendor/nim-faststreams index cf8d4d22..ce27581a 160000 --- a/vendor/nim-faststreams +++ b/vendor/nim-faststreams @@ -1 +1 @@ -Subproject commit cf8d4d22636b8e514caf17e49f9c786ac56b0e85 +Subproject commit ce27581a3e881f782f482cb66dc5b07a02bd615e diff --git a/vendor/nim-json-serialization b/vendor/nim-json-serialization index 6eadb6e9..a6dcf03e 160000 --- a/vendor/nim-json-serialization +++ b/vendor/nim-json-serialization @@ -1 +1 @@ -Subproject commit 6eadb6e939ffa7882ff5437033c11a9464d3385c +Subproject commit a6dcf03e04e179127a5fcb7e495d19a821d56c17 diff --git a/vendor/nim-protobuf-serialization b/vendor/nim-protobuf-serialization index 5a31137a..4d74e157 160000 --- a/vendor/nim-protobuf-serialization +++ b/vendor/nim-protobuf-serialization @@ -1 +1 @@ -Subproject commit 5a31137a82c2b6a989c9ed979bb636c7a49f570e +Subproject commit 4d74e157cdf1bdcd0ffd41519ebde740c4b80447 diff --git a/vendor/nim-serialization b/vendor/nim-serialization index 2086c996..b0f2fa32 160000 --- a/vendor/nim-serialization +++ b/vendor/nim-serialization @@ -1 +1 @@ -Subproject commit 2086c99608b4bf472e1ef5fe063710f280243396 +Subproject commit b0f2fa32960ea532a184394b0f27be37bd80248b diff --git a/vendor/nim-toml-serialization b/vendor/nim-toml-serialization index fea85b27..b5b387e6 160000 --- a/vendor/nim-toml-serialization +++ b/vendor/nim-toml-serialization @@ -1 +1 @@ -Subproject commit fea85b27f0badcf617033ca1bc05444b5fd8aa7a +Subproject commit b5b387e6fb2a7cc75d54a269b07cc6218361bd46 From 858101c74cd34ad8115ec995d32d920727b461b8 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Mon, 15 Dec 2025 11:00:51 +0100 Subject: [PATCH 17/20] chore: bump eth & networking (#1353) --- vendor/nim-bearssl | 2 +- vendor/nim-eth | 2 +- vendor/nim-ethers | 2 +- vendor/nim-json-rpc | 2 +- vendor/nim-libp2p | 2 +- vendor/nim-ngtcp2 | 2 +- vendor/nim-quic | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/vendor/nim-bearssl b/vendor/nim-bearssl index ff43a1da..f08d7220 160000 --- a/vendor/nim-bearssl +++ b/vendor/nim-bearssl @@ -1 +1 @@ -Subproject commit ff43a1dad1af1cc4bd928c243a1c0470a315aa2c +Subproject commit f08d72203f9e110c099c6f393e1c0640fcbe176f diff --git a/vendor/nim-eth b/vendor/nim-eth index dcfbc429..d9135e6c 160000 --- a/vendor/nim-eth +++ b/vendor/nim-eth @@ -1 +1 @@ -Subproject commit dcfbc4291d39b59563828c3e32be4d51a2f25931 +Subproject commit d9135e6c3c5d6d819afdfb566aa8d958756b73a8 diff --git a/vendor/nim-ethers b/vendor/nim-ethers index 30871c7b..965b8cd7 160000 --- a/vendor/nim-ethers +++ b/vendor/nim-ethers @@ -1 +1 @@ -Subproject commit 30871c7b1d5784e36c51223bd36ef6f1fffcc030 +Subproject commit 965b8cd752544df96b5effecbbd27a8f56a25d62 diff --git a/vendor/nim-json-rpc b/vendor/nim-json-rpc index cbe8edf6..b6e40a77 160000 --- a/vendor/nim-json-rpc +++ b/vendor/nim-json-rpc @@ -1 +1 @@ -Subproject commit cbe8edf69d743a787b76b1cd25bfc4eae89927f7 +Subproject commit b6e40a776fa2d00b97a9366761fb7da18f31ae5c diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p index aa8ce46f..e82080f7 160000 --- a/vendor/nim-libp2p +++ b/vendor/nim-libp2p @@ -1 +1 @@ -Subproject commit aa8ce46f782240cb99a7222c474022b8cfd24e52 +Subproject commit e82080f7b1aa61c6d35fa5311b873f41eff4bb52 diff --git a/vendor/nim-ngtcp2 b/vendor/nim-ngtcp2 index 6834f475..791eb859 160000 --- a/vendor/nim-ngtcp2 +++ b/vendor/nim-ngtcp2 @@ -1 +1 @@ -Subproject commit 6834f4756b6af58356ac9c4fef3d71db3c3ae5fe +Subproject commit 791eb859145f9f268eb23eb9cbe777bdd7699c4d diff --git a/vendor/nim-quic b/vendor/nim-quic index ddcb31ff..525842ae 160000 --- a/vendor/nim-quic +++ b/vendor/nim-quic @@ -1 +1 @@ -Subproject commit ddcb31ffb74b5460ab37fd13547eca90594248bc +Subproject commit 525842aeca6111fd5035568d0f59aa2b338cc29d From 99a78a41ee0ba5b156a51852812f4df342ac6647 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Mon, 15 Dec 2025 19:59:11 +0100 Subject: [PATCH 18/20] fix: address cross-thread safety issues For minimal correctness, we must ensure that buffers that cross the thread boundary are allocated and deallocated within the same thread and that there is no reference counting going on during the computation. To get there with minimal changes: * Preallocate a buffer for the outcome of the merkle tree computation * Pass pointers instead of `ref` types between threads * Avoid relying on isolation - this is an ORC-only feature * Add `SharedBuf` as a simple "view" type that allows working with a set of values while at the same time avoiding allocations and refcounts - the view checks for out-of-bounds acccess much like a seq, but the user is responsible for managing lifetime (which in this case is simple since all that needs to happen is for the task to complete) * In order not to upset the code too much, use a simple linear packer for the hashes that simply copies the values back and forth * Block cancellation and panic if the thread signalling mechanism fails - cancelling the task itself would require inserting cancellation points in the computation The worker task relies on a nuance, namely that calling a closure procedure does not count as a reference-counting event - while this works, it can be brittle in "general" code since it's easy to make copy of the closure itself by accident - the refactoring necessary for addressing this point is beyond the scope of this change however. --- codex/merkletree/codex/codex.nim | 35 ++++++---- codex/merkletree/merkletree.nim | 71 ++++++++++++++++----- codex/merkletree/poseidon2.nim | 42 ++++++++---- codex/utils/sharedbuf.nim | 24 +++++++ tests/codex/merkletree/generictreetests.nim | 7 ++ tests/codex/merkletree/testcodextree.nim | 26 ++++++++ 6 files changed, 168 insertions(+), 37 deletions(-) create mode 100644 codex/utils/sharedbuf.nim diff --git a/codex/merkletree/codex/codex.nim b/codex/merkletree/codex/codex.nim index 5da0d8c0..480c3b24 100644 --- a/codex/merkletree/codex/codex.nim +++ b/codex/merkletree/codex/codex.nim @@ -136,7 +136,8 @@ func init*( var self = CodexTree(mcodec: mcodec, compress: compressor, zero: Zero) - self.layers = ?merkleTreeWorker(self, leaves, isBottomLayer = true) + self.layers = ?merkleTreeWorker(self[], leaves, isBottomLayer = true) + success self proc init*( @@ -144,7 +145,7 @@ proc init*( tp: Taskpool, mcodec: MultiCodec = Sha256HashCodec, leaves: seq[ByteHash], -): Future[?!CodexTree] {.async: (raises: [CancelledError]).} = +): Future[?!CodexTree] {.async: (raises: []).} = if leaves.len == 0: return failure "Empty leaves" @@ -165,25 +166,37 @@ proc init*( var tree = CodexTree(mcodec: mcodec, compress: compressor, zero: Zero) - var task = - CodexTreeTask(tree: cast[ptr ByteTree](addr tree), leaves: leaves, signal: signal) + var + nodesPerLevel = nodesPerLevel(leaves.len) + hashes = nodesPerLevel.foldl(a + b, 0) + layers = newSeq[byte](hashes * digestSize) + + var task = CodexTreeTask( + tree: addr tree[], + leaves: SharedBuf.view(leaves), + layers: SharedBuf.view(layers), + signal: signal, + ) doAssert tp.numThreads > 1, "Must have at least one separate thread or signal will never be fired" tp.spawn merkleTreeWorker(addr task) - let threadFut = signal.wait() - - if err =? catch(await threadFut.join()).errorOption: - ?catch(await noCancel threadFut) - if err of CancelledError: - raise (ref CancelledError) err + # To support cancellation, we'd have to ensure the task we posted to taskpools + # exits early - since we're not doing that, block cancellation attempts + try: + await noCancel signal.wait() + except AsyncError as exc: + # Since we initialized the signal, the OS or chronos is misbehaving. In any + # case, it would mean the task is still running which would cause a memory + # a memory violation if we let it run - panic instead + raiseAssert "Could not wait for signal, was it initialized? " & exc.msg if not task.success.load(): return failure("merkle tree task failed") - tree.layers = extractValue(task.layers) + tree.layers = unpack[ByteHash](task.layers, leaves.len, digestSize) success tree diff --git a/codex/merkletree/merkletree.nim b/codex/merkletree/merkletree.nim index 5c12f198..a1a82b98 100644 --- a/codex/merkletree/merkletree.nim +++ b/codex/merkletree/merkletree.nim @@ -10,22 +10,27 @@ {.push raises: [].} import std/[bitops, atomics] +import stew/assign2 import pkg/questionable/results import pkg/taskpools import pkg/chronos/threadsync import ../errors -import ../utils/uniqueptr +import ../utils/sharedbuf + +export sharedbuf type CompressFn*[H, K] = proc(x, y: H, key: K): ?!H {.noSideEffect, raises: [].} - MerkleTree*[H, K] = ref object of RootObj + MerkleTreeObj*[H, K] = object of RootObj layers*: seq[seq[H]] compress*: CompressFn[H, K] zero*: H + MerkleTree*[H, K] = ref MerkleTreeObj[H, K] + MerkleProof*[H, K] = ref object of RootObj index*: int # linear index of the leaf, starting from 0 path*: seq[H] # order: from the bottom to the top @@ -34,10 +39,10 @@ type zero*: H # zero value MerkleTask*[H, K] = object - tree*: ptr MerkleTree[H, K] - leaves*: seq[H] + tree*: ptr MerkleTreeObj[H, K] + leaves*: SharedBuf[H] signal*: ThreadSignalPtr - layers*: UniquePtr[seq[seq[H]]] + layers*: SharedBuf[byte] success*: Atomic[bool] func depth*[H, K](self: MerkleTree[H, K]): int = @@ -49,6 +54,25 @@ func leavesCount*[H, K](self: MerkleTree[H, K]): int = func levels*[H, K](self: MerkleTree[H, K]): int = return self.layers.len +func nodesPerLevel*(nleaves: int): seq[int] = + ## Given a number of leaves, the number of nodes at each depth (from the + ## bottom/leaves to the root) + if nleaves <= 0: + return @[] + elif nleaves == 1: + return @[1, 1] # leaf and root + + var nodes: seq[int] = @[] + var m = nleaves + while true: + nodes.add(m) + if m == 1: + break + # Next layer size is ceil(m/2) + m = (m + 1) shr 1 + + nodes + func leaves*[H, K](self: MerkleTree[H, K]): seq[H] = return self.layers[0] @@ -133,7 +157,7 @@ func verify*[H, K](proof: MerkleProof[H, K], leaf: H, root: H): ?!bool = success bool(root == ?proof.reconstructRoot(leaf)) func merkleTreeWorker*[H, K]( - self: MerkleTree[H, K], xs: openArray[H], isBottomLayer: static bool + self: MerkleTreeObj[H, K], xs: openArray[H], isBottomLayer: static bool ): ?!seq[seq[H]] = let a = low(xs) let b = high(xs) @@ -162,21 +186,38 @@ func merkleTreeWorker*[H, K]( success @[@xs] & ?self.merkleTreeWorker(ys, isBottomLayer = false) +proc pack*[H](tgt: SharedBuf[byte], v: seq[seq[H]]) = + # Pack the given nested sequences into a flat buffer + var pos = 0 + for layer in v: + for h in layer: + assign(tgt.toOpenArray(pos, pos + h.len - 1), h) + pos += h.len + +proc unpack*[H](src: SharedBuf[byte], nleaves, digestSize: int): seq[seq[H]] = + # Given a flat buffer and the number of leaves, unpack the merkle tree from + # its flat storage + var + nodesPerLevel = nodesPerLevel(nleaves) + res = newSeq[seq[H]](nodesPerLevel.len) + pos = 0 + for i, layer in res.mpairs: + layer = newSeq[H](nodesPerLevel[i]) + for j, h in layer.mpairs: + assign(h, src.toOpenArray(pos, pos + digestSize - 1)) + pos += digestSize + res + proc merkleTreeWorker*[H, K](task: ptr MerkleTask[H, K]) {.gcsafe.} = defer: discard task[].signal.fireSync() - let res = merkleTreeWorker(task[].tree[], task[].leaves, isBottomLayer = true) - - if res.isErr: + let res = merkleTreeWorker( + task[].tree[], task[].leaves.toOpenArray(), isBottomLayer = true + ).valueOr: task[].success.store(false) return - var layers = res.get() - var newOuterSeq = newSeq[seq[H]](layers.len) - for i in 0 ..< layers.len: - var isoInner = isolate(layers[i]) - newOuterSeq[i] = extract(isoInner) + task.layers.pack(res) - task[].layers = newUniquePtr(newOuterSeq) task[].success.store(true) diff --git a/codex/merkletree/poseidon2.nim b/codex/merkletree/poseidon2.nim index 6feb9df4..553605d3 100644 --- a/codex/merkletree/poseidon2.nim +++ b/codex/merkletree/poseidon2.nim @@ -19,7 +19,6 @@ import pkg/constantine/platforms/abstractions import pkg/questionable/results import ../utils -import ../utils/uniqueptr import ../rng import ./merkletree @@ -79,12 +78,22 @@ func init*(_: type Poseidon2Tree, leaves: openArray[Poseidon2Hash]): ?!Poseidon2 var self = Poseidon2Tree(compress: compressor, zero: Poseidon2Zero) - self.layers = ?merkleTreeWorker(self, leaves, isBottomLayer = true) + self.layers = ?merkleTreeWorker(self[], leaves, isBottomLayer = true) success self +proc len*(v: Poseidon2Hash): int = + sizeof(v) + +proc assign*(v: var openArray[byte], h: Poseidon2Hash) = + doAssert v.len == sizeof(h) + copyMem(addr v[0], addr h, sizeof(h)) +proc assign*(h: var Poseidon2Hash, v: openArray[byte]) = + doAssert v.len == sizeof(h) + copyMem(addr h, addr v[0], sizeof(h)) + proc init*( _: type Poseidon2Tree, tp: Taskpool, leaves: seq[Poseidon2Hash] -): Future[?!Poseidon2Tree] {.async: (raises: [CancelledError]).} = +): Future[?!Poseidon2Tree] {.async: (raises: []).} = if leaves.len == 0: return failure "Empty leaves" @@ -100,8 +109,16 @@ proc init*( signal.close().expect("closing once works") var tree = Poseidon2Tree(compress: compressor, zero: Poseidon2Zero) + var + nodesPerLevel = nodesPerLevel(leaves.len) + hashes = nodesPerLevel.foldl(a + b, 0) + layers = newSeq[byte](hashes * sizeof(Poseidon2Hash)) + var task = Poseidon2TreeTask( - tree: cast[ptr Poseidon2Tree](addr tree), leaves: leaves, signal: signal + tree: addr tree[], + leaves: SharedBuf.view(leaves), + layers: SharedBuf.view(layers), + signal: signal, ) doAssert tp.numThreads > 1, @@ -109,17 +126,20 @@ proc init*( tp.spawn merkleTreeWorker(addr task) - let threadFut = signal.wait() - - if err =? catch(await threadFut.join()).errorOption: - ?catch(await noCancel threadFut) - if err of CancelledError: - raise (ref CancelledError) err + # To support cancellation, we'd have to ensure the task we posted to taskpools + # exits early - since we're not doing that, block cancellation attempts + try: + await noCancel signal.wait() + except AsyncError as exc: + # Since we initialized the signal, the OS or chronos is misbehaving. In any + # case, it would mean the task is still running which would cause a memory + # a memory violation if we let it run - panic instead + raiseAssert "Could not wait for signal, was it initialized? " & exc.msg if not task.success.load(): return failure("merkle tree task failed") - tree.layers = extractValue(task.layers) + tree.layers = unpack[Poseidon2Hash](task.layers, leaves.len, sizeof(Poseidon2Hash)) success tree diff --git a/codex/utils/sharedbuf.nim b/codex/utils/sharedbuf.nim new file mode 100644 index 00000000..186d7126 --- /dev/null +++ b/codex/utils/sharedbuf.nim @@ -0,0 +1,24 @@ +import stew/ptrops + +type SharedBuf*[T] = object + payload*: ptr UncheckedArray[T] + len*: int + +proc view*[T](_: type SharedBuf, v: openArray[T]): SharedBuf[T] = + if v.len > 0: + SharedBuf[T](payload: makeUncheckedArray(addr v[0]), len: v.len) + else: + default(SharedBuf[T]) + +template checkIdx(v: SharedBuf, i: int) = + doAssert i > 0 and i <= v.len + +proc `[]`*[T](v: SharedBuf[T], i: int): var T = + v.checkIdx(i) + v.payload[i] + +template toOpenArray*[T](v: SharedBuf[T]): var openArray[T] = + v.payload.toOpenArray(0, v.len - 1) + +template toOpenArray*[T](v: SharedBuf[T], s, e: int): var openArray[T] = + v.toOpenArray().toOpenArray(s, e) diff --git a/tests/codex/merkletree/generictreetests.nim b/tests/codex/merkletree/generictreetests.nim index 6244bc1c..e24cbad1 100644 --- a/tests/codex/merkletree/generictreetests.nim +++ b/tests/codex/merkletree/generictreetests.nim @@ -12,6 +12,13 @@ proc testGenericTree*[H, K, U]( let data = @data suite "Correctness tests - " & name: + test "Should build correct tree for single leaf": + let expectedRoot = compress(data[0], zero, K.KeyOddAndBottomLayer) + + let tree = makeTree(data[0 .. 0]) + check: + tree.root.tryGet == expectedRoot + test "Should build correct tree for even bottom layer": let expectedRoot = compress( compress( diff --git a/tests/codex/merkletree/testcodextree.nim b/tests/codex/merkletree/testcodextree.nim index 3f80c3bd..32e5dd09 100644 --- a/tests/codex/merkletree/testcodextree.nim +++ b/tests/codex/merkletree/testcodextree.nim @@ -96,6 +96,32 @@ suite "Test CodexTree": tree.get().leaves == expectedLeaves.mapIt(it.mhash.tryGet.digestBytes) tree.get().mcodec == sha256 + test "Should build tree the same tree sync and async": + var tp = Taskpool.new(numThreads = 2) + defer: + tp.shutdown() + + let expectedLeaves = data.mapIt( + Cid.init(CidVersion.CIDv1, BlockCodec, MultiHash.digest($sha256, it).tryGet).tryGet + ) + + let + atree = (await CodexTree.init(tp, leaves = expectedLeaves)) + stree = CodexTree.init(leaves = expectedLeaves) + + check: + toSeq(atree.get().nodes) == toSeq(stree.get().nodes) + atree.get().root == stree.get().root + + # Single-leaf trees have their root separately computed + let + atree1 = (await CodexTree.init(tp, leaves = expectedLeaves[0..0])) + stree1 = CodexTree.init(leaves = expectedLeaves[0..0]) + + check: + toSeq(atree.get().nodes) == toSeq(stree.get().nodes) + atree.get().root == stree.get().root + test "Should build from raw digestbytes (should not hash leaves)": let tree = CodexTree.init(sha256, leaves = data).tryGet From db8f81cd63f7250c9da0337556b1351977a2a9a3 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Wed, 17 Dec 2025 13:30:37 +0100 Subject: [PATCH 19/20] perf: flatten merkle tree A classic encoding of a merkle tree is to store the layers consecutively in memory breadth-first. This encoding has several advantages: * Good performance for accessing successive nodes, such as when constructing the tree or serializing it * Significantly lower memory usage - avoids the per-node allocation overhead which otherwise more than doubles the memory usage for "regular" 32-byte hashes * Less memory management - a single memory allocation can reserve memory for the whole tree meaning that there are fewer allocations to keep track of * Simplified buffer lifetimes - with all memory allocated up-front, there's no need for cross-thread memory management or transfers While we're here, we can clean up a few other things in the implementation: * Move async implementation to `merkletree` so that it doesn't have to be repeated * Factor tree construction into preparation and computation - the latter is the part offloaded onto a different thread * Simplify task posting - `threadpools` already creates a "task" from the worker function call * Deprecate several high-overhead accessors that presumably are only needed in tests --- codex/merkletree/codex/codex.nim | 90 +---- codex/merkletree/merkletree.nim | 327 ++++++++++++++----- codex/merkletree/poseidon2.nim | 95 ++---- tests/codex/merkletree/testposeidon2tree.nim | 1 - 4 files changed, 293 insertions(+), 220 deletions(-) diff --git a/codex/merkletree/codex/codex.nim b/codex/merkletree/codex/codex.nim index 480c3b24..9bbc66e3 100644 --- a/codex/merkletree/codex/codex.nim +++ b/codex/merkletree/codex/codex.nim @@ -21,12 +21,10 @@ import pkg/chronos/threadsync import ../../utils import ../../rng import ../../errors -import ../../blocktype +import ../../codextypes from ../../utils/digest import digestBytes -import ../../utils/uniqueptr - import ../merkletree export merkletree @@ -45,8 +43,6 @@ type ByteTree* = MerkleTree[ByteHash, ByteTreeKey] ByteProof* = MerkleProof[ByteHash, ByteTreeKey] - CodexTreeTask* = MerkleTask[ByteHash, ByteTreeKey] - CodexTree* = ref object of ByteTree mcodec*: MultiCodec @@ -119,9 +115,7 @@ func compress*(x, y: openArray[byte], key: ByteTreeKey, codec: MultiCodec): ?!By let digest = ?MultiHash.digest(codec, input).mapFailure success digest.digestBytes -func init*( - _: type CodexTree, mcodec: MultiCodec = Sha256HashCodec, leaves: openArray[ByteHash] -): ?!CodexTree = +func initTree(mcodec: MultiCodec, leaves: openArray[ByteHash]): ?!CodexTree = if leaves.len == 0: return failure "Empty leaves" @@ -134,70 +128,25 @@ func init*( if digestSize != leaves[0].len: return failure "Invalid hash length" - var self = CodexTree(mcodec: mcodec, compress: compressor, zero: Zero) - - self.layers = ?merkleTreeWorker(self[], leaves, isBottomLayer = true) - + var self = CodexTree(mcodec: mcodec) + ?self.prepare(compressor, Zero, leaves) success self +func init*( + _: type CodexTree, mcodec: MultiCodec = Sha256HashCodec, leaves: openArray[ByteHash] +): ?!CodexTree = + let tree = ?initTree(mcodec, leaves) + ?tree.compute() + success tree + proc init*( _: type CodexTree, tp: Taskpool, mcodec: MultiCodec = Sha256HashCodec, leaves: seq[ByteHash], -): Future[?!CodexTree] {.async: (raises: []).} = - if leaves.len == 0: - return failure "Empty leaves" - - let - compressor = proc(x, y: seq[byte], key: ByteTreeKey): ?!ByteHash {.noSideEffect.} = - compress(x, y, key, mcodec) - digestSize = ?mcodec.digestSize.mapFailure - Zero: ByteHash = newSeq[byte](digestSize) - - if digestSize != leaves[0].len: - return failure "Invalid hash length" - - without signal =? ThreadSignalPtr.new(): - return failure("Unable to create thread signal") - - defer: - signal.close().expect("closing once works") - - var tree = CodexTree(mcodec: mcodec, compress: compressor, zero: Zero) - - var - nodesPerLevel = nodesPerLevel(leaves.len) - hashes = nodesPerLevel.foldl(a + b, 0) - layers = newSeq[byte](hashes * digestSize) - - var task = CodexTreeTask( - tree: addr tree[], - leaves: SharedBuf.view(leaves), - layers: SharedBuf.view(layers), - signal: signal, - ) - - doAssert tp.numThreads > 1, - "Must have at least one separate thread or signal will never be fired" - - tp.spawn merkleTreeWorker(addr task) - - # To support cancellation, we'd have to ensure the task we posted to taskpools - # exits early - since we're not doing that, block cancellation attempts - try: - await noCancel signal.wait() - except AsyncError as exc: - # Since we initialized the signal, the OS or chronos is misbehaving. In any - # case, it would mean the task is still running which would cause a memory - # a memory violation if we let it run - panic instead - raiseAssert "Could not wait for signal, was it initialized? " & exc.msg - - if not task.success.load(): - return failure("merkle tree task failed") - - tree.layers = unpack[ByteHash](task.layers, leaves.len, digestSize) - +): Future[?!CodexTree] {.async: (raises: [CancelledError]).} = + let tree = ?initTree(mcodec, leaves) + ?await tree.compute(tp) success tree func init*(_: type CodexTree, leaves: openArray[MultiHash]): ?!CodexTree = @@ -262,15 +211,8 @@ proc fromNodes*( if digestSize != nodes[0].len: return failure "Invalid hash length" - var - self = CodexTree(compress: compressor, zero: Zero, mcodec: mcodec) - layer = nleaves - pos = 0 - - while pos < nodes.len: - self.layers.add(nodes[pos ..< (pos + layer)]) - pos += layer - layer = divUp(layer, 2) + var self = CodexTree(mcodec: mcodec) + ?self.fromNodes(compressor, Zero, nodes, nleaves) let index = Rng.instance.rand(nleaves - 1) diff --git a/codex/merkletree/merkletree.nim b/codex/merkletree/merkletree.nim index a1a82b98..2059d0d2 100644 --- a/codex/merkletree/merkletree.nim +++ b/codex/merkletree/merkletree.nim @@ -9,11 +9,12 @@ {.push raises: [].} -import std/[bitops, atomics] +import std/[bitops, atomics, sequtils] import stew/assign2 import pkg/questionable/results import pkg/taskpools +import pkg/chronos import pkg/chronos/threadsync import ../errors @@ -21,13 +22,43 @@ import ../utils/sharedbuf export sharedbuf +template nodeData( + data: openArray[byte], offsets: openArray[int], nodeSize, i, j: int +): openArray[byte] = + ## Bytes of the j'th entry of the i'th level in the tree, starting with the + ## leaves (at level 0). + let start = (offsets[i] + j) * nodeSize + data.toOpenArray(start, start + nodeSize - 1) + type + # TODO hash functions don't fail - removing the ?! from this function would + # significantly simplify the flow below CompressFn*[H, K] = proc(x, y: H, key: K): ?!H {.noSideEffect, raises: [].} + CompressData[H, K] = object + fn: CompressFn[H, K] + nodeSize: int + zero: H + MerkleTreeObj*[H, K] = object of RootObj - layers*: seq[seq[H]] - compress*: CompressFn[H, K] - zero*: H + store*: seq[byte] + ## Flattened merkle tree where hashes are assumed to be trivial bytes and + ## uniform in size. + ## + ## Each layer of the tree is stored serially starting with the leaves and + ## ending with the root. + ## + ## Beacuse the tree might not be balanced, `layerOffsets` contains the + ## index of the starting point of each level, for easy lookup. + layerOffsets*: seq[int] + ## Starting point of each level in the tree, starting from the leaves - + ## multiplied by the entry size, this is the offset in the payload where + ## the entries of that level start + ## + ## For example, a tree with 4 leaves will have [0, 4, 6] stored here. + ## + ## See nodesPerLevel function, from whic this sequence is derived + compress*: CompressData[H, K] MerkleTree*[H, K] = ref MerkleTreeObj[H, K] @@ -39,24 +70,35 @@ type zero*: H # zero value MerkleTask*[H, K] = object - tree*: ptr MerkleTreeObj[H, K] - leaves*: SharedBuf[H] + store*: SharedBuf[byte] + layerOffsets: SharedBuf[int] + compress*: ptr CompressData[H, K] signal*: ThreadSignalPtr - layers*: SharedBuf[byte] success*: Atomic[bool] +func levels*[H, K](self: MerkleTree[H, K]): int = + return self.layerOffsets.len + func depth*[H, K](self: MerkleTree[H, K]): int = - return self.layers.len - 1 + return self.levels() - 1 + +func nodesInLayer(offsets: openArray[int], layer: int): int = + if layer == offsets.high: + 1 + else: + offsets[layer + 1] - offsets[layer] + +func nodesInLayer(self: MerkleTree | MerkleTreeObj, layer: int): int = + self.layerOffsets.nodesInLayer(layer) func leavesCount*[H, K](self: MerkleTree[H, K]): int = - return self.layers[0].len + return self.nodesInLayer(0) -func levels*[H, K](self: MerkleTree[H, K]): int = - return self.layers.len - -func nodesPerLevel*(nleaves: int): seq[int] = - ## Given a number of leaves, the number of nodes at each depth (from the - ## bottom/leaves to the root) +func nodesPerLevel(nleaves: int): seq[int] = + ## Given a number of leaves, return a seq with the number of nodes at each + ## layer of the tree (from the bottom/leaves to the root) + ## + ## Ie For a tree of 4 leaves, return `[4, 2, 1]` if nleaves <= 0: return @[] elif nleaves == 1: @@ -73,24 +115,60 @@ func nodesPerLevel*(nleaves: int): seq[int] = nodes -func leaves*[H, K](self: MerkleTree[H, K]): seq[H] = - return self.layers[0] +func layerOffsets(nleaves: int): seq[int] = + ## Given a number of leaves, return a seq of the starting offsets of each + ## layer in the node store that results from flattening the binary tree + ## + ## Ie For a tree of 4 leaves, return `[0, 4, 6]` + let nodes = nodesPerLevel(nleaves) + var tot = 0 + let offsets = nodes.mapIt: + let cur = tot + tot += it + cur + offsets -iterator layers*[H, K](self: MerkleTree[H, K]): seq[H] = - for layer in self.layers: - yield layer +template nodeData(self: MerkleTreeObj, i, j: int): openArray[byte] = + ## Bytes of the j'th node of the i'th level in the tree, starting with the + ## leaves (at level 0). + self.store.nodeData(self.layerOffsets, self.compress.nodeSize, i, j) + +func layer*[H, K]( + self: MerkleTree[H, K], layer: int +): seq[H] {.deprecated: "Expensive".} = + var nodes = newSeq[H](self.nodesInLayer(layer)) + for i, h in nodes.mpairs: + assign(h, self[].nodeData(layer, i)) + return nodes + +func leaves*[H, K](self: MerkleTree[H, K]): seq[H] {.deprecated: "Expensive".} = + self.layer(0) + +iterator layers*[H, K](self: MerkleTree[H, K]): seq[H] {.deprecated: "Expensive".} = + for i in 0 ..< self.layerOffsets.len: + yield self.layer(i) + +proc layers*[H, K](self: MerkleTree[H, K]): seq[seq[H]] {.deprecated: "Expensive".} = + for l in self.layers(): + result.add l iterator nodes*[H, K](self: MerkleTree[H, K]): H = - for layer in self.layers: - for node in layer: + ## Iterate over the nodes of each layer starting with the leaves + var node: H + for i in 0 ..< self.layerOffsets.len: + let nodesInLayer = self.nodesInLayer(i) + for j in 0 ..< nodesInLayer: + assign(node, self[].nodeData(i, j)) yield node func root*[H, K](self: MerkleTree[H, K]): ?!H = - let last = self.layers[^1] - if last.len != 1: + mixin assign + if self.layerOffsets.len == 0: return failure "invalid tree" - return success last[0] + var h: H + assign(h, self[].nodeData(self.layerOffsets.high(), 0)) + return success h func getProof*[H, K]( self: MerkleTree[H, K], index: int, proof: MerkleProof[H, K] @@ -106,18 +184,19 @@ func getProof*[H, K]( var m = nleaves for i in 0 ..< depth: let j = k xor 1 - path[i] = - if (j < m): - self.layers[i][j] - else: - self.zero + + if (j < m): + assign(path[i], self[].nodeData(i, j)) + else: + path[i] = self.compress.zero + k = k shr 1 m = (m + 1) shr 1 proof.index = index proof.path = path proof.nleaves = nleaves - proof.compress = self.compress + proof.compress = self.compress.fn success() @@ -156,68 +235,170 @@ func reconstructRoot*[H, K](proof: MerkleProof[H, K], leaf: H): ?!H = func verify*[H, K](proof: MerkleProof[H, K], leaf: H, root: H): ?!bool = success bool(root == ?proof.reconstructRoot(leaf)) -func merkleTreeWorker*[H, K]( - self: MerkleTreeObj[H, K], xs: openArray[H], isBottomLayer: static bool -): ?!seq[seq[H]] = - let a = low(xs) - let b = high(xs) - let m = b - a + 1 +func fromNodes*[H, K]( + self: MerkleTree[H, K], + compressor: CompressFn, + zero: H, + nodes: openArray[H], + nleaves: int, +): ?!void = + mixin assign + + if nodes.len < 2: # At least leaf and root + return failure "Not enough nodes" + + if nleaves == 0: + return failure "No leaves" + + self.compress = CompressData[H, K](fn: compressor, nodeSize: nodes[0].len, zero: zero) + self.layerOffsets = layerOffsets(nleaves) + + if self.layerOffsets[^1] + 1 != nodes.len: + return failure "bad node count" + + self.store = newSeqUninit[byte](nodes.len * self.compress.nodeSize) + + for i in 0 ..< nodes.len: + assign( + self[].store.toOpenArray( + i * self.compress.nodeSize, (i + 1) * self.compress.nodeSize - 1 + ), + nodes[i], + ) + + success() + +func merkleTreeWorker[H, K]( + store: var openArray[byte], + offsets: openArray[int], + compress: CompressData[H, K], + layer: int, + isBottomLayer: static bool, +): ?!void = + ## Worker used to compute the merkle tree from the leaves that are assumed to + ## already be stored at the beginning of the `store`, as done by `prepare`. + + # Throughout, we use `assign` to convert from H to bytes and back, assuming + # this assignment can be done somewhat efficiently (ie memcpy) - because + # the code must work with multihash where len(H) is can differ, we cannot + # simply use a fixed-size array here. + mixin assign + + template nodeData(i, j: int): openArray[byte] = + # Pick out the bytes of node j in layer i + store.nodeData(offsets, compress.nodeSize, i, j) + + let m = offsets.nodesInLayer(layer) when not isBottomLayer: if m == 1: - return success @[@xs] + return success() let halfn: int = m div 2 let n: int = 2 * halfn let isOdd: bool = (n != m) - var ys: seq[H] - if not isOdd: - ys = newSeq[H](halfn) - else: - ys = newSeq[H](halfn + 1) + # Because the compression function we work with works with H and not bytes, + # we need to extract H from the raw data - a little abstraction tax that + # ensures that properties like alignment of H are respected. + var a, b, tmp: H for i in 0 ..< halfn: const key = when isBottomLayer: K.KeyBottomLayer else: K.KeyNone - ys[i] = ?self.compress(xs[a + 2 * i], xs[a + 2 * i + 1], key = key) + + assign(a, nodeData(layer, i * 2)) + assign(b, nodeData(layer, i * 2 + 1)) + + tmp = ?compress.fn(a, b, key = key) + + assign(nodeData(layer + 1, i), tmp) + if isOdd: const key = when isBottomLayer: K.KeyOddAndBottomLayer else: K.KeyOdd - ys[halfn] = ?self.compress(xs[n], self.zero, key = key) - success @[@xs] & ?self.merkleTreeWorker(ys, isBottomLayer = false) + assign(a, nodeData(layer, n)) -proc pack*[H](tgt: SharedBuf[byte], v: seq[seq[H]]) = - # Pack the given nested sequences into a flat buffer - var pos = 0 - for layer in v: - for h in layer: - assign(tgt.toOpenArray(pos, pos + h.len - 1), h) - pos += h.len + tmp = ?compress.fn(a, compress.zero, key = key) -proc unpack*[H](src: SharedBuf[byte], nleaves, digestSize: int): seq[seq[H]] = - # Given a flat buffer and the number of leaves, unpack the merkle tree from - # its flat storage - var - nodesPerLevel = nodesPerLevel(nleaves) - res = newSeq[seq[H]](nodesPerLevel.len) - pos = 0 - for i, layer in res.mpairs: - layer = newSeq[H](nodesPerLevel[i]) - for j, h in layer.mpairs: - assign(h, src.toOpenArray(pos, pos + digestSize - 1)) - pos += digestSize - res + assign(nodeData(layer + 1, halfn), tmp) -proc merkleTreeWorker*[H, K](task: ptr MerkleTask[H, K]) {.gcsafe.} = + merkleTreeWorker(store, offsets, compress, layer + 1, false) + +proc merkleTreeWorker[H, K](task: ptr MerkleTask[H, K]) {.gcsafe.} = defer: discard task[].signal.fireSync() let res = merkleTreeWorker( - task[].tree[], task[].leaves.toOpenArray(), isBottomLayer = true - ).valueOr: - task[].success.store(false) - return + task[].store.toOpenArray(), + task[].layerOffsets.toOpenArray(), + task[].compress[], + 0, + isBottomLayer = true, + ) - task.layers.pack(res) + task[].success.store(res.isOk()) - task[].success.store(true) +func prepare*[H, K]( + self: MerkleTree[H, K], compressor: CompressFn, zero: H, leaves: openArray[H] +): ?!void = + ## Prepare the instance for computing the merkle tree of the given leaves using + ## the given compression function. After preparation, `compute` should be + ## called to perform the actual computation. `leaves` will be copied into the + ## tree so they can be freed after the call. + + if leaves.len == 0: + return failure "No leaves" + + self.compress = + CompressData[H, K](fn: compressor, nodeSize: leaves[0].len, zero: zero) + self.layerOffsets = layerOffsets(leaves.len) + + self.store = newSeqUninit[byte]((self.layerOffsets[^1] + 1) * self.compress.nodeSize) + + for j in 0 ..< leaves.len: + assign(self[].nodeData(0, j), leaves[j]) + + return success() + +proc compute*[H, K](self: MerkleTree[H, K]): ?!void = + merkleTreeWorker( + self.store, self.layerOffsets, self.compress, 0, isBottomLayer = true + ) + +proc compute*[H, K]( + self: MerkleTree[H, K], tp: Taskpool +): Future[?!void] {.async: (raises: []).} = + if tp.numThreads == 1: + # With a single thread, there's no point creating a separate task + return self.compute() + + # TODO this signal would benefit from reuse across computations + without signal =? ThreadSignalPtr.new(): + return failure("Unable to create thread signal") + + defer: + signal.close().expect("closing once works") + + var task = MerkleTask[H, K]( + store: SharedBuf.view(self.store), + layerOffsets: SharedBuf.view(self.layerOffsets), + compress: addr self.compress, + signal: signal, + ) + + tp.spawn merkleTreeWorker(addr task) + + # To support cancellation, we'd have to ensure the task we posted to taskpools + # exits early - since we're not doing that, block cancellation attempts + try: + await noCancel signal.wait() + except AsyncError as exc: + # Since we initialized the signal, the OS or chronos is misbehaving. In any + # case, it would mean the task is still running which would cause a memory + # a memory violation if we let it run - panic instead + raiseAssert "Could not wait for signal, was it initialized? " & exc.msg + + if not task.success.load(): + return failure("merkle tree task failed") + + return success() diff --git a/codex/merkletree/poseidon2.nim b/codex/merkletree/poseidon2.nim index 553605d3..22c14a0e 100644 --- a/codex/merkletree/poseidon2.nim +++ b/codex/merkletree/poseidon2.nim @@ -46,7 +46,16 @@ type Poseidon2Tree* = MerkleTree[Poseidon2Hash, PoseidonKeysEnum] Poseidon2Proof* = MerkleProof[Poseidon2Hash, PoseidonKeysEnum] - Poseidon2TreeTask* = MerkleTask[Poseidon2Hash, PoseidonKeysEnum] +proc len*(v: Poseidon2Hash): int = + sizeof(v) + +proc assign*(v: var openArray[byte], h: Poseidon2Hash) = + doAssert v.len == sizeof(h) + copyMem(addr v[0], addr h, sizeof(h)) + +proc assign*(h: var Poseidon2Hash, v: openArray[byte]) = + doAssert v.len == sizeof(h) + copyMem(addr h, addr v[0], sizeof(h)) proc `$`*(self: Poseidon2Tree): string = let root = if self.root.isOk: self.root.get.toHex else: "none" @@ -67,7 +76,7 @@ converter toKey*(key: PoseidonKeysEnum): Poseidon2Hash = of KeyOdd: KeyOddF of KeyOddAndBottomLayer: KeyOddAndBottomLayerF -func init*(_: type Poseidon2Tree, leaves: openArray[Poseidon2Hash]): ?!Poseidon2Tree = +proc initTree(leaves: openArray[Poseidon2Hash]): ?!Poseidon2Tree = if leaves.len == 0: return failure "Empty leaves" @@ -76,72 +85,24 @@ func init*(_: type Poseidon2Tree, leaves: openArray[Poseidon2Hash]): ?!Poseidon2 ): ?!Poseidon2Hash {.noSideEffect.} = success compress(x, y, key.toKey) - var self = Poseidon2Tree(compress: compressor, zero: Poseidon2Zero) - - self.layers = ?merkleTreeWorker(self[], leaves, isBottomLayer = true) + var self = Poseidon2Tree() + ?self.prepare(compressor, Poseidon2Zero, leaves) success self -proc len*(v: Poseidon2Hash): int = - sizeof(v) +func init*(_: type Poseidon2Tree, leaves: openArray[Poseidon2Hash]): ?!Poseidon2Tree = + let self = ?initTree(leaves) + ?self.compute() -proc assign*(v: var openArray[byte], h: Poseidon2Hash) = - doAssert v.len == sizeof(h) - copyMem(addr v[0], addr h, sizeof(h)) -proc assign*(h: var Poseidon2Hash, v: openArray[byte]) = - doAssert v.len == sizeof(h) - copyMem(addr h, addr v[0], sizeof(h)) + success self proc init*( _: type Poseidon2Tree, tp: Taskpool, leaves: seq[Poseidon2Hash] -): Future[?!Poseidon2Tree] {.async: (raises: []).} = - if leaves.len == 0: - return failure "Empty leaves" +): Future[?!Poseidon2Tree] {.async: (raises: [CancelledError]).} = + let self = ?initTree(leaves) - let compressor = proc( - x, y: Poseidon2Hash, key: PoseidonKeysEnum - ): ?!Poseidon2Hash {.noSideEffect.} = - success compress(x, y, key.toKey) + ?await self.compute(tp) - without signal =? ThreadSignalPtr.new(): - return failure("Unable to create thread signal") - - defer: - signal.close().expect("closing once works") - - var tree = Poseidon2Tree(compress: compressor, zero: Poseidon2Zero) - var - nodesPerLevel = nodesPerLevel(leaves.len) - hashes = nodesPerLevel.foldl(a + b, 0) - layers = newSeq[byte](hashes * sizeof(Poseidon2Hash)) - - var task = Poseidon2TreeTask( - tree: addr tree[], - leaves: SharedBuf.view(leaves), - layers: SharedBuf.view(layers), - signal: signal, - ) - - doAssert tp.numThreads > 1, - "Must have at least one separate thread or signal will never be fired" - - tp.spawn merkleTreeWorker(addr task) - - # To support cancellation, we'd have to ensure the task we posted to taskpools - # exits early - since we're not doing that, block cancellation attempts - try: - await noCancel signal.wait() - except AsyncError as exc: - # Since we initialized the signal, the OS or chronos is misbehaving. In any - # case, it would mean the task is still running which would cause a memory - # a memory violation if we let it run - panic instead - raiseAssert "Could not wait for signal, was it initialized? " & exc.msg - - if not task.success.load(): - return failure("merkle tree task failed") - - tree.layers = unpack[Poseidon2Hash](task.layers, leaves.len, sizeof(Poseidon2Hash)) - - success tree + success self func init*(_: type Poseidon2Tree, leaves: openArray[array[31, byte]]): ?!Poseidon2Tree = Poseidon2Tree.init(leaves.mapIt(Poseidon2Hash.fromBytes(it))) @@ -154,23 +115,13 @@ proc init*( proc fromNodes*( _: type Poseidon2Tree, nodes: openArray[Poseidon2Hash], nleaves: int ): ?!Poseidon2Tree = - if nodes.len == 0: - return failure "Empty nodes" - let compressor = proc( x, y: Poseidon2Hash, key: PoseidonKeysEnum ): ?!Poseidon2Hash {.noSideEffect.} = success compress(x, y, key.toKey) - var - self = Poseidon2Tree(compress: compressor, zero: zero) - layer = nleaves - pos = 0 - - while pos < nodes.len: - self.layers.add(nodes[pos ..< (pos + layer)]) - pos += layer - layer = divUp(layer, 2) + let self = Poseidon2Tree() + ?self.fromNodes(compressor, Poseidon2Zero, nodes, nleaves) let index = Rng.instance.rand(nleaves - 1) diff --git a/tests/codex/merkletree/testposeidon2tree.nim b/tests/codex/merkletree/testposeidon2tree.nim index 45a727e5..8e7ce34d 100644 --- a/tests/codex/merkletree/testposeidon2tree.nim +++ b/tests/codex/merkletree/testposeidon2tree.nim @@ -72,7 +72,6 @@ suite "Test Poseidon2Tree": tree.leaves == expectedLeaves test "Build poseidon2 tree from byte leaves asynchronously": - echo "Build poseidon2 tree from byte leaves asynchronously" var tp = Taskpool.new() defer: tp.shutdown() From ac5e3c733ba3b02ae1919b3879cdea842c795932 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Wed, 17 Dec 2025 13:53:18 +0100 Subject: [PATCH 20/20] chore: bump taskpools .. to support generic workers which allows getting rid of MerkleTask --- codex/merkletree/merkletree.nim | 38 +++++++++++++-------------------- vendor/nim-taskpools | 2 +- 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/codex/merkletree/merkletree.nim b/codex/merkletree/merkletree.nim index 2059d0d2..465c5e8b 100644 --- a/codex/merkletree/merkletree.nim +++ b/codex/merkletree/merkletree.nim @@ -69,13 +69,6 @@ type compress*: CompressFn[H, K] # compress function zero*: H # zero value - MerkleTask*[H, K] = object - store*: SharedBuf[byte] - layerOffsets: SharedBuf[int] - compress*: ptr CompressData[H, K] - signal*: ThreadSignalPtr - success*: Atomic[bool] - func levels*[H, K](self: MerkleTree[H, K]): int = return self.layerOffsets.len @@ -324,19 +317,20 @@ func merkleTreeWorker[H, K]( merkleTreeWorker(store, offsets, compress, layer + 1, false) -proc merkleTreeWorker[H, K](task: ptr MerkleTask[H, K]) {.gcsafe.} = +proc merkleTreeWorker[H, K]( + store: SharedBuf[byte], + offsets: SharedBuf[int], + compress: ptr CompressData[H, K], + signal: ThreadSignalPtr, +): bool = defer: - discard task[].signal.fireSync() + discard signal.fireSync() let res = merkleTreeWorker( - task[].store.toOpenArray(), - task[].layerOffsets.toOpenArray(), - task[].compress[], - 0, - isBottomLayer = true, + store.toOpenArray(), offsets.toOpenArray(), compress[], 0, isBottomLayer = true ) - task[].success.store(res.isOk()) + return res.isOk() func prepare*[H, K]( self: MerkleTree[H, K], compressor: CompressFn, zero: H, leaves: openArray[H] @@ -379,15 +373,13 @@ proc compute*[H, K]( defer: signal.close().expect("closing once works") - var task = MerkleTask[H, K]( - store: SharedBuf.view(self.store), - layerOffsets: SharedBuf.view(self.layerOffsets), - compress: addr self.compress, - signal: signal, + let res = tp.spawn merkleTreeWorker( + SharedBuf.view(self.store), + SharedBuf.view(self.layerOffsets), + addr self.compress, + signal, ) - tp.spawn merkleTreeWorker(addr task) - # To support cancellation, we'd have to ensure the task we posted to taskpools # exits early - since we're not doing that, block cancellation attempts try: @@ -398,7 +390,7 @@ proc compute*[H, K]( # a memory violation if we let it run - panic instead raiseAssert "Could not wait for signal, was it initialized? " & exc.msg - if not task.success.load(): + if not res.sync(): return failure("merkle tree task failed") return success() diff --git a/vendor/nim-taskpools b/vendor/nim-taskpools index 4acdc6ef..97f76fae 160000 --- a/vendor/nim-taskpools +++ b/vendor/nim-taskpools @@ -1 +1 @@ -Subproject commit 4acdc6ef005a93dba09f902ed75197548cf7b451 +Subproject commit 97f76faef6ba64bc77d9808c27ec5e9917e7cfde