From 0595723f66bec374471aafab3a9412ea4f95371c Mon Sep 17 00:00:00 2001 From: Giuliano Mega Date: Tue, 4 Feb 2025 13:01:14 -0300 Subject: [PATCH 1/6] Minor improvements to download API (#1092) * chore: improve error messages in upload API * chore: remove unreachable (dead) code * fix: API integration test --- codex/rest/api.nim | 12 ++++++------ tests/integration/testrestapi.nim | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/codex/rest/api.nim b/codex/rest/api.nim index 134aa8d2..a64d26cf 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -179,7 +179,7 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute trace "Handling file upload" var bodyReader = request.getBodyReader() if bodyReader.isErr(): - return RestApiResponse.error(Http500) + return RestApiResponse.error(Http500, msg = bodyReader.error()) # Attempt to handle `Expect` header # some clients (curl), wait 1000ms @@ -190,10 +190,13 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute var mimetype = request.headers.getString(ContentTypeHeader).some if mimetype.get() != "": + let mimetypeVal = mimetype.get() var m = newMimetypes() - let extension = m.getExt(mimetype.get(), "") + let extension = m.getExt(mimetypeVal, "") if extension == "": - return RestApiResponse.error(Http422, "The MIME type is not valid.") + return RestApiResponse.error( + Http422, "The MIME type '" & mimetypeVal & "' is not valid." + ) else: mimetype = string.none @@ -231,9 +234,6 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute finally: await reader.closeWait() - trace "Something went wrong error" - return RestApiResponse.error(Http500) - router.api(MethodGet, "/api/codex/v1/data") do() -> RestApiResponse: let json = await formatManifestBlocks(node) return RestApiResponse.response($json, contentType = "application/json") diff --git a/tests/integration/testrestapi.nim b/tests/integration/testrestapi.nim index 52b722d6..f1f2299f 100644 --- a/tests/integration/testrestapi.nim +++ b/tests/integration/testrestapi.nim @@ -200,7 +200,7 @@ twonodessuite "REST API": let response = client1.uploadRaw("some file contents", headers) check response.status == "422 Unprocessable Entity" - check response.body == "The MIME type is not valid." + check response.body == "The MIME type 'hello/world' is not valid." test "node retrieve the metadata", twoNodesConfig: let headers = newHttpHeaders( From 54d499be419e6b82204f328f98c14508e0b2eabf Mon Sep 17 00:00:00 2001 From: Slava <20563034+veaceslavdoina@users.noreply.github.com> Date: Tue, 4 Feb 2025 23:22:34 +0200 Subject: [PATCH 2/6] docker: add BOOTSTRAP_NODE_URL to Docker entrypoint (#1098) --- docker/docker-entrypoint.sh | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 2ec840e9..d883f0eb 100644 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -9,6 +9,34 @@ if [[ -n "${ENV_PATH}" ]]; then set +a fi +# Bootstrap node from URL +if [[ -n "${BOOTSTRAP_NODE_URL}" ]]; then + BOOTSTRAP_NODE_URL="${BOOTSTRAP_NODE_URL}/api/codex/v1/spr" + WAIT=${BOOTSTRAP_NODE_URL_WAIT:-300} + SECONDS=0 + SLEEP=1 + # Run and retry if fail + while (( SECONDS < WAIT )); do + SPR=$(curl -s -f -m 5 -H 'Accept: text/plain' "${BOOTSTRAP_NODE_URL}") + # Check if exit code is 0 and returned value is not empty + if [[ $? -eq 0 && -n "${SPR}" ]]; then + export CODEX_BOOTSTRAP_NODE="${SPR}" + echo "Bootstrap node: CODEX_BOOTSTRAP_NODE=${CODEX_BOOTSTRAP_NODE}" + break + else + # Sleep and check again + echo "Can't get SPR from ${BOOTSTRAP_NODE_URL} - Retry in $SLEEP seconds / $((WAIT - SECONDS))" + sleep $SLEEP + fi + done +fi + +# Stop Codex run if unable to get SPR +if [[ -n "${BOOTSTRAP_NODE_URL}" && -z "${CODEX_BOOTSTRAP_NODE}" ]]; then + echo "Unable to get SPR from ${BOOTSTRAP_NODE_URL} in ${BOOTSTRAP_NODE_URL_WAIT} seconds - Stop Codex run" + exit 1 +fi + # Parameters if [[ -z "${CODEX_NAT}" ]]; then if [[ "${NAT_IP_AUTO}" == "true" && -z "${NAT_PUBLIC_IP_AUTO}" ]]; then From c05eec422c8d6835f9606c2fdb35f645cc20c35f Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Thu, 6 Feb 2025 16:21:12 +0100 Subject: [PATCH 3/6] fix dataset and slot size calculations in integration tests (#1095) * fixes datasetSize and slotSize helpers (and also RandomChunker.example) * adds overload for <> for seq[byte] * changes RandomChunker.example to return seq[byte] * fixes restapi tests after correcting RandomChunker.example * review: use string.fromBytes from nim-stew to convert seq[byte] to string --- tests/examples.nim | 5 ++--- tests/integration/codexclient.nim | 3 +++ tests/integration/marketplacesuite.nim | 10 +++++++--- tests/integration/testmarketplace.nim | 4 ++-- tests/integration/testproofs.nim | 4 +--- tests/integration/testrestapi.nim | 26 ++++++++++++++------------ tests/integration/testupdownload.nim | 2 +- 7 files changed, 30 insertions(+), 24 deletions(-) diff --git a/tests/examples.nim b/tests/examples.nim index bfb34cff..c96fefd6 100644 --- a/tests/examples.nim +++ b/tests/examples.nim @@ -86,8 +86,7 @@ proc example(_: type G2Point): G2Point = proc example*(_: type Groth16Proof): Groth16Proof = Groth16Proof(a: G1Point.example, b: G2Point.example, c: G1Point.example) -proc example*(_: type RandomChunker, blocks: int): Future[string] {.async.} = - # doAssert blocks >= 3, "must be more than 3 blocks" +proc example*(_: type RandomChunker, blocks: int): Future[seq[byte]] {.async.} = let rng = Rng.instance() let chunker = RandomChunker.new( rng, size = DefaultBlockSize * blocks.NBytes, chunkSize = DefaultBlockSize @@ -95,7 +94,7 @@ proc example*(_: type RandomChunker, blocks: int): Future[string] {.async.} = var data: seq[byte] while (let moar = await chunker.getBytes(); moar != []): data.add moar - return byteutils.toHex(data) + return data proc example*(_: type RandomChunker): Future[string] {.async.} = await RandomChunker.example(3) diff --git a/tests/integration/codexclient.nim b/tests/integration/codexclient.nim index 7826b151..d1191fb9 100644 --- a/tests/integration/codexclient.nim +++ b/tests/integration/codexclient.nim @@ -44,6 +44,9 @@ proc upload*(client: CodexClient, contents: string): ?!Cid = assert response.status == "200 OK" Cid.init(response.body).mapFailure +proc upload*(client: CodexClient, bytes: seq[byte]): ?!Cid = + client.upload(string.fromBytes(bytes)) + proc download*(client: CodexClient, cid: Cid, local = false): ?!string = let response = client.http.get( client.baseurl & "/data/" & $cid & (if local: "" else: "/network/stream") diff --git a/tests/integration/marketplacesuite.nim b/tests/integration/marketplacesuite.nim index 4d155186..68283ad1 100644 --- a/tests/integration/marketplacesuite.nim +++ b/tests/integration/marketplacesuite.nim @@ -4,6 +4,7 @@ from pkg/libp2p import Cid import pkg/codex/contracts/marketplace as mp import pkg/codex/periods import pkg/codex/utils/json +from pkg/codex/utils import roundUp, divUp import ./multinodes import ../contracts/time import ../contracts/deployment @@ -45,11 +46,14 @@ template marketplacesuite*(name: string, body: untyped) = proc periods(p: int): uint64 = p.uint64 * period - proc slotSize(blocks: int): UInt256 = - (DefaultBlockSize * blocks.NBytes).Natural.u256 + proc slotSize(blocks, nodes, tolerance: int): UInt256 = + let ecK = nodes - tolerance + let blocksRounded = roundUp(blocks, ecK) + let blocksPerSlot = divUp(blocksRounded, ecK) + (DefaultBlockSize * blocksPerSlot.NBytes).Natural.u256 proc datasetSize(blocks, nodes, tolerance: int): UInt256 = - (nodes + tolerance).u256 * slotSize(blocks) + return nodes.u256 * slotSize(blocks, nodes, tolerance) proc createAvailabilities( datasetSize: UInt256, diff --git a/tests/integration/testmarketplace.nim b/tests/integration/testmarketplace.nim index bc030a1d..7813485b 100644 --- a/tests/integration/testmarketplace.nim +++ b/tests/integration/testmarketplace.nim @@ -112,7 +112,7 @@ marketplacesuite "Marketplace": await ethProvider.advanceTime(duration) # Checking that the hosting node received reward for at least the time between - let slotSize = slotSize(blocks) + let slotSize = slotSize(blocks, ecNodes, ecTolerance) let pricePerSlotPerSecond = minPricePerBytePerSecond * slotSize check eventually (await token.balanceOf(hostAccount)) - startBalanceHost >= (duration - 5 * 60) * pricePerSlotPerSecond * ecNodes.u256 @@ -197,7 +197,7 @@ marketplacesuite "Marketplace payouts": await advanceToNextPeriod() - let slotSize = slotSize(blocks) + let slotSize = slotSize(blocks, ecNodes, ecTolerance) let pricePerSlotPerSecond = minPricePerBytePerSecond * slotSize check eventually ( diff --git a/tests/integration/testproofs.nim b/tests/integration/testproofs.nim index b25643ad..a547890b 100644 --- a/tests/integration/testproofs.nim +++ b/tests/integration/testproofs.nim @@ -1,7 +1,6 @@ from std/times import inMilliseconds import pkg/questionable import pkg/codex/logutils -import pkg/stew/byteutils import ../contracts/time import ../contracts/deployment import ../codex/helpers @@ -60,8 +59,7 @@ marketplacesuite "Hosts submit regular proofs": let purchase = client0.getPurchase(purchaseId).get check purchase.error == none string - let request = purchase.request.get - let slotSize = request.ask.slotSize + let slotSize = slotSize(blocks, ecNodes, ecTolerance) check eventually( client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000 diff --git a/tests/integration/testrestapi.nim b/tests/integration/testrestapi.nim index f1f2299f..8cbe9817 100644 --- a/tests/integration/testrestapi.nim +++ b/tests/integration/testrestapi.nim @@ -1,5 +1,6 @@ import std/httpclient import std/sequtils +import std/strformat from pkg/libp2p import `==` import pkg/codex/units import ./twonodes @@ -144,18 +145,19 @@ twonodessuite "REST API": check responseBefore.body == "Invalid parameters: `tolerance` cannot be greater than `nodes`" - test "request storage succeeds if nodes and tolerance within range", twoNodesConfig: - let data = await RandomChunker.example(blocks = 2) - let cid = client1.upload(data).get - let duration = 100.u256 - let pricePerBytePerSecond = 1.u256 - let proofProbability = 3.u256 - let expiry = 30.uint - let collateralPerByte = 1.u256 - let ecParams = @[(3, 1), (5, 2)] - - for ecParam in ecParams: - let (nodes, tolerance) = ecParam + for ecParams in @[ + (minBlocks: 2, nodes: 3, tolerance: 1), (minBlocks: 3, nodes: 5, tolerance: 2) + ]: + let (minBlocks, nodes, tolerance) = ecParams + test "request storage succeeds if nodes and tolerance within range " & + fmt"({minBlocks=}, {nodes=}, {tolerance=})", twoNodesConfig: + let data = await RandomChunker.example(blocks = minBlocks) + let cid = client1.upload(data).get + let duration = 100.u256 + let pricePerBytePerSecond = 1.u256 + let proofProbability = 3.u256 + let expiry = 30.uint + let collateralPerByte = 1.u256 var responseBefore = client1.requestStorageRaw( cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, diff --git a/tests/integration/testupdownload.nim b/tests/integration/testupdownload.nim index 74bee8c7..05d3a496 100644 --- a/tests/integration/testupdownload.nim +++ b/tests/integration/testupdownload.nim @@ -88,7 +88,7 @@ twonodessuite "Uploads and downloads": let cid = a.upload(data).get let response = b.download(cid).get check: - response == data + @response.mapIt(it.byte) == data for run in 0 .. 10: await transferTest(client1, client2) From e62a09d9b10a02d5f94ce558b3c245369ebda4ab Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Thu, 6 Feb 2025 22:36:14 +0100 Subject: [PATCH 4/6] add ccache and sccache to speed up CI (#1074) * add ccache and sccache to speed up CI Signed-off-by: Csaba Kiraly * include testname and nim version in cache separation Signed-off-by: Csaba Kiraly * Make sure ccache has precedence over custom clang/llvm Signed-off-by: Csaba Kiraly * enable ccache for windows Signed-off-by: Csaba Kiraly * ccache: evict old files Make sure old unused cache files are not lingering around for long Signed-off-by: Csaba Kiraly --------- Signed-off-by: Csaba Kiraly --- .../actions/nimbus-build-system/action.yml | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/.github/actions/nimbus-build-system/action.yml b/.github/actions/nimbus-build-system/action.yml index c7bdb627..219966db 100644 --- a/.github/actions/nimbus-build-system/action.yml +++ b/.github/actions/nimbus-build-system/action.yml @@ -97,6 +97,33 @@ runs: # Set GCC-14 as the default sudo update-alternatives --set gcc /usr/bin/gcc-14 + - name: Install ccache on Linux/Mac + if: inputs.os == 'linux' || inputs.os == 'macos' + uses: hendrikmuhs/ccache-action@v1.2 + with: + create-symlink: true + key: ${{ matrix.os }}-${{ matrix.builder }}-${{ matrix.cpu }}-${{ matrix.tests }}-${{ matrix.nim_version }} + evict-old-files: 7d + + - name: Install ccache on Windows + if: inputs.os == 'windows' + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ matrix.os }}-${{ matrix.builder }}-${{ matrix.cpu }}-${{ matrix.tests }}-${{ matrix.nim_version }} + evict-old-files: 7d + + - name: Enable ccache on Windows + if: inputs.os == 'windows' + shell: ${{ inputs.shell }} {0} + run: | + CCACHE_DIR=$(dirname $(which ccache))/ccached + mkdir ${CCACHE_DIR} + ln -s $(which ccache) ${CCACHE_DIR}/gcc.exe + ln -s $(which ccache) ${CCACHE_DIR}/g++.exe + ln -s $(which ccache) ${CCACHE_DIR}/cc.exe + ln -s $(which ccache) ${CCACHE_DIR}/c++.exe + echo "export PATH=${CCACHE_DIR}:\$PATH" >> $HOME/.bash_profile # prefix path in MSYS2 + - name: Derive environment variables shell: ${{ inputs.shell }} {0} run: | @@ -154,8 +181,11 @@ runs: llvm_bin_dir="${llvm_dir}/bin" llvm_lib_dir="${llvm_dir}/lib" echo "${llvm_bin_dir}" >> ${GITHUB_PATH} + # Make sure ccache has precedence (GITHUB_PATH is appending before) + echo "$(brew --prefix)/opt/ccache/libexec" >> ${GITHUB_PATH} + echo $PATH echo "LDFLAGS=${LDFLAGS} -L${libomp_lib_dir} -L${llvm_lib_dir} -Wl,-rpath,${llvm_lib_dir}" >> ${GITHUB_ENV} - NIMFLAGS="${NIMFLAGS} $(quote "-d:LeopardCmakeFlags='-DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=${llvm_bin_dir}/clang -DCMAKE_CXX_COMPILER=${llvm_bin_dir}/clang++' -d:LeopardExtraCompilerlags='-fopenmp' -d:LeopardExtraLinkerFlags='-fopenmp -L${libomp_lib_dir}'")" + NIMFLAGS="${NIMFLAGS} $(quote "-d:LeopardCmakeFlags='-DCMAKE_BUILD_TYPE=Release' -d:LeopardExtraCompilerFlags='-fopenmp' -d:LeopardExtraLinkerFlags='-fopenmp -L${libomp_lib_dir}'")" echo "NIMFLAGS=${NIMFLAGS}" >> $GITHUB_ENV fi @@ -191,6 +221,7 @@ runs: - name: Build Nim and Codex dependencies shell: ${{ inputs.shell }} {0} run: | + which gcc gcc --version make -j${ncpu} CI_CACHE=NimBinaries ${ARCH_OVERRIDE} QUICK_AND_DIRTY_COMPILER=1 update echo From 17d3f99f45bbbcadfb3d9c3de846ecd3f885f933 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 6 Feb 2025 15:36:35 -0600 Subject: [PATCH 5/6] use a case-of instead of if for better readability (#1063) --- codex/blockexchange/engine/engine.nim | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/codex/blockexchange/engine/engine.nim b/codex/blockexchange/engine/engine.nim index e1624e99..87335b0a 100644 --- a/codex/blockexchange/engine/engine.nim +++ b/codex/blockexchange/engine/engine.nim @@ -398,7 +398,8 @@ proc wantListHandler*(b: BlockExcEngine, peer: PeerId, wantList: WantList) {.asy have = await e.address in b.localStore price = @(b.pricing.get(Pricing(price: 0.u256)).price.toBytesBE) - if e.wantType == WantType.WantHave: + case e.wantType: + of WantType.WantHave: if have: presence.add( BlockPresence( @@ -415,17 +416,19 @@ proc wantListHandler*(b: BlockExcEngine, peer: PeerId, wantList: WantList) {.asy peerCtx.peerWants.add(e) codex_block_exchange_want_have_lists_received.inc() - elif e.wantType == WantType.WantBlock: + of WantType.WantBlock: peerCtx.peerWants.add(e) schedulePeer = true codex_block_exchange_want_block_lists_received.inc() else: # Updating existing entry in peer wants # peer doesn't want this block anymore if e.cancel: + trace "Canceling want for block", address = e.address peerCtx.peerWants.del(idx) else: # peer might want to ask for the same cid with # different want params + trace "Updating want for block", address = e.address peerCtx.peerWants[idx] = e # update entry if presence.len > 0: From dfa90a9981cc49859cda04c68b18956f57880495 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Thu, 6 Feb 2025 17:18:00 -0800 Subject: [PATCH 6/6] fix(build): compilation on macos when including nim-nat-traversal (#1084) * fix(build): compilation on macos when including nim-nat-traversal - removes the `VERSION` rename to `VERSION_temp` in the Makefile - instead, relies on `-iqoute` to include the `nim-nat-traversal/vendor/libnatpmp-upstream` directory in the search paths. `-iquote` will match the `vendor/libnatpmp-upstream/VERSION` file for `#include "version"` and not `#include `, the latter being what is included by the macos sdk and was causing issues with `-I`. The [gcc 14.2 docs](https://gcc.gnu.org/onlinedocs/gcc-14.2.0/cpp/Invocation.html#index-I) describe how `-iquote` alleviates this issue: > Directories specified with -iquote apply only to the quote form of the directive, #include "file". Directories specified with -I, -isystem, or -idirafter apply to lookup for both the #include "file" and #include directives. For more info, please see https://github.com/status-im/nim-nat-traversal/pull/34. * bump nim-nat-traversal Now that https://github.com/status-im/nim-nat-traversal/pull/34 has been merged, change back to master commit --- build.nims | 23 ++--------------------- vendor/nim-nat-traversal | 2 +- 2 files changed, 3 insertions(+), 22 deletions(-) diff --git a/build.nims b/build.nims index 5e190f6b..aa090e71 100644 --- a/build.nims +++ b/build.nims @@ -2,24 +2,8 @@ mode = ScriptMode.Verbose import std/os except commandLineParams -const VendorPath = "vendor/nim-nat-traversal/vendor/libnatpmp-upstream" -let - oldVersionFile = joinPath(VendorPath, "VERSION") - newVersionFile = joinPath(VendorPath, "VERSION_temp") - -proc renameFile(oldName, newName: string) = - if fileExists(oldName): - mvFile(oldName, newName) - else: - echo "File ", oldName, " does not exist" - - ### Helper functions proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") = - # This is a quick workaround to avoid VERSION file conflict on macOS - # More details here: https://github.com/codex-storage/nim-codex/issues/1059 - if defined(macosx): - renameFile(oldVersionFile, newVersionFile) if not dirExists "build": mkDir "build" @@ -37,11 +21,8 @@ proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") = # Place build output in 'build' folder, even if name includes a longer path. outName = os.lastPathPart(name) cmd = "nim " & lang & " --out:build/" & outName & " " & extra_params & " " & srcDir & name & ".nim" - try: - exec(cmd) - finally: - if defined(macosx): - renameFile(newVersionFile, oldVersionFile) + + exec(cmd) proc test(name: string, srcDir = "tests/", params = "", lang = "c") = buildBinary name, srcDir, params diff --git a/vendor/nim-nat-traversal b/vendor/nim-nat-traversal index 5e405974..6508ce75 160000 --- a/vendor/nim-nat-traversal +++ b/vendor/nim-nat-traversal @@ -1 +1 @@ -Subproject commit 5e4059746e9095e1731b02eeaecd62a70fbe664d +Subproject commit 6508ce75060878dfcdfa21f94721672c69a1823b