diff --git a/.github/actions/nimbus-build-system/action.yml b/.github/actions/nimbus-build-system/action.yml index 012881a3..e4f58209 100644 --- a/.github/actions/nimbus-build-system/action.yml +++ b/.github/actions/nimbus-build-system/action.yml @@ -78,6 +78,12 @@ runs: mingw-w64-i686-ntldd-git mingw-w64-i686-rust + - name: MSYS2 (Windows All) - Downgrade to gcc 13 + if: inputs.os == 'windows' + shell: ${{ inputs.shell }} {0} + run: | + pacman -U --noconfirm https://repo.msys2.org/mingw/ucrt64/mingw-w64-ucrt-x86_64-gcc-13.2.0-6-any.pkg.tar.zst https://repo.msys2.org/mingw/ucrt64/mingw-w64-ucrt-x86_64-gcc-libs-13.2.0-6-any.pkg.tar.zst + - name: Derive environment variables shell: ${{ inputs.shell }} {0} run: | diff --git a/.github/workflows/ci-reusable.yml b/.github/workflows/ci-reusable.yml index b83757d1..595ce99d 100644 --- a/.github/workflows/ci-reusable.yml +++ b/.github/workflows/ci-reusable.yml @@ -26,7 +26,7 @@ jobs: name: '${{ matrix.os }}-${{ matrix.cpu }}-${{ matrix.nim_version }}-${{ matrix.tests }}' runs-on: ${{ matrix.builder }} - timeout-minutes: 80 + timeout-minutes: 90 steps: - name: Checkout sources uses: actions/checkout@v4 @@ -53,7 +53,7 @@ jobs: node-version: 18.15 - name: Start Ethereum node with Codex contracts - if: matrix.tests == 'contract' || matrix.tests == 'integration' || matrix.tests == 'all' + if: matrix.tests == 'contract' || matrix.tests == 'integration' || matrix.tests == 'tools' || matrix.tests == 'all' working-directory: vendor/codex-contracts-eth env: MSYS2_PATH_TYPE: inherit @@ -79,6 +79,11 @@ jobs: path: tests/integration/logs/ retention-days: 1 + ## Part 4 Tools ## + - name: Tools tests + if: matrix.tests == 'tools' || matrix.tests == 'all' + run: make -j${ncpu} testTools + status: if: always() needs: [build] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4754e3ae..b3290657 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,7 @@ jobs: os {windows}, cpu {amd64}, builder {windows-latest}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {msys2} os {windows}, cpu {amd64}, builder {windows-latest}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {msys2} os {windows}, cpu {amd64}, builder {windows-latest}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {msys2} + os {windows}, cpu {amd64}, builder {windows-latest}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {msys2} build: needs: matrix diff --git a/.github/workflows/docker-dist-tests.yml b/.github/workflows/docker-dist-tests.yml index ae1e5e9e..2700e619 100644 --- a/.github/workflows/docker-dist-tests.yml +++ b/.github/workflows/docker-dist-tests.yml @@ -24,7 +24,7 @@ jobs: name: Build and Push uses: ./.github/workflows/docker-reusable.yml with: - nimflags: '-d:disableMarchNative -d:codex_enable_api_debug_peers=true -d:codex_enable_proof_failures=true -d:codex_enable_log_counter=true' + nimflags: '-d:disableMarchNative -d:codex_enable_api_debug_peers=true -d:codex_enable_proof_failures=true -d:codex_enable_log_counter=true -d:verify_circuit=true' nat_ip_auto: true tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }} tag_suffix: dist-tests diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c3e67d9f..01e90146 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,11 +8,13 @@ on: env: cache_nonce: 0 # Allows for easily busting actions/cache caches - nim_version: v1.6.14 + nim_version: pinned rust_version: 1.78.0 - binary_base: codex - nim_flags: '-d:verify_circuit=true' - upload_to_codex: false + codex_binary_base: codex + cirdl_binary_base: cirdl + build_dir: build + nim_flags: '' + windows_libs: 'libstdc++-6.dll libgomp-1.dll libgcc_s_seh-1.dll libwinpthread-1.dll' jobs: # Matrix @@ -69,19 +71,48 @@ jobs: macos*) os_name="darwin" ;; windows*) os_name="windows" ;; esac - binary="${{ env.binary_base }}-${{ github.ref_name }}-${os_name}-${{ matrix.cpu }}" - [[ ${os_name} == "windows" ]] && binary="${binary}.exe" - echo "binary=${binary}" >>$GITHUB_ENV + codex_binary="${{ env.codex_binary_base }}-${{ github.ref_name }}-${os_name}-${{ matrix.cpu }}" + cirdl_binary="${{ env.cirdl_binary_base }}-${{ github.ref_name }}-${os_name}-${{ matrix.cpu }}" + if [[ ${os_name} == "windows" ]]; then + codex_binary="${codex_binary}.exe" + cirdl_binary="${cirdl_binary}.exe" + fi + echo "codex_binary=${codex_binary}" >>$GITHUB_ENV + echo "cirdl_binary=${cirdl_binary}" >>$GITHUB_ENV - name: Release - Build run: | - make NIMFLAGS="--out:${{ env.binary }} ${{ env.nim_flags }}" - - - name: Release - Upload binaries + make NIMFLAGS="--out:${{ env.build_dir }}/${{ env.codex_binary }} ${{ env.nim_flags }}" + make cirdl NIMFLAGS="--out:${{ env.build_dir }}/${{ env.cirdl_binary }} ${{ env.nim_flags }}" + + - name: Release - Libraries + run: | + if [[ "${{ matrix.os }}" == "windows" ]]; then + for lib in ${{ env.windows_libs }}; do + cp -v "${MINGW_PREFIX}/bin/${lib}" "${{ env.build_dir }}" + done + fi + + - name: Release - Upload codex build artifacts uses: actions/upload-artifact@v4 with: - name: release-${{ env.binary }} - path: ${{ env.binary }} + name: release-${{ env.codex_binary }} + path: ${{ env.build_dir }}/${{ env.codex_binary_base }}* + retention-days: 1 + + - name: Release - Upload cirdl build artifacts + uses: actions/upload-artifact@v4 + with: + name: release-${{ env.cirdl_binary }} + path: ${{ env.build_dir }}/${{ env.cirdl_binary_base }}* + retention-days: 1 + + - name: Release - Upload windows libs + if: matrix.os == 'windows' + uses: actions/upload-artifact@v4 + with: + name: release-${{ matrix.os }}-libs + path: ${{ env.build_dir }}/*.dll retention-days: 1 # Release @@ -100,46 +131,47 @@ jobs: - name: Release - Compress and checksum run: | cd /tmp/release - prepare() { - # Checksum + checksum() { arc="${1}" sha256sum "${arc}" >"${arc}.sha256" - - # Upload to Codex - if [[ "${{ env.upload_to_codex }}" == "true" ]]; then - codex_endpoints="${{ secrets.CODEX_ENDPOINTS }}" - codex_username="${{ secrets.CODEX_USERNAME }}" - codex_password="${{ secrets.CODEX_PASSWORD }}" - - for endpoint in ${codex_endpoints}; do - echo "::add-mask::${endpoint}" - cid=$(curl -X POST \ - "${endpoint}/api/codex/v1/data" \ - -u "${codex_username}":"${codex_password}" \ - -H "content-type: application/octet-stream" \ - -T "${arc}") - - echo "${cid}" >"${arc}.cid" - done - fi } + # Compress and prepare - for file in *; do + for file in ${{ env.codex_binary_base }}* ${{ env.cirdl_binary_base }}*; do if [[ "${file}" == *".exe"* ]]; then + + # Windows - binary only arc="${file%.*}.zip" zip "${arc}" "${file}" + checksum "${arc}" + + # Windows - binary and libs + arc="${file%.*}-libs.zip" + zip "${arc}" "${file}" ${{ env.windows_libs }} rm -f "${file}" - prepare "${arc}" + checksum "${arc}" else + + # Linux/macOS arc="${file}.tar.gz" + chmod 755 "${file}" tar cfz "${arc}" "${file}" rm -f "${file}" - prepare "${arc}" + checksum "${arc}" fi done + rm -f ${{ env.windows_libs }} + + - name: Release - Upload compressed artifacts and checksums + uses: actions/upload-artifact@v4 + with: + name: archives-and-checksums + path: /tmp/release/ + retention-days: 1 - name: Release uses: softprops/action-gh-release@v2 + if: startsWith(github.ref, 'refs/tags/') with: files: | /tmp/release/* diff --git a/.gitignore b/.gitignore index c1866c8c..1b8885e0 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ !*.* *.exe +!LICENSE* !Makefile nimcache/ diff --git a/.gitmodules b/.gitmodules index 06d1b823..6842ddea 100644 --- a/.gitmodules +++ b/.gitmodules @@ -215,3 +215,6 @@ [submodule "vendor/nim-leveldbstatic"] path = vendor/nim-leveldbstatic url = https://github.com/codex-storage/nim-leveldb.git +[submodule "vendor/nim-zippy"] + path = vendor/nim-zippy + url = https://github.com/status-im/nim-zippy.git diff --git a/BUILDING.md b/BUILDING.md index 525fa160..91db823c 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -33,7 +33,7 @@ The current implementation of Codex's zero-knowledge proving circuit requires th On a bare bones installation of Debian (or a distribution derived from Debian, such as Ubuntu), run ```shell -apt-get update && apt-get install build-essential cmake curl git rustc cargo +$ apt-get update && apt-get install build-essential cmake curl git rustc cargo ``` Non-Debian distributions have different package managers: `apk`, `dnf`, `pacman`, `rpm`, `yum`, etc. @@ -157,6 +157,16 @@ In Bash run make test ``` +### Tools + +#### Circuit download tool + +To build the circuit download tool located in `tools/cirdl` run: + +```shell +make cirdl +``` + ### testAll #### Prerequisites diff --git a/LICENSE-APACHEv2 b/LICENSE-APACHEv2 new file mode 100644 index 00000000..cbd5c758 --- /dev/null +++ b/LICENSE-APACHEv2 @@ -0,0 +1,199 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 00000000..d13cc4b2 --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile index 54ac04d2..5e6054c1 100644 --- a/Makefile +++ b/Makefile @@ -74,6 +74,11 @@ all: | build deps echo -e $(BUILD_MSG) "build/$@" && \ $(ENV_SCRIPT) nim codex $(NIM_PARAMS) build.nims +# Build tools/cirdl +cirdl: | deps + echo -e $(BUILD_MSG) "build/$@" && \ + $(ENV_SCRIPT) nim toolsCirdl $(NIM_PARAMS) build.nims + # must be included after the default target -include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk @@ -124,7 +129,12 @@ testAll: | build deps # Builds and runs Taiko L2 tests testTaiko: | build deps echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim testTaiko $(NIM_PARAMS) codex.nims + $(ENV_SCRIPT) nim testTaiko $(NIM_PARAMS) build.nims + +# Builds and runs tool tests +testTools: | cirdl + echo -e $(BUILD_MSG) "build/$@" && \ + $(ENV_SCRIPT) nim testTools $(NIM_PARAMS) build.nims # nim-libbacktrace LIBBACKTRACE_MAKE_FLAGS := -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0 diff --git a/build.nims b/build.nims index 33d7b623..3d1a3cac 100644 --- a/build.nims +++ b/build.nims @@ -1,5 +1,6 @@ mode = ScriptMode.Verbose +import std/os except commandLineParams ### Helper functions proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") = @@ -14,7 +15,11 @@ proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") = for i in 2.. 0: warn "Failed to send block request cancellations to peers", peers = failed.len -proc getAnnouceCids(blocksDelivery: seq[BlockDelivery]): seq[Cid] = - var cids = initHashSet[Cid]() - for bd in blocksDelivery: - if bd.address.leaf: - cids.incl(bd.address.treeCid) - else: - without isM =? bd.address.cid.isManifest, err: - warn "Unable to determine if cid is manifest" - continue - if isM: - cids.incl(bd.address.cid) - return cids.toSeq - proc resolveBlocks*(b: BlockExcEngine, blocksDelivery: seq[BlockDelivery]) {.async.} = b.pendingBlocks.resolve(blocksDelivery) await b.scheduleTasks(blocksDelivery) - let announceCids = getAnnouceCids(blocksDelivery) await b.cancelBlocks(blocksDelivery.mapIt(it.address)) - b.discovery.queueProvideBlocksReq(announceCids) - proc resolveBlocks*(b: BlockExcEngine, blocks: seq[Block]) {.async.} = await b.resolveBlocks( blocks.mapIt( @@ -596,6 +584,7 @@ proc new*( wallet: WalletRef, network: BlockExcNetwork, discovery: DiscoveryEngine, + advertiser: Advertiser, peerStore: PeerCtxStore, pendingBlocks: PendingBlocksManager, concurrentTasks = DefaultConcurrentTasks, @@ -616,6 +605,7 @@ proc new*( concurrentTasks: concurrentTasks, taskQueue: newAsyncHeapQueue[BlockExcPeerCtx](DefaultTaskQueueSize), discovery: discovery, + advertiser: advertiser, blockFetchTimeout: blockFetchTimeout) proc peerEventHandler(peerId: PeerId, event: PeerEvent) {.async.} = diff --git a/codex/blocktype.nim b/codex/blocktype.nim index dd3b70e0..c44e4fd8 100644 --- a/codex/blocktype.nim +++ b/codex/blocktype.nim @@ -31,7 +31,7 @@ import ./codextypes export errors, logutils, units, codextypes type - Block* = object of RootObj + Block* = ref object of RootObj cid*: Cid data*: seq[byte] diff --git a/codex/codex.nim b/codex/codex.nim index 0b9182fb..ff7b9c55 100644 --- a/codex/codex.nim +++ b/codex/codex.nim @@ -110,7 +110,7 @@ proc bootstrapInteractions( quit QuitFailure let marketplace = Marketplace.new(marketplaceAddress, signer) - let market = OnChainMarket.new(marketplace) + let market = OnChainMarket.new(marketplace, config.rewardRecipient) let clock = OnChainClock.new(provider) var client: ?ClientInteractions @@ -268,36 +268,13 @@ proc new*( peerStore = PeerCtxStore.new() pendingBlocks = PendingBlocksManager.new() + advertiser = Advertiser.new(repoStore, discovery) blockDiscovery = DiscoveryEngine.new(repoStore, peerStore, network, discovery, pendingBlocks) - engine = BlockExcEngine.new(repoStore, wallet, network, blockDiscovery, peerStore, pendingBlocks) + engine = BlockExcEngine.new(repoStore, wallet, network, blockDiscovery, advertiser, peerStore, pendingBlocks) store = NetworkStore.new(engine, repoStore) prover = if config.prover: - if not fileAccessible($config.circomR1cs, {AccessFlags.Read}) and - endsWith($config.circomR1cs, ".r1cs"): - error "Circom R1CS file not accessible" - raise (ref Defect)( - msg: "r1cs file not readable, doesn't exist or wrong extension (.r1cs)") - - if not fileAccessible($config.circomWasm, {AccessFlags.Read}) and - endsWith($config.circomWasm, ".wasm"): - error "Circom wasm file not accessible" - raise (ref Defect)( - msg: "wasm file not readable, doesn't exist or wrong extension (.wasm)") - - let zkey = if not config.circomNoZkey: - if not fileAccessible($config.circomZkey, {AccessFlags.Read}) and - endsWith($config.circomZkey, ".zkey"): - error "Circom zkey file not accessible" - raise (ref Defect)( - msg: "zkey file not readable, doesn't exist or wrong extension (.zkey)") - - $config.circomZkey - else: "" - - some Prover.new( - store, - CircomCompat.init($config.circomR1cs, $config.circomWasm, zkey), - config.numProofSamples) + let backend = config.initializeBackend().expect("Unable to create prover backend.") + some Prover.new(store, backend, config.numProofSamples) else: none Prover diff --git a/codex/conf.nim b/codex/conf.nim index fb7548c7..27697a9b 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -62,6 +62,7 @@ const codex_enable_log_counter* {.booldefine.} = false DefaultDataDir* = defaultDataDir() + DefaultCircuitDir* = defaultDataDir() / "circuits" type StartUpCmd* {.pure.} = enum @@ -293,28 +294,40 @@ type name: "validator-max-slots" .}: int + rewardRecipient* {. + desc: "Address to send payouts to (eg rewards and refunds)" + name: "reward-recipient" + .}: Option[EthAddress] + case persistenceCmd* {. defaultValue: noCmd command }: PersistenceCmd of PersistenceCmd.prover: + circuitDir* {. + desc: "Directory where Codex will store proof circuit data" + defaultValue: DefaultCircuitDir + defaultValueDesc: $DefaultCircuitDir + abbr: "cd" + name: "circuit-dir" }: OutDir + circomR1cs* {. desc: "The r1cs file for the storage circuit" - defaultValue: $DefaultDataDir / "circuits" / "proof_main.r1cs" - defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.r1cs" + defaultValue: $DefaultCircuitDir / "proof_main.r1cs" + defaultValueDesc: $DefaultCircuitDir & "/proof_main.r1cs" name: "circom-r1cs" .}: InputFile circomWasm* {. desc: "The wasm file for the storage circuit" - defaultValue: $DefaultDataDir / "circuits" / "proof_main.wasm" + defaultValue: $DefaultCircuitDir / "proof_main.wasm" defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.wasm" name: "circom-wasm" .}: InputFile circomZkey* {. desc: "The zkey file for the storage circuit" - defaultValue: $DefaultDataDir / "circuits" / "proof_main.zkey" + defaultValue: $DefaultCircuitDir / "proof_main.zkey" defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.zkey" name: "circom-zkey" .}: InputFile diff --git a/codex/contracts/config.nim b/codex/contracts/config.nim index fd6a1fa8..76e00207 100644 --- a/codex/contracts/config.nim +++ b/codex/contracts/config.nim @@ -18,6 +18,10 @@ type timeout*: UInt256 # mark proofs as missing before the timeout (in seconds) downtime*: uint8 # ignore this much recent blocks for proof requirements zkeyHash*: string # hash of the zkey file which is linked to the verifier + # Ensures the pointer does not remain in downtime for many consecutive + # periods. For each period increase, move the pointer `pointerProduct` + # blocks. Should be a prime number to ensure there are no cycles. + downtimeProduct*: uint8 func fromTuple(_: type ProofConfig, tupl: tuple): ProofConfig = @@ -25,7 +29,8 @@ func fromTuple(_: type ProofConfig, tupl: tuple): ProofConfig = period: tupl[0], timeout: tupl[1], downtime: tupl[2], - zkeyHash: tupl[3] + zkeyHash: tupl[3], + downtimeProduct: tupl[4] ) func fromTuple(_: type CollateralConfig, tupl: tuple): CollateralConfig = diff --git a/codex/contracts/deployment.nim b/codex/contracts/deployment.nim index b5b2e311..6ab65425 100644 --- a/codex/contracts/deployment.nim +++ b/codex/contracts/deployment.nim @@ -19,6 +19,10 @@ const knownAddresses = { # Taiko Alpha-3 Testnet "167005": { "Marketplace": Address.init("0x948CF9291b77Bd7ad84781b9047129Addf1b894F") + }.toTable, + # Codex Testnet - Sept-2024 + "789987": { + "Marketplace": Address.init("0xCDef8d6884557be4F68dC265b6bB2E3e52a6C9d6") }.toTable }.toTable diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index c874d5db..1595b449 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -19,17 +19,24 @@ type OnChainMarket* = ref object of Market contract: Marketplace signer: Signer + rewardRecipient: ?Address MarketSubscription = market.Subscription EventSubscription = ethers.Subscription OnChainMarketSubscription = ref object of MarketSubscription eventSubscription: EventSubscription -func new*(_: type OnChainMarket, contract: Marketplace): OnChainMarket = +func new*( + _: type OnChainMarket, + contract: Marketplace, + rewardRecipient = Address.none): OnChainMarket = + without signer =? contract.signer: raiseAssert("Marketplace contract should have a signer") + OnChainMarket( contract: contract, signer: signer, + rewardRecipient: rewardRecipient ) proc raiseMarketError(message: string) {.raises: [MarketError].} = @@ -163,7 +170,23 @@ method fillSlot(market: OnChainMarket, method freeSlot*(market: OnChainMarket, slotId: SlotId) {.async.} = convertEthersError: - discard await market.contract.freeSlot(slotId).confirm(0) + var freeSlot: Future[?TransactionResponse] + if rewardRecipient =? market.rewardRecipient: + # If --reward-recipient specified, use it as the reward recipient, and use + # the SP's address as the collateral recipient + let collateralRecipient = await market.getSigner() + freeSlot = market.contract.freeSlot( + slotId, + rewardRecipient, # --reward-recipient + collateralRecipient) # SP's address + + else: + # Otherwise, use the SP's address as both the reward and collateral + # recipient (the contract will use msg.sender for both) + freeSlot = market.contract.freeSlot(slotId) + + discard await freeSlot.confirm(0) + method withdrawFunds(market: OnChainMarket, requestId: RequestId) {.async.} = @@ -347,9 +370,11 @@ method subscribeProofSubmission*(market: OnChainMarket, method unsubscribe*(subscription: OnChainMarketSubscription) {.async.} = await subscription.eventSubscription.unsubscribe() -method queryPastStorageRequests*(market: OnChainMarket, - blocksAgo: int): - Future[seq[PastStorageRequest]] {.async.} = +method queryPastEvents*[T: MarketplaceEvent]( + market: OnChainMarket, + _: type T, + blocksAgo: int): Future[seq[T]] {.async.} = + convertEthersError: let contract = market.contract let provider = contract.provider @@ -357,13 +382,6 @@ method queryPastStorageRequests*(market: OnChainMarket, let head = await provider.getBlockNumber() let fromBlock = BlockTag.init(head - blocksAgo.abs.u256) - let events = await contract.queryFilter(StorageRequested, - fromBlock, - BlockTag.latest) - return events.map(event => - PastStorageRequest( - requestId: event.requestId, - ask: event.ask, - expiry: event.expiry - ) - ) + return await contract.queryFilter(T, + fromBlock, + BlockTag.latest) diff --git a/codex/contracts/marketplace.nim b/codex/contracts/marketplace.nim index 301f8c25..f98b9a80 100644 --- a/codex/contracts/marketplace.nim +++ b/codex/contracts/marketplace.nim @@ -16,25 +16,6 @@ export requests type Marketplace* = ref object of Contract - StorageRequested* = object of Event - requestId*: RequestId - ask*: StorageAsk - expiry*: UInt256 - SlotFilled* = object of Event - requestId* {.indexed.}: RequestId - slotIndex*: UInt256 - SlotFreed* = object of Event - requestId* {.indexed.}: RequestId - slotIndex*: UInt256 - RequestFulfilled* = object of Event - requestId* {.indexed.}: RequestId - RequestCancelled* = object of Event - requestId* {.indexed.}: RequestId - RequestFailed* = object of Event - requestId* {.indexed.}: RequestId - ProofSubmitted* = object of Event - id*: SlotId - proc config*(marketplace: Marketplace): MarketplaceConfig {.contract, view.} proc token*(marketplace: Marketplace): Address {.contract, view.} @@ -45,7 +26,9 @@ proc minCollateralThreshold*(marketplace: Marketplace): UInt256 {.contract, view proc requestStorage*(marketplace: Marketplace, request: StorageRequest): ?TransactionResponse {.contract.} proc fillSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256, proof: Groth16Proof): ?TransactionResponse {.contract.} proc withdrawFunds*(marketplace: Marketplace, requestId: RequestId): ?TransactionResponse {.contract.} +proc withdrawFunds*(marketplace: Marketplace, requestId: RequestId, withdrawAddress: Address): ?TransactionResponse {.contract.} proc freeSlot*(marketplace: Marketplace, id: SlotId): ?TransactionResponse {.contract.} +proc freeSlot*(marketplace: Marketplace, id: SlotId, rewardRecipient: Address, collateralRecipient: Address): ?TransactionResponse {.contract.} proc getRequest*(marketplace: Marketplace, id: RequestId): StorageRequest {.contract, view.} proc getHost*(marketplace: Marketplace, id: SlotId): Address {.contract, view.} proc getActiveSlot*(marketplace: Marketplace, id: SlotId): Slot {.contract, view.} diff --git a/codex/contracts/requests.nim b/codex/contracts/requests.nim index 1363fb9d..d94baa17 100644 --- a/codex/contracts/requests.nim +++ b/codex/contracts/requests.nim @@ -163,12 +163,12 @@ func id*(request: StorageRequest): RequestId = let encoding = AbiEncoder.encode((request, )) RequestId(keccak256.digest(encoding).data) -func slotId*(requestId: RequestId, slot: UInt256): SlotId = - let encoding = AbiEncoder.encode((requestId, slot)) +func slotId*(requestId: RequestId, slotIndex: UInt256): SlotId = + let encoding = AbiEncoder.encode((requestId, slotIndex)) SlotId(keccak256.digest(encoding).data) -func slotId*(request: StorageRequest, slot: UInt256): SlotId = - slotId(request.id, slot) +func slotId*(request: StorageRequest, slotIndex: UInt256): SlotId = + slotId(request.id, slotIndex) func id*(slot: Slot): SlotId = slotId(slot.request, slot.slotIndex) diff --git a/codex/manifest/manifest.nim b/codex/manifest/manifest.nim index 5966feb3..93aa5ba5 100644 --- a/codex/manifest/manifest.nim +++ b/codex/manifest/manifest.nim @@ -29,7 +29,7 @@ import ../logutils # TODO: Manifest should be reworked to more concrete types, # perhaps using inheritance type - Manifest* = object of RootObj + Manifest* = ref object of RootObj treeCid {.serialize.}: Cid # Root of the merkle tree datasetSize {.serialize.}: NBytes # Total size of all blocks blockSize {.serialize.}: NBytes # Size of each contained block (might not be needed if blocks are len-prefixed) diff --git a/codex/market.nim b/codex/market.nim index b521c395..245c28d5 100644 --- a/codex/market.nim +++ b/codex/market.nim @@ -28,11 +28,28 @@ type OnRequestCancelled* = proc(requestId: RequestId) {.gcsafe, upraises:[].} OnRequestFailed* = proc(requestId: RequestId) {.gcsafe, upraises:[].} OnProofSubmitted* = proc(id: SlotId) {.gcsafe, upraises:[].} - PastStorageRequest* = object + ProofChallenge* = array[32, byte] + + # Marketplace events -- located here due to the Market abstraction + MarketplaceEvent* = Event + StorageRequested* = object of MarketplaceEvent requestId*: RequestId ask*: StorageAsk expiry*: UInt256 - ProofChallenge* = array[32, byte] + SlotFilled* = object of MarketplaceEvent + requestId* {.indexed.}: RequestId + slotIndex*: UInt256 + SlotFreed* = object of MarketplaceEvent + requestId* {.indexed.}: RequestId + slotIndex*: UInt256 + RequestFulfilled* = object of MarketplaceEvent + requestId* {.indexed.}: RequestId + RequestCancelled* = object of MarketplaceEvent + requestId* {.indexed.}: RequestId + RequestFailed* = object of MarketplaceEvent + requestId* {.indexed.}: RequestId + ProofSubmitted* = object of MarketplaceEvent + id*: SlotId method getZkeyHash*(market: Market): Future[?string] {.base, async.} = raiseAssert("not implemented") @@ -202,7 +219,8 @@ method subscribeProofSubmission*(market: Market, method unsubscribe*(subscription: Subscription) {.base, async, upraises:[].} = raiseAssert("not implemented") -method queryPastStorageRequests*(market: Market, - blocksAgo: int): - Future[seq[PastStorageRequest]] {.base, async.} = +method queryPastEvents*[T: MarketplaceEvent]( + market: Market, + _: type T, + blocksAgo: int): Future[seq[T]] {.base, async.} = raiseAssert("not implemented") diff --git a/codex/node.nim b/codex/node.nim index 19065c99..88305a08 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -366,9 +366,6 @@ proc store*( blocks = manifest.blocksCount, datasetSize = manifest.datasetSize - await self.discovery.provide(manifestBlk.cid) - await self.discovery.provide(treeCid) - return manifestBlk.cid.success proc iterateManifests*(self: CodexNodeRef, onManifest: OnManifest) {.async.} = diff --git a/codex/rest/api.nim b/codex/rest/api.nim index d44cffa4..1323654a 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -110,6 +110,20 @@ proc retrieveCid( proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRouter) = let allowedOrigin = router.allowedOrigin # prevents capture inside of api defintion + router.api( + MethodOptions, + "/api/codex/v1/data") do ( + resp: HttpResponseRef) -> RestApiResponse: + + if corsOrigin =? allowedOrigin: + resp.setHeader("Access-Control-Allow-Origin", corsOrigin) + resp.setHeader("Access-Control-Allow-Methods", "POST, OPTIONS") + resp.setHeader("Access-Control-Allow-Headers", "content-type") + resp.setHeader("Access-Control-Max-Age", "86400") + + resp.status = Http204 + await resp.sendBody("") + router.rawApi( MethodPost, "/api/codex/v1/data") do ( @@ -210,13 +224,15 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute return RestApiResponse.response($json, contentType="application/json") proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = + let allowedOrigin = router.allowedOrigin + router.api( MethodGet, "/api/codex/v1/sales/slots") do () -> RestApiResponse: ## Returns active slots for the host try: without contracts =? node.contracts.host: - return RestApiResponse.error(Http503, "Sales unavailable") + return RestApiResponse.error(Http503, "Persistence is not enabled") let json = %(await contracts.sales.mySlots()) return RestApiResponse.response($json, contentType="application/json") @@ -231,7 +247,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = ## slot is not active for the host. without contracts =? node.contracts.host: - return RestApiResponse.error(Http503, "Sales unavailable") + return RestApiResponse.error(Http503, "Persistence is not enabled") without slotId =? slotId.tryGet.catch, error: return RestApiResponse.error(Http400, error.msg) @@ -242,7 +258,9 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = let restAgent = RestSalesAgent( state: agent.state() |? "none", slotIndex: agent.data.slotIndex, - requestId: agent.data.requestId + requestId: agent.data.requestId, + request: agent.data.request, + reservation: agent.data.reservation, ) return RestApiResponse.response(restAgent.toJson, contentType="application/json") @@ -254,7 +272,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = try: without contracts =? node.contracts.host: - return RestApiResponse.error(Http503, "Sales unavailable") + return RestApiResponse.error(Http503, "Persistence is not enabled") without avails =? (await contracts.sales.context.reservations.all(Availability)), err: return RestApiResponse.error(Http500, err.msg) @@ -273,25 +291,32 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = ## ## totalSize - size of available storage in bytes ## duration - maximum time the storage should be sold for (in seconds) - ## minPrice - minimum price to be paid (in amount of tokens) + ## minPrice - minimal price paid (in amount of tokens) for the whole hosted request's slot for the request's duration ## maxCollateral - maximum collateral user is willing to pay per filled Slot (in amount of tokens) + var headers = newSeq[(string,string)]() + + if corsOrigin =? allowedOrigin: + headers.add(("Access-Control-Allow-Origin", corsOrigin)) + headers.add(("Access-Control-Allow-Methods", "POST, OPTIONS")) + headers.add(("Access-Control-Max-Age", "86400")) + try: without contracts =? node.contracts.host: - return RestApiResponse.error(Http503, "Sales unavailable") + return RestApiResponse.error(Http503, "Persistence is not enabled", headers = headers) let body = await request.getBody() without restAv =? RestAvailability.fromJson(body), error: - return RestApiResponse.error(Http400, error.msg) + return RestApiResponse.error(Http400, error.msg, headers = headers) let reservations = contracts.sales.context.reservations if restAv.totalSize == 0: - return RestApiResponse.error(Http400, "Total size must be larger then zero") + return RestApiResponse.error(Http400, "Total size must be larger then zero", headers = headers) if not reservations.hasAvailable(restAv.totalSize.truncate(uint)): - return RestApiResponse.error(Http422, "Not enough storage quota") + return RestApiResponse.error(Http422, "Not enough storage quota", headers = headers) without availability =? ( await reservations.createAvailability( @@ -300,14 +325,27 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = restAv.minPrice, restAv.maxCollateral) ), error: - return RestApiResponse.error(Http500, error.msg) + return RestApiResponse.error(Http500, error.msg, headers = headers) return RestApiResponse.response(availability.toJson, Http201, - contentType="application/json") + contentType="application/json", + headers = headers) except CatchableError as exc: trace "Excepting processing request", exc = exc.msg - return RestApiResponse.error(Http500) + return RestApiResponse.error(Http500, headers = headers) + + router.api( + MethodOptions, + "/api/codex/v1/sales/availability/{id}") do (id: AvailabilityId, resp: HttpResponseRef) -> RestApiResponse: + + if corsOrigin =? allowedOrigin: + resp.setHeader("Access-Control-Allow-Origin", corsOrigin) + resp.setHeader("Access-Control-Allow-Methods", "PATCH, OPTIONS") + resp.setHeader("Access-Control-Max-Age", "86400") + + resp.status = Http204 + await resp.sendBody("") router.rawApi( MethodPatch, @@ -323,7 +361,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = try: without contracts =? node.contracts.host: - return RestApiResponse.error(Http503, "Sales unavailable") + return RestApiResponse.error(Http503, "Persistence is not enabled") without id =? id.tryGet.catch, error: return RestApiResponse.error(Http400, error.msg) @@ -379,7 +417,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = try: without contracts =? node.contracts.host: - return RestApiResponse.error(Http503, "Sales unavailable") + return RestApiResponse.error(Http503, "Persistence is not enabled") without id =? id.tryGet.catch, error: return RestApiResponse.error(Http400, error.msg) @@ -387,6 +425,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = return RestApiResponse.error(Http400, error.msg) let reservations = contracts.sales.context.reservations + let market = contracts.sales.context.market if error =? (await reservations.get(keyId, Availability)).errorOption: if error of NotExistsError: @@ -404,6 +443,8 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = return RestApiResponse.error(Http500) proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) = + let allowedOrigin = router.allowedOrigin + router.rawApi( MethodPost, "/api/codex/v1/storage/request/{cid}") do (cid: Cid) -> RestApiResponse: @@ -418,37 +459,47 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) = ## tolerance - allowed number of nodes that can be lost before content is lost ## colateral - requested collateral from hosts when they fill slot + var headers = newSeq[(string,string)]() + + if corsOrigin =? allowedOrigin: + headers.add(("Access-Control-Allow-Origin", corsOrigin)) + headers.add(("Access-Control-Allow-Methods", "POST, OPTIONS")) + headers.add(("Access-Control-Max-Age", "86400")) + try: without contracts =? node.contracts.client: - return RestApiResponse.error(Http503, "Purchasing unavailable") + return RestApiResponse.error(Http503, "Persistence is not enabled", headers = headers) without cid =? cid.tryGet.catch, error: - return RestApiResponse.error(Http400, error.msg) + return RestApiResponse.error(Http400, error.msg, headers = headers) let body = await request.getBody() without params =? StorageRequestParams.fromJson(body), error: - return RestApiResponse.error(Http400, error.msg) + return RestApiResponse.error(Http400, error.msg, headers = headers) - let nodes = params.nodes |? 1 - let tolerance = params.tolerance |? 0 + let nodes = params.nodes |? 3 + let tolerance = params.tolerance |? 1 + + if tolerance == 0: + return RestApiResponse.error(Http400, "Tolerance needs to be bigger then zero", headers = headers) # prevent underflow if tolerance > nodes: - return RestApiResponse.error(Http400, "Invalid parameters: `tolerance` cannot be greater than `nodes`") + return RestApiResponse.error(Http400, "Invalid parameters: `tolerance` cannot be greater than `nodes`", headers = headers) let ecK = nodes - tolerance let ecM = tolerance # for readability # ensure leopard constrainst of 1 < K ≥ M if ecK <= 1 or ecK < ecM: - return RestApiResponse.error(Http400, "Invalid parameters: parameters must satify `1 < (nodes - tolerance) ≥ tolerance`") + return RestApiResponse.error(Http400, "Invalid parameters: parameters must satify `1 < (nodes - tolerance) ≥ tolerance`", headers = headers) without expiry =? params.expiry: - return RestApiResponse.error(Http400, "Expiry required") + return RestApiResponse.error(Http400, "Expiry required", headers = headers) if expiry <= 0 or expiry >= params.duration: - return RestApiResponse.error(Http400, "Expiry needs value bigger then zero and smaller then the request's duration") + return RestApiResponse.error(Http400, "Expiry needs value bigger then zero and smaller then the request's duration", headers = headers) without purchaseId =? await node.requestStorage( cid, @@ -463,14 +514,14 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) = if error of InsufficientBlocksError: return RestApiResponse.error(Http400, "Dataset too small for erasure parameters, need at least " & - $(ref InsufficientBlocksError)(error).minSize.int & " bytes") + $(ref InsufficientBlocksError)(error).minSize.int & " bytes", headers = headers) - return RestApiResponse.error(Http500, error.msg) + return RestApiResponse.error(Http500, error.msg, headers = headers) return RestApiResponse.response(purchaseId.toHex) except CatchableError as exc: trace "Excepting processing request", exc = exc.msg - return RestApiResponse.error(Http500) + return RestApiResponse.error(Http500, headers = headers) router.api( MethodGet, @@ -479,7 +530,7 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) = try: without contracts =? node.contracts.client: - return RestApiResponse.error(Http503, "Purchasing unavailable") + return RestApiResponse.error(Http503, "Persistence is not enabled") without id =? id.tryGet.catch, error: return RestApiResponse.error(Http400, error.msg) @@ -504,7 +555,7 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) = "/api/codex/v1/storage/purchases") do () -> RestApiResponse: try: without contracts =? node.contracts.client: - return RestApiResponse.error(Http503, "Purchasing unavailable") + return RestApiResponse.error(Http503, "Persistence is not enabled") let purchaseIds = contracts.purchasing.getPurchaseIds() return RestApiResponse.response($ %purchaseIds, contentType="application/json") diff --git a/codex/rest/json.nim b/codex/rest/json.nim index fba708be..24936b34 100644 --- a/codex/rest/json.nim +++ b/codex/rest/json.nim @@ -38,6 +38,8 @@ type state* {.serialize.}: string requestId* {.serialize.}: RequestId slotIndex* {.serialize.}: UInt256 + request* {.serialize.}: ?StorageRequest + reservation* {.serialize.}: ?Reservation RestContent* = object cid* {.serialize.}: Cid diff --git a/codex/sales.nim b/codex/sales.nim index c4fcb217..974d4f29 100644 --- a/codex/sales.nim +++ b/codex/sales.nim @@ -180,7 +180,7 @@ proc filled( processing.complete() proc processSlot(sales: Sales, item: SlotQueueItem, done: Future[void]) = - debug "processing slot from queue", requestId = item.requestId, + debug "Processing slot from queue", requestId = item.requestId, slot = item.slotIndex let agent = newSalesAgent( @@ -202,13 +202,17 @@ proc processSlot(sales: Sales, item: SlotQueueItem, done: Future[void]) = proc deleteInactiveReservations(sales: Sales, activeSlots: seq[Slot]) {.async.} = let reservations = sales.context.reservations without reservs =? await reservations.all(Reservation): - info "no unused reservations found for deletion" + return let unused = reservs.filter(r => ( let slotId = slotId(r.requestId, r.slotIndex) not activeSlots.any(slot => slot.id == slotId) )) - info "found unused reservations for deletion", unused = unused.len + + if unused.len == 0: + return + + info "Found unused reservations for deletion", unused = unused.len for reservation in unused: @@ -219,9 +223,9 @@ proc deleteInactiveReservations(sales: Sales, activeSlots: seq[Slot]) {.async.} if err =? (await reservations.deleteReservation( reservation.id, reservation.availabilityId )).errorOption: - error "failed to delete unused reservation", error = err.msg + error "Failed to delete unused reservation", error = err.msg else: - trace "deleted unused reservation" + trace "Deleted unused reservation" proc mySlots*(sales: Sales): Future[seq[Slot]] {.async.} = let market = sales.context.market diff --git a/codex/sales/reservations.nim b/codex/sales/reservations.nim index 0b5eaaf5..027bda95 100644 --- a/codex/sales/reservations.nim +++ b/codex/sales/reservations.nim @@ -16,7 +16,7 @@ ## |----------------------------------------| |--------------------------------------| ## | UInt256 | totalSize | | | UInt256 | size | | ## |----------------------------------------| |--------------------------------------| -## | UInt256 | freeSize | | | SlotId | slotId | | +## | UInt256 | freeSize | | | UInt256 | slotIndex | | ## |----------------------------------------| +--------------------------------------+ ## | UInt256 | duration | | ## |----------------------------------------| @@ -65,7 +65,7 @@ type totalSize* {.serialize.}: UInt256 freeSize* {.serialize.}: UInt256 duration* {.serialize.}: UInt256 - minPrice* {.serialize.}: UInt256 + minPrice* {.serialize.}: UInt256 # minimal price paid for the whole hosted slot for the request's duration maxCollateral* {.serialize.}: UInt256 Reservation* = ref object id* {.serialize.}: ReservationId diff --git a/codex/sales/states/preparing.nim b/codex/sales/states/preparing.nim index e5a441d3..d7f38f5b 100644 --- a/codex/sales/states/preparing.nim +++ b/codex/sales/states/preparing.nim @@ -69,11 +69,11 @@ method run*(state: SalePreparing, machine: Machine): Future[?State] {.async.} = request.ask.duration, request.ask.pricePerSlot, request.ask.collateral): - debug "no availability found for request, ignoring" + debug "No availability found for request, ignoring" return some State(SaleIgnored()) - info "availability found for request, creating reservation" + info "Availability found for request, creating reservation" without reservation =? await reservations.createReservation( availability.id, diff --git a/codex/slots/proofs.nim b/codex/slots/proofs.nim index 0b4ad667..4f7f01b5 100644 --- a/codex/slots/proofs.nim +++ b/codex/slots/proofs.nim @@ -1,4 +1,5 @@ import ./proofs/backends import ./proofs/prover +import ./proofs/backendfactory -export circomcompat, prover +export circomcompat, prover, backendfactory diff --git a/codex/slots/proofs/backendfactory.nim b/codex/slots/proofs/backendfactory.nim new file mode 100644 index 00000000..80dc1b8e --- /dev/null +++ b/codex/slots/proofs/backendfactory.nim @@ -0,0 +1,85 @@ +import os +import strutils +import pkg/chronos +import pkg/chronicles +import pkg/questionable +import pkg/confutils/defs +import pkg/stew/io2 +import pkg/ethers + +import ../../conf +import ./backends +import ./backendutils + +proc initializeFromConfig( + config: CodexConf, + utils: BackendUtils): ?!AnyBackend = + if not fileAccessible($config.circomR1cs, {AccessFlags.Read}) or + not endsWith($config.circomR1cs, ".r1cs"): + return failure("Circom R1CS file not accessible") + + if not fileAccessible($config.circomWasm, {AccessFlags.Read}) or + not endsWith($config.circomWasm, ".wasm"): + return failure("Circom wasm file not accessible") + + if not fileAccessible($config.circomZkey, {AccessFlags.Read}) or + not endsWith($config.circomZkey, ".zkey"): + return failure("Circom zkey file not accessible") + + trace "Initialized prover backend from cli config" + success(utils.initializeCircomBackend( + $config.circomR1cs, + $config.circomWasm, + $config.circomZkey)) + +proc r1csFilePath(config: CodexConf): string = + config.circuitDir / "proof_main.r1cs" + +proc wasmFilePath(config: CodexConf): string = + config.circuitDir / "proof_main.wasm" + +proc zkeyFilePath(config: CodexConf): string = + config.circuitDir / "proof_main.zkey" + +proc initializeFromCircuitDirFiles( + config: CodexConf, + utils: BackendUtils): ?!AnyBackend = + if fileExists(config.r1csFilePath) and + fileExists(config.wasmFilePath) and + fileExists(config.zkeyFilePath): + trace "Initialized prover backend from local files" + return success(utils.initializeCircomBackend( + config.r1csFilePath, + config.wasmFilePath, + config.zkeyFilePath)) + + failure("Circuit files not found") + +proc suggestDownloadTool(config: CodexConf) = + without address =? config.marketplaceAddress: + raise (ref Defect)(msg: "Proving backend initializing while marketplace address not set.") + + let + tokens = [ + "cirdl", + "\"" & $config.circuitDir & "\"", + config.ethProvider, + $address + ] + instructions = "'./" & tokens.join(" ") & "'" + + warn "Proving circuit files are not found. Please run the following to download them:", instructions + +proc initializeBackend*( + config: CodexConf, + utils: BackendUtils = BackendUtils()): ?!AnyBackend = + + without backend =? initializeFromConfig(config, utils), cliErr: + info "Could not initialize prover backend from CLI options...", msg = cliErr.msg + without backend =? initializeFromCircuitDirFiles(config, utils), localErr: + info "Could not initialize prover backend from circuit dir files...", msg = localErr.msg + suggestDownloadTool(config) + return failure("CircuitFilesNotFound") + # Unexpected: value of backend does not survive leaving each scope. (definition does though...) + return success(backend) + return success(backend) diff --git a/codex/slots/proofs/backends.nim b/codex/slots/proofs/backends.nim index 477ba140..3872d821 100644 --- a/codex/slots/proofs/backends.nim +++ b/codex/slots/proofs/backends.nim @@ -1,3 +1,6 @@ import ./backends/circomcompat export circomcompat + +type + AnyBackend* = CircomCompat diff --git a/codex/slots/proofs/backends/circomcompat.nim b/codex/slots/proofs/backends/circomcompat.nim index 70ff4dff..8619457a 100644 --- a/codex/slots/proofs/backends/circomcompat.nim +++ b/codex/slots/proofs/backends/circomcompat.nim @@ -9,17 +9,14 @@ {.push raises: [].} -import std/sequtils +import std/sugar import pkg/chronos import pkg/questionable/results import pkg/circomcompat -import pkg/poseidon2/io import ../../types import ../../../stores -import ../../../merkletree -import ../../../codextypes import ../../../contracts import ./converters @@ -39,6 +36,41 @@ type backendCfg : ptr CircomBn254Cfg vkp* : ptr CircomKey + NormalizedProofInputs*[H] {.borrow: `.`.} = distinct ProofInputs[H] + +func normalizeInput*[H](self: CircomCompat, input: ProofInputs[H]): + NormalizedProofInputs[H] = + ## Parameters in CIRCOM circuits are statically sized and must be properly + ## padded before they can be passed onto the circuit. This function takes + ## variable length parameters and performs that padding. + ## + ## The output from this function can be JSON-serialized and used as direct + ## inputs to the CIRCOM circuit for testing and debugging when one wishes + ## to bypass the Rust FFI. + + let normSamples = collect: + for sample in input.samples: + var merklePaths = sample.merklePaths + merklePaths.setLen(self.slotDepth) + Sample[H]( + cellData: sample.cellData, + merklePaths: merklePaths + ) + + var normSlotProof = input.slotProof + normSlotProof.setLen(self.datasetDepth) + + NormalizedProofInputs[H] ProofInputs[H]( + entropy: input.entropy, + datasetRoot: input.datasetRoot, + slotIndex: input.slotIndex, + slotRoot: input.slotRoot, + nCellsPerSlot: input.nCellsPerSlot, + nSlotsPerDataSet: input.nSlotsPerDataSet, + slotProof: normSlotProof, + samples: normSamples + ) + proc release*(self: CircomCompat) = ## Release the ctx ## @@ -49,27 +81,20 @@ proc release*(self: CircomCompat) = if not isNil(self.vkp): self.vkp.unsafeAddr.release_key() -proc prove*[H]( +proc prove[H]( self: CircomCompat, - input: ProofInputs[H]): ?!CircomProof = - ## Encode buffers using a ctx - ## + input: NormalizedProofInputs[H]): ?!CircomProof = - # NOTE: All inputs are statically sized per circuit - # and adjusted accordingly right before being passed - # to the circom ffi - `setLen` is used to adjust the - # sequence length to the correct size which also 0 pads - # to the correct length doAssert input.samples.len == self.numSamples, "Number of samples does not match" doAssert input.slotProof.len <= self.datasetDepth, - "Number of slot proofs does not match" + "Slot proof is too deep - dataset has more slots than what we can handle?" doAssert input.samples.allIt( block: (it.merklePaths.len <= self.slotDepth + self.blkDepth and - it.cellData.len <= self.cellElms * 32)), "Merkle paths length does not match" + it.cellData.len == self.cellElms)), "Merkle paths too deep or cells too big for circuit" # TODO: All parameters should match circom's static parametter var @@ -116,8 +141,7 @@ proc prove*[H]( var slotProof = input.slotProof.mapIt( it.toBytes ).concat - slotProof.setLen(self.datasetDepth) # zero pad inputs to correct size - + doAssert(slotProof.len == self.datasetDepth) # arrays are always flattened if ctx.pushInputU256Array( "slotProof".cstring, @@ -128,16 +152,14 @@ proc prove*[H]( for s in input.samples: var merklePaths = s.merklePaths.mapIt( it.toBytes ) - data = s.cellData + data = s.cellData.mapIt( @(it.toBytes) ).concat - merklePaths.setLen(self.slotDepth) # zero pad inputs to correct size if ctx.pushInputU256Array( "merklePaths".cstring, merklePaths[0].addr, uint (merklePaths[0].len * merklePaths.len)) != ERR_OK: return failure("Failed to push merkle paths") - data.setLen(self.cellElms * 32) # zero pad inputs to correct size if ctx.pushInputU256Array( "cellData".cstring, data[0].addr, @@ -162,6 +184,12 @@ proc prove*[H]( success proof +proc prove*[H]( + self: CircomCompat, + input: ProofInputs[H]): ?!CircomProof = + + self.prove(self.normalizeInput(input)) + proc verify*[H]( self: CircomCompat, proof: CircomProof, diff --git a/codex/slots/proofs/backendutils.nim b/codex/slots/proofs/backendutils.nim new file mode 100644 index 00000000..19740acb --- /dev/null +++ b/codex/slots/proofs/backendutils.nim @@ -0,0 +1,12 @@ +import ./backends + +type + BackendUtils* = ref object of RootObj + +method initializeCircomBackend*( + self: BackendUtils, + r1csFile: string, + wasmFile: string, + zKeyFile: string +): AnyBackend {.base.} = + CircomCompat.init(r1csFile, wasmFile, zKeyFile) diff --git a/codex/slots/proofs/prover.nim b/codex/slots/proofs/prover.nim index 9077c478..956270c2 100644 --- a/codex/slots/proofs/prover.nim +++ b/codex/slots/proofs/prover.nim @@ -21,11 +21,13 @@ import ../../merkletree import ../../stores import ../../market import ../../utils/poseidon2digest +import ../../conf import ../builder import ../sampler import ./backends +import ./backendfactory import ../types export backends @@ -34,7 +36,6 @@ logScope: topics = "codex prover" type - AnyBackend* = CircomCompat AnyProof* = CircomProof AnySampler* = Poseidon2Sampler @@ -86,7 +87,6 @@ proc verify*( inputs: AnyProofInputs): Future[?!bool] {.async.} = ## Prove a statement using backend. ## Returns a future that resolves to a proof. - self.backend.verify(proof, inputs) proc new*( @@ -96,6 +96,6 @@ proc new*( nSamples: int): Prover = Prover( - backend: backend, store: store, + backend: backend, nSamples: nSamples) diff --git a/codex/slots/sampler/sampler.nim b/codex/slots/sampler/sampler.nim index d22121c2..3270d55a 100644 --- a/codex/slots/sampler/sampler.nim +++ b/codex/slots/sampler/sampler.nim @@ -38,7 +38,7 @@ type func getCell*[T, H]( self: DataSampler[T, H], blkBytes: seq[byte], - blkCellIdx: Natural): seq[byte] = + blkCellIdx: Natural): seq[H] = let cellSize = self.builder.cellSize.uint64 @@ -47,7 +47,7 @@ func getCell*[T, H]( doAssert (dataEnd - dataStart) == cellSize, "Invalid cell size" - toInputData[H](blkBytes[dataStart ..< dataEnd]) + blkBytes[dataStart ..< dataEnd].elements(H).toSeq() proc getSample*[T, H]( self: DataSampler[T, H], diff --git a/codex/slots/sampler/utils.nim b/codex/slots/sampler/utils.nim index fa68e408..998f2cdc 100644 --- a/codex/slots/sampler/utils.nim +++ b/codex/slots/sampler/utils.nim @@ -7,23 +7,13 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. -import std/sugar import std/bitops -import std/sequtils import pkg/questionable/results -import pkg/poseidon2 -import pkg/poseidon2/io - import pkg/constantine/math/arithmetic -import pkg/constantine/math/io/io_fields - import ../../merkletree -func toInputData*[H](data: seq[byte]): seq[byte] = - return toSeq(data.elements(H)).mapIt( @(it.toBytes) ).concat - func extractLowBits*[n: static int](elm: BigInt[n], k: int): uint64 = doAssert( k > 0 and k <= 64 ) var r = 0'u64 @@ -39,6 +29,7 @@ func extractLowBits(fld: Poseidon2Hash, k: int): uint64 = return extractLowBits(elm, k); func floorLog2*(x : int) : int = + doAssert ( x > 0 ) var k = -1 var y = x while (y > 0): @@ -47,10 +38,8 @@ func floorLog2*(x : int) : int = return k func ceilingLog2*(x : int) : int = - if (x == 0): - return -1 - else: - return (floorLog2(x-1) + 1) + doAssert ( x > 0 ) + return (floorLog2(x - 1) + 1) func toBlkInSlot*(cell: Natural, numCells: Natural): Natural = let log2 = ceilingLog2(numCells) @@ -80,7 +69,7 @@ func cellIndices*( numCells: Natural, nSamples: Natural): seq[Natural] = var indices: seq[Natural] - while (indices.len < nSamples): - let idx = cellIndex(entropy, slotRoot, numCells, indices.len + 1) - indices.add(idx.Natural) + for i in 1..nSamples: + indices.add(cellIndex(entropy, slotRoot, numCells, i)) + indices diff --git a/codex/slots/types.nim b/codex/slots/types.nim index 04690adc..8703086e 100644 --- a/codex/slots/types.nim +++ b/codex/slots/types.nim @@ -9,7 +9,7 @@ type Sample*[H] = object - cellData*: seq[byte] + cellData*: seq[H] merklePaths*: seq[H] PublicInputs*[H] = object @@ -24,5 +24,5 @@ type slotRoot*: H nCellsPerSlot*: Natural nSlotsPerDataSet*: Natural - slotProof*: seq[H] - samples*: seq[Sample[H]] + slotProof*: seq[H] # inclusion proof that shows that the slot root (leaf) is part of the dataset (root) + samples*: seq[Sample[H]] # inclusion proofs which show that the selected cells (leafs) are part of the slot (roots) diff --git a/codex/stores/blockstore.nim b/codex/stores/blockstore.nim index 4921bebb..52f37517 100644 --- a/codex/stores/blockstore.nim +++ b/codex/stores/blockstore.nim @@ -29,7 +29,9 @@ type BlockType* {.pure.} = enum Manifest, Block, Both + CidCallback* = proc(cid: Cid): Future[void] {.gcsafe, raises:[].} BlockStore* = ref object of RootObj + onBlockStored*: ?CidCallback method getBlock*(self: BlockStore, cid: Cid): Future[?!Block] {.base.} = ## Get a block from the blockstore diff --git a/codex/stores/cachestore.nim b/codex/stores/cachestore.nim index d6623373..130d2ade 100644 --- a/codex/stores/cachestore.nim +++ b/codex/stores/cachestore.nim @@ -197,6 +197,9 @@ method putBlock*( return success() discard self.putBlockSync(blk) + if onBlock =? self.onBlockStored: + await onBlock(blk.cid) + return success() method putCidAndProof*( @@ -282,7 +285,8 @@ proc new*( cache: cache, cidAndProofCache: cidAndProofCache, currentSize: currentSize, - size: cacheSize) + size: cacheSize, + onBlockStored: CidCallback.none) for blk in blocks: discard store.putBlockSync(blk) diff --git a/codex/stores/repostore/store.nim b/codex/stores/repostore/store.nim index 7d629131..5ff99e64 100644 --- a/codex/stores/repostore/store.nim +++ b/codex/stores/repostore/store.nim @@ -189,6 +189,9 @@ method putBlock*( if err =? (await self.updateTotalBlocksCount(plusCount = 1)).errorOption: return failure(err) + + if onBlock =? self.onBlockStored: + await onBlock(blk.cid) else: trace "Block already exists" diff --git a/codex/stores/repostore/types.nim b/codex/stores/repostore/types.nim index 4338e63a..2f88183d 100644 --- a/codex/stores/repostore/types.nim +++ b/codex/stores/repostore/types.nim @@ -11,6 +11,7 @@ import pkg/chronos import pkg/datastore import pkg/datastore/typedds import pkg/libp2p/cid +import pkg/questionable import ../blockstore import ../../clock @@ -103,5 +104,6 @@ func new*( clock: clock, postFixLen: postFixLen, quotaMaxBytes: quotaMaxBytes, - blockTtl: blockTtl + blockTtl: blockTtl, + onBlockStored: CidCallback.none ) diff --git a/config.nims b/config.nims index 45830fad..b64aacbd 100644 --- a/config.nims +++ b/config.nims @@ -121,6 +121,9 @@ switch("define", "ctt_asm=false") # Allow the use of old-style case objects for nim config compatibility switch("define", "nimOldCaseObjects") +# Enable compat mode for Chronos V4 +switch("define", "chronosHandleException") + # begin Nimble config (version 1) when system.fileExists("nimble.paths"): include "nimble.paths" diff --git a/docker/codex.Dockerfile b/docker/codex.Dockerfile index f3ffb92e..7811a38f 100644 --- a/docker/codex.Dockerfile +++ b/docker/codex.Dockerfile @@ -24,9 +24,9 @@ RUN echo "export PATH=$PATH:$HOME/.cargo/bin" >> $BASH_ENV WORKDIR ${BUILD_HOME} COPY . . -RUN make clean RUN make -j ${MAKE_PARALLEL} update RUN make -j ${MAKE_PARALLEL} +RUN make -j ${MAKE_PARALLEL} cirdl # Create FROM ${IMAGE} @@ -35,10 +35,10 @@ ARG APP_HOME ARG NAT_IP_AUTO WORKDIR ${APP_HOME} -COPY --from=builder ${BUILD_HOME}/build/codex /usr/local/bin +COPY --from=builder ${BUILD_HOME}/build/* /usr/local/bin COPY --from=builder ${BUILD_HOME}/openapi.yaml . COPY --from=builder --chmod=0755 ${BUILD_HOME}/docker/docker-entrypoint.sh / -RUN apt-get update && apt-get install -y libgomp1 bash curl jq && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y libgomp1 curl jq && rm -rf /var/lib/apt/lists/* ENV NAT_IP_AUTO=${NAT_IP_AUTO} ENTRYPOINT ["/docker-entrypoint.sh"] CMD ["codex"] diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 9875d9ae..ab03e634 100644 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -50,6 +50,21 @@ if [ -n "${PRIV_KEY}" ]; then echo "Private key set" fi +# Circuit downloader +# cirdl [circuitPath] [rpcEndpoint] [marketplaceAddress] +if [[ "$@" == *"prover"* ]]; then + echo "Run Circuit downloader" + # Set circuits dir from CODEX_CIRCUIT_DIR variables if set + if [[ -z "${CODEX_CIRCUIT_DIR}" ]]; then + export CODEX_CIRCUIT_DIR="${CODEX_DATA_DIR}/circuits" + fi + # Download circuits + mkdir -p "${CODEX_CIRCUIT_DIR}" + chmod 700 "${CODEX_CIRCUIT_DIR}" + cirdl "${CODEX_CIRCUIT_DIR}" "${CODEX_ETH_PROVIDER}" "${CODEX_MARKETPLACE_ADDRESS}" + [[ $? -ne 0 ]] && { echo "Failed to download circuit files"; exit 1; } +fi + # Run echo "Run Codex node" exec "$@" diff --git a/openapi.yaml b/openapi.yaml index 94450bf3..9a2056ca 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -23,6 +23,8 @@ components: Id: type: string description: 32bits identifier encoded in hex-decimal string. + minLength: 66 + maxLength: 66 example: 0x... BigInt: @@ -136,7 +138,7 @@ components: $ref: "#/components/schemas/Duration" minPrice: type: string - description: Minimum price to be paid (in amount of tokens) as decimal string + description: Minimal price paid (in amount of tokens) for the whole hosted request's slot for the request's duration as decimal string maxCollateral: type: string description: Maximum collateral user is willing to pay per filled Slot (in amount of tokens) as decimal string @@ -168,7 +170,39 @@ components: $ref: "#/components/schemas/StorageRequest" slotIndex: type: string - description: Slot Index as hexadecimal string + description: Slot Index as decimal string + + SlotAgent: + type: object + properties: + id: + $ref: "#/components/schemas/SlotId" + slotIndex: + type: string + description: Slot Index as decimal string + requestId: + $ref: "#/components/schemas/Id" + request: + $ref: "#/components/schemas/StorageRequest" + reservation: + $ref: "#/components/schemas/Reservation" + state: + type: string + description: Description of the slot's + enum: + - SaleCancelled + - SaleDownloading + - SaleErrored + - SaleFailed + - SaleFilled + - SaleFilling + - SaleFinished + - SaleIgnored + - SaleInitialProving + - SalePayout + - SalePreparing + - SaleProving + - SaleUnknown Reservation: type: object @@ -183,7 +217,7 @@ components: $ref: "#/components/schemas/Id" slotIndex: type: string - description: Slot Index as hexadecimal string + description: Slot Index as decimal string StorageRequestCreation: type: object @@ -259,6 +293,15 @@ components: state: type: string description: Description of the Request's state + enum: + - cancelled + - error + - failed + - finished + - pending + - started + - submitted + - unknown error: type: string description: If Request failed, then here is presented the error message @@ -491,7 +534,7 @@ paths: $ref: "#/components/schemas/Slot" "503": - description: Sales are unavailable + description: Persistence is not enabled "/sales/slots/{slotId}": get: @@ -511,7 +554,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/Slot" + $ref: "#/components/schemas/SlotAgent" "400": description: Invalid or missing SlotId @@ -520,13 +563,13 @@ paths: description: Host is not in an active sale for the slot "503": - description: Sales are unavailable + description: Persistence is not enabled "/sales/availability": get: summary: "Returns storage that is for sale" tags: [ Marketplace ] - operationId: getOfferedStorage + operationId: getAvailabilities responses: "200": description: Retrieved storage availabilities of the node @@ -535,11 +578,11 @@ paths: schema: type: array items: - $ref: "#/components/schemas/SalesAvailability" + $ref: "#/components/schemas/SalesAvailabilityREAD" "500": description: Error getting unused availabilities "503": - description: Sales are unavailable + description: Persistence is not enabled post: summary: "Offers storage for sale" @@ -564,7 +607,7 @@ paths: "500": description: Error reserving availability "503": - description: Sales are unavailable + description: Persistence is not enabled "/sales/availability/{id}": patch: summary: "Updates availability" @@ -597,10 +640,10 @@ paths: "500": description: Error reserving availability "503": - description: Sales are unavailable + description: Persistence is not enabled "/sales/availability/{id}/reservations": - patch: + get: summary: "Get availability's reservations" description: Return's list of Reservations for ongoing Storage Requests that the node hosts. operationId: getReservations @@ -628,7 +671,7 @@ paths: "500": description: Error getting reservations "503": - description: Sales are unavailable + description: Persistence is not enabled "/storage/request/{cid}": post: @@ -659,7 +702,7 @@ paths: "404": description: Request ID not found "503": - description: Purchasing is unavailable + description: Persistence is not enabled "/storage/purchases": get: @@ -676,7 +719,7 @@ paths: items: type: string "503": - description: Purchasing is unavailable + description: Persistence is not enabled "/storage/purchases/{id}": get: @@ -702,7 +745,7 @@ paths: "404": description: Purchase not found "503": - description: Purchasing is unavailable + description: Persistence is not enabled "/node/spr": get: diff --git a/tests/circuits/fixtures/proof_main.r1cs b/tests/circuits/fixtures/proof_main.r1cs index a2b9c6b7..8b58ffa8 100644 Binary files a/tests/circuits/fixtures/proof_main.r1cs and b/tests/circuits/fixtures/proof_main.r1cs differ diff --git a/tests/circuits/fixtures/proof_main.wasm b/tests/circuits/fixtures/proof_main.wasm index 30fe6f0e..f908d4f7 100644 Binary files a/tests/circuits/fixtures/proof_main.wasm and b/tests/circuits/fixtures/proof_main.wasm differ diff --git a/tests/circuits/fixtures/proof_main.zkey b/tests/circuits/fixtures/proof_main.zkey index 999a12ab..5451ad0f 100644 Binary files a/tests/circuits/fixtures/proof_main.zkey and b/tests/circuits/fixtures/proof_main.zkey differ diff --git a/tests/codex/blockexchange/discovery/testdiscovery.nim b/tests/codex/blockexchange/discovery/testdiscovery.nim index 9ba29e5d..a136f89e 100644 --- a/tests/codex/blockexchange/discovery/testdiscovery.nim +++ b/tests/codex/blockexchange/discovery/testdiscovery.nim @@ -32,6 +32,7 @@ asyncchecksuite "Block Advertising and Discovery": peerStore: PeerCtxStore blockDiscovery: MockDiscovery discovery: DiscoveryEngine + advertiser: Advertiser wallet: WalletRef network: BlockExcNetwork localStore: CacheStore @@ -68,11 +69,17 @@ asyncchecksuite "Block Advertising and Discovery": pendingBlocks, minPeersPerBlock = 1) + advertiser = Advertiser.new( + localStore, + blockDiscovery + ) + engine = BlockExcEngine.new( localStore, wallet, network, discovery, + advertiser, peerStore, pendingBlocks) @@ -200,11 +207,17 @@ asyncchecksuite "E2E - Multiple Nodes Discovery": pendingBlocks, minPeersPerBlock = 1) + advertiser = Advertiser.new( + localStore, + blockDiscovery + ) + engine = BlockExcEngine.new( localStore, wallet, network, discovery, + advertiser, peerStore, pendingBlocks) networkStore = NetworkStore.new(engine, localStore) diff --git a/tests/codex/blockexchange/discovery/testdiscoveryengine.nim b/tests/codex/blockexchange/discovery/testdiscoveryengine.nim index ff6b60d2..61a6f09f 100644 --- a/tests/codex/blockexchange/discovery/testdiscoveryengine.nim +++ b/tests/codex/blockexchange/discovery/testdiscoveryengine.nim @@ -74,30 +74,6 @@ asyncchecksuite "Test Discovery Engine": await allFuturesThrowing(allFinished(wants)).wait(1.seconds) await discoveryEngine.stop() - test "Should Advertise Haves": - var - localStore = CacheStore.new(blocks.mapIt(it)) - discoveryEngine = DiscoveryEngine.new( - localStore, - peerStore, - network, - blockDiscovery, - pendingBlocks, - discoveryLoopSleep = 100.millis) - haves = collect(initTable): - for cid in @[manifestBlock.cid, manifest.treeCid]: - { cid: newFuture[void]() } - - blockDiscovery.publishBlockProvideHandler = - proc(d: MockDiscovery, cid: Cid) {.async, gcsafe.} = - if not haves[cid].finished: - haves[cid].complete - - await discoveryEngine.start() - await allFuturesThrowing( - allFinished(toSeq(haves.values))).wait(5.seconds) - await discoveryEngine.stop() - test "Should queue discovery request": var localStore = CacheStore.new() @@ -191,36 +167,3 @@ asyncchecksuite "Test Discovery Engine": reqs.complete() await discoveryEngine.stop() - - test "Should not request if there is already an inflight advertise request": - var - localStore = CacheStore.new() - discoveryEngine = DiscoveryEngine.new( - localStore, - peerStore, - network, - blockDiscovery, - pendingBlocks, - discoveryLoopSleep = 100.millis, - concurrentAdvReqs = 2) - reqs = newFuture[void]() - count = 0 - - blockDiscovery.publishBlockProvideHandler = - proc(d: MockDiscovery, cid: Cid) {.async, gcsafe.} = - check cid == blocks[0].cid - if count > 0: - check false - count.inc - - await reqs # queue the request - - await discoveryEngine.start() - discoveryEngine.queueProvideBlocksReq(@[blocks[0].cid]) - await sleepAsync(200.millis) - - discoveryEngine.queueProvideBlocksReq(@[blocks[0].cid]) - await sleepAsync(200.millis) - - reqs.complete() - await discoveryEngine.stop() diff --git a/tests/codex/blockexchange/engine/testadvertiser.nim b/tests/codex/blockexchange/engine/testadvertiser.nim new file mode 100644 index 00000000..4cbd2117 --- /dev/null +++ b/tests/codex/blockexchange/engine/testadvertiser.nim @@ -0,0 +1,106 @@ +import std/sequtils +import std/random + +import pkg/chronos +import pkg/libp2p/routing_record +import pkg/codexdht/discv5/protocol as discv5 + +import pkg/codex/blockexchange +import pkg/codex/stores +import pkg/codex/chunker +import pkg/codex/discovery +import pkg/codex/blocktype as bt +import pkg/codex/manifest + +import ../../../asynctest +import ../../helpers +import ../../helpers/mockdiscovery +import ../../examples + +asyncchecksuite "Advertiser": + var + blockDiscovery: MockDiscovery + localStore: BlockStore + advertiser: Advertiser + let + manifest = Manifest.new( + treeCid = Cid.example, + blockSize = 123.NBytes, + datasetSize = 234.NBytes) + manifestBlk = Block.new(data = manifest.encode().tryGet(), codec = ManifestCodec).tryGet() + + setup: + blockDiscovery = MockDiscovery.new() + localStore = CacheStore.new() + + advertiser = Advertiser.new( + localStore, + blockDiscovery + ) + + await advertiser.start() + + teardown: + await advertiser.stop() + + test "blockStored should queue manifest Cid for advertising": + (await localStore.putBlock(manifestBlk)).tryGet() + + check: + manifestBlk.cid in advertiser.advertiseQueue + + test "blockStored should queue tree Cid for advertising": + (await localStore.putBlock(manifestBlk)).tryGet() + + check: + manifest.treeCid in advertiser.advertiseQueue + + test "blockStored should not queue non-manifest non-tree CIDs for discovery": + let blk = bt.Block.example + + (await localStore.putBlock(blk)).tryGet() + + check: + blk.cid notin advertiser.advertiseQueue + + test "Should not queue if there is already an inflight advertise request": + var + reqs = newFuture[void]() + manifestCount = 0 + treeCount = 0 + + blockDiscovery.publishBlockProvideHandler = + proc(d: MockDiscovery, cid: Cid) {.async, gcsafe.} = + if cid == manifestBlk.cid: + inc manifestCount + if cid == manifest.treeCid: + inc treeCount + + await reqs # queue the request + + (await localStore.putBlock(manifestBlk)).tryGet() + (await localStore.putBlock(manifestBlk)).tryGet() + + reqs.complete() + check eventually manifestCount == 1 + check eventually treeCount == 1 + + test "Should advertise existing manifests and their trees": + let + newStore = CacheStore.new([manifestBlk]) + + await advertiser.stop() + advertiser = Advertiser.new( + newStore, + blockDiscovery + ) + await advertiser.start() + + check eventually manifestBlk.cid in advertiser.advertiseQueue + check eventually manifest.treeCid in advertiser.advertiseQueue + + test "Stop should clear onBlockStored callback": + await advertiser.stop() + + check: + localStore.onBlockStored.isNone() diff --git a/tests/codex/blockexchange/engine/testengine.nim b/tests/codex/blockexchange/engine/testengine.nim index 5bf02b1b..c22a1a6a 100644 --- a/tests/codex/blockexchange/engine/testengine.nim +++ b/tests/codex/blockexchange/engine/testengine.nim @@ -78,11 +78,17 @@ asyncchecksuite "NetworkStore engine basic": blockDiscovery, pendingBlocks) + advertiser = Advertiser.new( + localStore, + blockDiscovery + ) + engine = BlockExcEngine.new( localStore, wallet, network, discovery, + advertiser, peerStore, pendingBlocks) @@ -113,11 +119,17 @@ asyncchecksuite "NetworkStore engine basic": blockDiscovery, pendingBlocks) + advertiser = Advertiser.new( + localStore, + blockDiscovery + ) + engine = BlockExcEngine.new( localStore, wallet, network, discovery, + advertiser, peerStore, pendingBlocks) @@ -139,6 +151,7 @@ asyncchecksuite "NetworkStore engine handlers": network: BlockExcNetwork engine: BlockExcEngine discovery: DiscoveryEngine + advertiser: Advertiser peerCtx: BlockExcPeerCtx localStore: BlockStore blocks: seq[Block] @@ -176,11 +189,17 @@ asyncchecksuite "NetworkStore engine handlers": blockDiscovery, pendingBlocks) + advertiser = Advertiser.new( + localStore, + blockDiscovery + ) + engine = BlockExcEngine.new( localStore, wallet, network, discovery, + advertiser, peerStore, pendingBlocks) @@ -390,51 +409,6 @@ asyncchecksuite "NetworkStore engine handlers": discard await allFinished(pending) await allFuturesThrowing(cancellations.values().toSeq) - test "resolveBlocks should queue manifest CIDs for discovery": - engine.network = BlockExcNetwork( - request: BlockExcRequest(sendWantCancellations: NopSendWantCancellationsProc)) - - let - manifest = Manifest.new( - treeCid = Cid.example, - blockSize = 123.NBytes, - datasetSize = 234.NBytes - ) - - let manifestBlk = Block.new(data = manifest.encode().tryGet(), codec = ManifestCodec).tryGet() - let blks = @[manifestBlk] - - await engine.resolveBlocks(blks) - - check: - manifestBlk.cid in engine.discovery.advertiseQueue - - test "resolveBlocks should queue tree CIDs for discovery": - engine.network = BlockExcNetwork( - request: BlockExcRequest(sendWantCancellations: NopSendWantCancellationsProc)) - - let - tCid = Cid.example - delivery = BlockDelivery(blk: Block.example, address: BlockAddress(leaf: true, treeCid: tCid)) - - await engine.resolveBlocks(@[delivery]) - - check: - tCid in engine.discovery.advertiseQueue - - test "resolveBlocks should not queue non-manifest non-tree CIDs for discovery": - engine.network = BlockExcNetwork( - request: BlockExcRequest(sendWantCancellations: NopSendWantCancellationsProc)) - - let - blkCid = Cid.example - delivery = BlockDelivery(blk: Block.example, address: BlockAddress(leaf: false, cid: blkCid)) - - await engine.resolveBlocks(@[delivery]) - - check: - blkCid notin engine.discovery.advertiseQueue - asyncchecksuite "Task Handler": var rng: Rng @@ -448,6 +422,7 @@ asyncchecksuite "Task Handler": network: BlockExcNetwork engine: BlockExcEngine discovery: DiscoveryEngine + advertiser: Advertiser localStore: BlockStore peersCtx: seq[BlockExcPeerCtx] @@ -481,11 +456,17 @@ asyncchecksuite "Task Handler": blockDiscovery, pendingBlocks) + advertiser = Advertiser.new( + localStore, + blockDiscovery + ) + engine = BlockExcEngine.new( localStore, wallet, network, discovery, + advertiser, peerStore, pendingBlocks) peersCtx = @[] diff --git a/tests/codex/blockexchange/testengine.nim b/tests/codex/blockexchange/testengine.nim index 5277e027..9cd968ee 100644 --- a/tests/codex/blockexchange/testengine.nim +++ b/tests/codex/blockexchange/testengine.nim @@ -1,5 +1,6 @@ import ./engine/testengine import ./engine/testblockexc import ./engine/testpayments +import ./engine/testadvertiser {.warning[UnusedImport]: off.} diff --git a/tests/codex/helpers/mockmarket.nim b/tests/codex/helpers/mockmarket.nim index 7c36bee7..ed460adb 100644 --- a/tests/codex/helpers/mockmarket.nim +++ b/tests/codex/helpers/mockmarket.nim @@ -101,7 +101,8 @@ proc new*(_: type MockMarket): MockMarket = proofs: ProofConfig( period: 10.u256, timeout: 5.u256, - downtime: 64.uint8 + downtime: 64.uint8, + downtimeProduct: 67.uint8 ) ) MockMarket(signer: Address.example, config: config) @@ -419,16 +420,21 @@ method subscribeProofSubmission*(mock: MockMarket, mock.subscriptions.onProofSubmitted.add(subscription) return subscription -method queryPastStorageRequests*(market: MockMarket, - blocksAgo: int): - Future[seq[PastStorageRequest]] {.async.} = - # MockMarket does not have the concept of blocks, so simply return all - # previous events - return market.requested.map(request => - PastStorageRequest(requestId: request.id, +method queryPastEvents*[T: MarketplaceEvent]( + market: MockMarket, + _: type T, + blocksAgo: int): Future[seq[T]] {.async.} = + + if T of StorageRequested: + return market.requested.map(request => + StorageRequested(requestId: request.id, ask: request.ask, expiry: request.expiry) - ) + ) + elif T of SlotFilled: + return market.filled.map(slot => + SlotFilled(requestId: slot.requestId, slotIndex: slot.slotIndex) + ) method unsubscribe*(subscription: RequestSubscription) {.async.} = subscription.market.subscriptions.onRequest.keepItIf(it != subscription) diff --git a/tests/codex/helpers/nodeutils.nim b/tests/codex/helpers/nodeutils.nim index d8798b3d..8c624bb2 100644 --- a/tests/codex/helpers/nodeutils.nim +++ b/tests/codex/helpers/nodeutils.nim @@ -40,8 +40,9 @@ proc generateNodes*( localStore = CacheStore.new(blocks.mapIt( it )) peerStore = PeerCtxStore.new() pendingBlocks = PendingBlocksManager.new() + advertiser = Advertiser.new(localStore, discovery) blockDiscovery = DiscoveryEngine.new(localStore, peerStore, network, discovery, pendingBlocks) - engine = BlockExcEngine.new(localStore, wallet, network, blockDiscovery, peerStore, pendingBlocks) + engine = BlockExcEngine.new(localStore, wallet, network, blockDiscovery, advertiser, peerStore, pendingBlocks) networkStore = NetworkStore.new(engine, localStore) switch.mount(network) diff --git a/tests/codex/node/helpers.nim b/tests/codex/node/helpers.nim index 498ea45b..b3f89e5c 100644 --- a/tests/codex/node/helpers.nim +++ b/tests/codex/node/helpers.nim @@ -82,6 +82,7 @@ template setupAndTearDown*() {.dirty.} = peerStore: PeerCtxStore pendingBlocks: PendingBlocksManager discovery: DiscoveryEngine + advertiser: Advertiser taskpool: Taskpool let @@ -109,7 +110,8 @@ template setupAndTearDown*() {.dirty.} = peerStore = PeerCtxStore.new() pendingBlocks = PendingBlocksManager.new() discovery = DiscoveryEngine.new(localStore, peerStore, network, blockDiscovery, pendingBlocks) - engine = BlockExcEngine.new(localStore, wallet, network, discovery, peerStore, pendingBlocks) + advertiser = Advertiser.new(localStore, blockDiscovery) + engine = BlockExcEngine.new(localStore, wallet, network, discovery, advertiser, peerStore, pendingBlocks) store = NetworkStore.new(engine, localStore) taskpool = Taskpool.new(num_threads = countProcessors()) node = CodexNodeRef.new( @@ -120,8 +122,6 @@ template setupAndTearDown*() {.dirty.} = discovery = blockDiscovery, taskpool = taskpool) - await node.start() - teardown: close(file) await node.stop() diff --git a/tests/codex/node/testnode.nim b/tests/codex/node/testnode.nim index 3aa8a424..ab8317ec 100644 --- a/tests/codex/node/testnode.nim +++ b/tests/codex/node/testnode.nim @@ -49,6 +49,9 @@ privateAccess(CodexNodeRef) # enable access to private fields asyncchecksuite "Test Node - Basic": setupAndTearDown() + setup: + await node.start() + test "Fetch Manifest": let manifest = await storeDataGetManifest(localStore, chunker) diff --git a/tests/codex/sales/testsales.nim b/tests/codex/sales/testsales.nim index 543a7133..0fdf3bf9 100644 --- a/tests/codex/sales/testsales.nim +++ b/tests/codex/sales/testsales.nim @@ -323,8 +323,7 @@ asyncchecksuite "Sales": slot: UInt256, onBatch: BatchProc): Future[?!void] {.async.} = let blk = bt.Block.new( @[1.byte] ).get - onBatch( blk.repeat(request.ask.slotSize.truncate(int)) ) - return success() + await onBatch( blk.repeat(request.ask.slotSize.truncate(int)) ) createAvailability() await market.requestStorage(request) @@ -337,8 +336,8 @@ asyncchecksuite "Sales": onBatch: BatchProc): Future[?!void] {.async.} = slotIndex = slot let blk = bt.Block.new( @[1.byte] ).get - onBatch(@[ blk ]) - return success() + await onBatch(@[ blk ]) + let sold = newFuture[void]() sales.onSale = proc(request: StorageRequest, slotIndex: UInt256) = sold.complete() diff --git a/tests/codex/sales/testslotqueue.nim b/tests/codex/sales/testslotqueue.nim index 71ac8597..193751c8 100644 --- a/tests/codex/sales/testslotqueue.nim +++ b/tests/codex/sales/testslotqueue.nim @@ -524,7 +524,7 @@ suite "Slot queue": request.ask, request.expiry, seen = true) - queue.push(item) + check queue.push(item).isOk check eventually queue.paused check onProcessSlotCalledWith.len == 0 @@ -534,7 +534,7 @@ suite "Slot queue": let request = StorageRequest.example var items = SlotQueueItem.init(request) - queue.push(items) + check queue.push(items).isOk # check all items processed check eventually queue.len == 0 @@ -546,7 +546,7 @@ suite "Slot queue": request.expiry, seen = true) check queue.paused - queue.push(item0) + check queue.push(item0).isOk check queue.paused test "paused queue waits for unpause before continuing processing": @@ -558,7 +558,7 @@ suite "Slot queue": seen = false) check queue.paused # push causes unpause - queue.push(item) + check queue.push(item).isOk # check all items processed check eventually onProcessSlotCalledWith == @[ (item.requestId, item.slotIndex), @@ -576,8 +576,8 @@ suite "Slot queue": request.ask, request.expiry, seen = true) - queue.push(item0) - queue.push(item1) + check queue.push(item0).isOk + check queue.push(item1).isOk check queue[0].seen check queue[1].seen diff --git a/tests/codex/slots/backends/helpers.nim b/tests/codex/slots/backends/helpers.nim index e81762cf..85e8ae65 100644 --- a/tests/codex/slots/backends/helpers.nim +++ b/tests/codex/slots/backends/helpers.nim @@ -17,21 +17,6 @@ import pkg/codex/utils/json export types -func fromCircomData*(_: type Poseidon2Hash, cellData: seq[byte]): seq[Poseidon2Hash] = - var - pos = 0 - cellElms: seq[Bn254Fr] - while pos < cellData.len: - var - step = 32 - offset = min(pos + step, cellData.len) - data = cellData[pos..