Rework circuit downloader (#882)

* Introduces a start method to prover

* Moves backend creation into start method

* sets up three paths for backend initialization

* Extracts backend initialization to backend-factory

* Implements loading backend from cli files or previously downloaded local files

* Wires up downloading and unzipping

* functional implementation

* Fixes testprover.nim

* Sets up tests for backendfactory

* includes libzip-dev

* pulls in updated contracts

* removes integration cli tests for r1cs, wasm, and zkey file arguments.

* Fixes issue where inner-scope values are lost before returning

* sets local proof verification for dist-test images

* Adds two traces and bumps nim-ethers

* Adds separate path for circuit files

* Create circuit dir if not exists

* fix: make sure requestStorage is mined

* fix: correct place to plug confirm

* test: fixing contracts tests

* Restores gitmodules

* restores nim-datastore reference

* Sets up downloader exe

* sets up tool skeleton

* implements getting of circuit hash

* Implements downloader tool

* sets up test skeleton

* Implements test for cirdl

* includes testTools in testAll

* Cleanup building.md

* cleans up previous downloader implementation

* cleans up testbackendfactory

* moves start of prover into node.nim

* Fills in arguments in example command

* Initializes backend in prover constructor

* Restores tests

* Restores tests for cli instructions

* Review comments by Dmitriy, part 1

* Quotes path in download instruction.

* replaces curl with chronos http session

* Moves cirdl build output to 'build' folder.

* Fixes chronicles log output

* Add cirdl support to the codex Dockerfile

Signed-off-by: Slava <20563034+veaceslavdoina@users.noreply.github.com>

* Add cirdl support to the docker entrypoint

Signed-off-by: Slava <20563034+veaceslavdoina@users.noreply.github.com>

* Add cirdl support to the release workflow

Signed-off-by: Slava <20563034+veaceslavdoina@users.noreply.github.com>

* Disable verify_circuit flag for releases

Signed-off-by: Slava <20563034+veaceslavdoina@users.noreply.github.com>

* Removes backendFactory placeholder type

* wip

* Replaces zip library with status-im/zippy library (which supports zip and tar)

* Updates cirdl to not change circuitdir folder

* Switches from zip to tar.gz

* Review comments by Dmitriy

* updates codex-contracts-eth

* Adds testTools to CI

* Adds check for access to config.circuitdir

* Update fixture circuit zkey

* Update matrix to run tools tests on Windows

* Adds 'deps' dependency for cirdl

* Adjust docker-entrypoint.sh to use CODEX_CIRCUIT_DIR env var

* Review comments by Giuliano

---------

Signed-off-by: Slava <20563034+veaceslavdoina@users.noreply.github.com>
Co-authored-by: Adam Uhlíř <adam@uhlir.dev>
Co-authored-by: Veaceslav Doina <20563034+veaceslavdoina@users.noreply.github.com>

.github/workflows/ci-reusable.yml

@@ -53,7 +53,7 @@ jobs:
           node-version: 18.15
 
       - name: Start Ethereum node with Codex contracts
-        if: matrix.tests == 'contract' || matrix.tests == 'integration' || matrix.tests == 'all'
+        if: matrix.tests == 'contract' || matrix.tests == 'integration' || matrix.tests == 'tools' || matrix.tests == 'all'
         working-directory: vendor/codex-contracts-eth
         env:
           MSYS2_PATH_TYPE: inherit
@@ -79,6 +79,11 @@ jobs:
           path: tests/integration/logs/
           retention-days: 1
 
+      ## Part 4 Tools ##
+      - name: Tools tests
+        if: matrix.tests == 'tools' || matrix.tests == 'all'
+        run: make -j${ncpu} testTools
+
   status:
     if: always()
     needs: [build]

.github/workflows/ci.yml

@@ -33,6 +33,7 @@ jobs:
           os {windows}, cpu {amd64}, builder {windows-latest}, tests {unittest},    nim_version {${{ env.nim_version }}}, shell {msys2}
           os {windows}, cpu {amd64}, builder {windows-latest}, tests {contract},    nim_version {${{ env.nim_version }}}, shell {msys2}
           os {windows}, cpu {amd64}, builder {windows-latest}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {msys2}
+          os {windows}, cpu {amd64}, builder {windows-latest}, tests {tools},       nim_version {${{ env.nim_version }}}, shell {msys2}
 
   build:
     needs: matrix

.github/workflows/docker-dist-tests.yml

@@ -24,7 +24,7 @@ jobs:
     name: Build and Push
     uses: ./.github/workflows/docker-reusable.yml
     with:
-      nimflags: '-d:disableMarchNative -d:codex_enable_api_debug_peers=true -d:codex_enable_proof_failures=true -d:codex_enable_log_counter=true'
+      nimflags: '-d:disableMarchNative -d:codex_enable_api_debug_peers=true -d:codex_enable_proof_failures=true -d:codex_enable_log_counter=true -d:verify_circuit=true'
       nat_ip_auto: true
       tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
       tag_suffix: dist-tests

.github/workflows/release.yml

@@ -10,9 +10,10 @@ env:
   cache_nonce: 0 # Allows for easily busting actions/cache caches
   nim_version: pinned
   rust_version: 1.78.0
-  binary_base: codex
+  codex_binary_base: codex
+  cirdl_binary_base: cirdl
   build_dir: build
-  nim_flags: '-d:verify_circuit=true'
+  nim_flags: ''
   windows_libs: 'libstdc++-6.dll libgomp-1.dll libgcc_s_seh-1.dll libwinpthread-1.dll'
 
 jobs:
@@ -70,13 +71,19 @@ jobs:
             macos*)   os_name="darwin"  ;;
             windows*) os_name="windows" ;;
           esac
-          binary="${{ env.binary_base }}-${{ github.ref_name }}-${os_name}-${{ matrix.cpu }}"
-          [[ ${os_name} == "windows" ]] && binary="${binary}.exe"
-          echo "binary=${binary}" >>$GITHUB_ENV
+          codex_binary="${{ env.codex_binary_base }}-${{ github.ref_name }}-${os_name}-${{ matrix.cpu }}"
+          cirdl_binary="${{ env.cirdl_binary_base }}-${{ github.ref_name }}-${os_name}-${{ matrix.cpu }}"
+          if [[ ${os_name} == "windows" ]]; then
+            codex_binary="${codex_binary}.exe"
+            cirdl_binary="${cirdl_binary}.exe"
+          fi
+          echo "codex_binary=${codex_binary}" >>$GITHUB_ENV
+          echo "cirdl_binary=${cirdl_binary}" >>$GITHUB_ENV
 
       - name: Release - Build
         run: |
-          make NIMFLAGS="--out:${{ env.build_dir }}/${{ env.binary }} ${{ env.nim_flags }}"
+          make NIMFLAGS="--out:${{ env.build_dir }}/${{ env.codex_binary }} ${{ env.nim_flags }}"
+          make cirdl NIMFLAGS="--out:${{ env.build_dir }}/${{ env.cirdl_binary }} ${{ env.nim_flags }}"
 
       - name: Release - Libraries
         run: |
@@ -86,11 +93,26 @@ jobs:
             done
           fi
 
-      - name: Release - Upload build artifacts
+      - name: Release - Upload codex build artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: release-${{ env.binary }}
-          path: ${{ env.build_dir }}/
+          name: release-${{ env.codex_binary }}
+          path: ${{ env.build_dir }}/${{ env.codex_binary_base }}*
+          retention-days: 1
+
+      - name: Release - Upload cirdl build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: release-${{ env.cirdl_binary }}
+          path: ${{ env.build_dir }}/${{ env.cirdl_binary_base }}*
+          retention-days: 1
+
+      - name: Release - Upload windows libs
+        if: matrix.os == 'windows'
+        uses: actions/upload-artifact@v4
+        with:
+          name: release-${{ matrix.os }}-libs
+          path: ${{ env.build_dir }}/*.dll
           retention-days: 1
 
   # Release
@@ -115,32 +137,30 @@ jobs:
           }
 
           # Compress and prepare
-          for file in *; do
-            # Exclude libraries
-            if [[ "${file}" != *".dll"* ]]; then
-              if [[ "${file}" == *".exe"* ]]; then
+          for file in ${{ env.codex_binary_base }}* ${{ env.cirdl_binary_base }}*; do
+            if [[ "${file}" == *".exe"* ]]; then
 
-                # Windows - binary only
-                arc="${file%.*}.zip"
-                zip "${arc}" "${file}"
-                checksum "${arc}"
+              # Windows - binary only
+              arc="${file%.*}.zip"
+              zip "${arc}" "${file}"
+              checksum "${arc}"
 
-                # Windows - binary and libs
-                arc="${file%.*}-libs.zip"
-                zip "${arc}" "${file}" ${{ env.windows_libs }}
-                rm -f "${file}" ${{ env.windows_libs }}
-                checksum "${arc}"
-              else
+              # Windows - binary and libs
+              arc="${file%.*}-libs.zip"
+              zip "${arc}" "${file}" ${{ env.windows_libs }}
+              rm -f "${file}"
+              checksum "${arc}"
+            else
 
-                # Linux/macOS
-                arc="${file}.tar.gz"
-                chmod 755 "${file}"
-                tar cfz "${arc}" "${file}"
-                rm -f "${file}"
-                checksum "${arc}"
-              fi
+              # Linux/macOS
+              arc="${file}.tar.gz"
+              chmod 755 "${file}"
+              tar cfz "${arc}" "${file}"
+              rm -f "${file}"
+              checksum "${arc}"
             fi
           done
+          rm -f ${{ env.windows_libs }}
 
       - name: Release - Upload compressed artifacts and checksums
         uses: actions/upload-artifact@v4

.gitmodules

@@ -215,3 +215,6 @@
 [submodule "vendor/nim-leveldbstatic"]
 	path = vendor/nim-leveldbstatic
 	url = https://github.com/codex-storage/nim-leveldb.git
+[submodule "vendor/nim-zippy"]
+	path = vendor/nim-zippy
+	url = https://github.com/status-im/nim-zippy.git

BUILDING.md

@@ -33,7 +33,7 @@ The current implementation of Codex's zero-knowledge proving circuit requires th
 On a bare bones installation of Debian (or a distribution derived from Debian, such as Ubuntu), run
 
 ```shell
-apt-get update && apt-get install build-essential cmake curl git rustc cargo
+$ apt-get update && apt-get install build-essential cmake curl git rustc cargo
 ```
 
 Non-Debian distributions have different package managers: `apk`, `dnf`, `pacman`, `rpm`, `yum`, etc.
@@ -157,6 +157,16 @@ In Bash run
 make test
 ```
 
+### Tools
+
+#### Circuit download tool
+
+To build the circuit download tool located in `tools/cirdl` run:
+
+```shell
+make cirdl
+```
+
 ### testAll
 
 #### Prerequisites

Makefile

@@ -74,6 +74,11 @@ all: | build deps
 	echo -e $(BUILD_MSG) "build/$@" && \
 		$(ENV_SCRIPT) nim codex $(NIM_PARAMS) build.nims
 
+# Build tools/cirdl
+cirdl: | deps
+	echo -e $(BUILD_MSG) "build/$@" && \
+		$(ENV_SCRIPT) nim toolsCirdl $(NIM_PARAMS) build.nims
+
 # must be included after the default target
 -include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
 
@@ -124,7 +129,12 @@ testAll: | build deps
 # Builds and runs Taiko L2 tests
 testTaiko: | build deps
 	echo -e $(BUILD_MSG) "build/$@" && \
-		$(ENV_SCRIPT) nim testTaiko $(NIM_PARAMS) codex.nims
+		$(ENV_SCRIPT) nim testTaiko $(NIM_PARAMS) build.nims
+
+# Builds and runs tool tests
+testTools: | cirdl
+	echo -e $(BUILD_MSG) "build/$@" && \
+		$(ENV_SCRIPT) nim testTools $(NIM_PARAMS) build.nims
 
 # nim-libbacktrace
 LIBBACKTRACE_MAKE_FLAGS := -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0

build.nims

@@ -1,5 +1,6 @@
 mode = ScriptMode.Verbose
 
+import std/os except commandLineParams
 
 ### Helper functions
 proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") =
@@ -14,7 +15,11 @@ proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") =
     for i in 2..<paramCount():
       extra_params &= " " & paramStr(i)
 
-  let cmd = "nim " & lang & " --out:build/" & name & " " & extra_params & " " & srcDir & name & ".nim"
+  let
+    # Place build output in 'build' folder, even if name includes a longer path.
+    outName = os.lastPathPart(name)
+    cmd = "nim " & lang & " --out:build/" & outName & " " & extra_params & " " & srcDir & name & ".nim"
+
   exec(cmd)
 
 proc test(name: string, srcDir = "tests/", params = "", lang = "c") =
@@ -24,6 +29,9 @@ proc test(name: string, srcDir = "tests/", params = "", lang = "c") =
 task codex, "build codex binary":
   buildBinary "codex", params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE"
 
+task toolsCirdl, "build tools/cirdl binary":
+  buildBinary "tools/cirdl/cirdl"
+
 task testCodex, "Build & run Codex tests":
   test "testCodex", params = "-d:codex_enable_proof_failures=true"
 
@@ -40,10 +48,15 @@ task build, "build codex binary":
 task test, "Run tests":
   testCodexTask()
 
+task testTools, "Run Tools tests":
+  toolsCirdlTask()
+  test "testTools"
+
 task testAll, "Run all tests (except for Taiko L2 tests)":
   testCodexTask()
   testContractsTask()
   testIntegrationTask()
+  testToolsTask()
 
 task testTaiko, "Run Taiko L2 tests":
   codexTask()

codex.nim

@@ -67,6 +67,9 @@ when isMainModule:
     # permissions are insecure.
     quit QuitFailure
 
+  if config.prover() and not(checkAndCreateDataDir((config.circuitDir).string)):
+    quit QuitFailure
+
   trace "Data dir initialized", dir = $config.dataDir
 
   if not(checkAndCreateDataDir((config.dataDir / "repo"))):

codex/codex.nim

@@ -273,32 +273,8 @@ proc new*(
     engine = BlockExcEngine.new(repoStore, wallet, network, blockDiscovery, advertiser, peerStore, pendingBlocks)
     store = NetworkStore.new(engine, repoStore)
     prover = if config.prover:
-      if not fileAccessible($config.circomR1cs, {AccessFlags.Read}) and
-        endsWith($config.circomR1cs, ".r1cs"):
-        error "Circom R1CS file not accessible"
-        raise (ref Defect)(
-          msg: "r1cs file not readable, doesn't exist or wrong extension (.r1cs)")
-
-      if not fileAccessible($config.circomWasm, {AccessFlags.Read}) and
-        endsWith($config.circomWasm, ".wasm"):
-        error "Circom wasm file not accessible"
-        raise (ref Defect)(
-          msg: "wasm file not readable, doesn't exist or wrong extension (.wasm)")
-
-      let zkey = if not config.circomNoZkey:
-          if not fileAccessible($config.circomZkey, {AccessFlags.Read}) and
-            endsWith($config.circomZkey, ".zkey"):
-            error "Circom zkey file not accessible"
-            raise (ref Defect)(
-              msg: "zkey file not readable, doesn't exist or wrong extension (.zkey)")
-
-          $config.circomZkey
-        else: ""
-
-      some Prover.new(
-        store,
-        CircomCompat.init($config.circomR1cs, $config.circomWasm, zkey),
-        config.numProofSamples)
+      let backend = config.initializeBackend().expect("Unable to create prover backend.")
+      some Prover.new(store, backend, config.numProofSamples)
     else:
       none Prover
 

codex/conf.nim

@@ -62,6 +62,7 @@ const
   codex_enable_log_counter* {.booldefine.} = false
 
   DefaultDataDir* = defaultDataDir()
+  DefaultCircuitDir* = defaultDataDir() / "circuits"
 
 type
   StartUpCmd* {.pure.} = enum
@@ -303,23 +304,30 @@ type
         command }: PersistenceCmd
 
       of PersistenceCmd.prover:
+        circuitDir* {.
+          desc: "Directory where Codex will store proof circuit data"
+          defaultValue: DefaultCircuitDir
+          defaultValueDesc: $DefaultCircuitDir
+          abbr: "cd"
+          name: "circuit-dir" }: OutDir
+
         circomR1cs* {.
           desc: "The r1cs file for the storage circuit"
-          defaultValue: $DefaultDataDir / "circuits" / "proof_main.r1cs"
-          defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.r1cs"
+          defaultValue: $DefaultCircuitDir / "proof_main.r1cs"
+          defaultValueDesc: $DefaultCircuitDir & "/proof_main.r1cs"
           name: "circom-r1cs"
         .}: InputFile
 
         circomWasm* {.
           desc: "The wasm file for the storage circuit"
-          defaultValue: $DefaultDataDir / "circuits" / "proof_main.wasm"
+          defaultValue: $DefaultCircuitDir / "proof_main.wasm"
           defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.wasm"
           name: "circom-wasm"
         .}: InputFile
 
         circomZkey* {.
           desc: "The zkey file for the storage circuit"
-          defaultValue: $DefaultDataDir / "circuits" / "proof_main.zkey"
+          defaultValue: $DefaultCircuitDir / "proof_main.zkey"
           defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.zkey"
           name: "circom-zkey"
         .}: InputFile

codex/slots/proofs.nim

@@ -1,4 +1,5 @@
 import ./proofs/backends
 import ./proofs/prover
+import ./proofs/backendfactory
 
-export circomcompat, prover
+export circomcompat, prover, backendfactory

codex/slots/proofs/backendfactory.nim

@@ -0,0 +1,85 @@
+import os
+import strutils
+import pkg/chronos
+import pkg/chronicles
+import pkg/questionable
+import pkg/confutils/defs
+import pkg/stew/io2
+import pkg/ethers
+
+import ../../conf
+import ./backends
+import ./backendutils
+
+proc initializeFromConfig(
+  config: CodexConf,
+  utils: BackendUtils): ?!AnyBackend =
+  if not fileAccessible($config.circomR1cs, {AccessFlags.Read}) or
+    not endsWith($config.circomR1cs, ".r1cs"):
+    return failure("Circom R1CS file not accessible")
+
+  if not fileAccessible($config.circomWasm, {AccessFlags.Read}) or
+    not endsWith($config.circomWasm, ".wasm"):
+    return failure("Circom wasm file not accessible")
+
+  if not fileAccessible($config.circomZkey, {AccessFlags.Read}) or
+    not endsWith($config.circomZkey, ".zkey"):
+    return failure("Circom zkey file not accessible")
+
+  trace "Initialized prover backend from cli config"
+  success(utils.initializeCircomBackend(
+    $config.circomR1cs,
+    $config.circomWasm,
+    $config.circomZkey))
+
+proc r1csFilePath(config: CodexConf): string =
+  config.circuitDir / "proof_main.r1cs"
+
+proc wasmFilePath(config: CodexConf): string =
+  config.circuitDir / "proof_main.wasm"
+
+proc zkeyFilePath(config: CodexConf): string =
+  config.circuitDir / "proof_main.zkey"
+
+proc initializeFromCircuitDirFiles(
+  config: CodexConf,
+  utils: BackendUtils): ?!AnyBackend =
+  if fileExists(config.r1csFilePath) and
+    fileExists(config.wasmFilePath) and
+    fileExists(config.zkeyFilePath):
+    trace "Initialized prover backend from local files"
+    return success(utils.initializeCircomBackend(
+      config.r1csFilePath,
+      config.wasmFilePath,
+      config.zkeyFilePath))
+
+  failure("Circuit files not found")
+
+proc suggestDownloadTool(config: CodexConf) =
+  without address =? config.marketplaceAddress:
+    raise (ref Defect)(msg: "Proving backend initializing while marketplace address not set.")
+
+  let
+    tokens = [
+      "cirdl",
+      "\"" & $config.circuitDir & "\"",
+      config.ethProvider,
+      $address
+    ]
+    instructions = "'./" & tokens.join(" ") & "'"
+
+  warn "Proving circuit files are not found. Please run the following to download them:", instructions
+
+proc initializeBackend*(
+  config: CodexConf,
+  utils: BackendUtils = BackendUtils()): ?!AnyBackend =
+
+  without backend =? initializeFromConfig(config, utils), cliErr:
+    info "Could not initialize prover backend from CLI options...", msg = cliErr.msg
+    without backend =? initializeFromCircuitDirFiles(config, utils), localErr:
+      info "Could not initialize prover backend from circuit dir files...", msg = localErr.msg
+      suggestDownloadTool(config)
+      return failure("CircuitFilesNotFound")
+    # Unexpected: value of backend does not survive leaving each scope. (definition does though...)
+    return success(backend)
+  return success(backend)

codex/slots/proofs/backends.nim

@@ -1,3 +1,6 @@
 import ./backends/circomcompat
 
 export circomcompat
+
+type
+  AnyBackend* = CircomCompat

codex/slots/proofs/backendutils.nim

@@ -0,0 +1,12 @@
+import ./backends
+
+type
+  BackendUtils* = ref object of RootObj
+
+method initializeCircomBackend*(
+  self: BackendUtils,
+  r1csFile: string,
+  wasmFile: string,
+  zKeyFile: string
+): AnyBackend {.base.} =
+  CircomCompat.init(r1csFile, wasmFile, zKeyFile)

codex/slots/proofs/prover.nim

@@ -21,11 +21,13 @@ import ../../merkletree
 import ../../stores
 import ../../market
 import ../../utils/poseidon2digest
+import ../../conf
 
 import ../builder
 import ../sampler
 
 import ./backends
+import ./backendfactory
 import ../types
 
 export backends
@@ -34,7 +36,6 @@ logScope:
   topics = "codex prover"
 
 type
-  AnyBackend* = CircomCompat
   AnyProof* = CircomProof
 
   AnySampler* = Poseidon2Sampler
@@ -86,7 +87,6 @@ proc verify*(
   inputs: AnyProofInputs): Future[?!bool] {.async.} =
   ## Prove a statement using backend.
   ## Returns a future that resolves to a proof.
-
   self.backend.verify(proof, inputs)
 
 proc new*(
@@ -96,6 +96,6 @@ proc new*(
   nSamples: int): Prover =
 
   Prover(
-    backend: backend,
     store: store,
+    backend: backend,
     nSamples: nSamples)

docker/codex.Dockerfile

@@ -24,9 +24,9 @@ RUN echo "export PATH=$PATH:$HOME/.cargo/bin" >> $BASH_ENV
 
 WORKDIR ${BUILD_HOME}
 COPY . .
-RUN make clean
 RUN make -j ${MAKE_PARALLEL} update
 RUN make -j ${MAKE_PARALLEL}
+RUN make -j ${MAKE_PARALLEL} cirdl
 
 # Create
 FROM ${IMAGE}
@@ -35,10 +35,10 @@ ARG APP_HOME
 ARG NAT_IP_AUTO
 
 WORKDIR ${APP_HOME}
-COPY --from=builder ${BUILD_HOME}/build/codex /usr/local/bin
+COPY --from=builder ${BUILD_HOME}/build/* /usr/local/bin
 COPY --from=builder ${BUILD_HOME}/openapi.yaml .
 COPY --from=builder --chmod=0755 ${BUILD_HOME}/docker/docker-entrypoint.sh /
-RUN apt-get update && apt-get install -y libgomp1 bash curl jq && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y libgomp1 curl jq && rm -rf /var/lib/apt/lists/*
 ENV NAT_IP_AUTO=${NAT_IP_AUTO}
 ENTRYPOINT ["/docker-entrypoint.sh"]
 CMD ["codex"]

docker/docker-entrypoint.sh

@@ -50,6 +50,21 @@ if [ -n "${PRIV_KEY}" ]; then
   echo "Private key set"
 fi
 
+# Circuit downloader
+# cirdl [circuitPath] [rpcEndpoint] [marketplaceAddress]
+if [[ "$@" == *"prover"* ]]; then
+  echo "Run Circuit downloader"
+  # Set circuits dir from CODEX_CIRCUIT_DIR variables if set
+  if [[ -z "${CODEX_CIRCUIT_DIR}" ]]; then
+    export CODEX_CIRCUIT_DIR="${CODEX_DATA_DIR}/circuits"
+  fi
+  # Download circuits
+  mkdir -p "${CODEX_CIRCUIT_DIR}"
+  chmod 700 "${CODEX_CIRCUIT_DIR}"
+  cirdl "${CODEX_CIRCUIT_DIR}" "${CODEX_ETH_PROVIDER}" "${CODEX_MARKETPLACE_ADDRESS}"
+  [[ $? -ne 0 ]] && { echo "Failed to download circuit files"; exit 1; }
+fi
+
 # Run
 echo "Run Codex node"
 exec "$@"

tests/codex/slots/testbackendfactory.nim

@@ -0,0 +1,103 @@
+import os
+import ../../asynctest
+
+import pkg/chronos
+import pkg/confutils/defs
+import pkg/codex/conf
+import pkg/codex/slots/proofs/backends
+import pkg/codex/slots/proofs/backendfactory
+import pkg/codex/slots/proofs/backendutils
+
+import ../helpers
+import ../examples
+
+type
+  BackendUtilsMock = ref object of BackendUtils
+    argR1csFile: string
+    argWasmFile: string
+    argZKeyFile: string
+
+method initializeCircomBackend*(
+  self: BackendUtilsMock,
+  r1csFile: string,
+  wasmFile: string,
+  zKeyFile: string
+): AnyBackend =
+  self.argR1csFile = r1csFile
+  self.argWasmFile = wasmFile
+  self.argZKeyFile = zKeyFile
+  # We return a backend with *something* that's not nil that we can check for.
+  var
+    key = VerifyingKey(icLen: 123)
+    vkpPtr: ptr VerifyingKey = key.addr
+  return CircomCompat(vkp: vkpPtr)
+
+suite "Test BackendFactory":
+  let
+    utilsMock = BackendUtilsMock()
+    circuitDir = "testecircuitdir"
+
+  setup:
+    createDir(circuitDir)
+
+  teardown:
+    removeDir(circuitDir)
+
+  test "Should create backend from cli config":
+    let
+      config = CodexConf(
+        cmd: StartUpCmd.persistence,
+        nat: ValidIpAddress.init("127.0.0.1"),
+        discoveryIp: ValidIpAddress.init(IPv4_any()),
+        metricsAddress: ValidIpAddress.init("127.0.0.1"),
+        persistenceCmd: PersistenceCmd.prover,
+        marketplaceAddress: EthAddress.example.some,
+        circomR1cs: InputFile("tests/circuits/fixtures/proof_main.r1cs"),
+        circomWasm: InputFile("tests/circuits/fixtures/proof_main.wasm"),
+        circomZkey: InputFile("tests/circuits/fixtures/proof_main.zkey")
+      )
+      backend = config.initializeBackend(utilsMock).tryGet
+
+    check:
+      backend.vkp != nil
+      utilsMock.argR1csFile == $config.circomR1cs
+      utilsMock.argWasmFile == $config.circomWasm
+      utilsMock.argZKeyFile == $config.circomZkey
+
+  test "Should create backend from local files":
+    let
+      config = CodexConf(
+        cmd: StartUpCmd.persistence,
+        nat: ValidIpAddress.init("127.0.0.1"),
+        discoveryIp: ValidIpAddress.init(IPv4_any()),
+        metricsAddress: ValidIpAddress.init("127.0.0.1"),
+        persistenceCmd: PersistenceCmd.prover,
+        marketplaceAddress: EthAddress.example.some,
+
+        # Set the circuitDir such that the tests/circuits/fixtures/ files
+        # will be picked up as local files:
+        circuitDir: OutDir("tests/circuits/fixtures")
+      )
+      backend = config.initializeBackend(utilsMock).tryGet
+
+    check:
+      backend.vkp != nil
+      utilsMock.argR1csFile == config.circuitDir / "proof_main.r1cs"
+      utilsMock.argWasmFile == config.circuitDir / "proof_main.wasm"
+      utilsMock.argZKeyFile == config.circuitDir / "proof_main.zkey"
+
+  test "Should suggest usage of downloader tool when files not available":
+    let
+      config = CodexConf(
+        cmd: StartUpCmd.persistence,
+        nat: ValidIpAddress.init("127.0.0.1"),
+        discoveryIp: ValidIpAddress.init(IPv4_any()),
+        metricsAddress: ValidIpAddress.init("127.0.0.1"),
+        persistenceCmd: PersistenceCmd.prover,
+        marketplaceAddress: EthAddress.example.some,
+        circuitDir: OutDir(circuitDir)
+      )
+      backendResult = config.initializeBackend(utilsMock)
+
+    check:
+      backendResult.isErr

tests/codex/slots/testprover.nim

@@ -15,6 +15,8 @@ import pkg/codex/chunker
 import pkg/codex/blocktype as bt
 import pkg/codex/slots
 import pkg/codex/stores
+import pkg/codex/conf
+import pkg/confutils/defs
 import pkg/poseidon2/io
 import pkg/codex/utils/poseidon2digest
 
@@ -29,9 +31,6 @@ suite "Test Prover":
     cellSize = DefaultCellSize
     repoTmp = TempLevelDb.new()
     metaTmp = TempLevelDb.new()
-    r1cs = "tests/circuits/fixtures/proof_main.r1cs"
-    wasm = "tests/circuits/fixtures/proof_main.wasm"
-    circomBackend = CircomCompat.init(r1cs, wasm)
     challenge = 1234567.toF.toBytes.toArray32
 
   var
@@ -42,9 +41,21 @@ suite "Test Prover":
     let
       repoDs = repoTmp.newDb()
       metaDs = metaTmp.newDb()
+      config = CodexConf(
+        cmd: StartUpCmd.persistence,
+        nat: ValidIpAddress.init("127.0.0.1"),
+        discoveryIp: ValidIpAddress.init(IPv4_any()),
+        metricsAddress: ValidIpAddress.init("127.0.0.1"),
+        persistenceCmd: PersistenceCmd.prover,
+        circomR1cs: InputFile("tests/circuits/fixtures/proof_main.r1cs"),
+        circomWasm: InputFile("tests/circuits/fixtures/proof_main.wasm"),
+        circomZkey: InputFile("tests/circuits/fixtures/proof_main.zkey"),
+        numProofSamples: samples
+      )
+      backend = config.initializeBackend().tryGet()
 
     store = RepoStore.new(repoDs, metaDs)
-    prover = Prover.new(store, circomBackend, samples)
+    prover = Prover.new(store, backend, config.numProofSamples)
 
   teardown:
     await repoTmp.destroyDb()

tests/codex/testslots.nim

@@ -3,5 +3,6 @@ import ./slots/testsampler
 import ./slots/testconverters
 import ./slots/testbackends
 import ./slots/testprover
+import ./slots/testbackendfactory
 
 {.warning[UnusedImport]: off.}

tests/integration/testcli.nim

@@ -3,6 +3,7 @@ import std/tempfiles
 import codex/conf
 import codex/utils/fileutils
 import ./nodes
+import ../examples
 
 suite "Command line interface":
 
@@ -25,26 +26,32 @@ suite "Command line interface":
     node.stop()
     discard removeFile(unsafeKeyFile)
 
-  test "complains when persistence is enabled without accessible r1cs file":
-    let node = startNode(@["persistence", "prover"])
-    node.waitUntilOutput("r1cs file not readable, doesn't exist or wrong extension (.r1cs)")
+  let
+    marketplaceArg = "--marketplace-address=" & $EthAddress.example
+    expectedDownloadInstruction = "Proving circuit files are not found. Please run the following to download them:"
+
+  test "suggests downloading of circuit files when persistence is enabled without accessible r1cs file":
+    let node = startNode(@["persistence", "prover", marketplaceArg])
+    node.waitUntilOutput(expectedDownloadInstruction)
     node.stop()
 
-  test "complains when persistence is enabled without accessible wasm file":
+  test "suggests downloading of circuit files when persistence is enabled without accessible wasm file":
     let node = startNode(@[
       "persistence",
       "prover",
+      marketplaceArg,
       "--circom-r1cs=tests/circuits/fixtures/proof_main.r1cs"
     ])
-    node.waitUntilOutput("wasm file not readable, doesn't exist or wrong extension (.wasm)")
+    node.waitUntilOutput(expectedDownloadInstruction)
     node.stop()
 
-  test "complains when persistence is enabled without accessible zkey file":
+  test "suggests downloading of circuit files when persistence is enabled without accessible zkey file":
     let node = startNode(@[
       "persistence",
       "prover",
+      marketplaceArg,
       "--circom-r1cs=tests/circuits/fixtures/proof_main.r1cs",
       "--circom-wasm=tests/circuits/fixtures/proof_main.wasm"
     ])
-    node.waitUntilOutput("zkey file not readable, doesn't exist or wrong extension (.zkey)")
+    node.waitUntilOutput(expectedDownloadInstruction)
     node.stop()

tests/testTools.nim

@@ -0,0 +1,3 @@
+import ./tools/cirdl/testcirdl
+
+{.warning[UnusedImport]:off.}

tests/tools/cirdl/testcirdl.nim

@@ -0,0 +1,39 @@
+import std/os
+import std/osproc
+import std/options
+import pkg/chronos
+import codex/contracts
+import ../../integration/marketplacesuite
+
+marketplacesuite "tools/cirdl":
+  const
+    cirdl = "build" / "cirdl"
+    workdir = "."
+
+  test "circuit download tool":
+    let
+      circuitPath = "testcircuitpath"
+      rpcEndpoint = "ws://localhost:8545"
+      marketplaceAddress = $marketplace.address
+
+    discard existsOrCreateDir(circuitPath)
+
+    let args = [circuitPath, rpcEndpoint, marketplaceAddress]
+
+    let process = osproc.startProcess(
+      cirdl,
+      workdir,
+      args,
+      options={poParentStreams}
+    )
+
+    let returnCode = process.waitForExit()
+    check returnCode == 0
+
+    check:
+      fileExists(circuitPath/"proof_main_verification_key.json")
+      fileExists(circuitPath/"proof_main.r1cs")
+      fileExists(circuitPath/"proof_main.wasm")
+      fileExists(circuitPath/"proof_main.zkey")
+
+    removeDir(circuitPath)

tools/cirdl/cirdl.nim

@@ -0,0 +1,128 @@
+import std/os
+import std/streams
+import pkg/chronicles
+import pkg/chronos
+import pkg/ethers
+import pkg/questionable
+import pkg/questionable/results
+import pkg/zippy/tarballs
+import pkg/chronos/apps/http/httpclient
+import ../../codex/contracts/marketplace
+
+proc consoleLog(logLevel: LogLevel, msg: LogOutputStr) {.gcsafe.} =
+  try:
+    stdout.write(msg)
+    stdout.flushFile()
+  except IOError as err:
+    logLoggingFailure(cstring(msg), err)
+
+proc noOutput(logLevel: LogLevel, msg: LogOutputStr) = discard
+
+defaultChroniclesStream.outputs[0].writer = consoleLog
+defaultChroniclesStream.outputs[1].writer = noOutput
+defaultChroniclesStream.outputs[2].writer = noOutput
+
+proc printHelp() =
+  info "Usage: ./cirdl [circuitPath] [rpcEndpoint] [marketplaceAddress]"
+  info "  circuitPath: path where circuit files will be placed."
+  info "  rpcEndpoint: URL of web3 RPC endpoint."
+  info "  marketplaceAddress: Address of deployed Codex marketplace contracts."
+
+proc getCircuitHash(rpcEndpoint: string, marketplaceAddress: string): Future[?!string] {.async.} =
+  let provider = JsonRpcProvider.new(rpcEndpoint)
+  without address =? Address.init(marketplaceAddress):
+    return failure("Invalid address: " & marketplaceAddress)
+
+  let marketplace = Marketplace.new(address, provider)
+  let config = await marketplace.config()
+  return success config.proofs.zkeyHash
+
+proc formatUrl(hash: string): string =
+  "https://circuit.codex.storage/proving-key/" & hash
+
+proc retrieveUrl(uri: string): Future[seq[byte]] {.async.} =
+  let httpSession = HttpSessionRef.new()
+  try:
+    let resp = await httpSession.fetch(parseUri(uri))
+    return resp.data
+  finally:
+    await noCancel(httpSession.closeWait())
+
+proc downloadZipfile(url: string, filepath: string): Future[?!void] {.async.} =
+  try:
+    let file = await retrieveUrl(url)
+    var s = newFileStream(filepath, fmWrite)
+    for b in file:
+      s.write(b)
+    s.close()
+  except Exception as exc:
+    return failure(exc.msg)
+  success()
+
+proc unzip(zipfile: string, targetPath: string): ?!void =
+  try:
+    extractAll(zipfile, targetPath)
+  except Exception as exc:
+    return failure(exc.msg)
+  success()
+
+proc copyFiles(unpackDir: string, circuitPath: string): ?!void =
+  try:
+    for file in walkDir(unpackDir):
+      copyFileToDir(file.path, circuitPath)
+  except Exception as exc:
+    return failure(exc.msg)
+  success()
+
+proc main() {.async.} =
+  info "Codex Circuit Downloader, Aww yeah!"
+  let args = os.commandLineParams()
+  if args.len != 3:
+    printHelp()
+    return
+
+  let
+    circuitPath = args[0]
+    rpcEndpoint = args[1]
+    marketplaceAddress = args[2]
+    zipfile = "circuit.tar.gz"
+    unpackFolder = "." / "tempunpackfolder"
+
+  debug "Starting", circuitPath, rpcEndpoint, marketplaceAddress
+
+  if (dirExists(unpackFolder)):
+    removeDir(unpackFolder)
+
+  without circuitHash =? (await getCircuitHash(rpcEndpoint, marketplaceAddress)), err:
+    error "Failed to get circuit hash", msg = err.msg
+    return
+  debug "Got circuithash", circuitHash
+
+  let url = formatUrl(circuitHash)
+  if dlErr =? (await downloadZipfile(url, zipfile)).errorOption:
+    error "Failed to download circuit file", msg = dlErr.msg
+    return
+  debug "Download completed"
+
+  if err =? unzip(zipfile, unpackFolder).errorOption:
+    error "Failed to unzip file", msg = err.msg
+    return
+  debug "Unzip completed"
+
+  # Unpack library cannot unpack into existing directory. We also cannot
+  # delete the targer directory and have the library recreate it because
+  # Codex has likely created it and set correct permissions.
+  # So, we unpack to a temp folder and move the files.
+  if err =? copyFiles(unpackFolder, circuitPath).errorOption:
+    error "Failed to copy files", msg = err.msg
+    return
+  debug "Files copied"
+
+  removeFile(zipfile)
+  removeDir(unpackFolder)
+
+  debug "file and unpack folder removed"
+
+when isMainModule:
+  waitFor main()
+  info "Done!"

vendor/codex-contracts-eth

@@ -1 +1 @@
-Subproject commit 73a2ca0bd3ba90715ea7e818bafbd82208034a58
+Subproject commit 558bf645c3dc385437a3e695bba57e7dba1375fb

vendor/nim-circom-compat

@@ -1 +1 @@
-Subproject commit 4467e310b75aa0749ff28c1572a84ffce57d7c1c
+Subproject commit e710e4c333f367353aaa1ee82a55a47326b63a65

vendor/nim-zippy

@@ -0,0 +1 @@
+Subproject commit 8d6828f090325c5c015f66a438d31aa592a7045d

vendor/nimbus-build-system

@@ -1 +1 @@
-Subproject commit b2e1fb022f1ee800b439648953e92cc993c1264c
+Subproject commit fe9bc3f3759ae1add6bf8c899db2e75327f03782