Merge branch 'feature/ceremony-files' into async-profiling

.gitmodules

@@ -209,3 +209,6 @@
 	url = https://github.com/codex-storage/codex-storage-proofs-circuits.git
 	ignore = untracked
 	branch = master
+[submodule "vendor/zip"]
+	path = vendor/zip
+	url = https://github.com/nim-lang/zip.git

BUILDING.md

@@ -33,7 +33,7 @@ The current implementation of Codex's zero-knowledge proving circuit requires th
 On a bare bones installation of Debian (or a distribution derived from Debian, such as Ubuntu), run
 
 ```shell
-apt-get update && apt-get install build-essential cmake curl git rustc cargo
+$ apt-get update && apt-get install build-essential cmake curl git rustc cargo libzip-dev
 ```
 
 Non-Debian distributions have different package managers: `apk`, `dnf`, `pacman`, `rpm`, `yum`, etc.
@@ -41,7 +41,7 @@ Non-Debian distributions have different package managers: `apk`, `dnf`, `pacman`
 For example, on a bare bones installation of Fedora, run
 
 ```shell
-dnf install @development-tools cmake gcc-c++ rust cargo
+dnf install @development-tools cmake gcc-c++ libzip rust cargo
 ```
 
 ### macOS
@@ -53,7 +53,7 @@ xcode-select --install
 
 Install [Homebrew (`brew`)](https://brew.sh/) and in a new terminal run
 ```shell
-brew install bash cmake rust
+brew install bash cmake rust libzip
 ```
 
 Check that `PATH` is setup correctly

codex/codex.nim

@@ -23,6 +23,7 @@ import pkg/stew/shims/net as stewnet
 import pkg/datastore
 import pkg/ethers except Rng
 import pkg/stew/io2
+import pkg/questionable
 
 import ./node
 import ./conf
@@ -66,7 +67,7 @@ proc waitForSync(provider: Provider): Future[void] {.async.} =
       inc sleepTime
 
 proc bootstrapInteractions(
-  s: CodexServer): Future[void] {.async.} =
+  s: CodexServer): Future[?string] {.async.} =
   ## bootstrap interactions and return contracts
   ## using clients, hosts, validators pairings
   ##
@@ -139,6 +140,7 @@ proc bootstrapInteractions(
       validator = some ValidatorInteractions.new(clock, validation)
 
     s.codexNode.contracts = (client, host, validator)
+    return await market.getZkeyHash()
 
 proc start*(s: CodexServer) {.async.} =
   trace "Starting codex node", config = $s.config
@@ -173,7 +175,13 @@ proc start*(s: CodexServer) {.async.} =
   s.codexNode.discovery.updateAnnounceRecord(announceAddrs)
   s.codexNode.discovery.updateDhtRecord(s.config.nat, s.config.discoveryPort)
 
-  await s.bootstrapInteractions()
+  let proofCeremonyUrl = await s.bootstrapInteractions()
+
+  if prover =? s.codexNode.prover:
+    if err =? (await prover.start(s.config, proofCeremonyUrl)).errorOption:
+      error "Failed to start prover", msg = err.msg
+      return # should we abort start-up this way?
+
   await s.codexNode.start()
   s.restServer.start()
 
@@ -261,31 +269,8 @@ proc new*(
     engine = BlockExcEngine.new(repoStore, wallet, network, blockDiscovery, peerStore, pendingBlocks)
     store = NetworkStore.new(engine, repoStore)
     prover = if config.prover:
-      if not fileAccessible($config.circomR1cs, {AccessFlags.Read}) and
-        endsWith($config.circomR1cs, ".r1cs"):
-        error "Circom R1CS file not accessible"
-        raise (ref Defect)(
-          msg: "r1cs file not readable, doesn't exist or wrong extension (.r1cs)")
-
-      if not fileAccessible($config.circomWasm, {AccessFlags.Read}) and
-        endsWith($config.circomWasm, ".wasm"):
-        error "Circom wasm file not accessible"
-        raise (ref Defect)(
-          msg: "wasm file not readable, doesn't exist or wrong extension (.wasm)")
-
-      let zkey = if not config.circomNoZkey:
-          if not fileAccessible($config.circomZkey, {AccessFlags.Read}) and
-            endsWith($config.circomZkey, ".zkey"):
-            error "Circom zkey file not accessible"
-            raise (ref Defect)(
-              msg: "zkey file not readable, doesn't exist or wrong extension (.zkey)")
-
-          $config.circomZkey
-        else: ""
-
       some Prover.new(
         store,
-        CircomCompat.init($config.circomR1cs, $config.circomWasm, zkey),
         config.numProofSamples)
     else:
       none Prover

codex/node.nim

@@ -64,7 +64,7 @@ type
     networkId: PeerId
     networkStore: NetworkStore
     engine: BlockExcEngine
-    prover: ?Prover
+    prover*: ?Prover
     discovery: Discovery
     contracts*: Contracts
     clock*: Clock

codex/slots/proofs/backendfactory.nim

@@ -0,0 +1,106 @@
+import os
+import pkg/chronos
+import pkg/chronicles
+import pkg/questionable
+import pkg/confutils/defs
+import pkg/stew/io2
+
+import ../../conf
+import ./backends
+import ./backendutils
+
+proc initializeFromConfig(
+  config: CodexConf,
+  utils: BackendUtils): ?!AnyBackend =
+  if not fileAccessible($config.circomR1cs, {AccessFlags.Read}) or
+    not endsWith($config.circomR1cs, ".r1cs"):
+    return failure("Circom R1CS file not accessible")
+
+  if not fileAccessible($config.circomWasm, {AccessFlags.Read}) or
+    not endsWith($config.circomWasm, ".wasm"):
+    return failure("Circom wasm file not accessible")
+
+  if not fileAccessible($config.circomZkey, {AccessFlags.Read}) or
+    not endsWith($config.circomZkey, ".zkey"):
+    return failure("Circom zkey file not accessible")
+
+  trace "Initialized prover backend from cli config"
+  success(utils.initializeCircomBackend(
+    $config.circomR1cs,
+    $config.circomWasm,
+    $config.circomZkey))
+
+proc r1csFilePath(config: CodexConf): string =
+  config.dataDir / "proof_main.r1cs"
+
+proc wasmFilePath(config: CodexConf): string =
+  config.dataDir / "proof_main.wasm"
+
+proc zkeyFilePath(config: CodexConf): string =
+  config.dataDir / "proof_main.zkey"
+
+proc zipFilePath(config: CodexConf): string =
+  config.dataDir / "circuit.zip"
+
+proc initializeFromCeremonyFiles(
+  config: CodexConf,
+  utils: BackendUtils): ?!AnyBackend =
+  if fileExists(config.r1csFilePath) and
+    fileExists(config.wasmFilePath) and
+    fileExists(config.zkeyFilePath):
+    trace "Initialized prover backend from local files"
+    return success(utils.initializeCircomBackend(
+      config.r1csFilePath,
+      config.wasmFilePath,
+      config.zkeyFilePath))
+
+  failure("Ceremony files not found")
+
+proc downloadCeremony(
+  config: CodexConf,
+  ceremonyHash: string,
+  utils: BackendUtils
+): ?!void =
+  # TODO:
+  # In the future, the zip file will be stored in the Codex network
+  # instead of a url + ceremonyHash, we'll get a CID from the marketplace contract.
+
+  let url = "https://circuit.codex.storage/proving-key/" & ceremonyHash
+  trace "Downloading ceremony file", url, filepath = config.zipFilePath
+  return utils.downloadFile(url, config.zipFilePath)
+
+proc unzipCeremonyFile(
+  config: CodexConf,
+  utils: BackendUtils): ?!void =
+  trace "Unzipping..."
+  return utils.unzipFile(config.zipFilePath, $config.dataDir)
+
+proc initializeFromCeremonyHash(
+  config: CodexConf,
+  ceremonyHash: ?string,
+  utils: BackendUtils): Future[?!AnyBackend] {.async.} =
+
+  if hash =? ceremonyHash:
+    if dlErr =? downloadCeremony(config, hash, utils).errorOption:
+      return failure(dlErr)
+    if err =? unzipCeremonyFile(config, utils).errorOption:
+      return failure(err)
+    without backend =? initializeFromCeremonyFiles(config, utils), err:
+      return failure(err)
+    return success(backend)
+  else:
+    return failure("Ceremony URL not found")
+
+proc initializeBackend*(
+  config: CodexConf,
+  ceremonyHash: ?string,
+  utils: BackendUtils = BackendUtils()): Future[?!AnyBackend] {.async.} =
+
+  without backend =? initializeFromConfig(config, utils), cliErr:
+    info "Could not initialize prover backend from CLI options...", msg = cliErr.msg
+    without backend =? initializeFromCeremonyFiles(config, utils), localErr:
+      info "Could not initialize prover backend from local files...", msg = localErr.msg
+      without backend =? (await initializeFromCeremonyHash(config, ceremonyHash, utils)), urlErr:
+        warn "Could not initialize prover backend from ceremony url...", msg = urlErr.msg
+        return failure(urlErr)
+  return success(backend)

codex/slots/proofs/backends.nim

@@ -1,3 +1,6 @@
 import ./backends/circomcompat
 
 export circomcompat
+
+type
+  AnyBackend* = CircomCompat

codex/slots/proofs/backendutils.nim

@@ -0,0 +1,45 @@
+import os
+import zip/zipfiles
+import pkg/chronos
+import pkg/chronicles
+import pkg/questionable
+import pkg/questionable/results
+
+import ./backends
+
+type
+  BackendUtils* = ref object of RootObj
+
+method initializeCircomBackend*(
+  self: BackendUtils,
+  r1csFile: string,
+  wasmFile: string,
+  zKeyFile: string
+): AnyBackend {.base.} =
+  CircomCompat.init(r1csFile, wasmFile, zKeyFile)
+
+method downloadFile*(
+  self: BackendUtils,
+  url: string,
+  filepath: string
+): ?!void {.base.} =
+  try:
+    # Nim's default webclient does not support SSL on all platforms.
+    # Not without shipping additional binaries and cert-files... :(
+    # So we're using curl for now.
+    var rc = execShellCmd("curl -o " & filepath & " " & url)
+    if not rc == 0:
+      return failure("Download of '" & url & "' failed with return code: " & $rc)
+  except Exception as exc:
+    return failure(exc.msg)
+  success()
+
+method unzipFile*(
+  self: BackendUtils,
+  zipFile: string,
+  outputDir: string): ?!void {.base.} =
+  var z: ZipArchive
+  if not z.open(zipFile):
+    return failure("Unable to open zip file: " & zipFile)
+  z.extractAll(outputDir)
+  success()

codex/slots/proofs/prover.nim

@@ -21,11 +21,13 @@ import ../../merkletree
 import ../../stores
 import ../../market
 import ../../utils/poseidon2digest
+import ../../conf
 
 import ../builder
 import ../sampler
 
 import ./backends
+import ./backendfactory
 import ../types
 
 export backends
@@ -34,7 +36,6 @@ logScope:
   topics = "codex prover"
 
 type
-  AnyBackend* = CircomCompat
   AnyProof* = CircomProof
 
   AnySampler* = Poseidon2Sampler
@@ -42,7 +43,7 @@ type
 
   AnyProofInputs* = ProofInputs[Poseidon2Hash]
   Prover* = ref object of RootObj
-    backend: AnyBackend
+    backend: ?AnyBackend
     store: BlockStore
     nSamples: int
 
@@ -61,24 +62,27 @@ proc prove*(
 
   trace "Received proof challenge"
 
-  without builder =? AnyBuilder.new(self.store, manifest), err:
+  if backend =? self.backend:
-    error "Unable to create slots builder", err = err.msg
+    without builder =? AnyBuilder.new(self.store, manifest), err:
-    return failure(err)
+      error "Unable to create slots builder", err = err.msg
+      return failure(err)
 
-  without sampler =? AnySampler.new(slotIdx, self.store, builder), err:
+    without sampler =? AnySampler.new(slotIdx, self.store, builder), err:
-    error "Unable to create data sampler", err = err.msg
+      error "Unable to create data sampler", err = err.msg
-    return failure(err)
+      return failure(err)
 
-  without proofInput =? await sampler.getProofInput(challenge, self.nSamples), err:
+    without proofInput =? await sampler.getProofInput(challenge, self.nSamples), err:
-    error "Unable to get proof input for slot", err = err.msg
+      error "Unable to get proof input for slot", err = err.msg
-    return failure(err)
+      return failure(err)
 
-  # prove slot
+    # prove slot
-  without proof =? self.backend.prove(proofInput), err:
+    without proof =? backend.prove(proofInput), err:
-    error "Unable to prove slot", err = err.msg
+      error "Unable to prove slot", err = err.msg
-    return failure(err)
+      return failure(err)
 
-  success (proofInput, proof)
+    success (proofInput, proof)
+  else:
+    return failure("Prover was not started")
 
 proc verify*(
   self: Prover,
@@ -87,15 +91,29 @@ proc verify*(
   ## Prove a statement using backend.
   ## Returns a future that resolves to a proof.
 
-  self.backend.verify(proof, inputs)
+  if backend =? self.backend:
+    return backend.verify(proof, inputs)
+  else:
+    return failure("Prover was not started")
+
+proc start*(
+  self: Prover,
+  config: CodexConf,
+  ceremonyHash: ?string): Future[?!void] {.async.} =
+
+  without backend =? (await initializeBackend(config, ceremonyHash)), err:
+    error "Failed to initialize backend", msg = err.msg
+    return failure(err)
+
+  self.backend = some backend
+  return success()
 
 proc new*(
   _: type Prover,
   store: BlockStore,
-  backend: AnyBackend,
   nSamples: int): Prover =
 
   Prover(
-    backend: backend,
     store: store,
+    backend: none AnyBackend,
     nSamples: nSamples)

docker/codex.Dockerfile

@@ -30,7 +30,7 @@ ARG NAT_IP_AUTO
 WORKDIR ${APP_HOME}
 COPY --from=builder ${BUILD_HOME}/build/codex /usr/local/bin
 COPY --chmod=0755 docker/docker-entrypoint.sh /
-RUN apt-get update && apt-get install -y libgomp1 bash curl jq && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y libgomp1 bash curl jq libzip-dev && rm -rf /var/lib/apt/lists/*
 ENV NAT_IP_AUTO=${NAT_IP_AUTO}
 ENTRYPOINT ["/docker-entrypoint.sh"]
 CMD ["codex"]

tests/codex/slots/testbackendfactory.nim

@@ -0,0 +1,144 @@
+import os
+import std/strutils
+import std/sugar
+import std/math
+
+import ../../asynctest
+
+import pkg/chronos
+import pkg/confutils/defs
+import pkg/codex/conf
+import pkg/codex/slots/proofs/backends
+import pkg/codex/slots/proofs/backendfactory
+import pkg/codex/slots/proofs/backendutils
+
+import ./helpers
+import ../helpers
+
+type
+  BackendUtilsMock = ref object of BackendUtils
+    argR1csFile: string
+    argWasmFile: string
+    argZKeyFile: string
+    argUrl: string
+    argFilepath: string
+    argZipFile: string
+    argOutputDir: string
+
+method initializeCircomBackend*(
+  self: BackendUtilsMock,
+  r1csFile: string,
+  wasmFile: string,
+  zKeyFile: string
+): AnyBackend =
+  self.argR1csFile = r1csFile
+  self.argWasmFile = wasmFile
+  self.argZKeyFile = zKeyFile
+
+method downloadFile*(
+  self: BackendUtilsMock,
+  url: string,
+  filepath: string
+): ?!void =
+  self.argUrl = url
+  self.argFilepath = filepath
+  success()
+
+method unzipFile*(
+  self: BackendUtilsMock,
+  zipFile: string,
+  outputDir: string): ?!void =
+  self.argZipFile = zipFile
+  self.argOutputDir = outputDir
+  try:
+    writeFile(outputDir / "proof_main.r1cs", "r1cs_file")
+    writeFile(outputDir / "proof_main.wasm", "wasm_file")
+    writeFile(outputDir / "proof_main.zkey", "zkey_file")
+  except Exception as exc:
+    return failure(exc.msg)
+  success()
+
+suite "Test BackendFactory":
+  let
+    utilsMock = BackendUtilsMock()
+    datadir = "testdatadir"
+
+  setup:
+    createDir(datadir)
+
+  teardown:
+    removeDir(datadir)
+
+  test "Should create backend from cli config":
+    let
+      config = CodexConf(
+        cmd: StartUpCmd.persistence,
+        nat: ValidIpAddress.init("127.0.0.1"),
+        discoveryIp: ValidIpAddress.init(IPv4_any()),
+        metricsAddress: ValidIpAddress.init("127.0.0.1"),
+        persistenceCmd: PersistenceCmd.prover,
+        circomR1cs: InputFile("tests/circuits/fixtures/proof_main.r1cs"),
+        circomWasm: InputFile("tests/circuits/fixtures/proof_main.wasm"),
+        circomZkey: InputFile("tests/circuits/fixtures/proof_main.zkey")
+      )
+      ceremonyHash = string.none
+      backend = (await initializeBackend(config, ceremonyHash, utilsMock)).tryGet
+
+    check:
+      utilsMock.argR1csFile == $config.circomR1cs
+      utilsMock.argWasmFile == $config.circomWasm
+      utilsMock.argZKeyFile == $config.circomZkey
+      isEmptyOrWhitespace(utilsMock.argUrl)
+      isEmptyOrWhitespace(utilsMock.argFilepath)
+      isEmptyOrWhitespace(utilsMock.argZipFile)
+      isEmptyOrWhitespace(utilsMock.argOutputDir)
+
+  test "Should create backend from local files":
+    let
+      config = CodexConf(
+        cmd: StartUpCmd.persistence,
+        nat: ValidIpAddress.init("127.0.0.1"),
+        discoveryIp: ValidIpAddress.init(IPv4_any()),
+        metricsAddress: ValidIpAddress.init("127.0.0.1"),
+        persistenceCmd: PersistenceCmd.prover,
+
+        # Set the datadir such that the tests/circuits/fixtures/ files
+        # will be picked up as local files:
+        dataDir: OutDir("tests/circuits/fixtures")
+      )
+      ceremonyHash = string.none
+      backend = (await initializeBackend(config, ceremonyHash, utilsMock)).tryGet
+
+    check:
+      utilsMock.argR1csFile == config.dataDir / "proof_main.r1cs"
+      utilsMock.argWasmFile == config.dataDir / "proof_main.wasm"
+      utilsMock.argZKeyFile == config.dataDir / "proof_main.zkey"
+      isEmptyOrWhitespace(utilsMock.argUrl)
+      isEmptyOrWhitespace(utilsMock.argFilepath)
+      isEmptyOrWhitespace(utilsMock.argZipFile)
+      isEmptyOrWhitespace(utilsMock.argOutputDir)
+
+  test "Should download and unzip ceremony file if not available":
+    let
+      ceremonyHash = some "12345"
+      expectedZip = datadir / "circuit.zip"
+      expectedUrl = "https://circuit.codex.storage/proving-key/" & !ceremonyHash
+      config = CodexConf(
+        cmd: StartUpCmd.persistence,
+        nat: ValidIpAddress.init("127.0.0.1"),
+        discoveryIp: ValidIpAddress.init(IPv4_any()),
+        metricsAddress: ValidIpAddress.init("127.0.0.1"),
+        persistenceCmd: PersistenceCmd.prover,
+        dataDir: OutDir(datadir)
+      )
+
+      backend = (await initializeBackend(config, ceremonyHash, utilsMock)).tryGet
+
+    check:
+      utilsMock.argR1csFile == config.dataDir / "proof_main.r1cs"
+      utilsMock.argWasmFile == config.dataDir / "proof_main.wasm"
+      utilsMock.argZKeyFile == config.dataDir / "proof_main.zkey"
+      utilsMock.argUrl == expectedUrl
+      utilsMock.argFilepath == expectedZip
+      utilsMock.argZipFile == expectedZip
+      utilsMock.argOutputDir == datadir

tests/codex/slots/testprover.nim

@@ -15,6 +15,8 @@ import pkg/codex/chunker
 import pkg/codex/blocktype as bt
 import pkg/codex/slots
 import pkg/codex/stores
+import pkg/codex/conf
+import pkg/confutils/defs
 import pkg/poseidon2/io
 import pkg/codex/utils/poseidon2digest
 
@@ -57,13 +59,23 @@ suite "Test Prover":
 
   test "Should sample and prove a slot":
     let
-      r1cs = "tests/circuits/fixtures/proof_main.r1cs"
+      prover = Prover.new(store, samples)
-      wasm = "tests/circuits/fixtures/proof_main.wasm"
-
-      circomBackend = CircomCompat.init(r1cs, wasm)
-      prover = Prover.new(store, circomBackend, samples)
       challenge = 1234567.toF.toBytes.toArray32
-      (inputs, proof) = (await prover.prove(1, verifiable, challenge)).tryGet
+      config = CodexConf(
+        cmd: StartUpCmd.persistence,
+        nat: ValidIpAddress.init("127.0.0.1"),
+        discoveryIp: ValidIpAddress.init(IPv4_any()),
+        metricsAddress: ValidIpAddress.init("127.0.0.1"),
+        persistenceCmd: PersistenceCmd.prover,
+        circomR1cs: InputFile("tests/circuits/fixtures/proof_main.r1cs"),
+        circomWasm: InputFile("tests/circuits/fixtures/proof_main.wasm"),
+        circomZkey: InputFile("tests/circuits/fixtures/proof_main.zkey")
+      )
+      ceremonyHash = string.none
+
+    (await prover.start(config, ceremonyHash)).tryGet()
+
+    let (inputs, proof) = (await prover.prove(1, verifiable, challenge)).tryGet
 
     check:
       (await prover.verify(proof, inputs)).tryGet == true

tests/codex/testslots.nim

@@ -3,5 +3,6 @@ import ./slots/testsampler
 import ./slots/testconverters
 import ./slots/testbackends
 import ./slots/testprover
+import ./slots/testbackendfactory
 
 {.warning[UnusedImport]: off.}

tests/integration/testcli.nim

@@ -24,27 +24,3 @@ suite "Command line interface":
     node.waitUntilOutput("Ethereum private key file does not have safe file permissions")
     node.stop()
     discard removeFile(unsafeKeyFile)
-
-  test "complains when persistence is enabled without accessible r1cs file":
-    let node = startNode(@["persistence", "prover"])
-    node.waitUntilOutput("r1cs file not readable, doesn't exist or wrong extension (.r1cs)")
-    node.stop()
-
-  test "complains when persistence is enabled without accessible wasm file":
-    let node = startNode(@[
-      "persistence",
-      "prover",
-      "--circom-r1cs=tests/circuits/fixtures/proof_main.r1cs"
-    ])
-    node.waitUntilOutput("wasm file not readable, doesn't exist or wrong extension (.wasm)")
-    node.stop()
-
-  test "complains when persistence is enabled without accessible zkey file":
-    let node = startNode(@[
-      "persistence",
-      "prover",
-      "--circom-r1cs=tests/circuits/fixtures/proof_main.r1cs",
-      "--circom-wasm=tests/circuits/fixtures/proof_main.wasm"
-    ])
-    node.waitUntilOutput("zkey file not readable, doesn't exist or wrong extension (.zkey)")
-    node.stop()

vendor/codex-contracts-eth

@@ -1 +1 @@
-Subproject commit 118ee0b22b2d12c8fbf3376cf201d203d0a7cf97
+Subproject commit c3d7db345649d8a2dafabcede90edf5ff6b0bfc7

vendor/zip

@@ -0,0 +1 @@
+Subproject commit 06f5b0a0767b14c7595ed168611782be69e61543