wip rework cli to support circuit params

codex.nim

@@ -88,7 +88,11 @@ when isMainModule:
         config.dataDir / config.netPrivKeyFile
 
     privateKey = setupKey(keyPath).expect("Should setup private key!")
-    server = CodexServer.new(config, privateKey)
+    server = try:
+      CodexServer.new(config, privateKey)
+    except Exception as exc:
+      error "Failed to start Codex", msg = exc.msg
+      quit QuitFailure
 
   ## Ctrl+C handling
   proc controlCHandler() {.noconv.} =
@@ -125,8 +129,12 @@ when isMainModule:
 
   state = CodexStatus.Running
   while state == CodexStatus.Running:
-    # poll chronos
+    try:
-    chronos.poll()
+      # poll chronos
+      chronos.poll()
+    except Exception as exc:
+      error "Unhandled exception in async proc, aborting", msg = exc.msg
+      quit QuitFailure
 
   # wait fot futures to finish
   let res = waitFor allFinished(pendingFuts)

codex/codex.nim

@@ -22,6 +22,7 @@ import pkg/stew/io2
 import pkg/stew/shims/net as stewnet
 import pkg/datastore
 import pkg/ethers except Rng
+import pkg/stew/io2
 
 import ./node
 import ./conf
@@ -66,77 +67,70 @@ proc bootstrapInteractions(
     config = s.config
     repo = s.repoStore
 
-  if not config.persistence and not config.validator:
-    if config.ethAccount.isSome or config.ethPrivateKey.isSome:
-      warn "Ethereum account was set, but neither persistence nor validator is enabled"
-    return
-
-  if not config.ethAccount.isSome and not config.ethPrivateKey.isSome:
-    if config.persistence:
-      error "Persistence enabled, but no Ethereum account was set"
-    if config.validator:
-      error "Validator enabled, but no Ethereum account was set"
-    quit QuitFailure
-
-  let provider = JsonRpcProvider.new(config.ethProvider)
-  var signer: Signer
-  if account =? config.ethAccount:
-    signer = provider.getSigner(account)
-  elif keyFile =? config.ethPrivateKey:
-    without isSecure =? checkSecureFile(keyFile):
-      error "Could not check file permissions: does Ethereum private key file exist?"
-      quit QuitFailure
-    if not isSecure:
-      error "Ethereum private key file does not have safe file permissions"
-      quit QuitFailure
-    without key =? keyFile.readAllChars():
-      error "Unable to read Ethereum private key file"
-      quit QuitFailure
-    without wallet =? EthWallet.new(key.strip(), provider):
-      error "Invalid Ethereum private key in file"
-      quit QuitFailure
-    signer = wallet
-
-  let deploy = Deployment.new(provider, config)
-  without marketplaceAddress =? await deploy.address(Marketplace):
-    error "No Marketplace address was specified or there is no known address for the current network"
-    quit QuitFailure
-
-  let marketplace = Marketplace.new(marketplaceAddress, signer)
-  let market = OnChainMarket.new(marketplace)
-  let clock = OnChainClock.new(provider)
-
-  var client: ?ClientInteractions
-  var host: ?HostInteractions
-  var validator: ?ValidatorInteractions
-
-  if config.validator or config.persistence:
-    s.codexNode.clock = clock
-  else:
-    s.codexNode.clock = SystemClock()
-
   if config.persistence:
-    # This is used for simulation purposes. Normal nodes won't be compiled with this flag
+    if not config.ethAccount.isSome and not config.ethPrivateKey.isSome:
-    # and hence the proof failure will always be 0.
+      error "Persistence enabled, but no Ethereum account was set"
-    when codex_enable_proof_failures:
+      quit QuitFailure
-      let proofFailures = config.simulateProofFailures
+
-      if proofFailures > 0:
+    let provider = JsonRpcProvider.new(config.ethProvider)
-        warn "Enabling proof failure simulation!"
+    var signer: Signer
+    if account =? config.ethAccount:
+      signer = provider.getSigner(account)
+    elif keyFile =? config.ethPrivateKey:
+      without isSecure =? checkSecureFile(keyFile):
+        error "Could not check file permissions: does Ethereum private key file exist?"
+        quit QuitFailure
+      if not isSecure:
+        error "Ethereum private key file does not have safe file permissions"
+        quit QuitFailure
+      without key =? keyFile.readAllChars():
+        error "Unable to read Ethereum private key file"
+        quit QuitFailure
+      without wallet =? EthWallet.new(key.strip(), provider):
+        error "Invalid Ethereum private key in file"
+        quit QuitFailure
+      signer = wallet
+
+    let deploy = Deployment.new(provider, config)
+    without marketplaceAddress =? await deploy.address(Marketplace):
+      error "No Marketplace address was specified or there is no known address for the current network"
+      quit QuitFailure
+
+    let marketplace = Marketplace.new(marketplaceAddress, signer)
+    let market = OnChainMarket.new(marketplace)
+    let clock = OnChainClock.new(provider)
+
+    var client: ?ClientInteractions
+    var host: ?HostInteractions
+    var validator: ?ValidatorInteractions
+
+    if config.validator or config.persistence:
+      s.codexNode.clock = clock
     else:
-      let proofFailures = 0
+      s.codexNode.clock = SystemClock()
-      if config.simulateProofFailures > 0:
-        warn "Proof failure simulation is not enabled for this build! Configuration ignored"
 
-    let purchasing = Purchasing.new(market, clock)
+    if config.persistence:
-    let sales = Sales.new(market, clock, repo, proofFailures)
+      # This is used for simulation purposes. Normal nodes won't be compiled with this flag
-    client = some ClientInteractions.new(clock, purchasing)
+      # and hence the proof failure will always be 0.
-    host = some HostInteractions.new(clock, sales)
+      when codex_enable_proof_failures:
+        let proofFailures = config.simulateProofFailures
+        if proofFailures > 0:
+          warn "Enabling proof failure simulation!"
+      else:
+        let proofFailures = 0
+        if config.simulateProofFailures > 0:
+          warn "Proof failure simulation is not enabled for this build! Configuration ignored"
 
-  if config.validator:
+      let purchasing = Purchasing.new(market, clock)
-    let validation = Validation.new(clock, market, config.validatorMaxSlots)
+      let sales = Sales.new(market, clock, repo, proofFailures)
-    validator = some ValidatorInteractions.new(clock, validation)
+      client = some ClientInteractions.new(clock, purchasing)
+      host = some HostInteractions.new(clock, sales)
 
-  s.codexNode.contracts = (client, host, validator)
+    if config.validator:
+      let validation = Validation.new(clock, market, config.validatorMaxSlots)
+      validator = some ValidatorInteractions.new(clock, validation)
+
+    s.codexNode.contracts = (client, host, validator)
 
 proc start*(s: CodexServer) {.async.} =
   trace "Starting codex node", config = $s.config
@@ -265,11 +259,27 @@ proc new*(
     store = NetworkStore.new(engine, repoStore)
     erasure = Erasure.new(store, leoEncoderProvider, leoDecoderProvider)
     prover = if config.persistence:
-      let
+      if not fileAccessible($config.circomR1cs, {AccessFlags.Read}):
-        zkey = if config.circomNoZkey: "" else: config.circomZkey
+        error "Circom R1CS file not accessible"
-      some Prover.new(
+        raise (ref Defect)(
-        store,
+          msg: "r1cs file not readable or doesn't exist")
-        CircomCompat.init(config.circomR1cs, config.circomWasm, zkey))
+
+      if not fileAccessible($config.circomWasm, {AccessFlags.Read}):
+        error "Circom WASM file not accessible"
+        raise (ref Defect)(
+          msg: "wasm file not readable or doesn't exist")
+
+      let zkey = if not config.circomNoZkey:
+          if not fileAccessible($config.circomNoZkey, {AccessFlags.Read}):
+            error "Circom WASM file not accessible"
+            raise (ref Defect)(
+              msg: "wasm file not readable or doesn't exist")
+
+          $config.circomNoZkey
+        else:
+          ""
+
+      some Prover.new(store, CircomCompat.init($config.circomR1cs, $config.circomWasm, zkey))
     else:
       none Prover
 

codex/conf.nim

@@ -7,8 +7,6 @@
 ## This file may not be copied, modified, or distributed except according to
 ## those terms.
 
-import pkg/upraises
-
 {.push raises: [].}
 
 import std/os
@@ -48,18 +46,32 @@ export
   DefaultBlockMaintenanceInterval,
   DefaultNumberOfBlocksToMaintainPerInterval
 
+proc defaultDataDir*(): string =
+  let dataDir = when defined(windows):
+    "AppData" / "Roaming" / "Codex"
+  elif defined(macosx):
+    "Library" / "Application Support" / "Codex"
+  else:
+    ".cache" / "codex"
+
+  getHomeDir() / dataDir
+
 const
   codex_enable_api_debug_peers* {.booldefine.} = false
   codex_enable_proof_failures* {.booldefine.} = false
   codex_use_hardhat* {.booldefine.} = false
   codex_enable_log_counter* {.booldefine.} = false
 
+  DefaultDataDir* = defaultDataDir()
+
 type
   StartUpCmd* {.pure.} = enum
-    Persistence
+    noCmd
+    persistence
 
   PersistenceCmd* {.pure.} = enum
-    Validator
+    noCmd
+    validator
 
   LogKind* {.pure.} = enum
     Auto = "auto"
@@ -109,8 +121,8 @@ type
 
     dataDir* {.
       desc: "The directory where codex will store configuration and data"
-      defaultValue: defaultDataDir()
+      defaultValue: DefaultDataDir
-      defaultValueDesc: ""
+      defaultValueDesc: $DefaultDataDir
       abbr: "d"
       name: "data-dir" }: OutDir
 
@@ -225,16 +237,9 @@ type
       .}: Option[string]
 
     case cmd* {.
+      defaultValue: noCmd
       command }: StartUpCmd
-
+    of persistence:
-    of Persistence:
-
-      numProofSamples* {.
-        desc: "Number of samples to prove"
-        defaultValue: DefaultSamplesNum
-        defaultValueDesc: $DefaultSamplesNum
-        name: "proof-samples" }: int
-
       ethProvider* {.
         desc: "The URL of the JSON-RPC API of the Ethereum node"
         defaultValue: "ws://localhost:8545"
@@ -244,55 +249,94 @@ type
       ethAccount* {.
         desc: "The Ethereum account that is used for storage contracts"
         defaultValue: EthAddress.none
+        defaultValueDesc: ""
         name: "eth-account"
       .}: Option[EthAddress]
 
       ethPrivateKey* {.
         desc: "File containing Ethereum private key for storage contracts"
         defaultValue: string.none
+        defaultValueDesc: ""
         name: "eth-private-key"
       .}: Option[string]
 
       marketplaceAddress* {.
         desc: "Address of deployed Marketplace contract"
         defaultValue: EthAddress.none
+        defaultValueDesc: ""
         name: "marketplace-address"
       .}: Option[EthAddress]
 
       circomR1cs* {.
         desc: "The r1cs file for the storage circuit"
-        defaultValue: $defaultDataDir() / "circuits" / "proof_main.r1cs"
+        defaultValue: $DefaultDataDir / "circuits" / "proof_main.r1cs"
+        defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.r1cs"
         name: "circom-r1cs"
-      .}: string
+      .}: InputFile
 
       circomWasm* {.
         desc: "The wasm file for the storage circuit"
-        defaultValue: $defaultDataDir() / "circuits" / "proof_main.wasm"
+        defaultValue: $DefaultDataDir / "circuits" / "proof_main.wasm"
+        defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.wasm"
         name: "circom-wasm"
-      .}: string
+      .}: InputFile
 
       circomZkey* {.
         desc: "The zkey file for the storage circuit"
-        defaultValue: $defaultDataDir() / "circuits" / "proof_main.zkey"
+        defaultValue: $DefaultDataDir / "circuits" / "proof_main.zkey"
+        defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.zkey"
         name: "circom-zkey"
-      .}: string
+      .}: InputFile
 
+      # TODO: should probably be hidden and behind a feature flag
       circomNoZkey* {.
         desc: "Ignore the zkey file - use only for testing!"
         defaultValue: false
         name: "circom-no-zkey"
       .}: bool
 
+      numProofSamples* {.
+        desc: "Number of samples to prove"
+        defaultValue: DefaultSamplesNum
+        defaultValueDesc: $DefaultSamplesNum
+        name: "proof-samples" }: int
+
+      maxSlotDepth* {.
+        desc: "The maximum depth of the slot tree"
+        defaultValue: DefaultMaxSlotDepth
+        defaultValueDesc: $DefaultMaxSlotDepth
+        name: "max-slot-depth" }: int
+
+      maxDatasetDepth* {.
+        desc: "The maximum depth of the dataset tree"
+        defaultValue: DefaultMaxDatasetDepth
+        defaultValueDesc: $DefaultMaxDatasetDepth
+        name: "max-dataset-depth" }: int
+
+      maxBlockDepth* {.
+        desc: "The maximum depth of the network block merkle tree"
+        defaultValue: DefaultBlockDepth
+        defaultValueDesc: $DefaultBlockDepth
+        name: "max-block-depth" }: int
+
+      maxCellElms* {.
+        desc: "The maximum number of elements in a cell"
+        defaultValue: DefaultCellElms
+        defaultValueDesc: $DefaultCellElms
+        name: "max-cell-elements" }: int
+
       case persistenceCmd* {.
+        defaultValue: noCmd
         command }: PersistenceCmd
 
-      of Validator:
+      of validator:
         validatorMaxSlots* {.
           desc: "Maximum number of slots that the validator monitors"
           defaultValue: 1000
           name: "validator-max-slots"
         .}: int
 
+        # TODO: should go behind a feature flag
         simulateProofFailures* {.
             desc: "Simulates proof failures once every N proofs. 0 = disabled."
             defaultValue: 0
@@ -300,16 +344,22 @@ type
             hidden
           .}: int
 
+      of PersistenceCmd.noCmd:
+        discard # end of validator
+
+    of StartUpCmd.noCmd:
+      discard # end of persistence
+
   EthAddress* = ethers.Address
 
 logutils.formatIt(LogFormat.textLines, EthAddress): it.short0xHexLog
 logutils.formatIt(LogFormat.json, EthAddress): %it
 
 func persistence*(self: CodexConf): bool =
-  self.cmd == StartUpCmd.Persistence
+  self.cmd == StartUpCmd.persistence
 
 func validator*(self: CodexConf): bool =
-  self.persistenceCmd == PersistenceCmd.Validator
+  self.persistence and self.persistenceCmd == PersistenceCmd.validator
 
 proc getCodexVersion(): string =
   let tag = strip(staticExec("git tag"))
@@ -335,16 +385,6 @@ const
     "Codex revision: " & codexRevision & "\p" &
     nimBanner
 
-proc defaultDataDir*(): string =
-  let dataDir = when defined(windows):
-    "AppData" / "Roaming" / "Codex"
-  elif defined(macosx):
-    "Library" / "Application Support" / "Codex"
-  else:
-    ".cache" / "codex"
-
-  getHomeDir() / dataDir
-
 proc parseCmdArg*(T: typedesc[MultiAddress],
                   input: string): MultiAddress
                  {.upraises: [ValueError, LPError].} =
@@ -353,7 +393,7 @@ proc parseCmdArg*(T: typedesc[MultiAddress],
   if res.isOk:
     ma = res.get()
   else:
-    warn "Invalid MultiAddress", input=input, error=res.error()
+    warn "Invalid MultiAddress", input=input, error = res.error()
     quit QuitFailure
   ma
 
@@ -361,10 +401,10 @@ proc parseCmdArg*(T: type SignedPeerRecord, uri: string): T =
   var res: SignedPeerRecord
   try:
     if not res.fromURI(uri):
-      warn "Invalid SignedPeerRecord uri", uri=uri
+      warn "Invalid SignedPeerRecord uri", uri = uri
       quit QuitFailure
   except CatchableError as exc:
-    warn "Invalid SignedPeerRecord uri", uri=uri, error=exc.msg
+    warn "Invalid SignedPeerRecord uri", uri = uri, error = exc.msg
     quit QuitFailure
   res
 
@@ -375,7 +415,7 @@ proc parseCmdArg*(T: type NBytes, val: string): T =
   var num = 0'i64
   let count = parseSize(val, num, alwaysBin = true)
   if count == 0:
-      warn "Invalid number of bytes", nbytes=val
+      warn "Invalid number of bytes", nbytes = val
       quit QuitFailure
   NBytes(num)
 
@@ -383,7 +423,7 @@ proc parseCmdArg*(T: type Duration, val: string): T =
   var dur: Duration
   let count = parseDuration(val, dur)
   if count == 0:
-      warn "Invalid duration parse", dur=dur
+      warn "Cannot parse duration", dur = dur
       quit QuitFailure
   dur
 

codex/slots/proofs/backends/circomcompat.nim

@@ -110,10 +110,6 @@ proc prove*[H](
     "slotRoot".cstring, slotRoot[0].addr, slotRoot.len.uint32) != ERR_OK:
     return failure("Failed to push data set root")
 
-  if backend.pushInputU256Array(
-    "slotRoot".cstring, slotRoot[0].addr, slotRoot.len.uint32) != ERR_OK:
-    return failure("Failed to push data set root")
-
   if backend.pushInputU32(
     "nCellsPerSlot".cstring, input.nCellsPerSlot.uint32) != ERR_OK:
     return failure("Failed to push nCellsPerSlot")
@@ -176,8 +172,7 @@ proc prove*[H](
       if backend != nil:
         backend.addr.releaseCircomCompat()
 
-      if backend != nil:
+  success proof
-        backend.addr.releaseCircomCompat()
 
 proc verify*(
   self: CircomCompat,

codex/slots/proofs/prover.nim

@@ -25,10 +25,8 @@ import ../../utils/poseidon2digest
 import ../builder
 import ../sampler
 
-import ../../manifest
+import ./backends
-import ../../merkletree
+import ../types
-import ../../stores
-import ../../market
 
 export backends
 

config.nims

@@ -86,6 +86,8 @@ when (NimMajor, NimMinor) >= (1, 6):
 when (NimMajor, NimMinor, NimPatch) >= (1, 6, 11):
   --warning:"BareExcept:off"
 
+switch("define", "nimOldCaseObjects") # needed for confutils commands
+
 switch("define", "withoutPCRE")
 
 # the default open files limit is too low on macOS (512), breaking the

tests/codex/slots/backends/testcircomcompat.nim

@@ -16,10 +16,6 @@ import pkg/codex/codextypes
 import pkg/codex/manifest
 import pkg/codex/stores
 
-import pkg/constantine/math/arithmetic
-import pkg/constantine/math/io/io_fields
-import pkg/constantine/math/io/io_bigints
-
 import ./helpers
 import ../helpers
 
@@ -135,74 +131,3 @@ suite "Test Circom Compat Backend":
       proof = circom.prove(proofInput).tryGet
 
     check circom.verify(proof, publicInputs, verifyingKeyPtr[]).tryGet == false
-
-suite "Test Circom Compat Backend":
-  let
-    slotId = 3
-    samples = 5
-    ecK = 2
-    ecM = 2
-    numDatasetBlocks = 8
-    blockSize = DefaultBlockSize
-    cellSize = DefaultCellSize
-
-    r1cs = "tests/circuits/fixtures/proof_main.r1cs"
-    wasm = "tests/circuits/fixtures/proof_main.wasm"
-
-  var
-    store: BlockStore
-    manifest: Manifest
-    protected: Manifest
-    verifiable: Manifest
-    circom: CircomCompat
-    verifyingKeyPtr: ptr CircomKey
-    proofInput: ProofInput[Poseidon2Hash]
-    publicInputs: CircomInputs
-    challenge: array[32, byte]
-    builder: Poseidon2Builder
-    sampler: Poseidon2Sampler
-
-  setup:
-    let
-      repoDs = SQLiteDatastore.new(Memory).tryGet()
-      metaDs = SQLiteDatastore.new(Memory).tryGet()
-
-    store = RepoStore.new(repoDs, metaDs)
-
-    (manifest, protected, verifiable) =
-        await createVerifiableManifest(
-          store,
-          numDatasetBlocks,
-          ecK, ecM,
-          blockSize,
-          cellSize)
-
-    builder = Poseidon2Builder.new(store, verifiable).tryGet
-    sampler = Poseidon2Sampler.new(slotId, store, builder).tryGet
-
-    # circom = CircomCompat.init(r1cs, wasm, zkey)
-    circom = CircomCompat.init(r1cs, wasm)
-    verifyingKeyPtr = circom.getVerifyingKey().tryGet
-    challenge = 1234567.toF.toBytes.toArray32
-
-    proofInput = (await sampler.getProofInput(challenge, samples)).tryGet
-    publicInputs = proofInput.toPublicInputs.toCircomInputs
-
-  teardown:
-    publicInputs.releaseNimInputs()      # this is allocated by nim
-    verifyingKeyPtr.addr.releaseKey()    # this comes from the rust FFI
-    circom.release()                     # this comes from the rust FFI
-
-  test "Should verify with correct input":
-    var
-      proof = circom.prove(proofInput).tryGet
-
-    check circom.verify(proof, publicInputs, verifyingKeyPtr[]).tryGet
-
-  test "Should not verify with incorrect input":
-    proofInput.slotIndex = 1 # change slot index
-
-    let
-      proof = circom.prove(proofInput).tryGet
-
-    check circom.verify(proof, publicInputs, verifyingKeyPtr[]).tryGet == false