diff --git a/codex.nim b/codex.nim index 7749bdee..b534a0b3 100644 --- a/codex.nim +++ b/codex.nim @@ -10,7 +10,6 @@ import pkg/chronos import pkg/questionable import pkg/confutils -import pkg/confutils/defs import pkg/confutils/std/net import pkg/confutils/toml/defs as confTomlDefs import pkg/confutils/toml/std/net as confTomlNet diff --git a/codex/codex.nim b/codex/codex.nim index 234f2d7f..ccef82e1 100644 --- a/codex/codex.nim +++ b/codex/codex.nim @@ -208,9 +208,15 @@ proc new*( .withTcpTransport({ServerFlags.ReuseAddr}) .build() + let numThreads = + if int(config.numThreads) == 0: + countProcessors() + else: + int(config.numThreads) + var tp = try: - Taskpool.new(numThreads = int(config.numThreads)) + Taskpool.new(numThreads) except CatchableError as exc: raiseAssert("Failure in tp initialization:" & exc.msg) diff --git a/codex/conf.nim b/codex/conf.nim index 40e21c4b..44e6d3f4 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -18,6 +18,7 @@ import std/terminal # Is not used in tests import std/options import std/strutils import std/typetraits +import std/cpuinfo import pkg/chronos import pkg/chronicles/helpers @@ -54,9 +55,7 @@ export DefaultQuotaBytes, DefaultBlockTtl, DefaultBlockInterval, DefaultNumBlocksPerInterval, DefaultRequestCacheSize -type ThreadCount* = distinct Natural - -proc `==`*(a, b: ThreadCount): bool {.borrow.} +type ThreadCount* = range[0 .. 256] proc defaultDataDir*(): string = let dataDir = @@ -76,7 +75,6 @@ const DefaultDataDir* = defaultDataDir() DefaultCircuitDir* = defaultDataDir() / "circuits" - DefaultThreadCount* = ThreadCount(0) type StartUpCmd* {.pure.} = enum @@ -88,8 +86,11 @@ type prover ProverBackendCmd* {.pure.} = enum - nimGroth16 - circomCompat + nimgroth16 + circomcompat + + Curves* {.pure.} = enum + bn128 = "bn128" LogKind* {.pure.} = enum Auto = "auto" @@ -197,7 +198,8 @@ type numThreads* {. desc: "Number of worker threads (\"0\" = use as many threads as there are CPU cores available)", - defaultValue: DefaultThreadCount, + defaultValueDesc: "0", + defaultValue: ThreadCount(0), name: "num-threads" .}: ThreadCount @@ -384,6 +386,22 @@ type name: "circuit-dir" .}: OutDir + proverBackend* {. + desc: + "The backend to use for the prover. " & + "Must be one of: nimgroth16, circomcompat", + defaultValue: ProverBackendCmd.nimgroth16, + defaultValueDesc: "nimgroth16", + name: "prover-backend" + .}: ProverBackendCmd + + curve* {. + desc: "The curve to use for the storage circuit", + defaultValue: Curves.bn128, + defaultValueDesc: $Curves.bn128, + name: "curve" + .}: Curves + circomR1cs* {. desc: "The r1cs file for the storage circuit", defaultValue: $DefaultCircuitDir / "proof_main.r1cs", @@ -391,28 +409,21 @@ type name: "circom-r1cs" .}: InputFile - case proverBackendCmd*: ProverBackendCmd - of ProverBackendCmd.nimGroth16: - nimGroth16Curve* {. - desc: "The curve to use for the storage circuit", - defaultValue: "bn128", - name: "nim-groth16-curve" - .}: string + circomGraph* {. + desc: + "The graph file for the storage circuit (only used with nimgroth16 backend)", + defaultValue: $DefaultCircuitDir / "proof_main.bin", + defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.bin", + name: "circom-graph" + .}: InputFile - circomGraph* {. - desc: "The graph file for the storage circuit", - defaultValue: $DefaultCircuitDir / "graph.bin", - defaultValueDesc: $DefaultDataDir & "/circuits/graph.bin", - name: "nim-groth16-graph" - .}: InputFile - of ProverBackendCmd.circomCompat: - circomWasm* {. - desc: - "The wasm file for the storage circuit - DEPRECATED: use the default nimGroth16 backend", - defaultValue: $DefaultCircuitDir / "proof_main.wasm", - defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.wasm", - name: "circom-wasm" - .}: InputFile + circomWasm* {. + desc: + "The wasm file for the storage circuit (only used with circomcompat backend)", + defaultValue: $DefaultCircuitDir / "proof_main.wasm", + defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.wasm", + name: "circom-wasm" + .}: InputFile circomZkey* {. desc: "The zkey file for the storage circuit", @@ -421,11 +432,11 @@ type name: "circom-zkey" .}: InputFile - # TODO: should probably be hidden and behind a feature flag circomNoZkey* {. desc: "Ignore the zkey file - use only for testing!", defaultValue: false, - name: "circom-no-zkey" + name: "circom-no-zkey", + hidden .}: bool numProofSamples* {. @@ -516,7 +527,7 @@ const proc parseCmdArg*( T: typedesc[MultiAddress], input: string -): MultiAddress {.upraises: [ValueError].} = +): MultiAddress {.raises: [ValueError].} = var ma: MultiAddress try: let res = MultiAddress.init(input) @@ -530,12 +541,8 @@ proc parseCmdArg*( quit QuitFailure ma -proc parseCmdArg*(T: type ThreadCount, input: string): T {.upraises: [ValueError].} = - let count = parseInt(input) - if count != 0 and count < 2: - warn "Invalid number of threads", input = input - quit QuitFailure - ThreadCount(count) +proc parseCmdArg*(T: type ThreadCount, val: string): T {.raises: [ValueError].} = + ThreadCount(val.parseUInt()) proc parseCmdArg*(T: type SignedPeerRecord, uri: string): T = var res: SignedPeerRecord @@ -597,7 +604,7 @@ proc parseCmdArg*(T: type Duration, val: string): T = proc readValue*( r: var TomlReader, val: var EthAddress -) {.upraises: [SerializationError, IOError].} = +) {.raises: [SerializationError, IOError].} = val = EthAddress.init(r.readValue(string)).get() proc readValue*(r: var TomlReader, val: var SignedPeerRecord) = @@ -625,7 +632,7 @@ proc readValue*(r: var TomlReader, val: var MultiAddress) = proc readValue*( r: var TomlReader, val: var NBytes -) {.upraises: [SerializationError, IOError].} = +) {.raises: [SerializationError, IOError].} = var value = 0'i64 var str = r.readValue(string) let count = parseSize(str, value, alwaysBin = true) @@ -636,7 +643,7 @@ proc readValue*( proc readValue*( r: var TomlReader, val: var ThreadCount -) {.upraises: [SerializationError, IOError].} = +) {.raises: [SerializationError, IOError].} = var str = r.readValue(string) try: val = parseCmdArg(ThreadCount, str) @@ -645,7 +652,7 @@ proc readValue*( proc readValue*( r: var TomlReader, val: var Duration -) {.upraises: [SerializationError, IOError].} = +) {.raises: [SerializationError, IOError].} = var str = r.readValue(string) var dur: Duration let count = parseDuration(str, dur) @@ -712,7 +719,7 @@ proc stripAnsi*(v: string): string = res -proc updateLogLevel*(logLevel: string) {.upraises: [ValueError].} = +proc updateLogLevel*(logLevel: string) {.raises: [ValueError].} = # Updates log levels (without clearing old ones) let directives = logLevel.split(";") try: diff --git a/codex/slots/proofs/prover.nim b/codex/slots/proofs/prover.nim index 861ad627..b98aabe4 100644 --- a/codex/slots/proofs/prover.nim +++ b/codex/slots/proofs/prover.nim @@ -35,17 +35,14 @@ export backends logScope: topics = "codex prover" -type - Prover* = ref object - case backendKind: ProverBackendCmd - of ProverBackendCmd.nimGroth16: - groth16Backend*: NimGroth16BackendRef - of ProverBackendCmd.circomCompat: - circomCompatBackend*: CircomCompatBackendRef - nSamples: int - tp: Taskpool - - AnyBackend* = NimGroth16BackendRef or CircomCompatBackendRef +type Prover* = ref object + case backendKind: ProverBackendCmd + of ProverBackendCmd.nimgroth16: + groth16Backend*: NimGroth16BackendRef + of ProverBackendCmd.circomcompat: + circomCompatBackend*: CircomCompatBackendRef + nSamples: int + tp: Taskpool proc prove*[SomeSampler]( self: Prover, @@ -69,7 +66,7 @@ proc prove*[SomeSampler]( # prove slot case self.backendKind - of ProverBackendCmd.nimGroth16: + of ProverBackendCmd.nimgroth16: let proof = ?await self.groth16Backend.prove(proofInput) verified = @@ -78,7 +75,7 @@ proc prove*[SomeSampler]( else: bool.none return success (proof.toGroth16Proof, verified) - of ProverBackendCmd.circomCompat: + of ProverBackendCmd.circomcompat: let proof = ?await self.circomCompatBackend.prove(proofInput) verified = @@ -93,7 +90,7 @@ proc new*( ): Prover = Prover( circomCompatBackend: backend, - backendKind: ProverBackendCmd.circomCompat, + backendKind: ProverBackendCmd.circomcompat, nSamples: nSamples, tp: tp, ) @@ -103,7 +100,7 @@ proc new*( ): Prover = Prover( groth16Backend: backend, - backendKind: ProverBackendCmd.nimGroth16, + backendKind: ProverBackendCmd.nimgroth16, nSamples: nSamples, tp: tp, ) diff --git a/codex/slots/proofs/proverfactory.nim b/codex/slots/proofs/proverfactory.nim index 6f2795d7..6259b611 100644 --- a/codex/slots/proofs/proverfactory.nim +++ b/codex/slots/proofs/proverfactory.nim @@ -91,7 +91,7 @@ proc initializeNimGroth16Backend( $graphFile, $r1csFile, $zkeyFile, - config.nimGroth16Curve, + $config.curve, config.maxSlotDepth, config.maxDatasetDepth, config.maxBlockDepth, @@ -121,14 +121,14 @@ proc initializeCircomCompatBackend( proc initializeProver*(config: CodexConf, tp: Taskpool): ?!Prover = let prover = - case config.proverBackendCmd - of ProverBackendCmd.nimGroth16: + case config.proverBackend + of ProverBackendCmd.nimgroth16: without backend =? initializeNimGroth16Backend(config, tp), err: suggestDownloadTool(config) return failure("Unable to initialize NimGroth16 backend") Prover.new(backend, config.numProofSamples, tp) - of ProverBackendCmd.circomCompat: + of ProverBackendCmd.circomcompat: without backend =? initializeCircomCompatBackend(config, tp), err: suggestDownloadTool(config) return failure("Unable to initialize CircomCompat backend") diff --git a/tests/codex/slots/testproverfactory.nim b/tests/codex/slots/testproverfactory.nim index 4d4b5714..e3a3f211 100644 --- a/tests/codex/slots/testproverfactory.nim +++ b/tests/codex/slots/testproverfactory.nim @@ -29,7 +29,7 @@ suite "Test BackendFactory": metricsAddress: parseIpAddress("127.0.0.1"), persistenceCmd: PersistenceCmd.prover, marketplaceAddress: EthAddress.example.some, - proverBackendCmd: ProverBackendCmd.nimGroth16, + proverBackend: ProverBackendCmd.nimgroth16, circomGraph: InputFile("tests/circuits/fixtures/proof_main.bin"), circomR1cs: InputFile("tests/circuits/fixtures/proof_main.r1cs"), circomZkey: InputFile("tests/circuits/fixtures/proof_main.zkey"), @@ -47,7 +47,7 @@ suite "Test BackendFactory": metricsAddress: parseIpAddress("127.0.0.1"), persistenceCmd: PersistenceCmd.prover, marketplaceAddress: EthAddress.example.some, - proverBackendCmd: ProverBackendCmd.circomCompat, + proverBackend: ProverBackendCmd.circomcompat, circomWasm: InputFile("tests/circuits/fixtures/proof_main.wasm"), circomR1cs: InputFile("tests/circuits/fixtures/proof_main.r1cs"), circomZkey: InputFile("tests/circuits/fixtures/proof_main.zkey"), @@ -65,7 +65,7 @@ suite "Test BackendFactory": metricsAddress: parseIpAddress("127.0.0.1"), persistenceCmd: PersistenceCmd.prover, marketplaceAddress: EthAddress.example.some, - proverBackendCmd: ProverBackendCmd.circomCompat, + proverBackend: ProverBackendCmd.circomcompat, # Set the circuitDir such that the tests/circuits/fixtures/ files # will be picked up as local files: circuitDir: OutDir("tests/circuits/fixtures"), @@ -83,7 +83,7 @@ suite "Test BackendFactory": metricsAddress: parseIpAddress("127.0.0.1"), persistenceCmd: PersistenceCmd.prover, marketplaceAddress: EthAddress.example.some, - proverBackendCmd: ProverBackendCmd.nimGroth16, + proverBackend: ProverBackendCmd.nimgroth16, # Set the circuitDir such that the tests/circuits/fixtures/ files # will be picked up as local files: circuitDir: OutDir("tests/circuits/fixtures"), @@ -101,7 +101,7 @@ suite "Test BackendFactory": nat: NatConfig(hasExtIp: false, nat: NatNone), metricsAddress: parseIpAddress("127.0.0.1"), persistenceCmd: PersistenceCmd.prover, - proverBackendCmd: ProverBackendCmd.nimGroth16, + proverBackend: ProverBackendCmd.nimgroth16, marketplaceAddress: EthAddress.example.some, circuitDir: OutDir(circuitDir), )