diff --git a/codex/slots/proofs/backendfactory.nim b/codex/slots/proofs/backendfactory.nim index 7f2e159d..0f6a9b31 100644 --- a/codex/slots/proofs/backendfactory.nim +++ b/codex/slots/proofs/backendfactory.nim @@ -1,6 +1,4 @@ import os -import httpclient -import zip/zipfiles import pkg/chronos import pkg/chronicles import pkg/questionable @@ -9,30 +7,25 @@ import pkg/stew/io2 import ../../conf import ./backends - -proc initializeCircomBackend( - r1csFile: string, - wasmFile: string, - zKeyFile: string -): AnyBackend = - CircomCompat.init(r1csFile, wasmFile, zKeyFile) +import ./backendutils proc initializeFromConfig( - config: CodexConf): ?!AnyBackend = - if not fileAccessible($config.circomR1cs, {AccessFlags.Read}) and - endsWith($config.circomR1cs, ".r1cs"): + config: CodexConf, + utils: BackendUtils): ?!AnyBackend = + if not fileAccessible($config.circomR1cs, {AccessFlags.Read}) or + not endsWith($config.circomR1cs, ".r1cs"): return failure("Circom R1CS file not accessible") - if not fileAccessible($config.circomWasm, {AccessFlags.Read}) and - endsWith($config.circomWasm, ".wasm"): + if not fileAccessible($config.circomWasm, {AccessFlags.Read}) or + not endsWith($config.circomWasm, ".wasm"): return failure("Circom wasm file not accessible") - if not fileAccessible($config.circomZkey, {AccessFlags.Read}) and - endsWith($config.circomZkey, ".zkey"): + if not fileAccessible($config.circomZkey, {AccessFlags.Read}) or + not endsWith($config.circomZkey, ".zkey"): return failure("Circom zkey file not accessible") trace "Initialized prover backend from cli config" - success(initializeCircomBackend( + success(utils.initializeCircomBackend( $config.circomR1cs, $config.circomWasm, $config.circomZkey)) @@ -49,12 +42,14 @@ proc zkeyFilePath(config: CodexConf): string = proc zipFilePath(config: CodexConf): string = config.dataDir / "circuit.zip" -proc initializeFromCeremonyFiles(config: CodexConf): ?!AnyBackend = +proc initializeFromCeremonyFiles( + config: CodexConf, + utils: BackendUtils): ?!AnyBackend = if fileExists(config.r1csFilePath) and fileExists(config.wasmFilePath) and fileExists(config.zkeyFilePath): trace "Initialized prover backend from local files" - return success(initializeCircomBackend( + return success(utils.initializeCircomBackend( config.r1csFilePath, config.wasmFilePath, config.zkeyFilePath)) @@ -63,7 +58,8 @@ proc initializeFromCeremonyFiles(config: CodexConf): ?!AnyBackend = proc downloadCeremony( config: CodexConf, - ceremonyHash: string + ceremonyHash: string, + utils: BackendUtils ): ?!void = # TODO: # In the future, the zip file will be stored in the Codex network @@ -71,37 +67,25 @@ proc downloadCeremony( let url = "https://circuit.codex.storage/proving-key/" & ceremonyHash trace "Downloading ceremony file", url, filepath = config.zipFilePath - try: - # Nim's default webclient does not support SSL on all platforms. - # Not without shipping additional binaries and cert-files... :( - # So we're using curl for now. - var rc = execShellCmd("curl -o " & config.zipFilePath & " " & url) - if not rc == 0: - return failure("Download failed with return code: " & $rc) - except Exception as exc: - return failure(exc.msg) - trace "Download completed." - success() + return utils.downloadFile(url, config.zipFilePath) proc unzipCeremonyFile( - config: CodexConf): ?!void = + config: CodexConf, + utils: BackendUtils): ?!void = trace "Unzipping..." - var z: ZipArchive - if not z.open(config.zipFilePath): - return failure("Unable to open zip file: " & config.zipFilePath) - z.extractAll($config.dataDir) - success() + return utils.unzipFile(config.zipFilePath, $config.dataDir) proc initializeFromCeremonyHash( config: CodexConf, - ceremonyHash: ?string): Future[?!AnyBackend] {.async.} = + ceremonyHash: ?string, + utils: BackendUtils): Future[?!AnyBackend] {.async.} = if hash =? ceremonyHash: - if dlErr =? downloadCeremony(config, hash).errorOption: + if dlErr =? downloadCeremony(config, hash, utils).errorOption: return failure(dlErr) - if err =? unzipCeremonyFile(config).errorOption: + if err =? unzipCeremonyFile(config, utils).errorOption: return failure(err) - without backend =? initializeFromCeremonyFiles(config), err: + without backend =? initializeFromCeremonyFiles(config, utils), err: return failure(err) return success(backend) else: @@ -109,13 +93,14 @@ proc initializeFromCeremonyHash( proc initializeBackend*( config: CodexConf, - ceremonyHash: ?string): Future[?!AnyBackend] {.async.} = + ceremonyHash: ?string, + utils: BackendUtils = BackendUtils()): Future[?!AnyBackend] {.async.} = - without backend =? initializeFromConfig(config), cliErr: + without backend =? initializeFromConfig(config, utils), cliErr: info "Could not initialize prover backend from CLI options...", msg = cliErr.msg - without backend =? initializeFromCeremonyFiles(config), localErr: + without backend =? initializeFromCeremonyFiles(config, utils), localErr: info "Could not initialize prover backend from local files...", msg = localErr.msg - without backend =? (await initializeFromCeremonyHash(config, ceremonyHash)), urlErr: + without backend =? (await initializeFromCeremonyHash(config, ceremonyHash, utils)), urlErr: warn "Could not initialize prover backend from ceremony url...", msg = urlErr.msg return failure(urlErr) return success(backend) diff --git a/codex/slots/proofs/backendutils.nim b/codex/slots/proofs/backendutils.nim new file mode 100644 index 00000000..9614e76a --- /dev/null +++ b/codex/slots/proofs/backendutils.nim @@ -0,0 +1,45 @@ +import os +import zip/zipfiles +import pkg/chronos +import pkg/chronicles +import pkg/questionable +import pkg/questionable/results + +import ./backends + +type + BackendUtils* = ref object of RootObj + +method initializeCircomBackend*( + self: BackendUtils, + r1csFile: string, + wasmFile: string, + zKeyFile: string +): AnyBackend {.base.} = + CircomCompat.init(r1csFile, wasmFile, zKeyFile) + +method downloadFile*( + self: BackendUtils, + url: string, + filepath: string +): ?!void {.base.} = + try: + # Nim's default webclient does not support SSL on all platforms. + # Not without shipping additional binaries and cert-files... :( + # So we're using curl for now. + var rc = execShellCmd("curl -o " & filepath & " " & url) + if not rc == 0: + return failure("Download of '" & url & "' failed with return code: " & $rc) + except Exception as exc: + return failure(exc.msg) + success() + +method unzipFile*( + self: BackendUtils, + zipFile: string, + outputDir: string): ?!void {.base.} = + var z: ZipArchive + if not z.open(zipFile): + return failure("Unable to open zip file: " & zipFile) + z.extractAll(outputDir) + success() diff --git a/tests/codex/slots/testbackendfactory.nim b/tests/codex/slots/testbackendfactory.nim new file mode 100644 index 00000000..05a32e17 --- /dev/null +++ b/tests/codex/slots/testbackendfactory.nim @@ -0,0 +1,144 @@ +import os +import std/strutils +import std/sugar +import std/math + +import ../../asynctest + +import pkg/chronos +import pkg/confutils/defs +import pkg/codex/conf +import pkg/codex/slots/proofs/backends +import pkg/codex/slots/proofs/backendfactory +import pkg/codex/slots/proofs/backendutils + +import ./helpers +import ../helpers + +type + BackendUtilsMock = ref object of BackendUtils + argR1csFile: string + argWasmFile: string + argZKeyFile: string + argUrl: string + argFilepath: string + argZipFile: string + argOutputDir: string + +method initializeCircomBackend*( + self: BackendUtilsMock, + r1csFile: string, + wasmFile: string, + zKeyFile: string +): AnyBackend = + self.argR1csFile = r1csFile + self.argWasmFile = wasmFile + self.argZKeyFile = zKeyFile + +method downloadFile*( + self: BackendUtilsMock, + url: string, + filepath: string +): ?!void = + self.argUrl = url + self.argFilepath = filepath + success() + +method unzipFile*( + self: BackendUtilsMock, + zipFile: string, + outputDir: string): ?!void = + self.argZipFile = zipFile + self.argOutputDir = outputDir + try: + writeFile(outputDir / "proof_main.r1cs", "r1cs_file") + writeFile(outputDir / "proof_main.wasm", "wasm_file") + writeFile(outputDir / "proof_main.zkey", "zkey_file") + except Exception as exc: + return failure(exc.msg) + success() + +suite "Test BackendFactory": + let + utilsMock = BackendUtilsMock() + datadir = "testdatadir" + + setup: + createDir(datadir) + + teardown: + removeDir(datadir) + + test "Should create backend from cli config": + let + config = CodexConf( + cmd: StartUpCmd.persistence, + nat: ValidIpAddress.init("127.0.0.1"), + discoveryIp: ValidIpAddress.init(IPv4_any()), + metricsAddress: ValidIpAddress.init("127.0.0.1"), + persistenceCmd: PersistenceCmd.prover, + circomR1cs: InputFile("tests/circuits/fixtures/proof_main.r1cs"), + circomWasm: InputFile("tests/circuits/fixtures/proof_main.wasm"), + circomZkey: InputFile("tests/circuits/fixtures/proof_main.zkey") + ) + ceremonyHash = string.none + backend = (await initializeBackend(config, ceremonyHash, utilsMock)).tryGet + + check: + utilsMock.argR1csFile == $config.circomR1cs + utilsMock.argWasmFile == $config.circomWasm + utilsMock.argZKeyFile == $config.circomZkey + isEmptyOrWhitespace(utilsMock.argUrl) + isEmptyOrWhitespace(utilsMock.argFilepath) + isEmptyOrWhitespace(utilsMock.argZipFile) + isEmptyOrWhitespace(utilsMock.argOutputDir) + + test "Should create backend from local files": + let + config = CodexConf( + cmd: StartUpCmd.persistence, + nat: ValidIpAddress.init("127.0.0.1"), + discoveryIp: ValidIpAddress.init(IPv4_any()), + metricsAddress: ValidIpAddress.init("127.0.0.1"), + persistenceCmd: PersistenceCmd.prover, + + # Set the datadir such that the tests/circuits/fixtures/ files + # will be picked up as local files: + dataDir: OutDir("tests/circuits/fixtures") + ) + ceremonyHash = string.none + backend = (await initializeBackend(config, ceremonyHash, utilsMock)).tryGet + + check: + utilsMock.argR1csFile == config.dataDir / "proof_main.r1cs" + utilsMock.argWasmFile == config.dataDir / "proof_main.wasm" + utilsMock.argZKeyFile == config.dataDir / "proof_main.zkey" + isEmptyOrWhitespace(utilsMock.argUrl) + isEmptyOrWhitespace(utilsMock.argFilepath) + isEmptyOrWhitespace(utilsMock.argZipFile) + isEmptyOrWhitespace(utilsMock.argOutputDir) + + test "Should download and unzip ceremony file if not available": + let + ceremonyHash = some "12345" + expectedZip = datadir / "circuit.zip" + expectedUrl = "https://circuit.codex.storage/proving-key/" & !ceremonyHash + config = CodexConf( + cmd: StartUpCmd.persistence, + nat: ValidIpAddress.init("127.0.0.1"), + discoveryIp: ValidIpAddress.init(IPv4_any()), + metricsAddress: ValidIpAddress.init("127.0.0.1"), + persistenceCmd: PersistenceCmd.prover, + dataDir: OutDir(datadir) + ) + + backend = (await initializeBackend(config, ceremonyHash, utilsMock)).tryGet + + check: + utilsMock.argR1csFile == config.dataDir / "proof_main.r1cs" + utilsMock.argWasmFile == config.dataDir / "proof_main.wasm" + utilsMock.argZKeyFile == config.dataDir / "proof_main.zkey" + utilsMock.argUrl == expectedUrl + utilsMock.argFilepath == expectedZip + utilsMock.argZipFile == expectedZip + utilsMock.argOutputDir == datadir diff --git a/tests/codex/testslots.nim b/tests/codex/testslots.nim index 2292c4f8..059de7c2 100644 --- a/tests/codex/testslots.nim +++ b/tests/codex/testslots.nim @@ -3,5 +3,6 @@ import ./slots/testsampler import ./slots/testconverters import ./slots/testbackends import ./slots/testprover +import ./slots/testbackendfactory {.warning[UnusedImport]: off.}