nim-dagger/tools/cirdl/cirdl.nim

129 lines
3.8 KiB
Nim
Raw Permalink Normal View History

Rework circuit downloader (#882) * Introduces a start method to prover * Moves backend creation into start method * sets up three paths for backend initialization * Extracts backend initialization to backend-factory * Implements loading backend from cli files or previously downloaded local files * Wires up downloading and unzipping * functional implementation * Fixes testprover.nim * Sets up tests for backendfactory * includes libzip-dev * pulls in updated contracts * removes integration cli tests for r1cs, wasm, and zkey file arguments. * Fixes issue where inner-scope values are lost before returning * sets local proof verification for dist-test images * Adds two traces and bumps nim-ethers * Adds separate path for circuit files * Create circuit dir if not exists * fix: make sure requestStorage is mined * fix: correct place to plug confirm * test: fixing contracts tests * Restores gitmodules * restores nim-datastore reference * Sets up downloader exe * sets up tool skeleton * implements getting of circuit hash * Implements downloader tool * sets up test skeleton * Implements test for cirdl * includes testTools in testAll * Cleanup building.md * cleans up previous downloader implementation * cleans up testbackendfactory * moves start of prover into node.nim * Fills in arguments in example command * Initializes backend in prover constructor * Restores tests * Restores tests for cli instructions * Review comments by Dmitriy, part 1 * Quotes path in download instruction. * replaces curl with chronos http session * Moves cirdl build output to 'build' folder. * Fixes chronicles log output * Add cirdl support to the codex Dockerfile Signed-off-by: Slava <20563034+veaceslavdoina@users.noreply.github.com> * Add cirdl support to the docker entrypoint Signed-off-by: Slava <20563034+veaceslavdoina@users.noreply.github.com> * Add cirdl support to the release workflow Signed-off-by: Slava <20563034+veaceslavdoina@users.noreply.github.com> * Disable verify_circuit flag for releases Signed-off-by: Slava <20563034+veaceslavdoina@users.noreply.github.com> * Removes backendFactory placeholder type * wip * Replaces zip library with status-im/zippy library (which supports zip and tar) * Updates cirdl to not change circuitdir folder * Switches from zip to tar.gz * Review comments by Dmitriy * updates codex-contracts-eth * Adds testTools to CI * Adds check for access to config.circuitdir * Update fixture circuit zkey * Update matrix to run tools tests on Windows * Adds 'deps' dependency for cirdl * Adjust docker-entrypoint.sh to use CODEX_CIRCUIT_DIR env var * Review comments by Giuliano --------- Signed-off-by: Slava <20563034+veaceslavdoina@users.noreply.github.com> Co-authored-by: Adam Uhlíř <adam@uhlir.dev> Co-authored-by: Veaceslav Doina <20563034+veaceslavdoina@users.noreply.github.com>
2024-09-23 14:37:17 +00:00
import std/os
import std/streams
import pkg/chronicles
import pkg/chronos
import pkg/ethers
import pkg/questionable
import pkg/questionable/results
import pkg/zippy/tarballs
import pkg/chronos/apps/http/httpclient
import ../../codex/contracts/marketplace
proc consoleLog(logLevel: LogLevel, msg: LogOutputStr) {.gcsafe.} =
try:
stdout.write(msg)
stdout.flushFile()
except IOError as err:
logLoggingFailure(cstring(msg), err)
proc noOutput(logLevel: LogLevel, msg: LogOutputStr) = discard
defaultChroniclesStream.outputs[0].writer = consoleLog
defaultChroniclesStream.outputs[1].writer = noOutput
defaultChroniclesStream.outputs[2].writer = noOutput
proc printHelp() =
info "Usage: ./cirdl [circuitPath] [rpcEndpoint] [marketplaceAddress]"
info " circuitPath: path where circuit files will be placed."
info " rpcEndpoint: URL of web3 RPC endpoint."
info " marketplaceAddress: Address of deployed Codex marketplace contracts."
proc getCircuitHash(rpcEndpoint: string, marketplaceAddress: string): Future[?!string] {.async.} =
let provider = JsonRpcProvider.new(rpcEndpoint)
without address =? Address.init(marketplaceAddress):
return failure("Invalid address: " & marketplaceAddress)
let marketplace = Marketplace.new(address, provider)
let config = await marketplace.config()
return success config.proofs.zkeyHash
proc formatUrl(hash: string): string =
"https://circuit.codex.storage/proving-key/" & hash
proc retrieveUrl(uri: string): Future[seq[byte]] {.async.} =
let httpSession = HttpSessionRef.new()
try:
let resp = await httpSession.fetch(parseUri(uri))
return resp.data
finally:
await noCancel(httpSession.closeWait())
proc downloadZipfile(url: string, filepath: string): Future[?!void] {.async.} =
try:
let file = await retrieveUrl(url)
var s = newFileStream(filepath, fmWrite)
for b in file:
s.write(b)
s.close()
except Exception as exc:
return failure(exc.msg)
success()
proc unzip(zipfile: string, targetPath: string): ?!void =
try:
extractAll(zipfile, targetPath)
except Exception as exc:
return failure(exc.msg)
success()
proc copyFiles(unpackDir: string, circuitPath: string): ?!void =
try:
for file in walkDir(unpackDir):
copyFileToDir(file.path, circuitPath)
except Exception as exc:
return failure(exc.msg)
success()
proc main() {.async.} =
info "Codex Circuit Downloader, Aww yeah!"
let args = os.commandLineParams()
if args.len != 3:
printHelp()
return
let
circuitPath = args[0]
rpcEndpoint = args[1]
marketplaceAddress = args[2]
zipfile = "circuit.tar.gz"
unpackFolder = "." / "tempunpackfolder"
debug "Starting", circuitPath, rpcEndpoint, marketplaceAddress
if (dirExists(unpackFolder)):
removeDir(unpackFolder)
without circuitHash =? (await getCircuitHash(rpcEndpoint, marketplaceAddress)), err:
error "Failed to get circuit hash", msg = err.msg
return
debug "Got circuithash", circuitHash
let url = formatUrl(circuitHash)
if dlErr =? (await downloadZipfile(url, zipfile)).errorOption:
error "Failed to download circuit file", msg = dlErr.msg
return
debug "Download completed"
if err =? unzip(zipfile, unpackFolder).errorOption:
error "Failed to unzip file", msg = err.msg
return
debug "Unzip completed"
# Unpack library cannot unpack into existing directory. We also cannot
# delete the targer directory and have the library recreate it because
# Codex has likely created it and set correct permissions.
# So, we unpack to a temp folder and move the files.
if err =? copyFiles(unpackFolder, circuitPath).errorOption:
error "Failed to copy files", msg = err.msg
return
debug "Files copied"
removeFile(zipfile)
removeDir(unpackFolder)
debug "file and unpack folder removed"
when isMainModule:
waitFor main()
info "Done!"