codex-storage
/
nim-codex
mirror of https://github.com/codex-storage/nim-codex.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

wip rework cli to support circuit params

This commit is contained in:
Dmitriy Ryajov 2024-02-09 18:22:30 -06:00
parent 66df642661
commit 4f4cbe72b9
No known key found for this signature in database
GPG Key ID: DA8C680CE7C657A4
7 changed files with 176 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
codex.nim
View File

@ -88,7 +88,11 @@ when isMainModule:
config.dataDir / config.netPrivKeyFile
privateKey = setupKey(keyPath).expect("Should setup private key!")
server = CodexServer.new(config, privateKey)
server = try:
CodexServer.new(config, privateKey)
except Exception as exc:
error "Failed to start Codex", msg = exc.msg
quit QuitFailure
## Ctrl+C handling
proc controlCHandler() {.noconv.} =
@ -125,8 +129,12 @@ when isMainModule:
state = CodexStatus.Running
while state == CodexStatus.Running:
# poll chronos
chronos.poll()
try:
# poll chronos
chronos.poll()
except Exception as exc:
error "Unhandled exception in async proc, aborting", msg = exc.msg
quit QuitFailure
# wait fot futures to finish
let res = waitFor allFinished(pendingFuts)

152
codex/codex.nim
View File

@ -22,6 +22,7 @@ import pkg/stew/io2
import pkg/stew/shims/net as stewnet
import pkg/datastore
import pkg/ethers except Rng
import pkg/stew/io2
import ./node
import ./conf
@ -66,77 +67,70 @@ proc bootstrapInteractions(
config = s.config
repo = s.repoStore
if not config.persistence and not config.validator:
if config.ethAccount.isSome or config.ethPrivateKey.isSome:
warn "Ethereum account was set, but neither persistence nor validator is enabled"
return
if not config.ethAccount.isSome and not config.ethPrivateKey.isSome:
if config.persistence:
error "Persistence enabled, but no Ethereum account was set"
if config.validator:
error "Validator enabled, but no Ethereum account was set"
quit QuitFailure
let provider = JsonRpcProvider.new(config.ethProvider)
var signer: Signer
if account =? config.ethAccount:
signer = provider.getSigner(account)
elif keyFile =? config.ethPrivateKey:
without isSecure =? checkSecureFile(keyFile):
error "Could not check file permissions: does Ethereum private key file exist?"
quit QuitFailure
if not isSecure:
error "Ethereum private key file does not have safe file permissions"
quit QuitFailure
without key =? keyFile.readAllChars():
error "Unable to read Ethereum private key file"
quit QuitFailure
without wallet =? EthWallet.new(key.strip(), provider):
error "Invalid Ethereum private key in file"
quit QuitFailure
signer = wallet
let deploy = Deployment.new(provider, config)
without marketplaceAddress =? await deploy.address(Marketplace):
error "No Marketplace address was specified or there is no known address for the current network"
quit QuitFailure
let marketplace = Marketplace.new(marketplaceAddress, signer)
let market = OnChainMarket.new(marketplace)
let clock = OnChainClock.new(provider)
var client: ?ClientInteractions
var host: ?HostInteractions
var validator: ?ValidatorInteractions
if config.validator or config.persistence:
s.codexNode.clock = clock
else:
s.codexNode.clock = SystemClock()
if config.persistence:
# This is used for simulation purposes. Normal nodes won't be compiled with this flag
# and hence the proof failure will always be 0.
when codex_enable_proof_failures:
let proofFailures = config.simulateProofFailures
if proofFailures > 0:
warn "Enabling proof failure simulation!"
if not config.ethAccount.isSome and not config.ethPrivateKey.isSome:
error "Persistence enabled, but no Ethereum account was set"
quit QuitFailure
let provider = JsonRpcProvider.new(config.ethProvider)
var signer: Signer
if account =? config.ethAccount:
signer = provider.getSigner(account)
elif keyFile =? config.ethPrivateKey:
without isSecure =? checkSecureFile(keyFile):
error "Could not check file permissions: does Ethereum private key file exist?"
quit QuitFailure
if not isSecure:
error "Ethereum private key file does not have safe file permissions"
quit QuitFailure
without key =? keyFile.readAllChars():
error "Unable to read Ethereum private key file"
quit QuitFailure
without wallet =? EthWallet.new(key.strip(), provider):
error "Invalid Ethereum private key in file"
quit QuitFailure
signer = wallet
let deploy = Deployment.new(provider, config)
without marketplaceAddress =? await deploy.address(Marketplace):
error "No Marketplace address was specified or there is no known address for the current network"
quit QuitFailure
let marketplace = Marketplace.new(marketplaceAddress, signer)
let market = OnChainMarket.new(marketplace)
let clock = OnChainClock.new(provider)
var client: ?ClientInteractions
var host: ?HostInteractions
var validator: ?ValidatorInteractions
if config.validator or config.persistence:
s.codexNode.clock = clock
else:
let proofFailures = 0
if config.simulateProofFailures > 0:
warn "Proof failure simulation is not enabled for this build! Configuration ignored"
s.codexNode.clock = SystemClock()
let purchasing = Purchasing.new(market, clock)
let sales = Sales.new(market, clock, repo, proofFailures)
client = some ClientInteractions.new(clock, purchasing)
host = some HostInteractions.new(clock, sales)
if config.persistence:
# This is used for simulation purposes. Normal nodes won't be compiled with this flag
# and hence the proof failure will always be 0.
when codex_enable_proof_failures:
let proofFailures = config.simulateProofFailures
if proofFailures > 0:
warn "Enabling proof failure simulation!"
else:
let proofFailures = 0
if config.simulateProofFailures > 0:
warn "Proof failure simulation is not enabled for this build! Configuration ignored"
if config.validator:
let validation = Validation.new(clock, market, config.validatorMaxSlots)
validator = some ValidatorInteractions.new(clock, validation)
let purchasing = Purchasing.new(market, clock)
let sales = Sales.new(market, clock, repo, proofFailures)
client = some ClientInteractions.new(clock, purchasing)
host = some HostInteractions.new(clock, sales)
s.codexNode.contracts = (client, host, validator)
if config.validator:
let validation = Validation.new(clock, market, config.validatorMaxSlots)
validator = some ValidatorInteractions.new(clock, validation)
s.codexNode.contracts = (client, host, validator)
proc start*(s: CodexServer) {.async.} =
trace "Starting codex node", config = $s.config
@ -265,11 +259,27 @@ proc new*(
store = NetworkStore.new(engine, repoStore)
erasure = Erasure.new(store, leoEncoderProvider, leoDecoderProvider)
prover = if config.persistence:
let
zkey = if config.circomNoZkey: "" else: config.circomZkey
some Prover.new(
store,
CircomCompat.init(config.circomR1cs, config.circomWasm, zkey))
if not fileAccessible($config.circomR1cs, {AccessFlags.Read}):
error "Circom R1CS file not accessible"
raise (ref Defect)(
msg: "r1cs file not readable or doesn't exist")
if not fileAccessible($config.circomWasm, {AccessFlags.Read}):
error "Circom WASM file not accessible"
raise (ref Defect)(
msg: "wasm file not readable or doesn't exist")
let zkey = if not config.circomNoZkey:
if not fileAccessible($config.circomNoZkey, {AccessFlags.Read}):
error "Circom WASM file not accessible"
raise (ref Defect)(
msg: "wasm file not readable or doesn't exist")
$config.circomNoZkey
else:
""
some Prover.new(store, CircomCompat.init($config.circomR1cs, $config.circomWasm, zkey))
else:
none Prover

118
codex/conf.nim
View File

@ -7,8 +7,6 @@
## This file may not be copied, modified, or distributed except according to
## those terms.
import pkg/upraises
{.push raises: [].}
import std/os
@ -48,18 +46,32 @@ export
DefaultBlockMaintenanceInterval,
DefaultNumberOfBlocksToMaintainPerInterval
proc defaultDataDir*(): string =
let dataDir = when defined(windows):
"AppData" / "Roaming" / "Codex"
elif defined(macosx):
"Library" / "Application Support" / "Codex"
else:
".cache" / "codex"
getHomeDir() / dataDir
const
codex_enable_api_debug_peers* {.booldefine.} = false
codex_enable_proof_failures* {.booldefine.} = false
codex_use_hardhat* {.booldefine.} = false
codex_enable_log_counter* {.booldefine.} = false
DefaultDataDir* = defaultDataDir()
type
StartUpCmd* {.pure.} = enum
Persistence
noCmd
persistence
PersistenceCmd* {.pure.} = enum
Validator
noCmd
validator
LogKind* {.pure.} = enum
Auto = "auto"
@ -109,8 +121,8 @@ type
dataDir* {.
desc: "The directory where codex will store configuration and data"
defaultValue: defaultDataDir()
defaultValueDesc: ""
defaultValue: DefaultDataDir
defaultValueDesc: $DefaultDataDir
abbr: "d"
name: "data-dir" }: OutDir
@ -225,16 +237,9 @@ type
.}: Option[string]
case cmd* {.
defaultValue: noCmd
command }: StartUpCmd
of Persistence:
numProofSamples* {.
desc: "Number of samples to prove"
defaultValue: DefaultSamplesNum
defaultValueDesc: $DefaultSamplesNum
name: "proof-samples" }: int
of persistence:
ethProvider* {.
desc: "The URL of the JSON-RPC API of the Ethereum node"
defaultValue: "ws://localhost:8545"
@ -244,55 +249,94 @@ type
ethAccount* {.
desc: "The Ethereum account that is used for storage contracts"
defaultValue: EthAddress.none
defaultValueDesc: ""
name: "eth-account"
.}: Option[EthAddress]
ethPrivateKey* {.
desc: "File containing Ethereum private key for storage contracts"
defaultValue: string.none
defaultValueDesc: ""
name: "eth-private-key"
.}: Option[string]
marketplaceAddress* {.
desc: "Address of deployed Marketplace contract"
defaultValue: EthAddress.none
defaultValueDesc: ""
name: "marketplace-address"
.}: Option[EthAddress]
circomR1cs* {.
desc: "The r1cs file for the storage circuit"
defaultValue: $defaultDataDir() / "circuits" / "proof_main.r1cs"
defaultValue: $DefaultDataDir / "circuits" / "proof_main.r1cs"
defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.r1cs"
name: "circom-r1cs"
.}: string
.}: InputFile
circomWasm* {.
desc: "The wasm file for the storage circuit"
defaultValue: $defaultDataDir() / "circuits" / "proof_main.wasm"
defaultValue: $DefaultDataDir / "circuits" / "proof_main.wasm"
defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.wasm"
name: "circom-wasm"
.}: string
.}: InputFile
circomZkey* {.
desc: "The zkey file for the storage circuit"
defaultValue: $defaultDataDir() / "circuits" / "proof_main.zkey"
defaultValue: $DefaultDataDir / "circuits" / "proof_main.zkey"
defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.zkey"
name: "circom-zkey"
.}: string
.}: InputFile
# TODO: should probably be hidden and behind a feature flag
circomNoZkey* {.
desc: "Ignore the zkey file - use only for testing!"
defaultValue: false
name: "circom-no-zkey"
.}: bool
numProofSamples* {.
desc: "Number of samples to prove"
defaultValue: DefaultSamplesNum
defaultValueDesc: $DefaultSamplesNum
name: "proof-samples" }: int
maxSlotDepth* {.
desc: "The maximum depth of the slot tree"
defaultValue: DefaultMaxSlotDepth
defaultValueDesc: $DefaultMaxSlotDepth
name: "max-slot-depth" }: int
maxDatasetDepth* {.
desc: "The maximum depth of the dataset tree"
defaultValue: DefaultMaxDatasetDepth
defaultValueDesc: $DefaultMaxDatasetDepth
name: "max-dataset-depth" }: int
maxBlockDepth* {.
desc: "The maximum depth of the network block merkle tree"
defaultValue: DefaultBlockDepth
defaultValueDesc: $DefaultBlockDepth
name: "max-block-depth" }: int
maxCellElms* {.
desc: "The maximum number of elements in a cell"
defaultValue: DefaultCellElms
defaultValueDesc: $DefaultCellElms
name: "max-cell-elements" }: int
case persistenceCmd* {.
defaultValue: noCmd
command }: PersistenceCmd
of Validator:
of validator:
validatorMaxSlots* {.
desc: "Maximum number of slots that the validator monitors"
defaultValue: 1000
name: "validator-max-slots"
.}: int
# TODO: should go behind a feature flag
simulateProofFailures* {.
desc: "Simulates proof failures once every N proofs. 0 = disabled."
defaultValue: 0
@ -300,16 +344,22 @@ type
hidden
.}: int
of PersistenceCmd.noCmd:
discard # end of validator
of StartUpCmd.noCmd:
discard # end of persistence
EthAddress* = ethers.Address
logutils.formatIt(LogFormat.textLines, EthAddress): it.short0xHexLog
logutils.formatIt(LogFormat.json, EthAddress): %it
func persistence*(self: CodexConf): bool =
self.cmd == StartUpCmd.Persistence
self.cmd == StartUpCmd.persistence
func validator*(self: CodexConf): bool =
self.persistenceCmd == PersistenceCmd.Validator
self.persistence and self.persistenceCmd == PersistenceCmd.validator
proc getCodexVersion(): string =
let tag = strip(staticExec("git tag"))
@ -335,16 +385,6 @@ const
"Codex revision: " & codexRevision & "\p" &
nimBanner
proc defaultDataDir*(): string =
let dataDir = when defined(windows):
"AppData" / "Roaming" / "Codex"
elif defined(macosx):
"Library" / "Application Support" / "Codex"
else:
".cache" / "codex"
getHomeDir() / dataDir
proc parseCmdArg*(T: typedesc[MultiAddress],
input: string): MultiAddress
{.upraises: [ValueError, LPError].} =
@ -353,7 +393,7 @@ proc parseCmdArg*(T: typedesc[MultiAddress],
if res.isOk:
ma = res.get()
else:
warn "Invalid MultiAddress", input=input, error=res.error()
warn "Invalid MultiAddress", input=input, error = res.error()
quit QuitFailure
ma
@ -361,10 +401,10 @@ proc parseCmdArg*(T: type SignedPeerRecord, uri: string): T =
var res: SignedPeerRecord
try:
if not res.fromURI(uri):
warn "Invalid SignedPeerRecord uri", uri=uri
warn "Invalid SignedPeerRecord uri", uri = uri
quit QuitFailure
except CatchableError as exc:
warn "Invalid SignedPeerRecord uri", uri=uri, error=exc.msg
warn "Invalid SignedPeerRecord uri", uri = uri, error = exc.msg
quit QuitFailure
res
@ -375,7 +415,7 @@ proc parseCmdArg*(T: type NBytes, val: string): T =
var num = 0'i64
let count = parseSize(val, num, alwaysBin = true)
if count == 0:
warn "Invalid number of bytes", nbytes=val
warn "Invalid number of bytes", nbytes = val
quit QuitFailure
NBytes(num)
@ -383,7 +423,7 @@ proc parseCmdArg*(T: type Duration, val: string): T =
var dur: Duration
let count = parseDuration(val, dur)
if count == 0:
warn "Invalid duration parse", dur=dur
warn "Cannot parse duration", dur = dur
quit QuitFailure
dur

7
codex/slots/proofs/backends/circomcompat.nim
View File

@ -110,10 +110,6 @@ proc prove*[H](
"slotRoot".cstring, slotRoot[0].addr, slotRoot.len.uint32) != ERR_OK:
return failure("Failed to push data set root")
if backend.pushInputU256Array(
"slotRoot".cstring, slotRoot[0].addr, slotRoot.len.uint32) != ERR_OK:
return failure("Failed to push data set root")
if backend.pushInputU32(
"nCellsPerSlot".cstring, input.nCellsPerSlot.uint32) != ERR_OK:
return failure("Failed to push nCellsPerSlot")
@ -176,8 +172,7 @@ proc prove*[H](
if backend != nil:
backend.addr.releaseCircomCompat()
if backend != nil:
backend.addr.releaseCircomCompat()
success proof
proc verify*(
self: CircomCompat,

6
codex/slots/proofs/prover.nim
View File

@ -25,10 +25,8 @@ import ../../utils/poseidon2digest
import ../builder
import ../sampler
import ../../manifest
import ../../merkletree
import ../../stores
import ../../market
import ./backends
import ../types
export backends

2
config.nims
View File

@ -86,6 +86,8 @@ when (NimMajor, NimMinor) >= (1, 6):
when (NimMajor, NimMinor, NimPatch) >= (1, 6, 11):
--warning:"BareExcept:off"
switch("define", "nimOldCaseObjects") # needed for confutils commands
switch("define", "withoutPCRE")
# the default open files limit is too low on macOS (512), breaking the

75
tests/codex/slots/backends/testcircomcompat.nim
View File

@ -16,10 +16,6 @@ import pkg/codex/codextypes
import pkg/codex/manifest
import pkg/codex/stores
import pkg/constantine/math/arithmetic
import pkg/constantine/math/io/io_fields
import pkg/constantine/math/io/io_bigints
import ./helpers
import ../helpers
@ -135,74 +131,3 @@ suite "Test Circom Compat Backend":
proof = circom.prove(proofInput).tryGet
check circom.verify(proof, publicInputs, verifyingKeyPtr[]).tryGet == false
suite "Test Circom Compat Backend":
let
slotId = 3
samples = 5
ecK = 2
ecM = 2
numDatasetBlocks = 8
blockSize = DefaultBlockSize
cellSize = DefaultCellSize
r1cs = "tests/circuits/fixtures/proof_main.r1cs"
wasm = "tests/circuits/fixtures/proof_main.wasm"
var
store: BlockStore
manifest: Manifest
protected: Manifest
verifiable: Manifest
circom: CircomCompat
verifyingKeyPtr: ptr CircomKey
proofInput: ProofInput[Poseidon2Hash]
publicInputs: CircomInputs
challenge: array[32, byte]
builder: Poseidon2Builder
sampler: Poseidon2Sampler
setup:
let
repoDs = SQLiteDatastore.new(Memory).tryGet()
metaDs = SQLiteDatastore.new(Memory).tryGet()
store = RepoStore.new(repoDs, metaDs)
(manifest, protected, verifiable) =
await createVerifiableManifest(
store,
numDatasetBlocks,
ecK, ecM,
blockSize,
cellSize)
builder = Poseidon2Builder.new(store, verifiable).tryGet
sampler = Poseidon2Sampler.new(slotId, store, builder).tryGet
# circom = CircomCompat.init(r1cs, wasm, zkey)
circom = CircomCompat.init(r1cs, wasm)
verifyingKeyPtr = circom.getVerifyingKey().tryGet
challenge = 1234567.toF.toBytes.toArray32
proofInput = (await sampler.getProofInput(challenge, samples)).tryGet
publicInputs = proofInput.toPublicInputs.toCircomInputs
teardown:
publicInputs.releaseNimInputs() # this is allocated by nim
verifyingKeyPtr.addr.releaseKey() # this comes from the rust FFI
circom.release() # this comes from the rust FFI
test "Should verify with correct input":
var
proof = circom.prove(proofInput).tryGet
check circom.verify(proof, publicInputs, verifyingKeyPtr[]).tryGet
test "Should not verify with incorrect input":
proofInput.slotIndex = 1 # change slot index
let
proof = circom.prove(proofInput).tryGet
check circom.verify(proof, publicInputs, verifyingKeyPtr[]).tryGet == false