codex-storage
/
nim-codex
mirror of https://github.com/codex-storage/nim-codex.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

rework cli to accept circuit params

This commit is contained in:
Dmitriy Ryajov 2024-02-09 18:46:51 -06:00
parent 9e13d2251a
commit 2e0f087389
No known key found for this signature in database
GPG Key ID: DA8C680CE7C657A4
3 changed files with 389 additions and 297 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
codex.nim
View File

@ -23,6 +23,7 @@ import ./codex/codex
import ./codex/logutils
import ./codex/units
import ./codex/utils/keyutils
import ./codex/codextypes
export codex, conf, libp2p, chronos, logutils
@ -54,9 +55,6 @@ when isMainModule:
config.setupLogging()
config.setupMetrics()
case config.cmd:
of StartUpCommand.noCommand:
if config.nat == ValidIpAddress.init(IPv4_any()):
error "`--nat` cannot be set to the any (`0.0.0.0`) address"
quit QuitFailure
@ -90,7 +88,11 @@ when isMainModule:
config.dataDir / config.netPrivKeyFile
privateKey = setupKey(keyPath).expect("Should setup private key!")
server = CodexServer.new(config, privateKey)
server = try:
CodexServer.new(config, privateKey)
except Exception as exc:
error "Failed to start Codex", msg = exc.msg
quit QuitFailure
## Ctrl+C handling
proc controlCHandler() {.noconv.} =
@ -127,8 +129,12 @@ when isMainModule:
state = CodexStatus.Running
while state == CodexStatus.Running:
try:
# poll chronos
chronos.poll()
except Exception as exc:
error "Unhandled exception in async proc, aborting", msg = exc.msg
quit QuitFailure
# wait fot futures to finish
let res = waitFor allFinished(pendingFuts)
@ -139,6 +145,3 @@ when isMainModule:
quit QuitFailure
notice "Exited codex"
of StartUpCommand.initNode:
discard

42
codex/codex.nim
View File

@ -22,12 +22,14 @@ import pkg/stew/io2
import pkg/stew/shims/net as stewnet
import pkg/datastore
import pkg/ethers except Rng
import pkg/stew/io2
import ./node
import ./conf
import ./rng
import ./rest/api
import ./stores
import ./slots
import ./blockexchange
import ./utils/fileutils
import ./erasure
@ -65,17 +67,9 @@ proc bootstrapInteractions(
config = s.config
repo = s.repoStore
if not config.persistence and not config.validator:
if config.ethAccount.isSome or config.ethPrivateKey.isSome:
warn "Ethereum account was set, but neither persistence nor validator is enabled"
return
if not config.ethAccount.isSome and not config.ethPrivateKey.isSome:
if config.persistence:
if not config.ethAccount.isSome and not config.ethPrivateKey.isSome:
error "Persistence enabled, but no Ethereum account was set"
if config.validator:
error "Validator enabled, but no Ethereum account was set"
quit QuitFailure
let provider = JsonRpcProvider.new(config.ethProvider)
@ -131,6 +125,7 @@ proc bootstrapInteractions(
let sales = Sales.new(market, clock, repo, proofFailures)
client = some ClientInteractions.new(clock, purchasing)
host = some HostInteractions.new(clock, sales)
if config.validator:
let validation = Validation.new(clock, market, config.validatorMaxSlots)
validator = some ValidatorInteractions.new(clock, validation)
@ -262,11 +257,38 @@ proc new*(
blockDiscovery = DiscoveryEngine.new(repoStore, peerStore, network, discovery, pendingBlocks)
engine = BlockExcEngine.new(repoStore, wallet, network, blockDiscovery, peerStore, pendingBlocks)
store = NetworkStore.new(engine, repoStore)
erasure = Erasure.new(store, leoEncoderProvider, leoDecoderProvider)
prover = if config.persistence:
if not fileAccessible($config.circomR1cs, {AccessFlags.Read}):
error "Circom R1CS file not accessible"
raise (ref Defect)(
msg: "r1cs file not readable or doesn't exist")
if not fileAccessible($config.circomWasm, {AccessFlags.Read}):
error "Circom WASM file not accessible"
raise (ref Defect)(
msg: "wasm file not readable or doesn't exist")
let zkey = if not config.circomNoZkey:
if not fileAccessible($config.circomNoZkey, {AccessFlags.Read}):
error "Circom WASM file not accessible"
raise (ref Defect)(
msg: "wasm file not readable or doesn't exist")
$config.circomNoZkey
else:
""
some Prover.new(store, CircomCompat.init($config.circomR1cs, $config.circomWasm, zkey))
else:
none Prover
codexNode = CodexNodeRef.new(
switch = switch,
networkStore = store,
store = store,
engine = engine,
erasure = erasure,
prover = prover,
discovery = discovery)
restServer = RestServerRef.new(

161
codex/conf.nim
View File

@ -7,9 +7,7 @@
## This file may not be copied, modified, or distributed except according to
## those terms.
import pkg/upraises
push: {.upraises: [].}
{.push raises: [].}
import std/os
import std/terminal
@ -33,30 +31,47 @@ import pkg/ethers
import pkg/questionable
import pkg/questionable/results
import ./codextypes
import ./discovery
import ./logutils
import ./stores
import ./units
import ./utils
export units
export net
export units, net, codextypes
export
DefaultQuotaBytes,
DefaultBlockTtl,
DefaultBlockMaintenanceInterval,
DefaultNumberOfBlocksToMaintainPerInterval
proc defaultDataDir*(): string =
let dataDir = when defined(windows):
"AppData" / "Roaming" / "Codex"
elif defined(macosx):
"Library" / "Application Support" / "Codex"
else:
".cache" / "codex"
getHomeDir() / dataDir
const
codex_enable_api_debug_peers* {.booldefine.} = false
codex_enable_proof_failures* {.booldefine.} = false
codex_use_hardhat* {.booldefine.} = false
codex_enable_log_counter* {.booldefine.} = false
DefaultDataDir* = defaultDataDir()
type
StartUpCommand* {.pure.} = enum
noCommand,
initNode
StartUpCmd* {.pure.} = enum
noCmd
persistence
PersistenceCmd* {.pure.} = enum
noCmd
validator
LogKind* {.pure.} = enum
Auto = "auto"
@ -106,17 +121,11 @@ type
dataDir* {.
desc: "The directory where codex will store configuration and data"
defaultValue: defaultDataDir()
defaultValueDesc: ""
defaultValue: DefaultDataDir
defaultValueDesc: $DefaultDataDir
abbr: "d"
name: "data-dir" }: OutDir
case cmd* {.
command
defaultValue: noCommand }: StartUpCommand
of noCommand:
listenAddrs* {.
desc: "Multi Addresses to listen on"
defaultValue: @[
@ -220,12 +229,17 @@ type
name: "cache-size"
abbr: "c" }: NBytes
persistence* {.
desc: "Enables persistence mechanism, requires an Ethereum node"
defaultValue: false
name: "persistence"
.}: bool
logFile* {.
desc: "Logs to file"
defaultValue: string.none
name: "log-file"
hidden
.}: Option[string]
case cmd* {.
defaultValue: noCmd
command }: StartUpCmd
of persistence:
ethProvider* {.
desc: "The URL of the JSON-RPC API of the Ethereum node"
defaultValue: "ws://localhost:8545"
@ -235,33 +249,94 @@ type
ethAccount* {.
desc: "The Ethereum account that is used for storage contracts"
defaultValue: EthAddress.none
defaultValueDesc: ""
name: "eth-account"
.}: Option[EthAddress]
ethPrivateKey* {.
desc: "File containing Ethereum private key for storage contracts"
defaultValue: string.none
defaultValueDesc: ""
name: "eth-private-key"
.}: Option[string]
marketplaceAddress* {.
desc: "Address of deployed Marketplace contract"
defaultValue: EthAddress.none
defaultValueDesc: ""
name: "marketplace-address"
.}: Option[EthAddress]
validator* {.
desc: "Enables validator, requires an Ethereum node"
circomR1cs* {.
desc: "The r1cs file for the storage circuit"
defaultValue: $DefaultDataDir / "circuits" / "proof_main.r1cs"
defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.r1cs"
name: "circom-r1cs"
.}: InputFile
circomWasm* {.
desc: "The wasm file for the storage circuit"
defaultValue: $DefaultDataDir / "circuits" / "proof_main.wasm"
defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.wasm"
name: "circom-wasm"
.}: InputFile
circomZkey* {.
desc: "The zkey file for the storage circuit"
defaultValue: $DefaultDataDir / "circuits" / "proof_main.zkey"
defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.zkey"
name: "circom-zkey"
.}: InputFile
# TODO: should probably be hidden and behind a feature flag
circomNoZkey* {.
desc: "Ignore the zkey file - use only for testing!"
defaultValue: false
name: "validator"
name: "circom-no-zkey"
.}: bool
numProofSamples* {.
desc: "Number of samples to prove"
defaultValue: DefaultSamplesNum
defaultValueDesc: $DefaultSamplesNum
name: "proof-samples" }: int
maxSlotDepth* {.
desc: "The maximum depth of the slot tree"
defaultValue: DefaultMaxSlotDepth
defaultValueDesc: $DefaultMaxSlotDepth
name: "max-slot-depth" }: int
maxDatasetDepth* {.
desc: "The maximum depth of the dataset tree"
defaultValue: DefaultMaxDatasetDepth
defaultValueDesc: $DefaultMaxDatasetDepth
name: "max-dataset-depth" }: int
maxBlockDepth* {.
desc: "The maximum depth of the network block merkle tree"
defaultValue: DefaultBlockDepth
defaultValueDesc: $DefaultBlockDepth
name: "max-block-depth" }: int
maxCellElms* {.
desc: "The maximum number of elements in a cell"
defaultValue: DefaultCellElms
defaultValueDesc: $DefaultCellElms
name: "max-cell-elements" }: int
case persistenceCmd* {.
defaultValue: noCmd
command }: PersistenceCmd
of validator:
validatorMaxSlots* {.
desc: "Maximum number of slots that the validator monitors"
defaultValue: 1000
name: "validator-max-slots"
.}: int
# TODO: should go behind a feature flag
simulateProofFailures* {.
desc: "Simulates proof failures once every N proofs. 0 = disabled."
defaultValue: 0
@ -269,21 +344,23 @@ type
hidden
.}: int
logFile* {.
desc: "Logs to file"
defaultValue: string.none
name: "log-file"
hidden
.}: Option[string]
of PersistenceCmd.noCmd:
discard # end of validator
of initNode:
discard
of StartUpCmd.noCmd:
discard # end of persistence
EthAddress* = ethers.Address
logutils.formatIt(LogFormat.textLines, EthAddress): it.short0xHexLog
logutils.formatIt(LogFormat.json, EthAddress): %it
func persistence*(self: CodexConf): bool =
self.cmd == StartUpCmd.persistence
func validator*(self: CodexConf): bool =
self.persistence and self.persistenceCmd == PersistenceCmd.validator
proc getCodexVersion(): string =
let tag = strip(staticExec("git tag"))
if tag.isEmptyOrWhitespace:
@ -308,16 +385,6 @@ const
"Codex revision: " & codexRevision & "\p" &
nimBanner
proc defaultDataDir*(): string =
let dataDir = when defined(windows):
"AppData" / "Roaming" / "Codex"
elif defined(macosx):
"Library" / "Application Support" / "Codex"
else:
".cache" / "codex"
getHomeDir() / dataDir
proc parseCmdArg*(T: typedesc[MultiAddress],
input: string): MultiAddress
{.upraises: [ValueError, LPError].} =
@ -326,7 +393,7 @@ proc parseCmdArg*(T: typedesc[MultiAddress],
if res.isOk:
ma = res.get()
else:
warn "Invalid MultiAddress", input=input, error=res.error()
warn "Invalid MultiAddress", input=input, error = res.error()
quit QuitFailure
ma
@ -334,10 +401,10 @@ proc parseCmdArg*(T: type SignedPeerRecord, uri: string): T =
var res: SignedPeerRecord
try:
if not res.fromURI(uri):
warn "Invalid SignedPeerRecord uri", uri=uri
warn "Invalid SignedPeerRecord uri", uri = uri
quit QuitFailure
except CatchableError as exc:
warn "Invalid SignedPeerRecord uri", uri=uri, error=exc.msg
warn "Invalid SignedPeerRecord uri", uri = uri, error = exc.msg
quit QuitFailure
res
@ -348,7 +415,7 @@ proc parseCmdArg*(T: type NBytes, val: string): T =
var num = 0'i64
let count = parseSize(val, num, alwaysBin = true)
if count == 0:
warn "Invalid number of bytes", nbytes=val
warn "Invalid number of bytes", nbytes = val
quit QuitFailure
NBytes(num)
@ -356,7 +423,7 @@ proc parseCmdArg*(T: type Duration, val: string): T =
var dur: Duration
let count = parseDuration(val, dur)
if count == 0:
warn "Invalid duration parse", dur=dur
warn "Cannot parse duration", dur = dur
quit QuitFailure
dur