From ecb72fccd5e5d8b89fe620780a45a0b610139939 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 7 Feb 2024 21:27:46 -0600 Subject: [PATCH] wip --- codex.nim | 128 +++++++-------- codex/codex.nim | 1 - codex/conf.nim | 320 ++++++++++++++++++------------------- vendor/nimbus-build-system | 2 +- 4 files changed, 220 insertions(+), 231 deletions(-) diff --git a/codex.nim b/codex.nim index c996827e..6a044018 100644 --- a/codex.nim +++ b/codex.nim @@ -55,91 +55,85 @@ when isMainModule: config.setupLogging() config.setupMetrics() - case config.cmd: - of StartUpCommand.noCommand: + if config.nat == ValidIpAddress.init(IPv4_any()): + error "`--nat` cannot be set to the any (`0.0.0.0`) address" + quit QuitFailure - if config.nat == ValidIpAddress.init(IPv4_any()): - error "`--nat` cannot be set to the any (`0.0.0.0`) address" - quit QuitFailure + if config.nat == ValidIpAddress.init("127.0.0.1"): + warn "`--nat` is set to loopback, your node wont properly announce over the DHT" - if config.nat == ValidIpAddress.init("127.0.0.1"): - warn "`--nat` is set to loopback, your node wont properly announce over the DHT" + if not(checkAndCreateDataDir((config.dataDir).string)): + # We are unable to access/create data folder or data folder's + # permissions are insecure. + quit QuitFailure - if not(checkAndCreateDataDir((config.dataDir).string)): - # We are unable to access/create data folder or data folder's - # permissions are insecure. - quit QuitFailure + trace "Data dir initialized", dir = $config.dataDir - trace "Data dir initialized", dir = $config.dataDir + if not(checkAndCreateDataDir((config.dataDir / "repo"))): + # We are unable to access/create data folder or data folder's + # permissions are insecure. + quit QuitFailure - if not(checkAndCreateDataDir((config.dataDir / "repo"))): - # We are unable to access/create data folder or data folder's - # permissions are insecure. - quit QuitFailure + trace "Repo dir initialized", dir = config.dataDir / "repo" - trace "Repo dir initialized", dir = config.dataDir / "repo" + var + state: CodexStatus + pendingFuts: seq[Future[void]] - var - state: CodexStatus - pendingFuts: seq[Future[void]] + let + keyPath = + if isAbsolute(config.netPrivKeyFile): + config.netPrivKeyFile + else: + config.dataDir / config.netPrivKeyFile - let - keyPath = - if isAbsolute(config.netPrivKeyFile): - config.netPrivKeyFile - else: - config.dataDir / config.netPrivKeyFile + privateKey = setupKey(keyPath).expect("Should setup private key!") + server = CodexServer.new(config, privateKey) - privateKey = setupKey(keyPath).expect("Should setup private key!") - server = CodexServer.new(config, privateKey) + ## Ctrl+C handling + proc controlCHandler() {.noconv.} = + when defined(windows): + # workaround for https://github.com/nim-lang/Nim/issues/4057 + try: + setupForeignThreadGc() + except Exception as exc: raiseAssert exc.msg # shouldn't happen + notice "Shutting down after having received SIGINT" - ## Ctrl+C handling - proc controlCHandler() {.noconv.} = - when defined(windows): - # workaround for https://github.com/nim-lang/Nim/issues/4057 - try: - setupForeignThreadGc() - except Exception as exc: raiseAssert exc.msg # shouldn't happen - notice "Shutting down after having received SIGINT" + pendingFuts.add(server.stop()) + state = CodexStatus.Stopping + + notice "Stopping Codex" + + try: + setControlCHook(controlCHandler) + except Exception as exc: # TODO Exception + warn "Cannot set ctrl-c handler", msg = exc.msg + + # equivalent SIGTERM handler + when defined(posix): + proc SIGTERMHandler(signal: cint) {.noconv.} = + notice "Shutting down after having received SIGTERM" pendingFuts.add(server.stop()) state = CodexStatus.Stopping notice "Stopping Codex" - try: - setControlCHook(controlCHandler) - except Exception as exc: # TODO Exception - warn "Cannot set ctrl-c handler", msg = exc.msg + c_signal(ansi_c.SIGTERM, SIGTERMHandler) - # equivalent SIGTERM handler - when defined(posix): - proc SIGTERMHandler(signal: cint) {.noconv.} = - notice "Shutting down after having received SIGTERM" + pendingFuts.add(server.start()) - pendingFuts.add(server.stop()) - state = CodexStatus.Stopping + state = CodexStatus.Running + while state == CodexStatus.Running: + # poll chronos + chronos.poll() - notice "Stopping Codex" + # wait fot futures to finish + let res = waitFor allFinished(pendingFuts) + state = CodexStatus.Stopped - c_signal(ansi_c.SIGTERM, SIGTERMHandler) + if res.anyIt( it.failed ): + error "Codex didn't shutdown correctly" + quit QuitFailure - pendingFuts.add(server.start()) - - state = CodexStatus.Running - while state == CodexStatus.Running: - # poll chronos - chronos.poll() - - # wait fot futures to finish - let res = waitFor allFinished(pendingFuts) - state = CodexStatus.Stopped - - if res.anyIt( it.failed ): - error "Codex didn't shutdown correctly" - quit QuitFailure - - notice "Exited codex" - - of StartUpCommand.initNode: - discard + notice "Exited codex" diff --git a/codex/codex.nim b/codex/codex.nim index ea3c0068..bbaf0dfc 100644 --- a/codex/codex.nim +++ b/codex/codex.nim @@ -66,7 +66,6 @@ proc bootstrapInteractions( config = s.config repo = s.repoStore - if not config.persistence and not config.validator: if config.ethAccount.isSome or config.ethPrivateKey.isSome: warn "Ethereum account was set, but neither persistence nor validator is enabled" diff --git a/codex/conf.nim b/codex/conf.nim index c5f6f5d9..73010508 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -9,7 +9,7 @@ import pkg/upraises -push: {.upraises: [].} +{.push raises: [].} import std/os import std/terminal @@ -55,9 +55,11 @@ const codex_enable_log_counter* {.booldefine.} = false type - StartUpCommand* {.pure.} = enum - noCommand, - initNode + StartUpCmd* {.pure.} = enum + Persistence + + PersistenceCmd* {.pure.} = enum + Validator LogKind* {.pure.} = enum Auto = "auto" @@ -112,144 +114,120 @@ type abbr: "d" name: "data-dir" }: OutDir + listenAddrs* {. + desc: "Multi Addresses to listen on" + defaultValue: @[ + MultiAddress.init("/ip4/0.0.0.0/tcp/0") + .expect("Should init multiaddress")] + defaultValueDesc: "/ip4/0.0.0.0/tcp/0" + abbr: "i" + name: "listen-addrs" }: seq[MultiAddress] + + # TODO: change this once we integrate nat support + nat* {. + desc: "IP Addresses to announce behind a NAT" + defaultValue: ValidIpAddress.init("127.0.0.1") + defaultValueDesc: "127.0.0.1" + abbr: "a" + name: "nat" }: ValidIpAddress + + discoveryIp* {. + desc: "Discovery listen address" + defaultValue: ValidIpAddress.init(IPv4_any()) + defaultValueDesc: "0.0.0.0" + abbr: "e" + name: "disc-ip" }: ValidIpAddress + + discoveryPort* {. + desc: "Discovery (UDP) port" + defaultValue: 8090.Port + defaultValueDesc: "8090" + abbr: "u" + name: "disc-port" }: Port + + netPrivKeyFile* {. + desc: "Source of network (secp256k1) private key file path or name" + defaultValue: "key" + name: "net-privkey" }: string + + bootstrapNodes* {. + desc: "Specifies one or more bootstrap nodes to use when connecting to the network" + abbr: "b" + name: "bootstrap-node" }: seq[SignedPeerRecord] + + maxPeers* {. + desc: "The maximum number of peers to connect to" + defaultValue: 160 + name: "max-peers" }: int + + agentString* {. + defaultValue: "Codex" + desc: "Node agent string which is used as identifier in network" + name: "agent-string" }: string + + apiBindAddress* {. + desc: "The REST API bind address" + defaultValue: "127.0.0.1" + name: "api-bindaddr" + }: string + + apiPort* {. + desc: "The REST Api port", + defaultValue: 8080.Port + defaultValueDesc: "8080" + name: "api-port" + abbr: "p" }: Port + + repoKind* {. + desc: "Backend for main repo store (fs, sqlite)" + defaultValueDesc: "fs" + defaultValue: repoFS + name: "repo-kind" }: RepoKind + + storageQuota* {. + desc: "The size of the total storage quota dedicated to the node" + defaultValue: DefaultQuotaBytes + defaultValueDesc: $DefaultQuotaBytes + name: "storage-quota" + abbr: "q" }: NBytes + + blockTtl* {. + desc: "Default block timeout in seconds - 0 disables the ttl" + defaultValue: DefaultBlockTtl + defaultValueDesc: $DefaultBlockTtl + name: "block-ttl" + abbr: "t" }: Duration + + blockMaintenanceInterval* {. + desc: "Time interval in seconds - determines frequency of block maintenance cycle: how often blocks are checked for expiration and cleanup" + defaultValue: DefaultBlockMaintenanceInterval + defaultValueDesc: $DefaultBlockMaintenanceInterval + name: "block-mi" }: Duration + + blockMaintenanceNumberOfBlocks* {. + desc: "Number of blocks to check every maintenance cycle" + defaultValue: DefaultNumberOfBlocksToMaintainPerInterval + defaultValueDesc: $DefaultNumberOfBlocksToMaintainPerInterval + name: "block-mn" }: int + + cacheSize* {. + desc: "The size of the block cache, 0 disables the cache - might help on slow hardrives" + defaultValue: 0 + defaultValueDesc: "0" + name: "cache-size" + abbr: "c" }: NBytes + + logFile* {. + desc: "Logs to file" + defaultValue: string.none + name: "log-file" + hidden + .}: Option[string] + case cmd* {. - command - defaultValue: noCommand }: StartUpCommand + command }: StartUpCmd - of noCommand: - - listenAddrs* {. - desc: "Multi Addresses to listen on" - defaultValue: @[ - MultiAddress.init("/ip4/0.0.0.0/tcp/0") - .expect("Should init multiaddress")] - defaultValueDesc: "/ip4/0.0.0.0/tcp/0" - abbr: "i" - name: "listen-addrs" }: seq[MultiAddress] - - # TODO: change this once we integrate nat support - nat* {. - desc: "IP Addresses to announce behind a NAT" - defaultValue: ValidIpAddress.init("127.0.0.1") - defaultValueDesc: "127.0.0.1" - abbr: "a" - name: "nat" }: ValidIpAddress - - discoveryIp* {. - desc: "Discovery listen address" - defaultValue: ValidIpAddress.init(IPv4_any()) - defaultValueDesc: "0.0.0.0" - abbr: "e" - name: "disc-ip" }: ValidIpAddress - - discoveryPort* {. - desc: "Discovery (UDP) port" - defaultValue: 8090.Port - defaultValueDesc: "8090" - abbr: "u" - name: "disc-port" }: Port - - netPrivKeyFile* {. - desc: "Source of network (secp256k1) private key file path or name" - defaultValue: "key" - name: "net-privkey" }: string - - bootstrapNodes* {. - desc: "Specifies one or more bootstrap nodes to use when connecting to the network" - abbr: "b" - name: "bootstrap-node" }: seq[SignedPeerRecord] - - maxPeers* {. - desc: "The maximum number of peers to connect to" - defaultValue: 160 - name: "max-peers" }: int - - agentString* {. - defaultValue: "Codex" - desc: "Node agent string which is used as identifier in network" - name: "agent-string" }: string - - apiBindAddress* {. - desc: "The REST API bind address" - defaultValue: "127.0.0.1" - name: "api-bindaddr" - }: string - - apiPort* {. - desc: "The REST Api port", - defaultValue: 8080.Port - defaultValueDesc: "8080" - name: "api-port" - abbr: "p" }: Port - - repoKind* {. - desc: "Backend for main repo store (fs, sqlite)" - defaultValueDesc: "fs" - defaultValue: repoFS - name: "repo-kind" }: RepoKind - - storageQuota* {. - desc: "The size of the total storage quota dedicated to the node" - defaultValue: DefaultQuotaBytes - defaultValueDesc: $DefaultQuotaBytes - name: "storage-quota" - abbr: "q" }: NBytes - - blockTtl* {. - desc: "Default block timeout in seconds - 0 disables the ttl" - defaultValue: DefaultBlockTtl - defaultValueDesc: $DefaultBlockTtl - name: "block-ttl" - abbr: "t" }: Duration - - blockMaintenanceInterval* {. - desc: "Time interval in seconds - determines frequency of block maintenance cycle: how often blocks are checked for expiration and cleanup" - defaultValue: DefaultBlockMaintenanceInterval - defaultValueDesc: $DefaultBlockMaintenanceInterval - name: "block-mi" }: Duration - - blockMaintenanceNumberOfBlocks* {. - desc: "Number of blocks to check every maintenance cycle" - defaultValue: DefaultNumberOfBlocksToMaintainPerInterval - defaultValueDesc: $DefaultNumberOfBlocksToMaintainPerInterval - name: "block-mn" }: int - - cacheSize* {. - desc: "The size of the block cache, 0 disables the cache - might help on slow hardrives" - defaultValue: 0 - defaultValueDesc: "0" - name: "cache-size" - abbr: "c" }: NBytes - - persistence* {. - desc: "Enables persistence mechanism, requires an Ethereum node" - defaultValue: false - name: "persistence" - .}: bool - - circomR1cs* {. - desc: "The r1cs file for the storage circuit" - defaultValue: defaultDataDir() / "circuits" / "proof_main.r1cs" - name: "circom-r1cs" - .}: string - - circomWasm* {. - desc: "The wasm file for the storage circuit" - defaultValue: defaultDataDir() / "circuits" / "proof_main.wasm" - name: "circom-wasm" - .}: string - - circomZkey* {. - desc: "The zkey file for the storage circuit" - defaultValue: defaultDataDir() / "circuits" / "proof_main.zkey" - name: "circom-zkey" - .}: string - - circomNoZkey* {. - desc: "Ignore the zkey file - use only for testing!" - defaultValue: false - name: "circom-no-zkey" - .}: bool + of Persistence: numProofSamples* {. desc: "Number of samples to prove" @@ -281,40 +259,58 @@ type name: "marketplace-address" .}: Option[EthAddress] - validator* {. - desc: "Enables validator, requires an Ethereum node" + circomR1cs* {. + desc: "The r1cs file for the storage circuit" + defaultValue: $defaultDataDir() / "circuits" / "proof_main.r1cs" + name: "circom-r1cs" + .}: string + + circomWasm* {. + desc: "The wasm file for the storage circuit" + defaultValue: $defaultDataDir() / "circuits" / "proof_main.wasm" + name: "circom-wasm" + .}: string + + circomZkey* {. + desc: "The zkey file for the storage circuit" + defaultValue: $defaultDataDir() / "circuits" / "proof_main.zkey" + name: "circom-zkey" + .}: string + + circomNoZkey* {. + desc: "Ignore the zkey file - use only for testing!" defaultValue: false - name: "validator" + name: "circom-no-zkey" .}: bool - validatorMaxSlots* {. - desc: "Maximum number of slots that the validator monitors" - defaultValue: 1000 - name: "validator-max-slots" - .}: int + case persistenceCmd* {. + command }: PersistenceCmd - simulateProofFailures* {. - desc: "Simulates proof failures once every N proofs. 0 = disabled." - defaultValue: 0 - name: "simulate-proof-failures" - hidden + of Validator: + validatorMaxSlots* {. + desc: "Maximum number of slots that the validator monitors" + defaultValue: 1000 + name: "validator-max-slots" .}: int - logFile* {. - desc: "Logs to file" - defaultValue: string.none - name: "log-file" - hidden - .}: Option[string] - - of initNode: - discard + simulateProofFailures* {. + desc: "Simulates proof failures once every N proofs. 0 = disabled." + defaultValue: 0 + name: "simulate-proof-failures" + hidden + .}: int EthAddress* = ethers.Address logutils.formatIt(LogFormat.textLines, EthAddress): it.short0xHexLog logutils.formatIt(LogFormat.json, EthAddress): %it +func persistence*(self: CodexConf): bool = + self.cmd == StartUpCmd.Persistence + +func validator*(self: CodexConf): bool = + self.persistenceCmd == PersistenceCmd.Validator + proc getCodexVersion(): string = let tag = strip(staticExec("git tag")) if tag.isEmptyOrWhitespace: diff --git a/vendor/nimbus-build-system b/vendor/nimbus-build-system index 3866a8ab..fe9bc3f3 160000 --- a/vendor/nimbus-build-system +++ b/vendor/nimbus-build-system @@ -1 +1 @@ -Subproject commit 3866a8ab98fc6e0e6d406b88800aed72163d5fd4 +Subproject commit fe9bc3f3759ae1add6bf8c899db2e75327f03782