wip: serialize the config on disk

doesn't work because toml serialization is broken for writes and json serialization is broken for reads

codex.nim

@@ -11,9 +11,12 @@ import pkg/chronicles
 import pkg/chronos
 import pkg/confutils
 import pkg/libp2p
+import pkg/toml_serialization
+import pkg/json_serialization
 
 import ./codex/conf
 import ./codex/codex
+import ./codex/utils/serialization
 
 export codex, conf, libp2p, chronos, chronicles
 
@@ -22,34 +25,30 @@ when isMainModule:
 
   import pkg/confutils/defs
 
-  import ./codex/utils/fileutils
-
   when defined(posix):
     import system/ansi_c
 
-  let config = CodexConf.load(
+  let
-    version = codexFullVersion
+    config = CodexConf.load(
-  )
+      version = codexFullVersion,
+      secondarySources = proc (config: CodexConf, sources: auto) =
+        let
+          confFile = if config.confFile.isNone:
+              (config.dataDir / ConfFile).changeFileExt("toml")
+            else:
+              config.confFile.get.changeFileExt("toml")
+
+        if confFile.fileExists():
+          sources.addConfigFile(Toml, confFile.InputFile)
+    )
+
+  config.setupDataDir()
   config.setupLogging()
   config.setupMetrics()
 
   case config.cmd:
   of StartUpCommand.noCommand:
 
-    if not(checkAndCreateDataDir((config.dataDir).string)):
-      # We are unable to access/create data folder or data folder's
-      # permissions are insecure.
-      quit QuitFailure
-
-    trace "Data dir initialized", dir = $config.dataDir
-
-    if not(checkAndCreateDataDir((config.dataDir / "repo").string)):
-      # We are unable to access/create data folder or data folder's
-      # permissions are insecure.
-      quit QuitFailure
-
-    trace "Repo dir initialized", dir = config.dataDir / "repo"
-
     let server = CodexServer.new(config)
 
     ## Ctrl+C handling
@@ -77,4 +76,10 @@ when isMainModule:
 
     waitFor server.start()
   of StartUpCommand.initNode:
-    discard
+    let
+      confFile = if config.confFile.isSome:
+          config.confFile.get.string
+        else:
+          config.dataDir / ConfFile
+
+    Toml.saveFile(confFile.changeFileExt("toml"), config)

codex/conf.nim

@@ -21,16 +21,25 @@ import pkg/chronicles
 import pkg/chronicles/topics_registry
 import pkg/confutils/defs
 import pkg/confutils/std/net
+import confutils/toml/std/uri
 import pkg/metrics
 import pkg/metrics/chronos_httpserver
 import pkg/stew/shims/net as stewnet
 import pkg/libp2p
+import pkg/libp2p/crypto/secp
+import pkg/libp2p/crypto/crypto
 import pkg/ethers
+import pkg/stew/byteutils
 
 import ./discovery
 import ./stores/cachestore
+import ../codex/utils/fileutils
 
-export DefaultCacheSizeMiB, net
+export DefaultCacheSizeMiB, net, uri
+
+const
+  RepoDir* = "repo"
+  ConfFile* = "config"
 
 type
   StartUpCommand* {.pure.} = enum
@@ -48,6 +57,7 @@ type
     logLevel* {.
       defaultValue: LogLevel.INFO
       desc: "Sets the log level",
+      serializedFieldName: "log-level"
       name: "log-level" }: LogLevel
 
     logFormat* {.
@@ -55,116 +65,140 @@ type
       desc: "Specifies what kind of logs should be written to stdout (auto, colors, nocolors, json)"
       defaultValueDesc: "auto"
       defaultValue: LogKind.Auto
+      serializedFieldName: "log-format"
       name: "log-format" }: LogKind
 
     metricsEnabled* {.
       desc: "Enable the metrics server"
       defaultValue: false
+      serializedFieldName: "metrics"
       name: "metrics" }: bool
 
     metricsAddress* {.
       desc: "Listening address of the metrics server"
       defaultValue: ValidIpAddress.init("127.0.0.1")
       defaultValueDesc: "127.0.0.1"
+      serializedFieldName: "metrics-address"
       name: "metrics-address" }: ValidIpAddress
 
     metricsPort* {.
       desc: "Listening HTTP port of the metrics server"
       defaultValue: 8008
+      serializedFieldName: "metrics-port"
       name: "metrics-port" }: Port
 
     dataDir* {.
+      dontSerialize
       desc: "The directory where codex will store configuration and data."
       defaultValue: defaultDataDir()
       defaultValueDesc: ""
       abbr: "d"
+      serializedFieldName: "data-dir"
+      dontSerialize
       name: "data-dir" }: OutDir
 
+    listenAddrs* {.
+      desc: "MultiAddresses to listen on"
+      defaultValue: @[
+        MultiAddress.init("/ip4/0.0.0.0/tcp/0")
+        .expect("Should init multiaddress")]
+      defaultValueDesc: "/ip4/0.0.0.0/tcp/0"
+      abbr: "i"
+      serializedFieldName: "listen-addrs"
+      name: "listen-addrs" }: seq[MultiAddress]
+
+    announceAddrs* {.
+      desc: "MultiAddresses to announce behind a NAT"
+      defaultValue: @[]
+      defaultValueDesc: ""
+      abbr: "a"
+      serializedFieldName: "announce-addrs"
+      name: "announce-addrs" }: seq[MultiAddress]
+
+    discoveryPort* {.
+      desc: "Specify the discovery (UDP) port"
+      defaultValue: 8090.Port
+      defaultValueDesc: "8090"
+      serializedFieldName: "udp-port"
+      name: "udp-port" }: Port
+
+    netPrivKeyFile* {.
+      desc: "Source of network (secp256k1) private key file (random|<path>)"
+      defaultValue: "random"
+      serializedFieldName: "net-privkey"
+      dontSerialize
+      name: "net-privkey" }: string
+
+    bootstrapNodes* {.
+      desc: "Specifies one or more bootstrap nodes to use when connecting to the network."
+      abbr: "b"
+      serializedFieldName: "bootstrap-node"
+      name: "bootstrap-node" }: seq[SignedPeerRecord]
+
+    maxPeers* {.
+      desc: "The maximum number of peers to connect to"
+      defaultValue: 160
+      serializedFieldName: "max-peers"
+      name: "max-peers" }: int
+
+    agentString* {.
+      defaultValue: "Codex"
+      desc: "Node agent string which is used as identifier in network"
+      serializedFieldName: "agent-string"
+      name: "agent-string" }: string
+
+    apiPort* {.
+      desc: "The REST Api port",
+      defaultValue: 8080
+      defaultValueDesc: "8080"
+      serializedFieldName: "api-port"
+      name: "api-port"
+      abbr: "p" }: int
+
+    cacheSize* {.
+      desc: "The size in MiB of the block cache, 0 disables the cache"
+      defaultValue: DefaultCacheSizeMiB
+      defaultValueDesc: $DefaultCacheSizeMiB
+      serializedFieldName: "cache-size"
+      name: "cache-size"}: Natural
+
+    persistence* {.
+      desc: "Enables persistence mechanism, requires an Ethereum node"
+      defaultValue: false
+      name: "persistence".}: bool
+
+    ethProvider* {.
+      desc: "The URL of the JSON-RPC API of the Ethereum node"
+      defaultValue: "ws://localhost:8545"
+      serializedFieldName: "eth-provider"
+      name: "eth-provider".}: string
+
+    ethAccount* {.
+      desc: "The Ethereum account that is used for storage contracts"
+      defaultValue: EthAddress.none
+      serializedFieldName: "eth-account"
+      name: "eth-account"
+      dontSerialize.}: Option[EthAddress]
+
+    ethDeployment* {.
+      desc: "The json file describing the contract deployment"
+      defaultValue: string.none
+      serializedFieldName: "eth-deployment"
+      name: "eth-deployment".}: Option[string]
+
+    confFile* {.
+      desc: "The config file to be used, defaults to ``data-dir`/conf.toml`",
+      defaultValueDesc: ""
+      abbr: "c"
+      name: "conf"}: Option[string]
+
     case cmd* {.
+      dontSerialize
       command
       defaultValue: noCommand }: StartUpCommand
 
     of noCommand:
-      listenAddrs* {.
+      discard
-        desc: "Multi Addresses to listen on"
-        defaultValue: @[
-          MultiAddress.init("/ip4/0.0.0.0/tcp/0")
-          .expect("Should init multiaddress")]
-        defaultValueDesc: "/ip4/0.0.0.0/tcp/0"
-        abbr: "i"
-        name: "listen-addrs" }: seq[MultiAddress]
-
-      announceAddrs* {.
-        desc: "Multi Addresses to announce behind a NAT"
-        defaultValue: @[]
-        defaultValueDesc: ""
-        abbr: "a"
-        name: "announce-addrs" }: seq[MultiAddress]
-
-      discoveryPort* {.
-        desc: "Specify the discovery (UDP) port"
-        defaultValue: Port(8090)
-        defaultValueDesc: "8090"
-        name: "udp-port" }: Port
-
-      netPrivKeyFile* {.
-        desc: "Source of network (secp256k1) private key file (random|<path>)"
-        defaultValue: "random"
-        name: "net-privkey" }: string
-
-      bootstrapNodes* {.
-        desc: "Specifies one or more bootstrap nodes to use when connecting to the network."
-        abbr: "b"
-        name: "bootstrap-node" }: seq[SignedPeerRecord]
-
-      maxPeers* {.
-        desc: "The maximum number of peers to connect to"
-        defaultValue: 160
-        name: "max-peers" }: int
-
-      agentString* {.
-        defaultValue: "Codex"
-        desc: "Node agent string which is used as identifier in network"
-        name: "agent-string" }: string
-
-      apiPort* {.
-        desc: "The REST Api port",
-        defaultValue: 8080
-        defaultValueDesc: "8080"
-        name: "api-port"
-        abbr: "p" }: int
-
-      cacheSize* {.
-        desc: "The size in MiB of the block cache, 0 disables the cache"
-        defaultValue: DefaultCacheSizeMiB
-        defaultValueDesc: $DefaultCacheSizeMiB
-        name: "cache-size"
-        abbr: "c" }: Natural
-
-      persistence* {.
-        desc: "Enables persistence mechanism, requires an Ethereum node"
-        defaultValue: false
-        name: "persistence"
-      .}: bool
-
-      ethProvider* {.
-        desc: "The URL of the JSON-RPC API of the Ethereum node"
-        defaultValue: "ws://localhost:8545"
-        name: "eth-provider"
-      .}: string
-
-      ethAccount* {.
-        desc: "The Ethereum account that is used for storage contracts"
-        defaultValue: EthAddress.none
-        name: "eth-account"
-      .}: Option[EthAddress]
-
-      ethDeployment* {.
-        desc: "The json file describing the contract deployment"
-        defaultValue: string.none
-        name: "eth-deployment"
-      .}: Option[string]
-
     of initNode:
       discard
 
@@ -182,7 +216,6 @@ const
     "Codex build " & codexVersion & "\p" &
     nimBanner
 
-
 proc defaultDataDir*(): string =
   let dataDir = when defined(windows):
     "AppData" / "Roaming" / "Codex"
@@ -195,7 +228,7 @@ proc defaultDataDir*(): string =
 
 func parseCmdArg*(T: type MultiAddress, input: TaintedString): T
                  {.raises: [ValueError, LPError, Defect].} =
-  MultiAddress.init($input).tryGet()
+  MultiAddress.init($input).get()
 
 proc parseCmdArg*(T: type SignedPeerRecord, uri: TaintedString): T =
   var res: SignedPeerRecord
@@ -299,3 +332,19 @@ proc setupMetrics*(config: CodexConf) =
       raiseAssert exc.msg
     except Exception as exc:
       raiseAssert exc.msg # TODO fix metrics
+
+proc setupDataDir*(config: CodexConf) =
+  if not(checkAndCreateDataDir((config.dataDir).string)):
+    # We are unable to access/create data folder or data folder's
+    # permissions are insecure.
+    quit QuitFailure
+
+  trace "Data dir initialized", dir = $config.dataDir
+
+  let repoDir = config.dataDir / RepoDir
+  if not(checkAndCreateDataDir((repoDir).string)):
+    # We are unable to access/create data folder or data folder's
+    # permissions are insecure.
+    quit QuitFailure
+
+  trace "Repo dir initialized", dir = repoDir

codex/node.nim

@@ -152,7 +152,7 @@ proc retrieve*(
       try:
         await stream.pushData(blk.data)
       except CatchableError as exc:
-        trace "Unable to send block", cid
+        trace "Unable to send block", cid, err = exc.msg
         discard
       finally:
         await stream.pushEof()

codex/stores/networkstore.nim

@@ -38,7 +38,9 @@ method getBlock*(self: NetworkStore, cid: Cid): Future[?!bt.Block] {.async.} =
   trace "Getting block from local store or network", cid
 
   without blk =? await self.localStore.getBlock(cid), error:
-    if not (error of BlockNotFoundError): return failure error
+    if not (error of BlockNotFoundError):
+      return failure error
+
     trace "Block not in local store", cid
     # TODO: What if block isn't available in the engine too?
     # TODO: add retrieved block to the local store

codex/utils/serialization.nim

@@ -0,0 +1,219 @@
+## Nim-Codex
+## Copyright (c) 2022 Status Research & Development GmbH
+## Licensed under either of
+##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+## at your option.
+## This file may not be copied, modified, or distributed except according to
+## those terms.
+
+import pkg/upraises
+
+push: {.upraises: [].}
+
+import std/strutils
+
+import pkg/libp2p
+import pkg/libp2p/crypto/secp
+import pkg/libp2p/crypto/crypto
+import pkg/libp2pdht
+import pkg/confutils/defs
+import pkg/confutils/std/net
+import confutils/toml/std/uri
+import pkg/chronicles
+import pkg/toml_serialization
+import pkg/json_serialization
+import pkg/stew/byteutils
+import pkg/ethers
+
+import ../conf
+
+proc writeValue*(
+  writer: var TomlWriter,
+  value: SignedPeerRecord)
+  {.raises: [Defect, SerializationError, IOError].} =
+  writer.writeValue(value.toUri)
+
+proc readValue*(
+  r: var TomlReader,
+  value: var SignedPeerRecord)
+  {.raises: [Defect, SerializationError, IOError].} =
+  try:
+    discard value.fromURI(r.readValue(string))
+  except CatchableError as exc:
+    raise newException(Defect, exc.msg)
+
+proc writeValue*(
+  writer: var TomlWriter,
+  value: secp.SkPublicKey)
+  {.raises: [Defect, SerializationError, IOError].} =
+  writer.writeValue(value.getBytes().to0xHex)
+
+proc readValue*(
+  r: var TomlReader,
+  value: var secp.SkPublicKey)
+  {.raises: [Defect, SerializationError, IOError].} =
+    try:
+      value = secp.SkPublicKey
+      .init(r.readValue(string))
+      .expect("Hex encoded byte array expected for public key")
+    except ValueError as exc:
+      raise newException(SerializationError, exc.msg)
+
+proc writeValue*(
+  writer: var TomlWriter,
+  value: crypto.PublicKey)
+  {.raises: [Defect, SerializationError, IOError].} =
+  writer.writeValue(value.getBytes().get.to0xHex)
+
+proc readValue*(
+  r: var TomlReader,
+  value: var crypto.PublicKey)
+  {.raises: [Defect, SerializationError, IOError].} =
+    try:
+      value = crypto.PublicKey
+      .init(r.readValue(string))
+      .expect("Hex encoded byte array expected for public key")
+    except ValueError as exc:
+      raise newException(SerializationError, exc.msg)
+
+proc writeValue*(
+  writer: var TomlWriter,
+  value: MultiAddress)
+  {.raises: [Defect, SerializationError, IOError].} =
+  writer.writeValue($value)
+
+proc writeValue*(
+  writer: var TomlWriter,
+  value: seq[MultiAddress])
+  {.raises: [Defect, SerializationError, IOError].} =
+  writer.writeIterable(value)
+
+proc readValue*(
+  r: var TomlReader,
+  value: var MultiAddress)
+  {.raises: [Defect, SerializationError, IOError].} =
+  try:
+    value = MultiAddress.init(r.readValue(string)).get()
+  except ValueError as exc:
+    raise newException(SerializationError, exc.msg)
+
+proc writeValue*(
+  writer: var TomlWriter,
+  value: LogLevel)
+  {.raises: [Defect, SerializationError, IOError].} =
+  writer.writeValue($value)
+
+proc readValue*(
+  r: var TomlReader,
+  value: var LogLevel)
+  {.raises: [Defect, SerializationError, IOError].} =
+  try:
+    value = strutils.parseEnum[LogLevel](r.readValue(string))
+  except ValueError as exc:
+    raise newException(SerializationError, exc.msg)
+
+proc writeValue*(
+  writer: var TomlWriter,
+  value: LogKind)
+  {.raises: [Defect, SerializationError, IOError].} =
+  writer.writeValue($value)
+
+proc readValue*(
+  r: var TomlReader,
+  value: var LogKind)
+  {.raises: [Defect, SerializationError, IOError].} =
+  try:
+    value = strutils.parseEnum[LogKind](r.readValue(string))
+  except ValueError as exc:
+    raise newException(SerializationError, exc.msg)
+
+proc writeValue*(
+  writer: var TomlWriter,
+  value: ValidIpAddress)
+  {.raises: [Defect, SerializationError, IOError].} =
+  writeStackTrace()
+  writer.writeValue($value)
+
+proc readValue*(
+  r: var TomlReader,
+  value: var ValidIpAddress)
+  {.raises: [Defect, SerializationError, IOError].} =
+  try:
+    value = ValidIpAddress.init(r.readValue(string))
+  except ValueError as exc:
+    raise newException(SerializationError, exc.msg)
+
+proc writeValue*(
+  writer: var TomlWriter,
+  value: Option[string])
+  {.raises: [Defect, SerializationError, IOError].} =
+  if value.isSome:
+    writer.writeValue(value.get)
+  else:
+    writer.writeValue("")
+
+proc readValue*(
+  r: var TomlReader,
+  value: var Option[string])
+  {.raises: [Defect, SerializationError, IOError].} =
+  try:
+    value = r.readValue(string).some
+  except ValueError as exc:
+    raise newException(SerializationError, exc.msg)
+
+proc writeValue*(
+  writer: var TomlWriter,
+  value: Option[LogLevel])
+  {.raises: [Defect, SerializationError, IOError].} =
+  if value.isSome:
+    writer.writeValue($value.get)
+  else:
+    writer.writeValue("INFO")
+
+proc readValue*(
+  r: var TomlReader,
+  value: var Option[LogLevel])
+  {.raises: [Defect, SerializationError, IOError].} =
+  try:
+    value = strutils.parseEnum[LogLevel](r.readValue(string)).some
+  except ValueError as exc:
+    raise newException(SerializationError, exc.msg)
+
+proc writeValue*(
+  writer: var TomlWriter,
+  value: Port)
+  {.raises: [Defect, SerializationError, IOError].} =
+  writer.writeValue(value.int)
+
+proc readValue*(
+  r: var TomlReader,
+  value: var Port)
+  {.raises: [Defect, SerializationError, IOError].} =
+  try:
+    value = Port r.readValue(int)
+  except ValueError as exc:
+    raise newException(Defect, exc.msg)
+
+proc writeValue*(
+  writer: var TomlWriter,
+  value: EthAddress)
+  {.raises: [Defect, SerializationError, IOError].} =
+  writer.writeValue($value)
+
+proc readValue*(
+  r: var TomlReader,
+  value: var EthAddress)
+  {.raises: [Defect, SerializationError, IOError].} =
+  try:
+    value = EthAddress.init(r.readValue(string)).get()
+  except ValueError as exc:
+    raise newException(SerializationError, exc.msg)
+
+template writeValue*(writer: var TomlWriter,
+                     value: TypedInputFile|InputFile|InputDir|OutPath|OutDir|OutFile) =
+  writer.writeValue(string value)
+
+template readValue*(reader: var TomlReader,
+                     value: var TypedInputFile|InputFile|InputDir|OutPath|OutDir|OutFile) =
+  value = typeof(value) reader.readValue(string)