codex-storage
/
nim-codex
mirror of https://github.com/codex-storage/nim-codex.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

wip: serialize the config on disk 

doesn't work because toml serialization is broken for writes and json serialization is broken for reads
This commit is contained in:
Dmitriy Ryajov 2022-10-27 17:28:30 -06:00
parent 6e4a8b86ab
commit c6eea8c128
No known key found for this signature in database
GPG Key ID: DA8C680CE7C657A4
5 changed files with 380 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
codex.nim
View File

@ -11,9 +11,12 @@ import pkg/chronicles
import pkg/chronos
import pkg/confutils
import pkg/libp2p
import pkg/toml_serialization
import pkg/json_serialization
import ./codex/conf
import ./codex/codex
import ./codex/utils/serialization
export codex, conf, libp2p, chronos, chronicles
@ -22,34 +25,30 @@ when isMainModule:
import pkg/confutils/defs
import ./codex/utils/fileutils
when defined(posix):
import system/ansi_c
let config = CodexConf.load(
version = codexFullVersion
)
let
config = CodexConf.load(
version = codexFullVersion,
secondarySources = proc (config: CodexConf, sources: auto) =
let
confFile = if config.confFile.isNone:
(config.dataDir / ConfFile).changeFileExt("toml")
else:
config.confFile.get.changeFileExt("toml")
if confFile.fileExists():
sources.addConfigFile(Toml, confFile.InputFile)
)
config.setupDataDir()
config.setupLogging()
config.setupMetrics()
case config.cmd:
of StartUpCommand.noCommand:
if not(checkAndCreateDataDir((config.dataDir).string)):
# We are unable to access/create data folder or data folder's
# permissions are insecure.
quit QuitFailure
trace "Data dir initialized", dir = $config.dataDir
if not(checkAndCreateDataDir((config.dataDir / "repo").string)):
# We are unable to access/create data folder or data folder's
# permissions are insecure.
quit QuitFailure
trace "Repo dir initialized", dir = config.dataDir / "repo"
let server = CodexServer.new(config)
## Ctrl+C handling
@ -77,4 +76,10 @@ when isMainModule:
waitFor server.start()
of StartUpCommand.initNode:
discard
let
confFile = if config.confFile.isSome:
config.confFile.get.string
else:
config.dataDir / ConfFile
Toml.saveFile(confFile.changeFileExt("toml"), config)

215
codex/conf.nim
View File

@ -21,16 +21,25 @@ import pkg/chronicles
import pkg/chronicles/topics_registry
import pkg/confutils/defs
import pkg/confutils/std/net
import confutils/toml/std/uri
import pkg/metrics
import pkg/metrics/chronos_httpserver
import pkg/stew/shims/net as stewnet
import pkg/libp2p
import pkg/libp2p/crypto/secp
import pkg/libp2p/crypto/crypto
import pkg/ethers
import pkg/stew/byteutils
import ./discovery
import ./stores/cachestore
import ../codex/utils/fileutils
export DefaultCacheSizeMiB, net
export DefaultCacheSizeMiB, net, uri
const
RepoDir* = "repo"
ConfFile* = "config"
type
StartUpCommand* {.pure.} = enum
@ -48,6 +57,7 @@ type
logLevel* {.
defaultValue: LogLevel.INFO
desc: "Sets the log level",
serializedFieldName: "log-level"
name: "log-level" }: LogLevel
logFormat* {.
@ -55,116 +65,140 @@ type
desc: "Specifies what kind of logs should be written to stdout (auto, colors, nocolors, json)"
defaultValueDesc: "auto"
defaultValue: LogKind.Auto
serializedFieldName: "log-format"
name: "log-format" }: LogKind
metricsEnabled* {.
desc: "Enable the metrics server"
defaultValue: false
serializedFieldName: "metrics"
name: "metrics" }: bool
metricsAddress* {.
desc: "Listening address of the metrics server"
defaultValue: ValidIpAddress.init("127.0.0.1")
defaultValueDesc: "127.0.0.1"
serializedFieldName: "metrics-address"
name: "metrics-address" }: ValidIpAddress
metricsPort* {.
desc: "Listening HTTP port of the metrics server"
defaultValue: 8008
serializedFieldName: "metrics-port"
name: "metrics-port" }: Port
dataDir* {.
dontSerialize
desc: "The directory where codex will store configuration and data."
defaultValue: defaultDataDir()
defaultValueDesc: ""
abbr: "d"
serializedFieldName: "data-dir"
dontSerialize
name: "data-dir" }: OutDir
listenAddrs* {.
desc: "MultiAddresses to listen on"
defaultValue: @[
MultiAddress.init("/ip4/0.0.0.0/tcp/0")
.expect("Should init multiaddress")]
defaultValueDesc: "/ip4/0.0.0.0/tcp/0"
abbr: "i"
serializedFieldName: "listen-addrs"
name: "listen-addrs" }: seq[MultiAddress]
announceAddrs* {.
desc: "MultiAddresses to announce behind a NAT"
defaultValue: @[]
defaultValueDesc: ""
abbr: "a"
serializedFieldName: "announce-addrs"
name: "announce-addrs" }: seq[MultiAddress]
discoveryPort* {.
desc: "Specify the discovery (UDP) port"
defaultValue: 8090.Port
defaultValueDesc: "8090"
serializedFieldName: "udp-port"
name: "udp-port" }: Port
netPrivKeyFile* {.
desc: "Source of network (secp256k1) private key file (random|<path>)"
defaultValue: "random"
serializedFieldName: "net-privkey"
dontSerialize
name: "net-privkey" }: string
bootstrapNodes* {.
desc: "Specifies one or more bootstrap nodes to use when connecting to the network."
abbr: "b"
serializedFieldName: "bootstrap-node"
name: "bootstrap-node" }: seq[SignedPeerRecord]
maxPeers* {.
desc: "The maximum number of peers to connect to"
defaultValue: 160
serializedFieldName: "max-peers"
name: "max-peers" }: int
agentString* {.
defaultValue: "Codex"
desc: "Node agent string which is used as identifier in network"
serializedFieldName: "agent-string"
name: "agent-string" }: string
apiPort* {.
desc: "The REST Api port",
defaultValue: 8080
defaultValueDesc: "8080"
serializedFieldName: "api-port"
name: "api-port"
abbr: "p" }: int
cacheSize* {.
desc: "The size in MiB of the block cache, 0 disables the cache"
defaultValue: DefaultCacheSizeMiB
defaultValueDesc: $DefaultCacheSizeMiB
serializedFieldName: "cache-size"
name: "cache-size"}: Natural
persistence* {.
desc: "Enables persistence mechanism, requires an Ethereum node"
defaultValue: false
name: "persistence".}: bool
ethProvider* {.
desc: "The URL of the JSON-RPC API of the Ethereum node"
defaultValue: "ws://localhost:8545"
serializedFieldName: "eth-provider"
name: "eth-provider".}: string
ethAccount* {.
desc: "The Ethereum account that is used for storage contracts"
defaultValue: EthAddress.none
serializedFieldName: "eth-account"
name: "eth-account"
dontSerialize.}: Option[EthAddress]
ethDeployment* {.
desc: "The json file describing the contract deployment"
defaultValue: string.none
serializedFieldName: "eth-deployment"
name: "eth-deployment".}: Option[string]
confFile* {.
desc: "The config file to be used, defaults to ``data-dir`/conf.toml`",
defaultValueDesc: ""
abbr: "c"
name: "conf"}: Option[string]
case cmd* {.
dontSerialize
command
defaultValue: noCommand }: StartUpCommand
of noCommand:
listenAddrs* {.
desc: "Multi Addresses to listen on"
defaultValue: @[
MultiAddress.init("/ip4/0.0.0.0/tcp/0")
.expect("Should init multiaddress")]
defaultValueDesc: "/ip4/0.0.0.0/tcp/0"
abbr: "i"
name: "listen-addrs" }: seq[MultiAddress]
announceAddrs* {.
desc: "Multi Addresses to announce behind a NAT"
defaultValue: @[]
defaultValueDesc: ""
abbr: "a"
name: "announce-addrs" }: seq[MultiAddress]
discoveryPort* {.
desc: "Specify the discovery (UDP) port"
defaultValue: Port(8090)
defaultValueDesc: "8090"
name: "udp-port" }: Port
netPrivKeyFile* {.
desc: "Source of network (secp256k1) private key file (random|<path>)"
defaultValue: "random"
name: "net-privkey" }: string
bootstrapNodes* {.
desc: "Specifies one or more bootstrap nodes to use when connecting to the network."
abbr: "b"
name: "bootstrap-node" }: seq[SignedPeerRecord]
maxPeers* {.
desc: "The maximum number of peers to connect to"
defaultValue: 160
name: "max-peers" }: int
agentString* {.
defaultValue: "Codex"
desc: "Node agent string which is used as identifier in network"
name: "agent-string" }: string
apiPort* {.
desc: "The REST Api port",
defaultValue: 8080
defaultValueDesc: "8080"
name: "api-port"
abbr: "p" }: int
cacheSize* {.
desc: "The size in MiB of the block cache, 0 disables the cache"
defaultValue: DefaultCacheSizeMiB
defaultValueDesc: $DefaultCacheSizeMiB
name: "cache-size"
abbr: "c" }: Natural
persistence* {.
desc: "Enables persistence mechanism, requires an Ethereum node"
defaultValue: false
name: "persistence"
.}: bool
ethProvider* {.
desc: "The URL of the JSON-RPC API of the Ethereum node"
defaultValue: "ws://localhost:8545"
name: "eth-provider"
.}: string
ethAccount* {.
desc: "The Ethereum account that is used for storage contracts"
defaultValue: EthAddress.none
name: "eth-account"
.}: Option[EthAddress]
ethDeployment* {.
desc: "The json file describing the contract deployment"
defaultValue: string.none
name: "eth-deployment"
.}: Option[string]
discard
of initNode:
discard
@ -182,7 +216,6 @@ const
"Codex build " & codexVersion & "\p" &
nimBanner
proc defaultDataDir*(): string =
let dataDir = when defined(windows):
"AppData" / "Roaming" / "Codex"
@ -195,7 +228,7 @@ proc defaultDataDir*(): string =
func parseCmdArg*(T: type MultiAddress, input: TaintedString): T
{.raises: [ValueError, LPError, Defect].} =
MultiAddress.init($input).tryGet()
MultiAddress.init($input).get()
proc parseCmdArg*(T: type SignedPeerRecord, uri: TaintedString): T =
var res: SignedPeerRecord
@ -299,3 +332,19 @@ proc setupMetrics*(config: CodexConf) =
raiseAssert exc.msg
except Exception as exc:
raiseAssert exc.msg # TODO fix metrics
proc setupDataDir*(config: CodexConf) =
if not(checkAndCreateDataDir((config.dataDir).string)):
# We are unable to access/create data folder or data folder's
# permissions are insecure.
quit QuitFailure
trace "Data dir initialized", dir = $config.dataDir
let repoDir = config.dataDir / RepoDir
if not(checkAndCreateDataDir((repoDir).string)):
# We are unable to access/create data folder or data folder's
# permissions are insecure.
quit QuitFailure
trace "Repo dir initialized", dir = repoDir

2
codex/node.nim
View File

@ -152,7 +152,7 @@ proc retrieve*(
try:
await stream.pushData(blk.data)
except CatchableError as exc:
trace "Unable to send block", cid
trace "Unable to send block", cid, err = exc.msg
discard
finally:
await stream.pushEof()

4
codex/stores/networkstore.nim
View File

@ -38,7 +38,9 @@ method getBlock*(self: NetworkStore, cid: Cid): Future[?!bt.Block] {.async.} =
trace "Getting block from local store or network", cid
without blk =? await self.localStore.getBlock(cid), error:
if not (error of BlockNotFoundError): return failure error
if not (error of BlockNotFoundError):
return failure error
trace "Block not in local store", cid
# TODO: What if block isn't available in the engine too?
# TODO: add retrieved block to the local store

219
codex/utils/serialization.nim Normal file
View File

@ -0,0 +1,219 @@
## Nim-Codex
## Copyright (c) 2022 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
import pkg/upraises
push: {.upraises: [].}
import std/strutils
import pkg/libp2p
import pkg/libp2p/crypto/secp
import pkg/libp2p/crypto/crypto
import pkg/libp2pdht
import pkg/confutils/defs
import pkg/confutils/std/net
import confutils/toml/std/uri
import pkg/chronicles
import pkg/toml_serialization
import pkg/json_serialization
import pkg/stew/byteutils
import pkg/ethers
import ../conf
proc writeValue*(
writer: var TomlWriter,
value: SignedPeerRecord)
{.raises: [Defect, SerializationError, IOError].} =
writer.writeValue(value.toUri)
proc readValue*(
r: var TomlReader,
value: var SignedPeerRecord)
{.raises: [Defect, SerializationError, IOError].} =
try:
discard value.fromURI(r.readValue(string))
except CatchableError as exc:
raise newException(Defect, exc.msg)
proc writeValue*(
writer: var TomlWriter,
value: secp.SkPublicKey)
{.raises: [Defect, SerializationError, IOError].} =
writer.writeValue(value.getBytes().to0xHex)
proc readValue*(
r: var TomlReader,
value: var secp.SkPublicKey)
{.raises: [Defect, SerializationError, IOError].} =
try:
value = secp.SkPublicKey
.init(r.readValue(string))
.expect("Hex encoded byte array expected for public key")
except ValueError as exc:
raise newException(SerializationError, exc.msg)
proc writeValue*(
writer: var TomlWriter,
value: crypto.PublicKey)
{.raises: [Defect, SerializationError, IOError].} =
writer.writeValue(value.getBytes().get.to0xHex)
proc readValue*(
r: var TomlReader,
value: var crypto.PublicKey)
{.raises: [Defect, SerializationError, IOError].} =
try:
value = crypto.PublicKey
.init(r.readValue(string))
.expect("Hex encoded byte array expected for public key")
except ValueError as exc:
raise newException(SerializationError, exc.msg)
proc writeValue*(
writer: var TomlWriter,
value: MultiAddress)
{.raises: [Defect, SerializationError, IOError].} =
writer.writeValue($value)
proc writeValue*(
writer: var TomlWriter,
value: seq[MultiAddress])
{.raises: [Defect, SerializationError, IOError].} =
writer.writeIterable(value)
proc readValue*(
r: var TomlReader,
value: var MultiAddress)
{.raises: [Defect, SerializationError, IOError].} =
try:
value = MultiAddress.init(r.readValue(string)).get()
except ValueError as exc:
raise newException(SerializationError, exc.msg)
proc writeValue*(
writer: var TomlWriter,
value: LogLevel)
{.raises: [Defect, SerializationError, IOError].} =
writer.writeValue($value)
proc readValue*(
r: var TomlReader,
value: var LogLevel)
{.raises: [Defect, SerializationError, IOError].} =
try:
value = strutils.parseEnum[LogLevel](r.readValue(string))
except ValueError as exc:
raise newException(SerializationError, exc.msg)
proc writeValue*(
writer: var TomlWriter,
value: LogKind)
{.raises: [Defect, SerializationError, IOError].} =
writer.writeValue($value)
proc readValue*(
r: var TomlReader,
value: var LogKind)
{.raises: [Defect, SerializationError, IOError].} =
try:
value = strutils.parseEnum[LogKind](r.readValue(string))
except ValueError as exc:
raise newException(SerializationError, exc.msg)
proc writeValue*(
writer: var TomlWriter,
value: ValidIpAddress)
{.raises: [Defect, SerializationError, IOError].} =
writeStackTrace()
writer.writeValue($value)
proc readValue*(
r: var TomlReader,
value: var ValidIpAddress)
{.raises: [Defect, SerializationError, IOError].} =
try:
value = ValidIpAddress.init(r.readValue(string))
except ValueError as exc:
raise newException(SerializationError, exc.msg)
proc writeValue*(
writer: var TomlWriter,
value: Option[string])
{.raises: [Defect, SerializationError, IOError].} =
if value.isSome:
writer.writeValue(value.get)
else:
writer.writeValue("")
proc readValue*(
r: var TomlReader,
value: var Option[string])
{.raises: [Defect, SerializationError, IOError].} =
try:
value = r.readValue(string).some
except ValueError as exc:
raise newException(SerializationError, exc.msg)
proc writeValue*(
writer: var TomlWriter,
value: Option[LogLevel])
{.raises: [Defect, SerializationError, IOError].} =
if value.isSome:
writer.writeValue($value.get)
else:
writer.writeValue("INFO")
proc readValue*(
r: var TomlReader,
value: var Option[LogLevel])
{.raises: [Defect, SerializationError, IOError].} =
try:
value = strutils.parseEnum[LogLevel](r.readValue(string)).some
except ValueError as exc:
raise newException(SerializationError, exc.msg)
proc writeValue*(
writer: var TomlWriter,
value: Port)
{.raises: [Defect, SerializationError, IOError].} =
writer.writeValue(value.int)
proc readValue*(
r: var TomlReader,
value: var Port)
{.raises: [Defect, SerializationError, IOError].} =
try:
value = Port r.readValue(int)
except ValueError as exc:
raise newException(Defect, exc.msg)
proc writeValue*(
writer: var TomlWriter,
value: EthAddress)
{.raises: [Defect, SerializationError, IOError].} =
writer.writeValue($value)
proc readValue*(
r: var TomlReader,
value: var EthAddress)
{.raises: [Defect, SerializationError, IOError].} =
try:
value = EthAddress.init(r.readValue(string)).get()
except ValueError as exc:
raise newException(SerializationError, exc.msg)
template writeValue*(writer: var TomlWriter,
value: TypedInputFile|InputFile|InputDir|OutPath|OutDir|OutFile) =
writer.writeValue(string value)
template readValue*(reader: var TomlReader,
value: var TypedInputFile|InputFile|InputDir|OutPath|OutDir|OutFile) =
value = typeof(value) reader.readValue(string)