551 lines
16 KiB
Nim
551 lines
16 KiB
Nim
## Nim-Codex
|
|
## Copyright (c) 2021 Status Research & Development GmbH
|
|
## Licensed under either of
|
|
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
|
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
|
## at your option.
|
|
## This file may not be copied, modified, or distributed except according to
|
|
## those terms.
|
|
|
|
import pkg/upraises
|
|
|
|
push: {.upraises: [].}
|
|
|
|
import std/os
|
|
import std/terminal
|
|
import std/options
|
|
import std/strutils
|
|
import std/typetraits
|
|
|
|
import pkg/chronos
|
|
import pkg/chronicles/helpers
|
|
import pkg/chronicles/topics_registry
|
|
import pkg/confutils/defs
|
|
import pkg/confutils/std/net
|
|
import pkg/toml_serialization
|
|
import pkg/metrics
|
|
import pkg/metrics/chronos_httpserver
|
|
import pkg/stew/shims/net as stewnet
|
|
import pkg/stew/shims/parseutils
|
|
import pkg/stew/byteutils
|
|
import pkg/libp2p
|
|
import pkg/ethers
|
|
import pkg/questionable
|
|
import pkg/questionable/results
|
|
|
|
import ./discovery
|
|
import ./logutils
|
|
import ./stores
|
|
import ./units
|
|
import ./utils
|
|
|
|
export units
|
|
export net
|
|
export
|
|
DefaultQuotaBytes,
|
|
DefaultBlockTtl,
|
|
DefaultBlockMaintenanceInterval,
|
|
DefaultNumberOfBlocksToMaintainPerInterval
|
|
|
|
const
|
|
codex_enable_api_debug_peers* {.booldefine.} = false
|
|
codex_enable_proof_failures* {.booldefine.} = false
|
|
codex_use_hardhat* {.booldefine.} = false
|
|
codex_enable_log_counter* {.booldefine.} = false
|
|
|
|
type
|
|
StartUpCommand* {.pure.} = enum
|
|
noCommand,
|
|
initNode
|
|
|
|
LogKind* {.pure.} = enum
|
|
Auto = "auto"
|
|
Colors = "colors"
|
|
NoColors = "nocolors"
|
|
Json = "json"
|
|
None = "none"
|
|
|
|
RepoKind* = enum
|
|
repoFS = "fs"
|
|
repoSQLite = "sqlite"
|
|
|
|
CodexConf* = object
|
|
configFile* {.
|
|
desc: "Loads the configuration from a TOML file"
|
|
defaultValueDesc: "none"
|
|
defaultValue: InputFile.none
|
|
name: "config-file" }: Option[InputFile]
|
|
|
|
logLevel* {.
|
|
defaultValue: "info"
|
|
desc: "Sets the log level",
|
|
name: "log-level" }: string
|
|
|
|
logFormat* {.
|
|
hidden
|
|
desc: "Specifies what kind of logs should be written to stdout (auto, colors, nocolors, json)"
|
|
defaultValueDesc: "auto"
|
|
defaultValue: LogKind.Auto
|
|
name: "log-format" }: LogKind
|
|
|
|
metricsEnabled* {.
|
|
desc: "Enable the metrics server"
|
|
defaultValue: false
|
|
name: "metrics" }: bool
|
|
|
|
metricsAddress* {.
|
|
desc: "Listening address of the metrics server"
|
|
defaultValue: ValidIpAddress.init("127.0.0.1")
|
|
defaultValueDesc: "127.0.0.1"
|
|
name: "metrics-address" }: ValidIpAddress
|
|
|
|
metricsPort* {.
|
|
desc: "Listening HTTP port of the metrics server"
|
|
defaultValue: 8008
|
|
name: "metrics-port" }: Port
|
|
|
|
dataDir* {.
|
|
desc: "The directory where codex will store configuration and data."
|
|
defaultValue: defaultDataDir()
|
|
defaultValueDesc: ""
|
|
abbr: "d"
|
|
name: "data-dir" }: OutDir
|
|
|
|
case cmd* {.
|
|
command
|
|
defaultValue: noCommand }: StartUpCommand
|
|
|
|
of noCommand:
|
|
|
|
listenAddrs* {.
|
|
desc: "Multi Addresses to listen on"
|
|
defaultValue: @[
|
|
MultiAddress.init("/ip4/0.0.0.0/tcp/0")
|
|
.expect("Should init multiaddress")]
|
|
defaultValueDesc: "/ip4/0.0.0.0/tcp/0"
|
|
abbr: "i"
|
|
name: "listen-addrs" }: seq[MultiAddress]
|
|
|
|
# TODO: change this once we integrate nat support
|
|
nat* {.
|
|
desc: "IP Addresses to announce behind a NAT"
|
|
defaultValue: ValidIpAddress.init("127.0.0.1")
|
|
defaultValueDesc: "127.0.0.1"
|
|
abbr: "a"
|
|
name: "nat" }: ValidIpAddress
|
|
|
|
discoveryIp* {.
|
|
desc: "Discovery listen address"
|
|
defaultValue: ValidIpAddress.init(IPv4_any())
|
|
defaultValueDesc: "0.0.0.0"
|
|
abbr: "e"
|
|
name: "disc-ip" }: ValidIpAddress
|
|
|
|
discoveryPort* {.
|
|
desc: "Discovery (UDP) port"
|
|
defaultValue: 8090.Port
|
|
defaultValueDesc: "8090"
|
|
abbr: "u"
|
|
name: "disc-port" }: Port
|
|
|
|
netPrivKeyFile* {.
|
|
desc: "Source of network (secp256k1) private key file path or name"
|
|
defaultValue: "key"
|
|
name: "net-privkey" }: string
|
|
|
|
bootstrapNodes* {.
|
|
desc: "Specifies one or more bootstrap nodes to use when connecting to the network."
|
|
abbr: "b"
|
|
name: "bootstrap-node" }: seq[SignedPeerRecord]
|
|
|
|
maxPeers* {.
|
|
desc: "The maximum number of peers to connect to"
|
|
defaultValue: 160
|
|
name: "max-peers" }: int
|
|
|
|
agentString* {.
|
|
defaultValue: "Codex"
|
|
desc: "Node agent string which is used as identifier in network"
|
|
name: "agent-string" }: string
|
|
|
|
apiBindAddress* {.
|
|
desc: "The REST API bind address"
|
|
defaultValue: "127.0.0.1"
|
|
name: "api-bindaddr"
|
|
}: string
|
|
|
|
apiPort* {.
|
|
desc: "The REST Api port",
|
|
defaultValue: 8080.Port
|
|
defaultValueDesc: "8080"
|
|
name: "api-port"
|
|
abbr: "p" }: Port
|
|
|
|
repoKind* {.
|
|
desc: "backend for main repo store (fs, sqlite)"
|
|
defaultValueDesc: "fs"
|
|
defaultValue: repoFS
|
|
name: "repo-kind" }: RepoKind
|
|
|
|
storageQuota* {.
|
|
desc: "The size of the total storage quota dedicated to the node"
|
|
defaultValue: DefaultQuotaBytes
|
|
defaultValueDesc: $DefaultQuotaBytes
|
|
name: "storage-quota"
|
|
abbr: "q" }: NBytes
|
|
|
|
blockTtl* {.
|
|
desc: "Default block timeout in seconds - 0 disables the ttl"
|
|
defaultValue: DefaultBlockTtl
|
|
defaultValueDesc: $DefaultBlockTtl
|
|
name: "block-ttl"
|
|
abbr: "t" }: Duration
|
|
|
|
blockMaintenanceInterval* {.
|
|
desc: "Time interval in seconds - determines frequency of block maintenance cycle: how often blocks are checked for expiration and cleanup."
|
|
defaultValue: DefaultBlockMaintenanceInterval
|
|
defaultValueDesc: $DefaultBlockMaintenanceInterval
|
|
name: "block-mi" }: Duration
|
|
|
|
blockMaintenanceNumberOfBlocks* {.
|
|
desc: "Number of blocks to check every maintenance cycle."
|
|
defaultValue: DefaultNumberOfBlocksToMaintainPerInterval
|
|
defaultValueDesc: $DefaultNumberOfBlocksToMaintainPerInterval
|
|
name: "block-mn" }: int
|
|
|
|
cacheSize* {.
|
|
desc: "The size of the block cache, 0 disables the cache - might help on slow hardrives"
|
|
defaultValue: 0
|
|
defaultValueDesc: "0"
|
|
name: "cache-size"
|
|
abbr: "c" }: NBytes
|
|
|
|
persistence* {.
|
|
desc: "Enables persistence mechanism, requires an Ethereum node"
|
|
defaultValue: false
|
|
name: "persistence"
|
|
.}: bool
|
|
|
|
ethProvider* {.
|
|
desc: "The URL of the JSON-RPC API of the Ethereum node"
|
|
defaultValue: "ws://localhost:8545"
|
|
name: "eth-provider"
|
|
.}: string
|
|
|
|
ethAccount* {.
|
|
desc: "The Ethereum account that is used for storage contracts"
|
|
defaultValue: EthAddress.none
|
|
name: "eth-account"
|
|
.}: Option[EthAddress]
|
|
|
|
ethPrivateKey* {.
|
|
desc: "File containing Ethereum private key for storage contracts"
|
|
defaultValue: string.none
|
|
name: "eth-private-key"
|
|
.}: Option[string]
|
|
|
|
marketplaceAddress* {.
|
|
desc: "Address of deployed Marketplace contract"
|
|
defaultValue: EthAddress.none
|
|
name: "marketplace-address"
|
|
.}: Option[EthAddress]
|
|
|
|
validator* {.
|
|
desc: "Enables validator, requires an Ethereum node"
|
|
defaultValue: false
|
|
name: "validator"
|
|
.}: bool
|
|
|
|
validatorMaxSlots* {.
|
|
desc: "Maximum number of slots that the validator monitors"
|
|
defaultValue: 1000
|
|
name: "validator-max-slots"
|
|
.}: int
|
|
|
|
simulateProofFailures* {.
|
|
desc: "Simulates proof failures once every N proofs. 0 = disabled."
|
|
defaultValue: 0
|
|
name: "simulate-proof-failures"
|
|
hidden
|
|
.}: int
|
|
|
|
logFile* {.
|
|
desc: "Logs to file"
|
|
defaultValue: string.none
|
|
name: "log-file"
|
|
hidden
|
|
.}: Option[string]
|
|
|
|
of initNode:
|
|
discard
|
|
|
|
EthAddress* = ethers.Address
|
|
|
|
logutils.formatIt(LogFormat.textLines, EthAddress): it.short0xHexLog
|
|
logutils.formatIt(LogFormat.json, EthAddress): %it
|
|
|
|
proc getCodexVersion(): string =
|
|
let tag = strip(staticExec("git tag"))
|
|
if tag.isEmptyOrWhitespace:
|
|
return "untagged build"
|
|
return tag
|
|
|
|
proc getCodexRevision(): string =
|
|
# using a slice in a static context breaks nimsuggest for some reason
|
|
var res = strip(staticExec("git rev-parse --short HEAD"))
|
|
res.setLen(6)
|
|
return res
|
|
|
|
proc getNimBanner(): string =
|
|
staticExec("nim --version | grep Version")
|
|
|
|
const
|
|
codexVersion* = getCodexVersion()
|
|
codexRevision* = getCodexRevision()
|
|
nimBanner* = getNimBanner()
|
|
|
|
codexFullVersion* =
|
|
"Codex version: " & codexVersion & "\p" &
|
|
"Codex revision: " & codexRevision & "\p" &
|
|
nimBanner
|
|
|
|
proc defaultDataDir*(): string =
|
|
let dataDir = when defined(windows):
|
|
"AppData" / "Roaming" / "Codex"
|
|
elif defined(macosx):
|
|
"Library" / "Application Support" / "Codex"
|
|
else:
|
|
".cache" / "codex"
|
|
|
|
getHomeDir() / dataDir
|
|
|
|
proc parseCmdArg*(T: typedesc[MultiAddress],
|
|
input: string): MultiAddress
|
|
{.upraises: [ValueError, LPError].} =
|
|
var ma: MultiAddress
|
|
let res = MultiAddress.init(input)
|
|
if res.isOk:
|
|
ma = res.get()
|
|
else:
|
|
warn "Invalid MultiAddress", input=input, error=res.error()
|
|
quit QuitFailure
|
|
ma
|
|
|
|
proc parseCmdArg*(T: type SignedPeerRecord, uri: string): T =
|
|
var res: SignedPeerRecord
|
|
try:
|
|
if not res.fromURI(uri):
|
|
warn "Invalid SignedPeerRecord uri", uri=uri
|
|
quit QuitFailure
|
|
except CatchableError as exc:
|
|
warn "Invalid SignedPeerRecord uri", uri=uri, error=exc.msg
|
|
quit QuitFailure
|
|
res
|
|
|
|
proc parseCmdArg*(T: type EthAddress, address: string): T =
|
|
EthAddress.init($address).get()
|
|
|
|
proc parseCmdArg*(T: type NBytes, val: string): T =
|
|
var num = 0'i64
|
|
let count = parseSize(val, num, alwaysBin = true)
|
|
if count == 0:
|
|
warn "Invalid number of bytes", nbytes=val
|
|
quit QuitFailure
|
|
NBytes(num)
|
|
|
|
proc parseCmdArg*(T: type Duration, val: string): T =
|
|
var dur: Duration
|
|
let count = parseDuration(val, dur)
|
|
if count == 0:
|
|
warn "Invalid duration parse", dur=dur
|
|
quit QuitFailure
|
|
dur
|
|
|
|
proc readValue*(r: var TomlReader, val: var EthAddress)
|
|
{.upraises: [SerializationError, IOError].} =
|
|
val = EthAddress.init(r.readValue(string)).get()
|
|
|
|
proc readValue*(r: var TomlReader, val: var SignedPeerRecord) =
|
|
without uri =? r.readValue(string).catch, err:
|
|
error "invalid SignedPeerRecord configuration value", error = err.msg
|
|
quit QuitFailure
|
|
|
|
val = SignedPeerRecord.parseCmdArg(uri)
|
|
|
|
proc readValue*(r: var TomlReader, val: var MultiAddress) =
|
|
without input =? r.readValue(string).catch, err:
|
|
error "invalid MultiAddress configuration value", error = err.msg
|
|
quit QuitFailure
|
|
|
|
let res = MultiAddress.init(input)
|
|
if res.isOk:
|
|
val = res.get()
|
|
else:
|
|
warn "Invalid MultiAddress", input=input, error=res.error()
|
|
quit QuitFailure
|
|
|
|
proc readValue*(r: var TomlReader, val: var NBytes)
|
|
{.upraises: [SerializationError, IOError].} =
|
|
var value = 0'i64
|
|
var str = r.readValue(string)
|
|
let count = parseSize(str, value, alwaysBin = true)
|
|
if count == 0:
|
|
error "invalid number of bytes for configuration value", value = str
|
|
quit QuitFailure
|
|
val = NBytes(value)
|
|
|
|
proc readValue*(r: var TomlReader, val: var Duration)
|
|
{.upraises: [SerializationError, IOError].} =
|
|
var str = r.readValue(string)
|
|
var dur: Duration
|
|
let count = parseDuration(str, dur)
|
|
if count == 0:
|
|
error "Invalid duration parse", value = str
|
|
quit QuitFailure
|
|
val = dur
|
|
|
|
# no idea why confutils needs this:
|
|
proc completeCmdArg*(T: type EthAddress; val: string): seq[string] =
|
|
discard
|
|
|
|
proc completeCmdArg*(T: type NBytes; val: string): seq[string] =
|
|
discard
|
|
|
|
proc completeCmdArg*(T: type Duration; val: string): seq[string] =
|
|
discard
|
|
|
|
# silly chronicles, colors is a compile-time property
|
|
proc stripAnsi*(v: string): string =
|
|
var
|
|
res = newStringOfCap(v.len)
|
|
i: int
|
|
|
|
while i < v.len:
|
|
let c = v[i]
|
|
if c == '\x1b':
|
|
var
|
|
x = i + 1
|
|
found = false
|
|
|
|
while x < v.len: # look for [..m
|
|
let c2 = v[x]
|
|
if x == i + 1:
|
|
if c2 != '[':
|
|
break
|
|
else:
|
|
if c2 in {'0'..'9'} + {';'}:
|
|
discard # keep looking
|
|
elif c2 == 'm':
|
|
i = x + 1
|
|
found = true
|
|
break
|
|
else:
|
|
break
|
|
inc x
|
|
|
|
if found: # skip adding c
|
|
continue
|
|
res.add c
|
|
inc i
|
|
|
|
res
|
|
|
|
proc updateLogLevel*(logLevel: string) {.upraises: [ValueError].} =
|
|
# Updates log levels (without clearing old ones)
|
|
let directives = logLevel.split(";")
|
|
try:
|
|
setLogLevel(parseEnum[LogLevel](directives[0].toUpperAscii))
|
|
except ValueError:
|
|
raise (ref ValueError)(msg: "Please specify one of: trace, debug, info, notice, warn, error or fatal")
|
|
|
|
if directives.len > 1:
|
|
for topicName, settings in parseTopicDirectives(directives[1..^1]):
|
|
if not setTopicState(topicName, settings.state, settings.logLevel):
|
|
warn "Unrecognized logging topic", topic = topicName
|
|
|
|
proc setupLogging*(conf: CodexConf) =
|
|
when defaultChroniclesStream.outputs.type.arity != 3:
|
|
warn "Logging configuration options not enabled in the current build"
|
|
else:
|
|
var logFile: ?IoHandle
|
|
proc noOutput(logLevel: LogLevel, msg: LogOutputStr) = discard
|
|
proc writeAndFlush(f: File, msg: LogOutputStr) =
|
|
try:
|
|
f.write(msg)
|
|
f.flushFile()
|
|
except IOError as err:
|
|
logLoggingFailure(cstring(msg), err)
|
|
|
|
proc stdoutFlush(logLevel: LogLevel, msg: LogOutputStr) =
|
|
writeAndFlush(stdout, msg)
|
|
|
|
proc noColorsFlush(logLevel: LogLevel, msg: LogOutputStr) =
|
|
writeAndFlush(stdout, stripAnsi(msg))
|
|
|
|
proc fileFlush(logLevel: LogLevel, msg: LogOutputStr) =
|
|
if file =? logFile:
|
|
if error =? file.writeFile(stripAnsi(msg).toBytes).errorOption:
|
|
error "failed to write to log file", errorCode = $error
|
|
|
|
defaultChroniclesStream.outputs[2].writer = noOutput
|
|
if logFilePath =? conf.logFile and logFilePath.len > 0:
|
|
let logFileHandle = openFile(
|
|
logFilePath,
|
|
{OpenFlags.Write, OpenFlags.Create, OpenFlags.Truncate}
|
|
)
|
|
if logFileHandle.isErr:
|
|
error "failed to open log file",
|
|
path = logFilePath,
|
|
errorCode = $logFileHandle.error
|
|
else:
|
|
logFile = logFileHandle.option
|
|
defaultChroniclesStream.outputs[2].writer = fileFlush
|
|
|
|
defaultChroniclesStream.outputs[1].writer = noOutput
|
|
|
|
let writer =
|
|
case conf.logFormat:
|
|
of LogKind.Auto:
|
|
if isatty(stdout):
|
|
stdoutFlush
|
|
else:
|
|
noColorsFlush
|
|
of LogKind.Colors: stdoutFlush
|
|
of LogKind.NoColors: noColorsFlush
|
|
of LogKind.Json:
|
|
defaultChroniclesStream.outputs[1].writer = stdoutFlush
|
|
noOutput
|
|
of LogKind.None:
|
|
noOutput
|
|
|
|
when codex_enable_log_counter:
|
|
var counter = 0.uint64
|
|
proc numberedWriter(logLevel: LogLevel, msg: LogOutputStr) =
|
|
inc(counter)
|
|
let withoutNewLine = msg[0..^2]
|
|
writer(logLevel, withoutNewLine & " count=" & $counter & "\n")
|
|
defaultChroniclesStream.outputs[0].writer = numberedWriter
|
|
else:
|
|
defaultChroniclesStream.outputs[0].writer = writer
|
|
|
|
try:
|
|
updateLogLevel(conf.logLevel)
|
|
except ValueError as err:
|
|
try:
|
|
stderr.write "Invalid value for --log-level. " & err.msg & "\n"
|
|
except IOError:
|
|
echo "Invalid value for --log-level. " & err.msg
|
|
quit QuitFailure
|
|
|
|
proc setupMetrics*(config: CodexConf) =
|
|
if config.metricsEnabled:
|
|
let metricsAddress = config.metricsAddress
|
|
notice "Starting metrics HTTP server",
|
|
url = "http://" & $metricsAddress & ":" & $config.metricsPort & "/metrics"
|
|
try:
|
|
startMetricsHttpServer($metricsAddress, config.metricsPort)
|
|
except CatchableError as exc:
|
|
raiseAssert exc.msg
|
|
except Exception as exc:
|
|
raiseAssert exc.msg # TODO fix metrics
|