diff --git a/codexcrawler.nim b/codexcrawler.nim index ba1589e..9c3ad81 100644 --- a/codexcrawler.nim +++ b/codexcrawler.nim @@ -2,6 +2,8 @@ import pkg/chronicles import pkg/chronos import ./codexcrawler/main +import ./codexcrawler/config + when defined(posix): import system/ansi_c @@ -15,9 +17,12 @@ type status: ApplicationStatus proc run(app: Application) = + let config = parseConfig() + info "Loaded configuration", config + app.status = ApplicationStatus.Running - waitFor runApplication() + waitFor startApplication() while app.status == ApplicationStatus.Running: try: diff --git a/codexcrawler.nimble b/codexcrawler.nimble index 0c8d60b..7fa17af 100644 --- a/codexcrawler.nimble +++ b/codexcrawler.nimble @@ -21,6 +21,7 @@ requires "stint#3236fa68394f1e3a06e2bc34218aacdd2d675923" requires "https://github.com/codex-storage/nim-datastore >= 0.1.0 & < 0.2.0" requires "questionable >= 0.10.15 & < 0.11.0" requires "https://github.com/codex-storage/nim-codex-dht#4bd3a39e0030f8ee269ef217344b6b59ec2be6dc" # 7 Jan 2024 - Support for Nim 2.0.14 +requires "docopt >= 0.7.1 & < 1.0.0" task test, "Run tests": exec "nimble install -d -y" diff --git a/codexcrawler/config.nim b/codexcrawler/config.nim new file mode 100644 index 0000000..d144290 --- /dev/null +++ b/codexcrawler/config.nim @@ -0,0 +1,49 @@ +import std/net +import ./version + +let doc = """ +Codex Network Crawler. Generates network metrics. + +Usage: + codexcrawler [--logLevel=] [--metricsAddress=] [--metricsPort=

] [--dataDir=

] [--discoveryPort=

] + +Options: + --logLevel= Sets log level [default: TRACE] + --metricsAddress= Listen address of the metrics server [default: 127.0.0.1] + --metricsPort=

Listen HTTP port of the metrics server [default: 8008] + --dataDir=

Directory for storing data [default: crawler_data] + --discoveryPort=

Port used for DHT [default: 8090] +""" + +import strutils +import docopt + +type + CrawlerConfig* = ref object + logLevel*: string + metricsAddress*: IpAddress + metricsPort*: Port + dataDir*: string + discPort*: Port + +proc `$`*(config: CrawlerConfig): string = + "CrawlerConfig:" & + " logLevel=" & config.logLevel & + " metricsAddress=" & $config.metricsAddress & + " metricsPort=" & $config.metricsPort & + " dataDir=" & config.dataDir & + " discPort=" & $config.discPort + +proc parseConfig*(): CrawlerConfig = + let args = docopt(doc, version = crawlerFullVersion) + + proc get(name: string): string = + $args[name] + + return CrawlerConfig( + logLevel: get("--logLevel"), + metricsAddress: parseIpAddress(get("--metricsAddress")), + metricsPort: Port(parseInt(get("--metricsPort"))), + dataDir: get("--dataDir"), + discPort: Port(parseInt(get("--discoveryPort"))), + ) diff --git a/codexcrawler/logging.nim b/codexcrawler/logging.nim new file mode 100644 index 0000000..213c2a3 --- /dev/null +++ b/codexcrawler/logging.nim @@ -0,0 +1,18 @@ +import pkg/chronicles +import pkg/chronicles/helpers +import pkg/chronicles/topics_registry + +proc updateLogLevel*(logLevel: string) {.upraises: [ValueError].} = + let directives = logLevel.split(";") + try: + setLogLevel(parseEnum[LogLevel](directives[0].toUpperAscii)) + except ValueError: + raise (ref ValueError)( + msg: + "Please specify one of: trace, debug, " & "info, notice, warn, error or fatal" + ) + + if directives.len > 1: + for topicName, settings in parseTopicDirectives(directives[1 ..^ 1]): + if not setTopicState(topicName, settings.state, settings.logLevel): + warn "Unrecognized logging topic", topic = topicName diff --git a/codexcrawler/main.nim b/codexcrawler/main.nim index 72260a9..89460db 100644 --- a/codexcrawler/main.nim +++ b/codexcrawler/main.nim @@ -4,7 +4,7 @@ import pkg/chronos logScope: topics = "main" -proc runApplication*() {.async.} = +proc startApplication*() {.async.} = proc aaa() {.async.} = while true: notice "a" diff --git a/codexcrawler/metrics.nim b/codexcrawler/metrics.nim new file mode 100644 index 0000000..7f6b54e --- /dev/null +++ b/codexcrawler/metrics.nim @@ -0,0 +1,14 @@ +import pkg/chronicles +import pkg/metrics +import pkg/metrics/chronos_httpserver + +proc setupMetrics*(metricsAddress: IpAddress, metricsPort: Port) = + let metricsAddress = metricsAddress + notice "Starting metrics HTTP server", + url = "http://" & $metricsAddress & ":" & $metricsPort & "/metrics" + try: + startMetricsHttpServer($metricsAddress, metricsPort) + except CatchableError as exc: + raiseAssert exc.msg + except Exception as exc: + raiseAssert exc.msg # TODO fix metrics diff --git a/codexcrawler/utils.nim b/codexcrawler/utils.nim new file mode 100644 index 0000000..205bb07 --- /dev/null +++ b/codexcrawler/utils.nim @@ -0,0 +1,46 @@ +import std/parseutils +import pkg/chronos + +when not declared(parseDuration): # Odd code formatting to minimize diff v. mainLine + const Whitespace = {' ', '\t', '\v', '\r', '\l', '\f'} + + func toLowerAscii(c: char): char = + if c in {'A' .. 'Z'}: + char(uint8(c) xor 0b0010_0000'u8) + else: + c + + func parseDuration*(s: string, size: var Duration): int = + ## Parse a size qualified by simple time into `Duration`. + ## + runnableExamples: + var res: Duration # caller must still know if 'b' refers to bytes|bits + doAssert parseDuration("10H", res) == 3 + doAssert res == initDuration(hours = 10) + doAssert parseDuration("64m", res) == 6 + doAssert res == initDuration(minutes = 64) + doAssert parseDuration("7m/block", res) == 2 # '/' stops parse + doAssert res == initDuration(minutes = 7) # 1 shl 30, forced binary metric + doAssert parseDuration("3d", res) == 2 # '/' stops parse + doAssert res == initDuration(days = 3) # 1 shl 30, forced binary metric + + const prefix = "s" & "mhdw" # byte|bit & lowCase metric-ish prefixes + const timeScale = [1.0, 60.0, 3600.0, 86_400.0, 604_800.0] + + var number: float + var scale = 1.0 + result = parseFloat(s, number) + if number < 0: # While parseFloat accepts negatives .. + result = 0 #.. we do not since sizes cannot be < 0 + else: + let start = result # Save spot to maybe unwind white to EOS + while result < s.len and s[result] in Whitespace: + inc result + if result < s.len: # Illegal starting char => unity + if (let si = prefix.find(s[result].toLowerAscii); si >= 0): + inc result # Now parse the scale + scale = timeScale[si] + else: # Unwind result advancement when there.. + result = start #..is no unit to the end of `s`. + var sizeF = number * scale + 0.5 # Saturate to int64.high when too big + size = seconds(int(sizeF)) diff --git a/codexcrawler/version.nim b/codexcrawler/version.nim new file mode 100644 index 0000000..da2c320 --- /dev/null +++ b/codexcrawler/version.nim @@ -0,0 +1,24 @@ +import std/strutils + +proc getCrawlerVersion(): string = + let tag = strip(staticExec("git tag")) + if tag.isEmptyOrWhitespace: + return "untagged build" + return tag + +proc getCrawlerRevision(): string = + # using a slice in a static context breaks nimsuggest for some reason + var res = strip(staticExec("git rev-parse --short HEAD")) + return res + +proc getNimBanner(): string = + staticExec("nim --version | grep Version") + +const + crawlerVersion* = getCrawlerVersion() + crawlerRevision* = getCrawlerRevision() + nimBanner* = getNimBanner() + + crawlerFullVersion* = + "CodexNetworkCrawler version: " & crawlerVersion & "\p" & "CodexNetworkCrawler revision: " & crawlerRevision & "\p" & + nimBanner