diff --git a/codexcrawler.nim b/codexcrawler.nim index 07832ad..3f22aff 100644 --- a/codexcrawler.nim +++ b/codexcrawler.nim @@ -1,8 +1,4 @@ -import std/os import pkg/chronicles -import pkg/chronos -import pkg/questionable -import pkg/questionable/results import ./codexcrawler/application diff --git a/codexcrawler/application.nim b/codexcrawler/application.nim index 6e0f89d..5ef5fab 100644 --- a/codexcrawler/application.nim +++ b/codexcrawler/application.nim @@ -4,11 +4,18 @@ import pkg/chronos import pkg/questionable import pkg/questionable/results +import pkg/datastore +import pkg/datastore/typedds +import pkg/metrics + import ./config import ./logging import ./metrics +import ./list -import ./main +declareGauge(todoNodesGauge, "DHT nodes to be visited") +declareGauge(okNodesGauge, "DHT nodes successfully contacted") +declareGauge(nokNodesGauge, "DHT nodes failed to contact") type ApplicationStatus* {.pure.} = enum @@ -18,26 +25,72 @@ type Application* = ref object status: ApplicationStatus + config*: CrawlerConfig + todoList*: List + okNodes*: List + nokNodes*: List + +proc createDatastore(app: Application): ?!TypedDatastore = + without store =? LevelDbDatastore.new(app.config.dataDir), err: + error "Failed to create datastore" + return failure(err) + return success(TypedDatastore.init(store)) + +proc initializeLists(app: Application): Future[?!void] {.async.} = + without store =? app.createDatastore(), err: + return failure(err) + + # We can't extract this into a function because gauges cannot be passed as argument. + # The use of global state in nim-metrics is not pleasant. + proc onTodoMetric(value: int64) = + todoNodesGauge.set(value) + proc onOkMetric(value: int64) = + okNodesGauge.set(value) + proc onNokMetric(value: int64) = + nokNodesGauge.set(value) + + app.todoList = List.new("todo", store, onTodoMetric) + app.okNodes = List.new("ok", store, onOkMetric) + app.nokNodes = List.new("nok", store, onNokMetric) + + if err =? (await app.todoList.load()).errorOption: + return failure(err) + if err =? (await app.okNodes.load()).errorOption: + return failure(err) + if err =? (await app.nokNodes.load()).errorOption: + return failure(err) + + return success() + +proc initializeApp(app: Application): Future[?!void] {.async.} = + if err =? (await app.initializeLists()).errorOption: + error "Failed to initialize lists", err = err.msg + return failure(err) + return success() + +proc stop*(app: Application) = + app.status = ApplicationStatus.Stopping proc run*(app: Application) = - let config = parseConfig() - info "Loaded configuration", config + app.config = parseConfig() + info "Loaded configuration", config = app.config # Configure loglevel - updateLogLevel(config.logLevel) + updateLogLevel(app.config.logLevel) # Ensure datadir path exists: - if not existsDir(config.dataDir): - createDir(config.dataDir) + if not existsDir(app.config.dataDir): + createDir(app.config.dataDir) - setupMetrics(config.metricsAddress, config.metricsPort) + setupMetrics(app.config.metricsAddress, app.config.metricsPort) info "Metrics endpoint initialized" info "Starting application" app.status = ApplicationStatus.Running - if err =? (waitFor startApplication(config)).errorOption: + if err =? (waitFor app.initializeApp()).errorOption: app.status = ApplicationStatus.Stopping error "Failed to start application", err = err.msg + return while app.status == ApplicationStatus.Running: try: @@ -46,6 +99,3 @@ proc run*(app: Application) = error "Unhandled exception", msg = exc.msg quit QuitFailure notice "Application closed" - -proc stop*(app: Application) = - app.status = ApplicationStatus.Stopping diff --git a/codexcrawler/list.nim b/codexcrawler/list.nim index 19ae13f..25ddfb8 100644 --- a/codexcrawler/list.nim +++ b/codexcrawler/list.nim @@ -3,9 +3,12 @@ import pkg/chronicles import pkg/metrics import pkg/datastore import pkg/datastore/typedds +import pkg/stew/byteutils +import pkg/stew/endians2 import pkg/questionable import pkg/questionable/results +import std/strutils import std/os import ./nodeentry @@ -22,6 +25,20 @@ type items: seq[NodeEntry] onMetric: OnUpdateMetric +proc encode(s: NodeEntry): seq[byte] = + (s.id & ";" & s.value).toBytes() + +proc decode(T: type NodeEntry, bytes: seq[byte]): ?!T = + let s = string.fromBytes(bytes) + if s.len == 0: + return success(NodeEntry(id: "", value: "")) + + let tokens = s.split(";") + if tokens.len != 2: + return failure("expected 2 tokens") + + success(NodeEntry(id: tokens[0], value: tokens[1])) + proc saveItem(this: List, item: NodeEntry): Future[?!void] {.async.} = without itemKey =? Key.init(this.name / item.id), err: return failure(err) diff --git a/codexcrawler/main.nim b/codexcrawler/main.nim deleted file mode 100644 index 5acf0a4..0000000 --- a/codexcrawler/main.nim +++ /dev/null @@ -1,46 +0,0 @@ -import std/os -import pkg/chronicles -import pkg/chronos -import pkg/questionable -import pkg/questionable/results - -import pkg/datastore -import pkg/datastore/typedds -import pkg/metrics - -import ./config -import ./list - -logScope: - topics = "main" - -declareGauge(example, "testing") - -proc startApplication*(config: CrawlerConfig): Future[?!void] {.async.} = - without exampleStore =? LevelDbDatastore.new(config.dataDir / "example"): - error "Failed to create datastore" - return failure("Failed to create datastore") - - let typedDs = TypedDatastore.init(exampleStore) - - proc onExampleMetric(value: int64) = - example.set(value) - - var exampleList = List.new("example", typedDs, onExampleMetric) - if err =? (await exampleList.load()).errorOption: - return failure(err) - - proc aaa() {.async.} = - var i = 0 - while true: - trace "a" - await sleepAsync(1000) - discard await exampleList.add(Entry(id: $i, value: "str!")) - inc i - - asyncSpawn aaa() - - await sleepAsync(1000) - - notice "b" - return success() diff --git a/codexcrawler/nodeentry.nim b/codexcrawler/nodeentry.nim index 470d03d..c2b1a54 100644 --- a/codexcrawler/nodeentry.nim +++ b/codexcrawler/nodeentry.nim @@ -9,17 +9,3 @@ type NodeEntry* = object proc `$`*(entry: NodeEntry): string = entry.id & ":" & entry.value - -proc encode(s: NodeEntry): seq[byte] = - (s.id & ";" & s.value).toBytes() - -proc decode(T: type NodeEntry, bytes: seq[byte]): ?!T = - let s = string.fromBytes(bytes) - if s.len == 0: - return success(NodeEntry(id: "", value: "")) - - let tokens = s.split(";") - if tokens.len != 2: - return failure("expected 2 tokens") - - success(NodeEntry(id: tokens[0], value: tokens[1])) diff --git a/config.nims b/config.nims index d3a9671..0b00ba9 100644 --- a/config.nims +++ b/config.nims @@ -1,3 +1,4 @@ ---define:metrics +--define: + metrics # switch("define", "chronicles_runtime_filtering=true") switch("define", "chronicles_log_level=TRACE")