From f10cf41e4d69d9db4bf43719ba8a120c12520df0 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Fri, 29 Sep 2023 20:02:30 +1000 Subject: [PATCH] Improve multinode suite for better debug options, including logging to file There is a 503 "sales unavailable" error --- tests/integration/codexclient.nim | 2 +- tests/integration/multinodes.nim | 172 +++++++++++++++++++----------- tests/integration/testproofs.nim | 35 ++++-- 3 files changed, 134 insertions(+), 75 deletions(-) diff --git a/tests/integration/codexclient.nim b/tests/integration/codexclient.nim index 99825dd7..ea23854f 100644 --- a/tests/integration/codexclient.nim +++ b/tests/integration/codexclient.nim @@ -83,7 +83,7 @@ proc postAvailability*( "maxCollateral": maxCollateral, } let response = client.http.post(url, $json) - assert response.status == "200 OK" + doAssert response.status == "200 OK", "expected 200 OK, got " & response.status & ", body: " & response.body Availability.fromJson(response.body.parseJson) proc getAvailabilities*(client: CodexClient): ?!seq[Availability] = diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index 631c8ff9..4012425c 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -6,6 +6,7 @@ import std/sequtils import std/strutils import std/sequtils import std/strutils +import std/sugar import pkg/chronicles import ../ethertest import ./codexclient @@ -23,18 +24,23 @@ type datadir*: string ethAccount*: Address StartNodes* = object - clients*: uint - providers*: uint - validators*: uint - DebugNodes* = object - client*: bool - provider*: bool - validator*: bool - topics*: string + clients*: StartNodeConfig + providers*: StartNodeConfig + validators*: StartNodeConfig + StartNodeConfig* = object + numNodes*: int + cliOptions*: seq[CliOption] + logFile*: ?string + logTopics*: seq[string] + debugEnabled*: bool Role* {.pure.} = enum Client, Provider, Validator + CliOption* = object of RootObj + nodeIdx*: ?int + key*: string + value*: string proc new*(_: type RunningNode, role: Role, @@ -48,29 +54,67 @@ proc new*(_: type RunningNode, datadir: datadir, ethAccount: ethAccount) -proc init*(_: type StartNodes, - clients, providers, validators: uint): StartNodes = - StartNodes(clients: clients, providers: providers, validators: validators) +proc nodes*(config: StartNodeConfig, numNodes: int): StartNodeConfig = + if numNodes < 0: + raise newException(ValueError, "numNodes must be >= 0") -proc init*(_: type DebugNodes, - client, provider, validator: bool, - topics: string = "validator,proving,market"): DebugNodes = - DebugNodes(client: client, provider: provider, validator: validator, - topics: topics) + var startConfig = config + startConfig.numNodes = numNodes + return startConfig + +proc simulateProofFailuresFor*( + config: StartNodeConfig, + providerIdx: int, + failEveryNProofs: int +): StartNodeConfig = + + if providerIdx > config.numNodes - 1: + raise newException(ValueError, "provider index out of bounds") + + var startConfig = config + startConfig.cliOptions.add( + CliOption( + nodeIdx: some providerIdx, + key: "--simulate-proof-failures", + value: $failEveryNProofs + ) + ) + return startConfig + +proc debug*(config: StartNodeConfig, enabled = true): StartNodeConfig = + ## output log in stdout + var startConfig = config + startConfig.debugEnabled = enabled + return startConfig + +proc withLogFile*( + config: StartNodeConfig, + file: bool | string +): StartNodeConfig = + + var startConfig = config + when file is bool: + if not file: startConfig.logFile = none string + else: startConfig.logFile = + some currentSourcePath.parentDir() / "codex" & $index & ".log" + else: + if file.len <= 0: + raise newException(ValueError, "file path length must be > 0") + startConfig.logFile = some file + + return startConfig + +proc withLogTopics*( + config: StartNodeConfig, + topics: varargs[string] +): StartNodeConfig = + + var startConfig = config + startConfig.logTopics = startConfig.logTopics.concat(@topics) + return startConfig template multinodesuite*(name: string, - startNodes: StartNodes, debugConfig: DebugConfig, body: untyped) = - - if (debugConfig.client or debugConfig.provider) and - (enabledLogLevel > LogLevel.TRACE or - enabledLogLevel == LogLevel.NONE): - echo "" - echo "More test debug logging is available by running the tests with " & - "'-d:chronicles_log_level=TRACE " & - "-d:chronicles_disabled_topics=websock " & - "-d:chronicles_default_output_device=stdout " & - "-d:chronicles_sinks=textlines'" - echo "" + startNodes: StartNodes, body: untyped) = ethersuite name: @@ -78,26 +122,28 @@ template multinodesuite*(name: string, var bootstrap: string proc newNodeProcess(index: int, - addlOptions: seq[string], - debug: bool): (NodeProcess, string, Address) = + config: StartNodeConfig + ): (NodeProcess, string, Address) = if index > accounts.len - 1: raiseAssert("Cannot start node at index " & $index & ", not enough eth accounts.") let datadir = getTempDir() / "Codex" & $index - let logdir = currentSourcePath.parentDir() + # let logdir = currentSourcePath.parentDir() var options = @[ "--api-port=" & $(8080 + index), "--data-dir=" & datadir, "--nat=127.0.0.1", "--disc-ip=127.0.0.1", "--disc-port=" & $(8090 + index), - "--eth-account=" & $accounts[index], - "--log-file=" & (logdir / "codex" & $index & ".log")] - .concat(addlOptions) - if debug: options.add "--log-level=INFO;TRACE: " & debugConfig.topics - let node = startNode(options, debug = debug) + "--eth-account=" & $accounts[index]] + if logFile =? config.logFile: + options.add "--log-file=" & logFile + if config.logTopics.len > 0: + options.add "--log-level=INFO;TRACE: " & config.logTopics.join(",") + + let node = startNode(options, config.debugEnabled) node.waitUntilStarted() (node, datadir, accounts[index]) @@ -106,48 +152,47 @@ template multinodesuite*(name: string, proc startClientNode() = let index = running.len - let (node, datadir, account) = newNodeProcess( - index, @["--persistence"], debugConfig.client) + var config = startNodes.clients + config.cliOptions.add CliOption(key: "--persistence") + let (node, datadir, account) = newNodeProcess(index, config) let restClient = newCodexClient(index) running.add RunningNode.new(Role.Client, node, restClient, datadir, account) - if debugConfig.client: + if config.debugEnabled: debug "started new client node and codex client", restApiPort = 8080 + index, discPort = 8090 + index, account - proc startProviderNode(cliOptions: seq[CliOption]) = + proc startProviderNode(cliOptions: seq[CliOption] = @[]) = let index = running.len - var options = @[ - "--bootstrap-node=" & bootstrap, - "--persistence" - ] + var config = startNodes.providers + config.cliOptions = config.cliOptions.concat(cliOptions) + config.cliOptions.add CliOption(key: "--bootstrap-node", value: bootstrap) + config.cliOptions.add CliOption(key: "--persistence") - for cliOption in cliOptions: - var option = cliOption.key - if cliOption.value.len > 0: - option &= "=" & cliOption.value - options.add option + config.cliOptions = config.cliOptions.filter( + o => (let idx = o.nodeIdx |? index; idx == index) + ) - let (node, datadir, account) = newNodeProcess(index, options, - debugConfig.provider) + let (node, datadir, account) = newNodeProcess(index, config) let restClient = newCodexClient(index) running.add RunningNode.new(Role.Provider, node, restClient, datadir, account) - if debugConfig.provider: + if config.debugEnabled: debug "started new provider node and codex client", restApiPort = 8080 + index, discPort = 8090 + index, account, - cliOptions = options.join(",") + cliOptions = config.cliOptions.join(",") proc startValidatorNode() = let index = running.len - let (node, datadir, account) = newNodeProcess(index, @[ - "--bootstrap-node=" & bootstrap, - "--validator"], - debugConfig.validator) + var config = startNodes.validators + config.cliOptions.add CliOption(key: "--bootstrap-node", value: bootstrap) + config.cliOptions.add CliOption(key: "--validator") + + let (node, datadir, account) = newNodeProcess(index, config) let restClient = newCodexClient(index) running.add RunningNode.new(Role.Validator, node, restClient, datadir, account) - if debugConfig.validator: + if config.debugEnabled: debug "started new validator node and codex client", restApiPort = 8080 + index, discPort = 8090 + index, account @@ -161,18 +206,15 @@ template multinodesuite*(name: string, running.filter(proc(r: RunningNode): bool = r.role == Role.Validator) setup: - for i in 0..