From 4119dd6a53ff93ee3b565bf9dae1d30fced53023 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Wed, 5 Feb 2025 18:31:40 +1100 Subject: [PATCH] update Make flag to simply debug DEBUG -> enables DebugCodexNodes, DebugTestHarness, and ShowContinuousStatusUpdates DEBUG_HARDHAT -> enables DebugHardhat PARALLEL -> enables EnableParallelTests Additionally, when DEBUG is enabled, all integration tests debug configs are enabled for Codex nodes, the Codex node output is printed with the test output (not interleaved), and the Codex node output is logged to file in `tests/integrations/logs/__IntegrationTests////_.log`. When DEBUG_HARDHAT is enabled, all hardhat output is printed with the test output (not interleaved), and the output is also written to a log file in `tests/integrations/logs/__IntegrationTests//hardhat.log --- .github/workflows/ci-reusable.yml | 2 +- Makefile | 16 ++---- tests/integration/multinodes.nim | 36 ++++-------- tests/integration/testcli.nim | 43 ++++++++++++-- tests/integration/testmanager.nim | 94 ++++++++++++++++++++++++------- tests/integration/utils.nim | 31 +++++++++- tests/testIntegration.nim | 24 +------- 7 files changed, 159 insertions(+), 87 deletions(-) diff --git a/.github/workflows/ci-reusable.yml b/.github/workflows/ci-reusable.yml index cd0d8d7e..dcb32107 100644 --- a/.github/workflows/ci-reusable.yml +++ b/.github/workflows/ci-reusable.yml @@ -81,7 +81,7 @@ jobs: - name: Parallel integration tests if: matrix.tests == 'integration-parallel' - run: make -j${ncpu} DEBUG_CODEXNODES=1 DEBUG_TESTHARNESS=1 DEBUG_UPDATES=1 testIntegration + run: make -j${ncpu} DEBUG=1 testIntegration - name: Upload integration tests log files uses: actions/upload-artifact@v4 diff --git a/Makefile b/Makefile index 6e1c81a8..732ea21b 100644 --- a/Makefile +++ b/Makefile @@ -142,23 +142,19 @@ testContracts: | build deps $(ENV_SCRIPT) nim testContracts $(NIM_PARAMS) build.nims TEST_PARAMS := -ifdef DEBUG_TESTHARNESS - TEST_PARAMS := $(TEST_PARAMS) -d:DebugTestHarness=$(DEBUG_TESTHARNESS) +ifdef DEBUG + TEST_PARAMS := $(TEST_PARAMS) -d:DebugTestHarness=$(DEBUG) + TEST_PARAMS := $(TEST_PARAMS) -d:DebugCodexNodes=$(DEBUG) + TEST_PARAMS := $(TEST_PARAMS) -d:ShowContinuousStatusUpdates=$(DEBUG) endif ifdef DEBUG_HARDHAT TEST_PARAMS := $(TEST_PARAMS) -d:DebugHardhat=$(DEBUG_HARDHAT) endif -ifdef DEBUG_CODEXNODES # true by default - TEST_PARAMS := $(TEST_PARAMS) -d:DebugCodexNodes=$(DEBUG_CODEXNODES) -endif -ifdef DEBUG_UPDATES - TEST_PARAMS := $(TEST_PARAMS) -d:ShowContinuousStatusUpdates=$(DEBUG_UPDATES) -endif ifdef TEST_TIMEOUT TEST_PARAMS := $(TEST_PARAMS) -d:TestTimeout=$(TEST_TIMEOUT) endif -ifdef ENABLE_PARALLEL_TESTS - TEST_PARAMS := $(TEST_PARAMS) -d:EnableParallelTests=$(ENABLE_PARALLEL_TESTS) +ifdef PARALLEL + TEST_PARAMS := $(TEST_PARAMS) -d:EnableParallelTests=$(PARALLEL) endif # Builds and runs the integration tests diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index 2a989b4f..daec6066 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -42,6 +42,8 @@ const HardhatPort {.intdefine.}: int = 8545 const CodexApiPort {.intdefine.}: int = 8080 const CodexDiscPort {.intdefine.}: int = 8090 const TestId {.strdefine.}: string = "TestId" +const DebugCodexNodes {.booldefine.}: bool = false +const LogsDir {.strdefine.}: string = "" proc raiseMultiNodeSuiteError(msg: string) = raise newException(MultiNodeSuiteError, msg) @@ -87,29 +89,6 @@ template multinodesuite*(name: string, body: untyped) = test tname: tbody - proc sanitize(pathSegment: string): string = - var sanitized = pathSegment - for invalid in invalidFilenameChars.items: - sanitized = sanitized.replace(invalid, '_').replace(' ', '_') - sanitized - - proc getLogFile(role: Role, index: ?int): string = - # create log file path, format: - # tests/integration/logs/ //_.log - - var logDir = - currentSourcePath.parentDir() / "logs" / sanitize($starttime & "__" & name) / - sanitize($currentTestName) - createDir(logDir) - - var fn = $role - if idx =? index: - fn &= "_" & $idx - fn &= ".log" - - let fileName = logDir / fn - return fileName - proc updatePort(url: var string, port: int) = let parts = url.split(':') url = @[parts[0], parts[1], $port].join(":") @@ -119,7 +98,8 @@ template multinodesuite*(name: string, body: untyped) = ): Future[NodeProcess] {.async.} = var args: seq[string] = @[] if config.logFile: - let updatedLogFile = getLogFile(role, none int) + let updatedLogFile = + getLogFile(LogsDir, starttime, name, currentTestName, $role, none int) args.add "--log-file=" & updatedLogFile let port = await nextFreePort(lastUsedHardhatPort) @@ -151,10 +131,14 @@ template multinodesuite*(name: string, body: untyped) = sanitize($role & "_" & $roleIdx) try: - if config.logFile.isSome: - let updatedLogFile = getLogFile(role, some roleIdx) + if config.logFile.isSome or DebugCodexNodes: + let updatedLogFile = + getLogFile(LogsDir, starttime, name, currentTestName, $role, some roleIdx) config.withLogFile(updatedLogFile) + if DebugCodexNodes: + config.debugEnabled = true + let apiPort = await nextFreePort(lastUsedCodexApiPort + nodeIdx) let discPort = await nextFreePort(lastUsedCodexDiscPort + nodeIdx) config.addCliOption("--api-port", $apiPort) diff --git a/tests/integration/testcli.nim b/tests/integration/testcli.nim index eaaa569b..0ea4aca9 100644 --- a/tests/integration/testcli.nim +++ b/tests/integration/testcli.nim @@ -1,4 +1,5 @@ import std/tempfiles +import std/times import codex/conf import codex/utils/fileutils import ../asynctest @@ -9,22 +10,52 @@ import ./utils import ../examples const HardhatPort {.intdefine.}: int = 8545 +const CodexApiPort {.intdefine.}: int = 8080 +const CodexDiscPort {.intdefine.}: int = 8090 +const DebugCodexNodes {.booldefine.}: bool = false +const LogsDir {.strdefine.}: string = "" asyncchecksuite "Command line interface": + let startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") let key = "4242424242424242424242424242424242424242424242424242424242424242" - var nodeCount = -1 - proc startCodex(args: seq[string]): Future[CodexProcess] {.async.} = - return await CodexProcess.startNode(args, false, "cli-test-node") + var currentTestName = "" + var testCount = 0 + var nodeCount = 0 + + template test(tname, tbody) = + inc testCount + currentTestName = tname + test tname: + tbody + + proc addLogFile(args: seq[string]): seq[string] = + when DebugCodexNodes: + return args.concat @[ + "--log-file=" & + getLogFile( + LogsDir, + startTime, + "Command line interface", + currentTestName, + "Client", + some nodeCount mod testCount, + ) + ] + else: + return args + + proc startCodex(arguments: seq[string]): Future[CodexProcess] {.async.} = inc nodeCount + let args = arguments.addLogFile return await CodexProcess.startNode( args.concat( @[ - "--api-port=" & $(await nextFreePort(8080 + nodeCount)), - "--disc-port=" & $(await nextFreePort(8090 + nodeCount)), + "--api-port=" & $(await nextFreePort(CodexApiPort + nodeCount)), + "--disc-port=" & $(await nextFreePort(CodexDiscPort + nodeCount)), ] ), - debug = false, + debug = DebugCodexNodes, "cli-test-node", ) diff --git a/tests/integration/testmanager.nim b/tests/integration/testmanager.nim index cab03ecc..4ce54098 100644 --- a/tests/integration/testmanager.nim +++ b/tests/integration/testmanager.nim @@ -1,6 +1,7 @@ import std/os import std/strformat import std/terminal +from std/times import fromUnix, format, now from std/unicode import toUpper import std/unittest import pkg/chronos @@ -38,6 +39,7 @@ type # Shows test status updates at regular time intervals. Useful for running # locally while attended. Set to false for unattended runs, eg CI. showContinuousStatusUpdates: bool + logsDir: string timeStart: ?Moment timeEnd: ?Moment codexPortLock: AsyncLock @@ -73,6 +75,7 @@ type testId: string # when used in datadir path, prevents data dir clashes status: IntegrationTestStatus command: string + logsDir: string TestManagerError* = object of CatchableError @@ -194,6 +197,8 @@ proc startHardhat( args.add("--port") args.add($port) + if test.manager.debugHardhat: + args.add("--log-file=" & test.logsDir / "hardhat.log") trace "starting hardhat process on port ", port try: @@ -255,9 +260,9 @@ proc printResult( of IntegrationTestStatus.Failed: if output =? test.output: if printStdErr: #manager.debugTestHarness - test.printOutputMarker(MarkerPosition.Start, "test harness errors (stderr)") + test.printOutputMarker(MarkerPosition.Start, "test file errors (stderr)") echo output.stdError - test.printOutputMarker(MarkerPosition.Finish, "test harness errors (stderr)") + test.printOutputMarker(MarkerPosition.Finish, "test file errors (stderr)") if printStdOut: test.printOutputMarker(MarkerPosition.Start, "codex node output (stdout)") echo output.stdOutput @@ -286,21 +291,25 @@ proc printStart(test: IntegrationTest) = proc buildCommand( test: IntegrationTest, hardhatPort: ?int ): Future[string] {.async: (raises: [CancelledError, TestManagerError]).} = - let logging = - if not test.manager.debugTestHarness: - "" - else: + var logging = string.none + if test.manager.debugTestHarness: + #!fmt: off + logging = some( "-d:chronicles_log_level=TRACE " & - "-d:chronicles_disabled_topics=websock,JSONRPC-HTTP-CLIENT,JSONRPC-WS-CLIENT " & - "-d:chronicles_default_output_device=stdout " & "-d:chronicles_sinks=textlines" + "-d:chronicles_disabled_topics=websock,JSONRPC-HTTP-CLIENT,JSONRPC-WS-CLIENT " & + "-d:chronicles_default_output_device=stdout " & + "-d:chronicles_sinks=textlines") + #!fmt: on - let strHardhatPort = - if not test.config.startHardhat: - "" - else: - without port =? hardhatPort: - raiseTestManagerError "hardhatPort required when 'config.startHardhat' is true" - "-d:HardhatPort=" & $port + var hhPort = string.none + if test.config.startHardhat: + without port =? hardhatPort: + raiseTestManagerError "hardhatPort required when 'config.startHardhat' is true" + hhPort = some "-d:HardhatPort=" & $port + + var logDir = string.none + if test.manager.debugCodexNodes: + logDir = some "-d:LogsDir=" & test.logsDir var testFile: string try: @@ -324,12 +333,25 @@ proc buildCommand( withLock(test.manager.hardhatPortLock): try: return - "nim c " & &"-d:CodexApiPort={apiPort} " & &"-d:CodexDiscPort={discPort} " & - &"{strHardhatPort} " & &"-d:TestId={test.testId} " & &"{logging} " & - "--verbosity:0 " & "--hints:off " & "-d:release " & "-r " & &"{testFile}" + #!fmt: off + "nim c " & + &"-d:CodexApiPort={apiPort} " & + &"-d:CodexDiscPort={discPort} " & + &"-d:DebugCodexNodes={test.manager.debugCodexNodes} " & + &"-d:DebugHardhat={test.manager.debugHardhat} " & + (logDir |? "") & " " & + (hhPort |? "") & " " & + &"-d:TestId={test.testId} " & + (logging |? "") & " " & + "--verbosity:0 " & + "--hints:off " & + "-d:release " & + "-r " & + &"{testFile}" + #!fmt: on except ValueError as parent: raiseTestManagerError "bad command --\n" & ", apiPort: " & $apiPort & - ", discPort: " & $discPort & ", logging: " & logging & ", testFile: " & + ", discPort: " & $discPort & ", logging: " & logging |? "" & ", testFile: " & testFile & ", error: " & parent.msg, parent proc setup( @@ -394,6 +416,13 @@ proc start(test: IntegrationTest) {.async: (raises: []).} = trace "Running test" + if test.manager.debugCodexNodes: + test.logsDir = test.manager.logsDir / sanitize(test.config.name) + try: + createDir(test.logsDir) + except CatchableError as e: + error "failed to create test log dir", logDir = test.logsDir, error = e.msg + test.timeStart = some Moment.now() test.status = IntegrationTestStatus.Running @@ -501,7 +530,7 @@ proc runTests(manager: TestManager) {.async: (raises: [CancelledError]).} = asyncSpawn futRun try: - # if runTests is cancelled, await allFutures will be cancelled, but allFutures + # if runTests is cancelled, await allFutures will be cancelled, but allFutures # does not propagate the cancellation to the futures it's waiting on, so we # need to cancel them here await allFutures testFutures @@ -581,6 +610,31 @@ proc printResult(manager: TestManager) {.raises: [TestManagerError].} = proc start*( manager: TestManager ) {.async: (raises: [CancelledError, TestManagerError]).} = + try: + if manager.debugCodexNodes: + let startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") + let logsDir = + currentSourcePath.parentDir() / "logs" / + sanitize(startTime & "__IntegrationTests") + createDir(logsDir) + manager.logsDir = logsDir + #!fmt: off + echoStyled bgWhite, fgBlack, styleBright, + "\n\n ", + styleUnderscore, + "ℹ️ LOGS AVAILABLE ℹ️\n\n", + resetStyle, bgWhite, fgBlack, styleBright, + """ Logs for this run will be available at:""", + resetStyle, bgWhite, fgBlack, + &"\n\n {logsDir}\n\n", + resetStyle, bgWhite, fgBlack, styleBright, + " NOTE: For CI runs, logs will be attached as artefacts\n" + #!fmt: on + except IOError as e: + raiseTestManagerError "failed to create hardhat log directory: " & e.msg, e + except OSError as e: + raiseTestManagerError "failed to create hardhat log directory: " & e.msg, e + if manager.showContinuousStatusUpdates: let fut = manager.continuallyShowUpdates() manager.trackedFutures.track fut diff --git a/tests/integration/utils.nim b/tests/integration/utils.nim index 6b5e536b..bcdff4ed 100644 --- a/tests/integration/utils.nim +++ b/tests/integration/utils.nim @@ -1,8 +1,11 @@ +import std/os import pkg/chronos import pkg/codex/logutils +{.push raises: [].} + proc nextFreePort*(startPort: int): Future[int] {.async: (raises: [CancelledError]).} = - proc client(server: StreamServer, transp: StreamTransport) {.async.} = + proc client(server: StreamServer, transp: StreamTransport) {.async: (raises: []).} = await transp.closeWait() var port = startPort @@ -22,3 +25,29 @@ proc nextFreePort*(startPort: int): Future[int] {.async: (raises: [CancelledErro inc port except TransportAddressError: raiseAssert "bad address" + +proc sanitize*(pathSegment: string): string = + var sanitized = pathSegment + for invalid in invalidFilenameChars.items: + sanitized = sanitized.replace(invalid, '_').replace(' ', '_') + sanitized + +proc getLogFile*( + logDir, startTime, suiteName, testName, role: string, index = int.none +): string {.raises: [IOError, OSError].} = + let logsDir = + if logDir == "": + currentSourcePath.parentDir() / "logs" / sanitize(startTime & "__" & suiteName) / + sanitize(testName) + else: + logDir / sanitize(suiteName) / sanitize(testName) + + createDir(logsDir) + + var fn = $role + if idx =? index: + fn &= "_" & $idx + fn &= ".log" + + let fileName = logsDir / fn + return fileName diff --git a/tests/testIntegration.nim b/tests/testIntegration.nim index 9ed70a48..7f18a481 100644 --- a/tests/testIntegration.nim +++ b/tests/testIntegration.nim @@ -27,7 +27,7 @@ const DebugHardhat {.booldefine.} = false # Echoes stdout from the integration test file process. Codex process logs can # also be output if a test uses a multinodesuite, requires CodexConfig.debug # to be enabled -const DebugCodexNodes {.booldefine.} = true +const DebugCodexNodes {.booldefine.} = false # Shows test status updates at time intervals. Useful for running locally with # active terminal interaction. Set to false for unattended runs, eg CI. const ShowContinuousStatusUpdates {.booldefine.} = false @@ -37,28 +37,6 @@ const TestTimeout {.intdefine.} = 60 const EnableParallelTests {.booldefine.} = true proc run() {.async.} = - when DebugTestHarness and enabledLogLevel != LogLevel.TRACE: - styledEcho bgWhite, - fgBlack, styleBright, "\n\n ", styleUnderscore, - "ℹ️ ADDITIONAL LOGGING AVAILABLE ℹ️\n\n", resetStyle, bgWhite, fgBlack, - styleBright, - """ - More integration test harness logs available by running with - -d:chronicles_log_level=TRACE, eg:""", - resetStyle, bgWhite, fgBlack, - "\n\n nim c -d:chronicles_log_level=TRACE -r ./testIntegration.nim\n\n" - - when DebugCodexNodes: - styledEcho bgWhite, - fgBlack, styleBright, "\n\n ", styleUnderscore, - "⚠️ ENABLE CODEX LOGGING ⚠️\n\n", resetStyle, bgWhite, fgBlack, - styleBright, - """ - For integration test suites that are multinodesuites, or for - tests launching a CodexProcess, ensure that CodexConfig.debug - is enabled to see chronicles logs. - """ - let manager = TestManager.new( configs = TestConfigs, DebugTestHarness,