diff --git a/.github/workflows/ci-reusable.yml b/.github/workflows/ci-reusable.yml index cd0d8d7e..dcb32107 100644 --- a/.github/workflows/ci-reusable.yml +++ b/.github/workflows/ci-reusable.yml @@ -81,7 +81,7 @@ jobs: - name: Parallel integration tests if: matrix.tests == 'integration-parallel' - run: make -j${ncpu} DEBUG_CODEXNODES=1 DEBUG_TESTHARNESS=1 DEBUG_UPDATES=1 testIntegration + run: make -j${ncpu} DEBUG=1 testIntegration - name: Upload integration tests log files uses: actions/upload-artifact@v4 diff --git a/Makefile b/Makefile index 6e1c81a8..732ea21b 100644 --- a/Makefile +++ b/Makefile @@ -142,23 +142,19 @@ testContracts: | build deps $(ENV_SCRIPT) nim testContracts $(NIM_PARAMS) build.nims TEST_PARAMS := -ifdef DEBUG_TESTHARNESS - TEST_PARAMS := $(TEST_PARAMS) -d:DebugTestHarness=$(DEBUG_TESTHARNESS) +ifdef DEBUG + TEST_PARAMS := $(TEST_PARAMS) -d:DebugTestHarness=$(DEBUG) + TEST_PARAMS := $(TEST_PARAMS) -d:DebugCodexNodes=$(DEBUG) + TEST_PARAMS := $(TEST_PARAMS) -d:ShowContinuousStatusUpdates=$(DEBUG) endif ifdef DEBUG_HARDHAT TEST_PARAMS := $(TEST_PARAMS) -d:DebugHardhat=$(DEBUG_HARDHAT) endif -ifdef DEBUG_CODEXNODES # true by default - TEST_PARAMS := $(TEST_PARAMS) -d:DebugCodexNodes=$(DEBUG_CODEXNODES) -endif -ifdef DEBUG_UPDATES - TEST_PARAMS := $(TEST_PARAMS) -d:ShowContinuousStatusUpdates=$(DEBUG_UPDATES) -endif ifdef TEST_TIMEOUT TEST_PARAMS := $(TEST_PARAMS) -d:TestTimeout=$(TEST_TIMEOUT) endif -ifdef ENABLE_PARALLEL_TESTS - TEST_PARAMS := $(TEST_PARAMS) -d:EnableParallelTests=$(ENABLE_PARALLEL_TESTS) +ifdef PARALLEL + TEST_PARAMS := $(TEST_PARAMS) -d:EnableParallelTests=$(PARALLEL) endif # Builds and runs the integration tests diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index 2a989b4f..daec6066 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -42,6 +42,8 @@ const HardhatPort {.intdefine.}: int = 8545 const CodexApiPort {.intdefine.}: int = 8080 const CodexDiscPort {.intdefine.}: int = 8090 const TestId {.strdefine.}: string = "TestId" +const DebugCodexNodes {.booldefine.}: bool = false +const LogsDir {.strdefine.}: string = "" proc raiseMultiNodeSuiteError(msg: string) = raise newException(MultiNodeSuiteError, msg) @@ -87,29 +89,6 @@ template multinodesuite*(name: string, body: untyped) = test tname: tbody - proc sanitize(pathSegment: string): string = - var sanitized = pathSegment - for invalid in invalidFilenameChars.items: - sanitized = sanitized.replace(invalid, '_').replace(' ', '_') - sanitized - - proc getLogFile(role: Role, index: ?int): string = - # create log file path, format: - # tests/integration/logs/ //_.log - - var logDir = - currentSourcePath.parentDir() / "logs" / sanitize($starttime & "__" & name) / - sanitize($currentTestName) - createDir(logDir) - - var fn = $role - if idx =? index: - fn &= "_" & $idx - fn &= ".log" - - let fileName = logDir / fn - return fileName - proc updatePort(url: var string, port: int) = let parts = url.split(':') url = @[parts[0], parts[1], $port].join(":") @@ -119,7 +98,8 @@ template multinodesuite*(name: string, body: untyped) = ): Future[NodeProcess] {.async.} = var args: seq[string] = @[] if config.logFile: - let updatedLogFile = getLogFile(role, none int) + let updatedLogFile = + getLogFile(LogsDir, starttime, name, currentTestName, $role, none int) args.add "--log-file=" & updatedLogFile let port = await nextFreePort(lastUsedHardhatPort) @@ -151,10 +131,14 @@ template multinodesuite*(name: string, body: untyped) = sanitize($role & "_" & $roleIdx) try: - if config.logFile.isSome: - let updatedLogFile = getLogFile(role, some roleIdx) + if config.logFile.isSome or DebugCodexNodes: + let updatedLogFile = + getLogFile(LogsDir, starttime, name, currentTestName, $role, some roleIdx) config.withLogFile(updatedLogFile) + if DebugCodexNodes: + config.debugEnabled = true + let apiPort = await nextFreePort(lastUsedCodexApiPort + nodeIdx) let discPort = await nextFreePort(lastUsedCodexDiscPort + nodeIdx) config.addCliOption("--api-port", $apiPort) diff --git a/tests/integration/testcli.nim b/tests/integration/testcli.nim index eaaa569b..0ea4aca9 100644 --- a/tests/integration/testcli.nim +++ b/tests/integration/testcli.nim @@ -1,4 +1,5 @@ import std/tempfiles +import std/times import codex/conf import codex/utils/fileutils import ../asynctest @@ -9,22 +10,52 @@ import ./utils import ../examples const HardhatPort {.intdefine.}: int = 8545 +const CodexApiPort {.intdefine.}: int = 8080 +const CodexDiscPort {.intdefine.}: int = 8090 +const DebugCodexNodes {.booldefine.}: bool = false +const LogsDir {.strdefine.}: string = "" asyncchecksuite "Command line interface": + let startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") let key = "4242424242424242424242424242424242424242424242424242424242424242" - var nodeCount = -1 - proc startCodex(args: seq[string]): Future[CodexProcess] {.async.} = - return await CodexProcess.startNode(args, false, "cli-test-node") + var currentTestName = "" + var testCount = 0 + var nodeCount = 0 + + template test(tname, tbody) = + inc testCount + currentTestName = tname + test tname: + tbody + + proc addLogFile(args: seq[string]): seq[string] = + when DebugCodexNodes: + return args.concat @[ + "--log-file=" & + getLogFile( + LogsDir, + startTime, + "Command line interface", + currentTestName, + "Client", + some nodeCount mod testCount, + ) + ] + else: + return args + + proc startCodex(arguments: seq[string]): Future[CodexProcess] {.async.} = inc nodeCount + let args = arguments.addLogFile return await CodexProcess.startNode( args.concat( @[ - "--api-port=" & $(await nextFreePort(8080 + nodeCount)), - "--disc-port=" & $(await nextFreePort(8090 + nodeCount)), + "--api-port=" & $(await nextFreePort(CodexApiPort + nodeCount)), + "--disc-port=" & $(await nextFreePort(CodexDiscPort + nodeCount)), ] ), - debug = false, + debug = DebugCodexNodes, "cli-test-node", ) diff --git a/tests/integration/testmanager.nim b/tests/integration/testmanager.nim index cab03ecc..4ce54098 100644 --- a/tests/integration/testmanager.nim +++ b/tests/integration/testmanager.nim @@ -1,6 +1,7 @@ import std/os import std/strformat import std/terminal +from std/times import fromUnix, format, now from std/unicode import toUpper import std/unittest import pkg/chronos @@ -38,6 +39,7 @@ type # Shows test status updates at regular time intervals. Useful for running # locally while attended. Set to false for unattended runs, eg CI. showContinuousStatusUpdates: bool + logsDir: string timeStart: ?Moment timeEnd: ?Moment codexPortLock: AsyncLock @@ -73,6 +75,7 @@ type testId: string # when used in datadir path, prevents data dir clashes status: IntegrationTestStatus command: string + logsDir: string TestManagerError* = object of CatchableError @@ -194,6 +197,8 @@ proc startHardhat( args.add("--port") args.add($port) + if test.manager.debugHardhat: + args.add("--log-file=" & test.logsDir / "hardhat.log") trace "starting hardhat process on port ", port try: @@ -255,9 +260,9 @@ proc printResult( of IntegrationTestStatus.Failed: if output =? test.output: if printStdErr: #manager.debugTestHarness - test.printOutputMarker(MarkerPosition.Start, "test harness errors (stderr)") + test.printOutputMarker(MarkerPosition.Start, "test file errors (stderr)") echo output.stdError - test.printOutputMarker(MarkerPosition.Finish, "test harness errors (stderr)") + test.printOutputMarker(MarkerPosition.Finish, "test file errors (stderr)") if printStdOut: test.printOutputMarker(MarkerPosition.Start, "codex node output (stdout)") echo output.stdOutput @@ -286,21 +291,25 @@ proc printStart(test: IntegrationTest) = proc buildCommand( test: IntegrationTest, hardhatPort: ?int ): Future[string] {.async: (raises: [CancelledError, TestManagerError]).} = - let logging = - if not test.manager.debugTestHarness: - "" - else: + var logging = string.none + if test.manager.debugTestHarness: + #!fmt: off + logging = some( "-d:chronicles_log_level=TRACE " & - "-d:chronicles_disabled_topics=websock,JSONRPC-HTTP-CLIENT,JSONRPC-WS-CLIENT " & - "-d:chronicles_default_output_device=stdout " & "-d:chronicles_sinks=textlines" + "-d:chronicles_disabled_topics=websock,JSONRPC-HTTP-CLIENT,JSONRPC-WS-CLIENT " & + "-d:chronicles_default_output_device=stdout " & + "-d:chronicles_sinks=textlines") + #!fmt: on - let strHardhatPort = - if not test.config.startHardhat: - "" - else: - without port =? hardhatPort: - raiseTestManagerError "hardhatPort required when 'config.startHardhat' is true" - "-d:HardhatPort=" & $port + var hhPort = string.none + if test.config.startHardhat: + without port =? hardhatPort: + raiseTestManagerError "hardhatPort required when 'config.startHardhat' is true" + hhPort = some "-d:HardhatPort=" & $port + + var logDir = string.none + if test.manager.debugCodexNodes: + logDir = some "-d:LogsDir=" & test.logsDir var testFile: string try: @@ -324,12 +333,25 @@ proc buildCommand( withLock(test.manager.hardhatPortLock): try: return - "nim c " & &"-d:CodexApiPort={apiPort} " & &"-d:CodexDiscPort={discPort} " & - &"{strHardhatPort} " & &"-d:TestId={test.testId} " & &"{logging} " & - "--verbosity:0 " & "--hints:off " & "-d:release " & "-r " & &"{testFile}" + #!fmt: off + "nim c " & + &"-d:CodexApiPort={apiPort} " & + &"-d:CodexDiscPort={discPort} " & + &"-d:DebugCodexNodes={test.manager.debugCodexNodes} " & + &"-d:DebugHardhat={test.manager.debugHardhat} " & + (logDir |? "") & " " & + (hhPort |? "") & " " & + &"-d:TestId={test.testId} " & + (logging |? "") & " " & + "--verbosity:0 " & + "--hints:off " & + "-d:release " & + "-r " & + &"{testFile}" + #!fmt: on except ValueError as parent: raiseTestManagerError "bad command --\n" & ", apiPort: " & $apiPort & - ", discPort: " & $discPort & ", logging: " & logging & ", testFile: " & + ", discPort: " & $discPort & ", logging: " & logging |? "" & ", testFile: " & testFile & ", error: " & parent.msg, parent proc setup( @@ -394,6 +416,13 @@ proc start(test: IntegrationTest) {.async: (raises: []).} = trace "Running test" + if test.manager.debugCodexNodes: + test.logsDir = test.manager.logsDir / sanitize(test.config.name) + try: + createDir(test.logsDir) + except CatchableError as e: + error "failed to create test log dir", logDir = test.logsDir, error = e.msg + test.timeStart = some Moment.now() test.status = IntegrationTestStatus.Running @@ -501,7 +530,7 @@ proc runTests(manager: TestManager) {.async: (raises: [CancelledError]).} = asyncSpawn futRun try: - # if runTests is cancelled, await allFutures will be cancelled, but allFutures + # if runTests is cancelled, await allFutures will be cancelled, but allFutures # does not propagate the cancellation to the futures it's waiting on, so we # need to cancel them here await allFutures testFutures @@ -581,6 +610,31 @@ proc printResult(manager: TestManager) {.raises: [TestManagerError].} = proc start*( manager: TestManager ) {.async: (raises: [CancelledError, TestManagerError]).} = + try: + if manager.debugCodexNodes: + let startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") + let logsDir = + currentSourcePath.parentDir() / "logs" / + sanitize(startTime & "__IntegrationTests") + createDir(logsDir) + manager.logsDir = logsDir + #!fmt: off + echoStyled bgWhite, fgBlack, styleBright, + "\n\n ", + styleUnderscore, + "ℹ️ LOGS AVAILABLE ℹ️\n\n", + resetStyle, bgWhite, fgBlack, styleBright, + """ Logs for this run will be available at:""", + resetStyle, bgWhite, fgBlack, + &"\n\n {logsDir}\n\n", + resetStyle, bgWhite, fgBlack, styleBright, + " NOTE: For CI runs, logs will be attached as artefacts\n" + #!fmt: on + except IOError as e: + raiseTestManagerError "failed to create hardhat log directory: " & e.msg, e + except OSError as e: + raiseTestManagerError "failed to create hardhat log directory: " & e.msg, e + if manager.showContinuousStatusUpdates: let fut = manager.continuallyShowUpdates() manager.trackedFutures.track fut diff --git a/tests/integration/utils.nim b/tests/integration/utils.nim index 6b5e536b..bcdff4ed 100644 --- a/tests/integration/utils.nim +++ b/tests/integration/utils.nim @@ -1,8 +1,11 @@ +import std/os import pkg/chronos import pkg/codex/logutils +{.push raises: [].} + proc nextFreePort*(startPort: int): Future[int] {.async: (raises: [CancelledError]).} = - proc client(server: StreamServer, transp: StreamTransport) {.async.} = + proc client(server: StreamServer, transp: StreamTransport) {.async: (raises: []).} = await transp.closeWait() var port = startPort @@ -22,3 +25,29 @@ proc nextFreePort*(startPort: int): Future[int] {.async: (raises: [CancelledErro inc port except TransportAddressError: raiseAssert "bad address" + +proc sanitize*(pathSegment: string): string = + var sanitized = pathSegment + for invalid in invalidFilenameChars.items: + sanitized = sanitized.replace(invalid, '_').replace(' ', '_') + sanitized + +proc getLogFile*( + logDir, startTime, suiteName, testName, role: string, index = int.none +): string {.raises: [IOError, OSError].} = + let logsDir = + if logDir == "": + currentSourcePath.parentDir() / "logs" / sanitize(startTime & "__" & suiteName) / + sanitize(testName) + else: + logDir / sanitize(suiteName) / sanitize(testName) + + createDir(logsDir) + + var fn = $role + if idx =? index: + fn &= "_" & $idx + fn &= ".log" + + let fileName = logsDir / fn + return fileName diff --git a/tests/testIntegration.nim b/tests/testIntegration.nim index 9ed70a48..7f18a481 100644 --- a/tests/testIntegration.nim +++ b/tests/testIntegration.nim @@ -27,7 +27,7 @@ const DebugHardhat {.booldefine.} = false # Echoes stdout from the integration test file process. Codex process logs can # also be output if a test uses a multinodesuite, requires CodexConfig.debug # to be enabled -const DebugCodexNodes {.booldefine.} = true +const DebugCodexNodes {.booldefine.} = false # Shows test status updates at time intervals. Useful for running locally with # active terminal interaction. Set to false for unattended runs, eg CI. const ShowContinuousStatusUpdates {.booldefine.} = false @@ -37,28 +37,6 @@ const TestTimeout {.intdefine.} = 60 const EnableParallelTests {.booldefine.} = true proc run() {.async.} = - when DebugTestHarness and enabledLogLevel != LogLevel.TRACE: - styledEcho bgWhite, - fgBlack, styleBright, "\n\n ", styleUnderscore, - "ℹ️ ADDITIONAL LOGGING AVAILABLE ℹ️\n\n", resetStyle, bgWhite, fgBlack, - styleBright, - """ - More integration test harness logs available by running with - -d:chronicles_log_level=TRACE, eg:""", - resetStyle, bgWhite, fgBlack, - "\n\n nim c -d:chronicles_log_level=TRACE -r ./testIntegration.nim\n\n" - - when DebugCodexNodes: - styledEcho bgWhite, - fgBlack, styleBright, "\n\n ", styleUnderscore, - "⚠️ ENABLE CODEX LOGGING ⚠️\n\n", resetStyle, bgWhite, fgBlack, - styleBright, - """ - For integration test suites that are multinodesuites, or for - tests launching a CodexProcess, ensure that CodexConfig.debug - is enabled to see chronicles logs. - """ - let manager = TestManager.new( configs = TestConfigs, DebugTestHarness,