update Make flag to simply debug

DEBUG -> enables DebugCodexNodes, DebugTestHarness, and ShowContinuousStatusUpdates
DEBUG_HARDHAT -> enables DebugHardhat
PARALLEL -> enables EnableParallelTests

Additionally, when DEBUG is enabled, all integration tests debug configs are enabled for Codex nodes, the Codex node output is printed with the test output (not interleaved), and the Codex node output is logged to file in `tests/integrations/logs/<starttime>__IntegrationTests/<integration_test_name>/<suite_name>/<testname>/<role>_<idx>.log`.
When DEBUG_HARDHAT is enabled, all hardhat output is printed with the test output (not interleaved), and the output is also written to a log file in `tests/integrations/logs/<starttime>__IntegrationTests/<integration_test_name>/hardhat.log

.github/workflows/ci-reusable.yml

@@ -81,7 +81,7 @@ jobs:
 
       - name: Parallel integration tests
         if: matrix.tests == 'integration-parallel'
-        run: make -j${ncpu} DEBUG_CODEXNODES=1 DEBUG_TESTHARNESS=1 DEBUG_UPDATES=1 testIntegration
+        run: make -j${ncpu} DEBUG=1 testIntegration
 
       - name: Upload integration tests log files
         uses: actions/upload-artifact@v4

Makefile

@@ -142,23 +142,19 @@ testContracts: | build deps
 		$(ENV_SCRIPT) nim testContracts $(NIM_PARAMS) build.nims
 
 TEST_PARAMS :=
-ifdef DEBUG_TESTHARNESS
+ifdef DEBUG
-	TEST_PARAMS := $(TEST_PARAMS) -d:DebugTestHarness=$(DEBUG_TESTHARNESS)
+	TEST_PARAMS := $(TEST_PARAMS) -d:DebugTestHarness=$(DEBUG)
+  TEST_PARAMS := $(TEST_PARAMS) -d:DebugCodexNodes=$(DEBUG)
+  TEST_PARAMS := $(TEST_PARAMS) -d:ShowContinuousStatusUpdates=$(DEBUG)
 endif
 ifdef DEBUG_HARDHAT
   TEST_PARAMS := $(TEST_PARAMS) -d:DebugHardhat=$(DEBUG_HARDHAT)
 endif
-ifdef DEBUG_CODEXNODES # true by default
-  TEST_PARAMS := $(TEST_PARAMS) -d:DebugCodexNodes=$(DEBUG_CODEXNODES)
-endif
-ifdef DEBUG_UPDATES
-  TEST_PARAMS := $(TEST_PARAMS) -d:ShowContinuousStatusUpdates=$(DEBUG_UPDATES)
-endif
 ifdef TEST_TIMEOUT
   TEST_PARAMS := $(TEST_PARAMS) -d:TestTimeout=$(TEST_TIMEOUT)
 endif
-ifdef ENABLE_PARALLEL_TESTS
+ifdef PARALLEL
-  TEST_PARAMS := $(TEST_PARAMS) -d:EnableParallelTests=$(ENABLE_PARALLEL_TESTS)
+  TEST_PARAMS := $(TEST_PARAMS) -d:EnableParallelTests=$(PARALLEL)
 endif
 
 # Builds and runs the integration tests

tests/integration/multinodes.nim

@@ -42,6 +42,8 @@ const HardhatPort {.intdefine.}: int = 8545
 const CodexApiPort {.intdefine.}: int = 8080
 const CodexDiscPort {.intdefine.}: int = 8090
 const TestId {.strdefine.}: string = "TestId"
+const DebugCodexNodes {.booldefine.}: bool = false
+const LogsDir {.strdefine.}: string = ""
 
 proc raiseMultiNodeSuiteError(msg: string) =
   raise newException(MultiNodeSuiteError, msg)
@@ -87,29 +89,6 @@ template multinodesuite*(name: string, body: untyped) =
       test tname:
         tbody
 
-    proc sanitize(pathSegment: string): string =
-      var sanitized = pathSegment
-      for invalid in invalidFilenameChars.items:
-        sanitized = sanitized.replace(invalid, '_').replace(' ', '_')
-      sanitized
-
-    proc getLogFile(role: Role, index: ?int): string =
-      # create log file path, format:
-      # tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
-
-      var logDir =
-        currentSourcePath.parentDir() / "logs" / sanitize($starttime & "__" & name) /
-        sanitize($currentTestName)
-      createDir(logDir)
-
-      var fn = $role
-      if idx =? index:
-        fn &= "_" & $idx
-      fn &= ".log"
-
-      let fileName = logDir / fn
-      return fileName
-
     proc updatePort(url: var string, port: int) =
       let parts = url.split(':')
       url = @[parts[0], parts[1], $port].join(":")
@@ -119,7 +98,8 @@ template multinodesuite*(name: string, body: untyped) =
     ): Future[NodeProcess] {.async.} =
       var args: seq[string] = @[]
       if config.logFile:
-        let updatedLogFile = getLogFile(role, none int)
+        let updatedLogFile =
+          getLogFile(LogsDir, starttime, name, currentTestName, $role, none int)
         args.add "--log-file=" & updatedLogFile
 
       let port = await nextFreePort(lastUsedHardhatPort)
@@ -151,10 +131,14 @@ template multinodesuite*(name: string, body: untyped) =
         sanitize($role & "_" & $roleIdx)
 
       try:
-        if config.logFile.isSome:
+        if config.logFile.isSome or DebugCodexNodes:
-          let updatedLogFile = getLogFile(role, some roleIdx)
+          let updatedLogFile =
+            getLogFile(LogsDir, starttime, name, currentTestName, $role, some roleIdx)
           config.withLogFile(updatedLogFile)
 
+        if DebugCodexNodes:
+          config.debugEnabled = true
+
         let apiPort = await nextFreePort(lastUsedCodexApiPort + nodeIdx)
         let discPort = await nextFreePort(lastUsedCodexDiscPort + nodeIdx)
         config.addCliOption("--api-port", $apiPort)

tests/integration/testcli.nim

@@ -1,4 +1,5 @@
 import std/tempfiles
+import std/times
 import codex/conf
 import codex/utils/fileutils
 import ../asynctest
@@ -9,22 +10,52 @@ import ./utils
 import ../examples
 
 const HardhatPort {.intdefine.}: int = 8545
+const CodexApiPort {.intdefine.}: int = 8080
+const CodexDiscPort {.intdefine.}: int = 8090
+const DebugCodexNodes {.booldefine.}: bool = false
+const LogsDir {.strdefine.}: string = ""
 
 asyncchecksuite "Command line interface":
+  let startTime = now().format("yyyy-MM-dd'_'HH:mm:ss")
   let key = "4242424242424242424242424242424242424242424242424242424242424242"
 
-  var nodeCount = -1
+  var currentTestName = ""
-  proc startCodex(args: seq[string]): Future[CodexProcess] {.async.} =
+  var testCount = 0
-    return await CodexProcess.startNode(args, false, "cli-test-node")
+  var nodeCount = 0
+
+  template test(tname, tbody) =
+    inc testCount
+    currentTestName = tname
+    test tname:
+      tbody
+
+  proc addLogFile(args: seq[string]): seq[string] =
+    when DebugCodexNodes:
+      return args.concat @[
+        "--log-file=" &
+          getLogFile(
+            LogsDir,
+            startTime,
+            "Command line interface",
+            currentTestName,
+            "Client",
+            some nodeCount mod testCount,
+          )
+      ]
+    else:
+      return args
+
+  proc startCodex(arguments: seq[string]): Future[CodexProcess] {.async.} =
     inc nodeCount
+    let args = arguments.addLogFile
     return await CodexProcess.startNode(
       args.concat(
         @[
-          "--api-port=" & $(await nextFreePort(8080 + nodeCount)),
+          "--api-port=" & $(await nextFreePort(CodexApiPort + nodeCount)),
-          "--disc-port=" & $(await nextFreePort(8090 + nodeCount)),
+          "--disc-port=" & $(await nextFreePort(CodexDiscPort + nodeCount)),
         ]
       ),
-      debug = false,
+      debug = DebugCodexNodes,
       "cli-test-node",
     )
 

tests/integration/testmanager.nim

@@ -1,6 +1,7 @@
 import std/os
 import std/strformat
 import std/terminal
+from std/times import fromUnix, format, now
 from std/unicode import toUpper
 import std/unittest
 import pkg/chronos
@@ -38,6 +39,7 @@ type
     # Shows test status updates at regular time intervals. Useful for running
     # locally while attended. Set to false for unattended runs, eg CI.
     showContinuousStatusUpdates: bool
+    logsDir: string
     timeStart: ?Moment
     timeEnd: ?Moment
     codexPortLock: AsyncLock
@@ -73,6 +75,7 @@ type
     testId: string # when used in datadir path, prevents data dir clashes
     status: IntegrationTestStatus
     command: string
+    logsDir: string
 
   TestManagerError* = object of CatchableError
 
@@ -194,6 +197,8 @@ proc startHardhat(
 
   args.add("--port")
   args.add($port)
+  if test.manager.debugHardhat:
+    args.add("--log-file=" & test.logsDir / "hardhat.log")
 
   trace "starting hardhat process on port ", port
   try:
@@ -255,9 +260,9 @@ proc printResult(
   of IntegrationTestStatus.Failed:
     if output =? test.output:
       if printStdErr: #manager.debugTestHarness
-        test.printOutputMarker(MarkerPosition.Start, "test harness errors (stderr)")
+        test.printOutputMarker(MarkerPosition.Start, "test file errors (stderr)")
         echo output.stdError
-        test.printOutputMarker(MarkerPosition.Finish, "test harness errors (stderr)")
+        test.printOutputMarker(MarkerPosition.Finish, "test file errors (stderr)")
       if printStdOut:
         test.printOutputMarker(MarkerPosition.Start, "codex node output (stdout)")
         echo output.stdOutput
@@ -286,21 +291,25 @@ proc printStart(test: IntegrationTest) =
 proc buildCommand(
     test: IntegrationTest, hardhatPort: ?int
 ): Future[string] {.async: (raises: [CancelledError, TestManagerError]).} =
-  let logging =
+  var logging = string.none
-    if not test.manager.debugTestHarness:
+  if test.manager.debugTestHarness:
-      ""
+    #!fmt: off
-    else:
+    logging = some(
       "-d:chronicles_log_level=TRACE " &
-        "-d:chronicles_disabled_topics=websock,JSONRPC-HTTP-CLIENT,JSONRPC-WS-CLIENT " &
+      "-d:chronicles_disabled_topics=websock,JSONRPC-HTTP-CLIENT,JSONRPC-WS-CLIENT " &
-        "-d:chronicles_default_output_device=stdout " & "-d:chronicles_sinks=textlines"
+      "-d:chronicles_default_output_device=stdout " &
+      "-d:chronicles_sinks=textlines")
+    #!fmt: on
 
-  let strHardhatPort =
+  var hhPort = string.none
-    if not test.config.startHardhat:
+  if test.config.startHardhat:
-      ""
+    without port =? hardhatPort:
-    else:
+      raiseTestManagerError "hardhatPort required when 'config.startHardhat' is true"
-      without port =? hardhatPort:
+    hhPort = some "-d:HardhatPort=" & $port
-        raiseTestManagerError "hardhatPort required when 'config.startHardhat' is true"
+
-      "-d:HardhatPort=" & $port
+  var logDir = string.none
+  if test.manager.debugCodexNodes:
+    logDir = some "-d:LogsDir=" & test.logsDir
 
   var testFile: string
   try:
@@ -324,12 +333,25 @@ proc buildCommand(
     withLock(test.manager.hardhatPortLock):
       try:
         return
-          "nim c " & &"-d:CodexApiPort={apiPort} " & &"-d:CodexDiscPort={discPort} " &
+          #!fmt: off
-          &"{strHardhatPort} " & &"-d:TestId={test.testId} " & &"{logging} " &
+          "nim c " &
-          "--verbosity:0 " & "--hints:off " & "-d:release " & "-r " & &"{testFile}"
+            &"-d:CodexApiPort={apiPort} " &
+            &"-d:CodexDiscPort={discPort} " &
+            &"-d:DebugCodexNodes={test.manager.debugCodexNodes} " &
+            &"-d:DebugHardhat={test.manager.debugHardhat} " &
+            (logDir |? "") & " " &
+            (hhPort |? "") & " " &
+            &"-d:TestId={test.testId} " &
+            (logging |? "") & " " &
+            "--verbosity:0 " &
+            "--hints:off " &
+            "-d:release " &
+          "-r " &
+            &"{testFile}"
+          #!fmt: on
       except ValueError as parent:
         raiseTestManagerError "bad command --\n" & ", apiPort: " & $apiPort &
-          ", discPort: " & $discPort & ", logging: " & logging & ", testFile: " &
+          ", discPort: " & $discPort & ", logging: " & logging |? "" & ", testFile: " &
           testFile & ", error: " & parent.msg, parent
 
 proc setup(
@@ -394,6 +416,13 @@ proc start(test: IntegrationTest) {.async: (raises: []).} =
 
   trace "Running test"
 
+  if test.manager.debugCodexNodes:
+    test.logsDir = test.manager.logsDir / sanitize(test.config.name)
+    try:
+      createDir(test.logsDir)
+    except CatchableError as e:
+      error "failed to create test log dir", logDir = test.logsDir, error = e.msg
+
   test.timeStart = some Moment.now()
   test.status = IntegrationTestStatus.Running
 
@@ -501,7 +530,7 @@ proc runTests(manager: TestManager) {.async: (raises: [CancelledError]).} =
     asyncSpawn futRun
 
   try:
-    # if runTests is cancelled, await allFutures will be cancelled, but allFutures 
+    # if runTests is cancelled, await allFutures will be cancelled, but allFutures
     # does not propagate the cancellation to the futures it's waiting on, so we
     # need to cancel them here
     await allFutures testFutures
@@ -581,6 +610,31 @@ proc printResult(manager: TestManager) {.raises: [TestManagerError].} =
 proc start*(
     manager: TestManager
 ) {.async: (raises: [CancelledError, TestManagerError]).} =
+  try:
+    if manager.debugCodexNodes:
+      let startTime = now().format("yyyy-MM-dd'_'HH:mm:ss")
+      let logsDir =
+        currentSourcePath.parentDir() / "logs" /
+        sanitize(startTime & "__IntegrationTests")
+      createDir(logsDir)
+      manager.logsDir = logsDir
+      #!fmt: off
+      echoStyled bgWhite, fgBlack, styleBright,
+        "\n\n  ",
+        styleUnderscore,
+        "ℹ️  LOGS AVAILABLE ℹ️\n\n",
+        resetStyle, bgWhite, fgBlack, styleBright,
+        """  Logs for this run will be available at:""",
+        resetStyle, bgWhite, fgBlack,
+        &"\n\n  {logsDir}\n\n",
+        resetStyle, bgWhite, fgBlack, styleBright,
+        "  NOTE: For CI runs, logs will be attached as artefacts\n"
+      #!fmt: on
+  except IOError as e:
+    raiseTestManagerError "failed to create hardhat log directory: " & e.msg, e
+  except OSError as e:
+    raiseTestManagerError "failed to create hardhat log directory: " & e.msg, e
+
   if manager.showContinuousStatusUpdates:
     let fut = manager.continuallyShowUpdates()
     manager.trackedFutures.track fut

tests/integration/utils.nim

@@ -1,8 +1,11 @@
+import std/os
 import pkg/chronos
 import pkg/codex/logutils
 
+{.push raises: [].}
+
 proc nextFreePort*(startPort: int): Future[int] {.async: (raises: [CancelledError]).} =
-  proc client(server: StreamServer, transp: StreamTransport) {.async.} =
+  proc client(server: StreamServer, transp: StreamTransport) {.async: (raises: []).} =
     await transp.closeWait()
 
   var port = startPort
@@ -22,3 +25,29 @@ proc nextFreePort*(startPort: int): Future[int] {.async: (raises: [CancelledErro
       inc port
     except TransportAddressError:
       raiseAssert "bad address"
+
+proc sanitize*(pathSegment: string): string =
+  var sanitized = pathSegment
+  for invalid in invalidFilenameChars.items:
+    sanitized = sanitized.replace(invalid, '_').replace(' ', '_')
+  sanitized
+
+proc getLogFile*(
+    logDir, startTime, suiteName, testName, role: string, index = int.none
+): string {.raises: [IOError, OSError].} =
+  let logsDir =
+    if logDir == "":
+      currentSourcePath.parentDir() / "logs" / sanitize(startTime & "__" & suiteName) /
+        sanitize(testName)
+    else:
+      logDir / sanitize(suiteName) / sanitize(testName)
+
+  createDir(logsDir)
+
+  var fn = $role
+  if idx =? index:
+    fn &= "_" & $idx
+  fn &= ".log"
+
+  let fileName = logsDir / fn
+  return fileName

tests/testIntegration.nim

@@ -27,7 +27,7 @@ const DebugHardhat {.booldefine.} = false
 # Echoes stdout from the integration test file process. Codex process logs can
 # also be output if a test uses a multinodesuite, requires CodexConfig.debug
 # to be enabled
-const DebugCodexNodes {.booldefine.} = true
+const DebugCodexNodes {.booldefine.} = false
 # Shows test status updates at time intervals. Useful for running locally with
 # active terminal interaction. Set to false for unattended runs, eg CI.
 const ShowContinuousStatusUpdates {.booldefine.} = false
@@ -37,28 +37,6 @@ const TestTimeout {.intdefine.} = 60
 const EnableParallelTests {.booldefine.} = true
 
 proc run() {.async.} =
-  when DebugTestHarness and enabledLogLevel != LogLevel.TRACE:
-    styledEcho bgWhite,
-      fgBlack, styleBright, "\n\n  ", styleUnderscore,
-      "ℹ️  ADDITIONAL LOGGING AVAILABLE ℹ️\n\n", resetStyle, bgWhite, fgBlack,
-      styleBright,
-      """
-  More integration test harness logs available by running with
-  -d:chronicles_log_level=TRACE, eg:""",
-      resetStyle, bgWhite, fgBlack,
-      "\n\n  nim c -d:chronicles_log_level=TRACE -r ./testIntegration.nim\n\n"
-
-  when DebugCodexNodes:
-    styledEcho bgWhite,
-      fgBlack, styleBright, "\n\n  ", styleUnderscore,
-      "⚠️ ENABLE CODEX LOGGING ⚠️\n\n", resetStyle, bgWhite, fgBlack,
-      styleBright,
-      """
-  For integration test suites that are multinodesuites, or for
-  tests launching a CodexProcess, ensure that CodexConfig.debug
-  is enabled to see chronicles logs.
-  """
-
   let manager = TestManager.new(
     configs = TestConfigs,
     DebugTestHarness,