Updates after rebase

.github/workflows/ci-reusable.yml

@@ -43,46 +43,6 @@ jobs:
           nim_version: ${{ matrix.nim_version }}
           coverage: false
 
-      - name: Check runner resources for parallel integration tests
-        run: |
-          echo "Determining runner"
-          case "${{ matrix.os }}" in
-            linux)    CPU=$(nproc --all)
-                      RAM=$(awk '/MemTotal/ {print int($2 / 1024 / 1024 + 0.5)}' /proc/meminfo)
-                      ;;
-            macos)    CPU=$(sysctl -n hw.ncpu)
-                      RAM=$(sysctl -n hw.memsize | awk '{print $0/1073741824}')
-                      sysctl -n hw.ncpu
-                      ;;
-            windows)  CPU=$NUMBER_OF_PROCESSORS
-                      RAM=$(systeminfo | awk '/Total Physical Memory:/ { gsub(/,/,"."); print int($4 + 0.5) }')
-                      ;;
-            *)        CPU=2
-                      RAM=8
-                      echo "Unknown runner"
-                      ;;
-          esac
-          echo "CPU=${CPU}" >> $GITHUB_ENV
-          echo "RAM=${RAM}" >> $GITHUB_ENV
-          echo "TYPE=${RUNNER_ENVIRONMENT}" >> $GITHUB_ENV
-
-          # Set PARALLEL=1 if the runner has enough resources
-          if [[ ("${{ matrix.os }}" == "linux" || "${{ matrix.os }}" == "windows") && "${CPU}" -ge 16 ]]; then
-            echo "PARALLEL=1" >> $GITHUB_ENV
-          elif [[ "${{ matrix.os }}" == "macos" && "${CPU}" -ge 6 ]]; then
-            echo "PARALLEL=1" >> $GITHUB_ENV
-          else
-            echo "PARALLEL=0" >> $GITHUB_ENV
-          fi
-
-      - name: Show runner information
-        run: |
-          echo "OS: ${{ matrix.os }}"
-          echo "CPU: ${{ env.CPU }}"
-          echo "RAM: ${{ env.RAM }} GB"
-          echo "TYPE: ${{ env.TYPE }}"
-          echo "PARALLEL: ${{ env.PARALLEL }}"
-
       ## Part 1 Tests ##
       - name: Unit tests
         if: matrix.tests == 'unittest' || matrix.tests == 'all'
@@ -100,10 +60,10 @@ jobs:
         env:
           MSYS2_PATH_TYPE: inherit
         run: |
-          npm install
+          npm ci
 
       - name: Run Ethereum node with Codex contracts
-        if: matrix.tests == 'contract' || (matrix.tests == 'integration' && env.PARALLEL != 1) || matrix.tests == 'tools' || matrix.tests == 'all'
+        if: matrix.tests == 'contract' || matrix.tests == 'integration' || matrix.tests == 'tools' || matrix.tests == 'all'
         working-directory: vendor/codex-contracts-eth
         env:
           MSYS2_PATH_TYPE: inherit
@@ -122,7 +82,7 @@ jobs:
         if: matrix.tests == 'integration' || matrix.tests == 'all'
         env:
           CODEX_INTEGRATION_TEST_INCLUDES: ${{ matrix.includes }}
-        run: make -j${ncpu} PARALLEL=${{ env.PARALLEL }} DEBUG=${{ runner.debug }} testIntegration
+        run: make -j${ncpu} DEBUG=${{ runner.debug }} testIntegration
 
       - name: Upload integration tests log files
         uses: actions/upload-artifact@v4

.github/workflows/ci.yml

@@ -10,10 +10,6 @@ on:
 env:
   cache_nonce: 0 # Allows for easily busting actions/cache caches
   nim_version: v2.2.4
-  builder_integration_from_vars: true
-  builder_integration_linux: runner-node-01-linux-03-eu-hel1
-  builder_integration_macos: macos-14-xlarge
-  builder_integration_windows: windows-latest-amd64-32vcpu
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref || github.run_id }}

codex/blockexchange/engine/engine.nim

@@ -68,12 +68,6 @@ const
   DefaultMaxPeersPerRequest* = 10
   DefaultTaskQueueSize = 100
   DefaultConcurrentTasks = 10
-    # DefaultMaxRetries = 3
-    # DefaultConcurrentDiscRequests = 10
-    # DefaultConcurrentAdvertRequests = 10
-    # DefaultDiscoveryTimeout = 1.minutes
-    # DefaultMaxQueriedBlocksCache = 1000
-    # DefaultMinPeersPerBlock = 3
 
 type
   TaskHandler* = proc(task: BlockExcPeerCtx): Future[void] {.gcsafe.}

tests/integration/30_minutes/testproofs.nim.ignore

@@ -7,7 +7,6 @@ import ../../codex/helpers
 import ../../examples
 import ../marketplacesuite
 import ../nodeconfigs
-from ../../helpers import eventuallySafe
 
 export logutils
 

tests/integration/5_minutes/testsales.nim.ignore

@@ -8,7 +8,6 @@ import ../../contracts/time
 import ../codexconfig
 import ../codexclient
 import ../nodeconfigs
-from ../../helpers import eventuallySafe
 import ../marketplacesuite
 
 proc findItem[T](items: seq[T], item: T): ?!T =

tests/integration/hardhatprocess.nim

@@ -79,11 +79,14 @@ method start*(
   let poptions = node.processOptions + {AsyncProcessOption.StdErrToStdOut}
 
   trace "starting node",
-    args, executable, workingDir = node.workingDir, processOptions = poptions
+    args = node.arguments,
+    executable = node.executable,
+    workingDir = node.workingDir,
+    processOptions = poptions
 
   try:
     node.process = await startProcess(
-      executable,
+      node.executable,
       node.workingDir,
       @["node"].concat(node.arguments),
       options = poptions,

tests/testIntegration.nim

@@ -1,127 +1,17 @@
 import std/os
-import std/strformat
-import std/terminal
-from std/times import format, now
-import std/terminal
-import std/typetraits
-import pkg/chronos
-import pkg/codex/conf
-import pkg/codex/logutils
-import ./integration/testmanager
-import ./integration/utils
+import std/strutils
+import ./imports
+
+## Limit which integration tests to run by setting the
+## environment variable during compilation. For example:
+## CODEX_INTEGRATION_TEST_INCLUDES="testFoo.nim,testBar.nim"
+const includes = getEnv("CODEX_INTEGRATION_TEST_INCLUDES")
+
+when includes != "":
+  # import only the specified tests
+  importAll(includes.split(","))
+else:
+  # import all tests in the integration/ directory
+  importTests(currentSourcePath().parentDir() / "integration")
 
 {.warning[UnusedImport]: off.}
-{.push raises: [].}
-
-const TestConfigs =
-  @[
-    IntegrationTestConfig.init("./integration/testcli", startHardhat = true),
-    IntegrationTestConfig.init("./integration/testrestapi", startHardhat = true),
-    IntegrationTestConfig.init("./integration/testupdownload", startHardhat = true),
-    IntegrationTestConfig.init("./integration/testsales", startHardhat = true),
-    IntegrationTestConfig.init("./integration/testpurchasing", startHardhat = true),
-    IntegrationTestConfig.init("./integration/testblockexpiration", startHardhat = true),
-    IntegrationTestConfig.init("./integration/testmarketplace", startHardhat = true),
-    IntegrationTestConfig.init("./integration/testproofs", startHardhat = true),
-    IntegrationTestConfig.init("./integration/testvalidator", startHardhat = true),
-    IntegrationTestConfig.init("./integration/testecbug", startHardhat = true),
-    IntegrationTestConfig.init(
-      "./integration/testrestapivalidation", startHardhat = true
-    ),
-  ]
-
-# Echoes stdout from Hardhat process
-const DebugHardhat {.booldefine.} = false
-# When true, shows all TRACE logs in Codex nodes' chronicles logs
-const NoCodexLogFilters {.booldefine.} = false
-# Shows test status updates at time intervals. Useful for running locally with
-# active terminal interaction. Set to false for unattended runs, eg CI.
-const ShowContinuousStatusUpdates {.booldefine.} = false
-# Timeout duration (in minutes) for EACH integration test file.
-const TestTimeout {.intdefine.} = 60
-
-const EnableParallelTests {.booldefine.} = true
-
-proc setupLogging(logFile: string) =
-  try:
-    let success = defaultChroniclesStream.outputs[0].open(logFile, fmAppend)
-    doAssert success, "Failed to open log file: " & logFile
-  except IOError, OSError:
-    let error = getCurrentException()
-    fatal "Failed to open log file", error = error.msg
-    raiseAssert "Could not open test manager log file: " & error.msg
-
-proc run(): Future[bool] {.async: (raises: []).} =
-  let startTime = now().format("yyyy-MM-dd'_'HH-mm-ss")
-  let logsDir =
-    currentSourcePath.parentDir() / "integration" / "logs" /
-    sanitize(startTime & "-IntegrationTests")
-  try:
-    createDir(logsDir)
-    #!fmt: off
-    styledEcho bgWhite, fgBlack, styleBright,
-      "\n\n  ",
-      styleUnderscore,
-      "ℹ️  LOGS AVAILABLE ℹ️\n\n",
-      resetStyle, bgWhite, fgBlack, styleBright,
-      """  Logs for this run will be available at:""",
-      resetStyle, bgWhite, fgBlack,
-      &"\n\n  {logsDir}\n\n",
-      resetStyle, bgWhite, fgBlack, styleBright,
-      "  NOTE: For CI runs, logs will be attached as artefacts\n"
-      #!fmt: on
-  except IOError as e:
-    raiseAssert "Failed to create log directory and echo log message: " & e.msg
-  except OSError as e:
-    raiseAssert "Failed to create log directory and echo log message: " & e.msg
-
-  setupLogging(TestManager.logFile(logsDir))
-
-  let manager = TestManager.new(
-    config = TestManagerConfig(
-      debugHardhat: DebugHardhat,
-      noCodexLogFilters: NoCodexLogFilters,
-      showContinuousStatusUpdates: ShowContinuousStatusUpdates,
-      logsDir: logsDir,
-      testTimeout: TestTimeout.minutes,
-    ),
-    testConfigs = TestConfigs,
-  )
-  try:
-    trace "starting test manager"
-    await manager.start()
-  except TestManagerError as e:
-    error "Failed to run test manager", error = e.msg
-    return false
-  except CancelledError:
-    return false
-  finally:
-    trace "Stopping test manager"
-    await manager.stop()
-    trace "Test manager stopped"
-
-  without wasSuccessful =? manager.allTestsPassed, error:
-    raiseAssert "Failed to get test status: " & error.msg
-
-  return wasSuccessful
-
-when isMainModule:
-  when EnableParallelTests:
-    let wasSuccessful = waitFor run()
-    if wasSuccessful:
-      quit(QuitSuccess)
-    else:
-      quit(QuitFailure) # indicate with a non-zero exit code that the tests failed
-  else:
-    # run tests serially
-    import ./integration/testcli
-    import ./integration/testrestapi
-    import ./integration/testupdownload
-    import ./integration/testsales
-    import ./integration/testpurchasing
-    import ./integration/testblockexpiration
-    import ./integration/testmarketplace
-    import ./integration/testproofs
-    import ./integration/testvalidator
-    import ./integration/testecbug
-    import ./integration/testrestapivalidation