mirror of
https://github.com/status-im/nim-codex.git
synced 2025-02-22 07:28:32 +00:00
update Make flag to simply debug
DEBUG -> enables DebugCodexNodes, DebugTestHarness, and ShowContinuousStatusUpdates DEBUG_HARDHAT -> enables DebugHardhat PARALLEL -> enables EnableParallelTests Additionally, when DEBUG is enabled, all integration tests debug configs are enabled for Codex nodes, the Codex node output is printed with the test output (not interleaved), and the Codex node output is logged to file in `tests/integrations/logs/<starttime>__IntegrationTests/<integration_test_name>/<suite_name>/<testname>/<role>_<idx>.log`. When DEBUG_HARDHAT is enabled, all hardhat output is printed with the test output (not interleaved), and the output is also written to a log file in `tests/integrations/logs/<starttime>__IntegrationTests/<integration_test_name>/hardhat.log
This commit is contained in:
Eric
parent
22b4847179
commit
4119dd6a53
2
.github/workflows/ci-reusable.yml
vendored
2
.github/workflows/ci-reusable.yml
vendored
@ -81,7 +81,7 @@ jobs:
|
||||
|
||||
- name: Parallel integration tests
|
||||
if: matrix.tests == 'integration-parallel'
|
||||
run: make -j${ncpu} DEBUG_CODEXNODES=1 DEBUG_TESTHARNESS=1 DEBUG_UPDATES=1 testIntegration
|
||||
run: make -j${ncpu} DEBUG=1 testIntegration
|
||||
|
||||
- name: Upload integration tests log files
|
||||
uses: actions/upload-artifact@v4
|
||||
|
||||
16
Makefile
16
Makefile
@ -142,23 +142,19 @@ testContracts: | build deps
|
||||
$(ENV_SCRIPT) nim testContracts $(NIM_PARAMS) build.nims
|
||||
|
||||
TEST_PARAMS :=
|
||||
ifdef DEBUG_TESTHARNESS
|
||||
TEST_PARAMS := $(TEST_PARAMS) -d:DebugTestHarness=$(DEBUG_TESTHARNESS)
|
||||
ifdef DEBUG
|
||||
TEST_PARAMS := $(TEST_PARAMS) -d:DebugTestHarness=$(DEBUG)
|
||||
TEST_PARAMS := $(TEST_PARAMS) -d:DebugCodexNodes=$(DEBUG)
|
||||
TEST_PARAMS := $(TEST_PARAMS) -d:ShowContinuousStatusUpdates=$(DEBUG)
|
||||
endif
|
||||
ifdef DEBUG_HARDHAT
|
||||
TEST_PARAMS := $(TEST_PARAMS) -d:DebugHardhat=$(DEBUG_HARDHAT)
|
||||
endif
|
||||
ifdef DEBUG_CODEXNODES # true by default
|
||||
TEST_PARAMS := $(TEST_PARAMS) -d:DebugCodexNodes=$(DEBUG_CODEXNODES)
|
||||
endif
|
||||
ifdef DEBUG_UPDATES
|
||||
TEST_PARAMS := $(TEST_PARAMS) -d:ShowContinuousStatusUpdates=$(DEBUG_UPDATES)
|
||||
endif
|
||||
ifdef TEST_TIMEOUT
|
||||
TEST_PARAMS := $(TEST_PARAMS) -d:TestTimeout=$(TEST_TIMEOUT)
|
||||
endif
|
||||
ifdef ENABLE_PARALLEL_TESTS
|
||||
TEST_PARAMS := $(TEST_PARAMS) -d:EnableParallelTests=$(ENABLE_PARALLEL_TESTS)
|
||||
ifdef PARALLEL
|
||||
TEST_PARAMS := $(TEST_PARAMS) -d:EnableParallelTests=$(PARALLEL)
|
||||
endif
|
||||
|
||||
# Builds and runs the integration tests
|
||||
|
||||
@ -42,6 +42,8 @@ const HardhatPort {.intdefine.}: int = 8545
|
||||
const CodexApiPort {.intdefine.}: int = 8080
|
||||
const CodexDiscPort {.intdefine.}: int = 8090
|
||||
const TestId {.strdefine.}: string = "TestId"
|
||||
const DebugCodexNodes {.booldefine.}: bool = false
|
||||
const LogsDir {.strdefine.}: string = ""
|
||||
|
||||
proc raiseMultiNodeSuiteError(msg: string) =
|
||||
raise newException(MultiNodeSuiteError, msg)
|
||||
@ -87,29 +89,6 @@ template multinodesuite*(name: string, body: untyped) =
|
||||
test tname:
|
||||
tbody
|
||||
|
||||
proc sanitize(pathSegment: string): string =
|
||||
var sanitized = pathSegment
|
||||
for invalid in invalidFilenameChars.items:
|
||||
sanitized = sanitized.replace(invalid, '_').replace(' ', '_')
|
||||
sanitized
|
||||
|
||||
proc getLogFile(role: Role, index: ?int): string =
|
||||
# create log file path, format:
|
||||
# tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
|
||||
|
||||
var logDir =
|
||||
currentSourcePath.parentDir() / "logs" / sanitize($starttime & "__" & name) /
|
||||
sanitize($currentTestName)
|
||||
createDir(logDir)
|
||||
|
||||
var fn = $role
|
||||
if idx =? index:
|
||||
fn &= "_" & $idx
|
||||
fn &= ".log"
|
||||
|
||||
let fileName = logDir / fn
|
||||
return fileName
|
||||
|
||||
proc updatePort(url: var string, port: int) =
|
||||
let parts = url.split(':')
|
||||
url = @[parts[0], parts[1], $port].join(":")
|
||||
@ -119,7 +98,8 @@ template multinodesuite*(name: string, body: untyped) =
|
||||
): Future[NodeProcess] {.async.} =
|
||||
var args: seq[string] = @[]
|
||||
if config.logFile:
|
||||
let updatedLogFile = getLogFile(role, none int)
|
||||
let updatedLogFile =
|
||||
getLogFile(LogsDir, starttime, name, currentTestName, $role, none int)
|
||||
args.add "--log-file=" & updatedLogFile
|
||||
|
||||
let port = await nextFreePort(lastUsedHardhatPort)
|
||||
@ -151,10 +131,14 @@ template multinodesuite*(name: string, body: untyped) =
|
||||
sanitize($role & "_" & $roleIdx)
|
||||
|
||||
try:
|
||||
if config.logFile.isSome:
|
||||
let updatedLogFile = getLogFile(role, some roleIdx)
|
||||
if config.logFile.isSome or DebugCodexNodes:
|
||||
let updatedLogFile =
|
||||
getLogFile(LogsDir, starttime, name, currentTestName, $role, some roleIdx)
|
||||
config.withLogFile(updatedLogFile)
|
||||
|
||||
if DebugCodexNodes:
|
||||
config.debugEnabled = true
|
||||
|
||||
let apiPort = await nextFreePort(lastUsedCodexApiPort + nodeIdx)
|
||||
let discPort = await nextFreePort(lastUsedCodexDiscPort + nodeIdx)
|
||||
config.addCliOption("--api-port", $apiPort)
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import std/tempfiles
|
||||
import std/times
|
||||
import codex/conf
|
||||
import codex/utils/fileutils
|
||||
import ../asynctest
|
||||
@ -9,22 +10,52 @@ import ./utils
|
||||
import ../examples
|
||||
|
||||
const HardhatPort {.intdefine.}: int = 8545
|
||||
const CodexApiPort {.intdefine.}: int = 8080
|
||||
const CodexDiscPort {.intdefine.}: int = 8090
|
||||
const DebugCodexNodes {.booldefine.}: bool = false
|
||||
const LogsDir {.strdefine.}: string = ""
|
||||
|
||||
asyncchecksuite "Command line interface":
|
||||
let startTime = now().format("yyyy-MM-dd'_'HH:mm:ss")
|
||||
let key = "4242424242424242424242424242424242424242424242424242424242424242"
|
||||
|
||||
var nodeCount = -1
|
||||
proc startCodex(args: seq[string]): Future[CodexProcess] {.async.} =
|
||||
return await CodexProcess.startNode(args, false, "cli-test-node")
|
||||
var currentTestName = ""
|
||||
var testCount = 0
|
||||
var nodeCount = 0
|
||||
|
||||
template test(tname, tbody) =
|
||||
inc testCount
|
||||
currentTestName = tname
|
||||
test tname:
|
||||
tbody
|
||||
|
||||
proc addLogFile(args: seq[string]): seq[string] =
|
||||
when DebugCodexNodes:
|
||||
return args.concat @[
|
||||
"--log-file=" &
|
||||
getLogFile(
|
||||
LogsDir,
|
||||
startTime,
|
||||
"Command line interface",
|
||||
currentTestName,
|
||||
"Client",
|
||||
some nodeCount mod testCount,
|
||||
)
|
||||
]
|
||||
else:
|
||||
return args
|
||||
|
||||
proc startCodex(arguments: seq[string]): Future[CodexProcess] {.async.} =
|
||||
inc nodeCount
|
||||
let args = arguments.addLogFile
|
||||
return await CodexProcess.startNode(
|
||||
args.concat(
|
||||
@[
|
||||
"--api-port=" & $(await nextFreePort(8080 + nodeCount)),
|
||||
"--disc-port=" & $(await nextFreePort(8090 + nodeCount)),
|
||||
"--api-port=" & $(await nextFreePort(CodexApiPort + nodeCount)),
|
||||
"--disc-port=" & $(await nextFreePort(CodexDiscPort + nodeCount)),
|
||||
]
|
||||
),
|
||||
debug = false,
|
||||
debug = DebugCodexNodes,
|
||||
"cli-test-node",
|
||||
)
|
||||
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import std/os
|
||||
import std/strformat
|
||||
import std/terminal
|
||||
from std/times import fromUnix, format, now
|
||||
from std/unicode import toUpper
|
||||
import std/unittest
|
||||
import pkg/chronos
|
||||
@ -38,6 +39,7 @@ type
|
||||
# Shows test status updates at regular time intervals. Useful for running
|
||||
# locally while attended. Set to false for unattended runs, eg CI.
|
||||
showContinuousStatusUpdates: bool
|
||||
logsDir: string
|
||||
timeStart: ?Moment
|
||||
timeEnd: ?Moment
|
||||
codexPortLock: AsyncLock
|
||||
@ -73,6 +75,7 @@ type
|
||||
testId: string # when used in datadir path, prevents data dir clashes
|
||||
status: IntegrationTestStatus
|
||||
command: string
|
||||
logsDir: string
|
||||
|
||||
TestManagerError* = object of CatchableError
|
||||
|
||||
@ -194,6 +197,8 @@ proc startHardhat(
|
||||
|
||||
args.add("--port")
|
||||
args.add($port)
|
||||
if test.manager.debugHardhat:
|
||||
args.add("--log-file=" & test.logsDir / "hardhat.log")
|
||||
|
||||
trace "starting hardhat process on port ", port
|
||||
try:
|
||||
@ -255,9 +260,9 @@ proc printResult(
|
||||
of IntegrationTestStatus.Failed:
|
||||
if output =? test.output:
|
||||
if printStdErr: #manager.debugTestHarness
|
||||
test.printOutputMarker(MarkerPosition.Start, "test harness errors (stderr)")
|
||||
test.printOutputMarker(MarkerPosition.Start, "test file errors (stderr)")
|
||||
echo output.stdError
|
||||
test.printOutputMarker(MarkerPosition.Finish, "test harness errors (stderr)")
|
||||
test.printOutputMarker(MarkerPosition.Finish, "test file errors (stderr)")
|
||||
if printStdOut:
|
||||
test.printOutputMarker(MarkerPosition.Start, "codex node output (stdout)")
|
||||
echo output.stdOutput
|
||||
@ -286,21 +291,25 @@ proc printStart(test: IntegrationTest) =
|
||||
proc buildCommand(
|
||||
test: IntegrationTest, hardhatPort: ?int
|
||||
): Future[string] {.async: (raises: [CancelledError, TestManagerError]).} =
|
||||
let logging =
|
||||
if not test.manager.debugTestHarness:
|
||||
""
|
||||
else:
|
||||
var logging = string.none
|
||||
if test.manager.debugTestHarness:
|
||||
#!fmt: off
|
||||
logging = some(
|
||||
"-d:chronicles_log_level=TRACE " &
|
||||
"-d:chronicles_disabled_topics=websock,JSONRPC-HTTP-CLIENT,JSONRPC-WS-CLIENT " &
|
||||
"-d:chronicles_default_output_device=stdout " & "-d:chronicles_sinks=textlines"
|
||||
"-d:chronicles_disabled_topics=websock,JSONRPC-HTTP-CLIENT,JSONRPC-WS-CLIENT " &
|
||||
"-d:chronicles_default_output_device=stdout " &
|
||||
"-d:chronicles_sinks=textlines")
|
||||
#!fmt: on
|
||||
|
||||
let strHardhatPort =
|
||||
if not test.config.startHardhat:
|
||||
""
|
||||
else:
|
||||
without port =? hardhatPort:
|
||||
raiseTestManagerError "hardhatPort required when 'config.startHardhat' is true"
|
||||
"-d:HardhatPort=" & $port
|
||||
var hhPort = string.none
|
||||
if test.config.startHardhat:
|
||||
without port =? hardhatPort:
|
||||
raiseTestManagerError "hardhatPort required when 'config.startHardhat' is true"
|
||||
hhPort = some "-d:HardhatPort=" & $port
|
||||
|
||||
var logDir = string.none
|
||||
if test.manager.debugCodexNodes:
|
||||
logDir = some "-d:LogsDir=" & test.logsDir
|
||||
|
||||
var testFile: string
|
||||
try:
|
||||
@ -324,12 +333,25 @@ proc buildCommand(
|
||||
withLock(test.manager.hardhatPortLock):
|
||||
try:
|
||||
return
|
||||
"nim c " & &"-d:CodexApiPort={apiPort} " & &"-d:CodexDiscPort={discPort} " &
|
||||
&"{strHardhatPort} " & &"-d:TestId={test.testId} " & &"{logging} " &
|
||||
"--verbosity:0 " & "--hints:off " & "-d:release " & "-r " & &"{testFile}"
|
||||
#!fmt: off
|
||||
"nim c " &
|
||||
&"-d:CodexApiPort={apiPort} " &
|
||||
&"-d:CodexDiscPort={discPort} " &
|
||||
&"-d:DebugCodexNodes={test.manager.debugCodexNodes} " &
|
||||
&"-d:DebugHardhat={test.manager.debugHardhat} " &
|
||||
(logDir |? "") & " " &
|
||||
(hhPort |? "") & " " &
|
||||
&"-d:TestId={test.testId} " &
|
||||
(logging |? "") & " " &
|
||||
"--verbosity:0 " &
|
||||
"--hints:off " &
|
||||
"-d:release " &
|
||||
"-r " &
|
||||
&"{testFile}"
|
||||
#!fmt: on
|
||||
except ValueError as parent:
|
||||
raiseTestManagerError "bad command --\n" & ", apiPort: " & $apiPort &
|
||||
", discPort: " & $discPort & ", logging: " & logging & ", testFile: " &
|
||||
", discPort: " & $discPort & ", logging: " & logging |? "" & ", testFile: " &
|
||||
testFile & ", error: " & parent.msg, parent
|
||||
|
||||
proc setup(
|
||||
@ -394,6 +416,13 @@ proc start(test: IntegrationTest) {.async: (raises: []).} =
|
||||
|
||||
trace "Running test"
|
||||
|
||||
if test.manager.debugCodexNodes:
|
||||
test.logsDir = test.manager.logsDir / sanitize(test.config.name)
|
||||
try:
|
||||
createDir(test.logsDir)
|
||||
except CatchableError as e:
|
||||
error "failed to create test log dir", logDir = test.logsDir, error = e.msg
|
||||
|
||||
test.timeStart = some Moment.now()
|
||||
test.status = IntegrationTestStatus.Running
|
||||
|
||||
@ -501,7 +530,7 @@ proc runTests(manager: TestManager) {.async: (raises: [CancelledError]).} =
|
||||
asyncSpawn futRun
|
||||
|
||||
try:
|
||||
# if runTests is cancelled, await allFutures will be cancelled, but allFutures
|
||||
# if runTests is cancelled, await allFutures will be cancelled, but allFutures
|
||||
# does not propagate the cancellation to the futures it's waiting on, so we
|
||||
# need to cancel them here
|
||||
await allFutures testFutures
|
||||
@ -581,6 +610,31 @@ proc printResult(manager: TestManager) {.raises: [TestManagerError].} =
|
||||
proc start*(
|
||||
manager: TestManager
|
||||
) {.async: (raises: [CancelledError, TestManagerError]).} =
|
||||
try:
|
||||
if manager.debugCodexNodes:
|
||||
let startTime = now().format("yyyy-MM-dd'_'HH:mm:ss")
|
||||
let logsDir =
|
||||
currentSourcePath.parentDir() / "logs" /
|
||||
sanitize(startTime & "__IntegrationTests")
|
||||
createDir(logsDir)
|
||||
manager.logsDir = logsDir
|
||||
#!fmt: off
|
||||
echoStyled bgWhite, fgBlack, styleBright,
|
||||
"\n\n ",
|
||||
styleUnderscore,
|
||||
"ℹ️ LOGS AVAILABLE ℹ️\n\n",
|
||||
resetStyle, bgWhite, fgBlack, styleBright,
|
||||
""" Logs for this run will be available at:""",
|
||||
resetStyle, bgWhite, fgBlack,
|
||||
&"\n\n {logsDir}\n\n",
|
||||
resetStyle, bgWhite, fgBlack, styleBright,
|
||||
" NOTE: For CI runs, logs will be attached as artefacts\n"
|
||||
#!fmt: on
|
||||
except IOError as e:
|
||||
raiseTestManagerError "failed to create hardhat log directory: " & e.msg, e
|
||||
except OSError as e:
|
||||
raiseTestManagerError "failed to create hardhat log directory: " & e.msg, e
|
||||
|
||||
if manager.showContinuousStatusUpdates:
|
||||
let fut = manager.continuallyShowUpdates()
|
||||
manager.trackedFutures.track fut
|
||||
|
||||
@ -1,8 +1,11 @@
|
||||
import std/os
|
||||
import pkg/chronos
|
||||
import pkg/codex/logutils
|
||||
|
||||
{.push raises: [].}
|
||||
|
||||
proc nextFreePort*(startPort: int): Future[int] {.async: (raises: [CancelledError]).} =
|
||||
proc client(server: StreamServer, transp: StreamTransport) {.async.} =
|
||||
proc client(server: StreamServer, transp: StreamTransport) {.async: (raises: []).} =
|
||||
await transp.closeWait()
|
||||
|
||||
var port = startPort
|
||||
@ -22,3 +25,29 @@ proc nextFreePort*(startPort: int): Future[int] {.async: (raises: [CancelledErro
|
||||
inc port
|
||||
except TransportAddressError:
|
||||
raiseAssert "bad address"
|
||||
|
||||
proc sanitize*(pathSegment: string): string =
|
||||
var sanitized = pathSegment
|
||||
for invalid in invalidFilenameChars.items:
|
||||
sanitized = sanitized.replace(invalid, '_').replace(' ', '_')
|
||||
sanitized
|
||||
|
||||
proc getLogFile*(
|
||||
logDir, startTime, suiteName, testName, role: string, index = int.none
|
||||
): string {.raises: [IOError, OSError].} =
|
||||
let logsDir =
|
||||
if logDir == "":
|
||||
currentSourcePath.parentDir() / "logs" / sanitize(startTime & "__" & suiteName) /
|
||||
sanitize(testName)
|
||||
else:
|
||||
logDir / sanitize(suiteName) / sanitize(testName)
|
||||
|
||||
createDir(logsDir)
|
||||
|
||||
var fn = $role
|
||||
if idx =? index:
|
||||
fn &= "_" & $idx
|
||||
fn &= ".log"
|
||||
|
||||
let fileName = logsDir / fn
|
||||
return fileName
|
||||
|
||||
@ -27,7 +27,7 @@ const DebugHardhat {.booldefine.} = false
|
||||
# Echoes stdout from the integration test file process. Codex process logs can
|
||||
# also be output if a test uses a multinodesuite, requires CodexConfig.debug
|
||||
# to be enabled
|
||||
const DebugCodexNodes {.booldefine.} = true
|
||||
const DebugCodexNodes {.booldefine.} = false
|
||||
# Shows test status updates at time intervals. Useful for running locally with
|
||||
# active terminal interaction. Set to false for unattended runs, eg CI.
|
||||
const ShowContinuousStatusUpdates {.booldefine.} = false
|
||||
@ -37,28 +37,6 @@ const TestTimeout {.intdefine.} = 60
|
||||
const EnableParallelTests {.booldefine.} = true
|
||||
|
||||
proc run() {.async.} =
|
||||
when DebugTestHarness and enabledLogLevel != LogLevel.TRACE:
|
||||
styledEcho bgWhite,
|
||||
fgBlack, styleBright, "\n\n ", styleUnderscore,
|
||||
"ℹ️ ADDITIONAL LOGGING AVAILABLE ℹ️\n\n", resetStyle, bgWhite, fgBlack,
|
||||
styleBright,
|
||||
"""
|
||||
More integration test harness logs available by running with
|
||||
-d:chronicles_log_level=TRACE, eg:""",
|
||||
resetStyle, bgWhite, fgBlack,
|
||||
"\n\n nim c -d:chronicles_log_level=TRACE -r ./testIntegration.nim\n\n"
|
||||
|
||||
when DebugCodexNodes:
|
||||
styledEcho bgWhite,
|
||||
fgBlack, styleBright, "\n\n ", styleUnderscore,
|
||||
"⚠️ ENABLE CODEX LOGGING ⚠️\n\n", resetStyle, bgWhite, fgBlack,
|
||||
styleBright,
|
||||
"""
|
||||
For integration test suites that are multinodesuites, or for
|
||||
tests launching a CodexProcess, ensure that CodexConfig.debug
|
||||
is enabled to see chronicles logs.
|
||||
"""
|
||||
|
||||
let manager = TestManager.new(
|
||||
configs = TestConfigs,
|
||||
DebugTestHarness,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user