2023-08-01 16:47:57 -07:00
|
|
|
mode = ScriptMode.Verbose
|
|
|
|
|
|
2024-09-23 16:37:17 +02:00
|
|
|
import std/os except commandLineParams
|
2025-01-22 19:23:08 +11:00
|
|
|
import std/strutils
|
2023-08-01 16:47:57 -07:00
|
|
|
|
|
|
|
|
### Helper functions
|
2025-01-22 19:23:08 +11:00
|
|
|
proc truthy(val: string): bool =
|
|
|
|
|
const truthySwitches = @["yes", "1", "on", "true"]
|
|
|
|
|
return val in truthySwitches
|
|
|
|
|
|
2023-08-01 16:47:57 -07:00
|
|
|
proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") =
|
|
|
|
|
if not dirExists "build":
|
|
|
|
|
mkDir "build"
|
2025-01-09 23:41:22 +05:30
|
|
|
|
2023-08-01 16:47:57 -07:00
|
|
|
# allow something like "nim nimbus --verbosity:0 --hints:off nimbus.nims"
|
|
|
|
|
var extra_params = params
|
|
|
|
|
when compiles(commandLineParams):
|
|
|
|
|
for param in commandLineParams():
|
|
|
|
|
extra_params &= " " & param
|
|
|
|
|
else:
|
2025-02-11 16:00:05 -03:00
|
|
|
for i in 2 ..< paramCount():
|
2023-08-01 16:47:57 -07:00
|
|
|
extra_params &= " " & paramStr(i)
|
|
|
|
|
|
2024-09-23 16:37:17 +02:00
|
|
|
let
|
|
|
|
|
# Place build output in 'build' folder, even if name includes a longer path.
|
|
|
|
|
outName = os.lastPathPart(name)
|
2025-02-11 16:00:05 -03:00
|
|
|
cmd =
|
|
|
|
|
"nim " & lang & " --out:build/" & outName & " " & extra_params & " " & srcDir &
|
|
|
|
|
name & ".nim"
|
2025-02-06 17:18:00 -08:00
|
|
|
|
|
|
|
|
exec(cmd)
|
2023-08-01 16:47:57 -07:00
|
|
|
|
|
|
|
|
proc test(name: string, srcDir = "tests/", params = "", lang = "c") =
|
|
|
|
|
buildBinary name, srcDir, params
|
|
|
|
|
exec "build/" & name
|
|
|
|
|
|
|
|
|
|
task codex, "build codex binary":
|
2025-02-11 16:00:05 -03:00
|
|
|
buildBinary "codex",
|
|
|
|
|
params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE"
|
2023-08-01 16:47:57 -07:00
|
|
|
|
2024-09-23 16:37:17 +02:00
|
|
|
task toolsCirdl, "build tools/cirdl binary":
|
|
|
|
|
buildBinary "tools/cirdl/cirdl"
|
|
|
|
|
|
2023-08-01 16:47:57 -07:00
|
|
|
task testCodex, "Build & run Codex tests":
|
2024-03-19 22:08:54 +11:00
|
|
|
test "testCodex", params = "-d:codex_enable_proof_failures=true"
|
2023-08-01 16:47:57 -07:00
|
|
|
|
|
|
|
|
task testContracts, "Build & run Codex Contract tests":
|
|
|
|
|
test "testContracts"
|
|
|
|
|
|
|
|
|
|
task testIntegration, "Run integration tests":
|
2025-02-11 16:00:05 -03:00
|
|
|
buildBinary "codex",
|
|
|
|
|
params =
|
2025-03-04 13:05:03 +11:00
|
|
|
"-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:chronicles_disabled_topics=JSONRPC-HTTP-CLIENT,websock,libp2p,discv5 -d:codex_enable_proof_failures=true"
|
refactor: debug and logging
- Move test manager values to config object
- Increase codex port separation (between api and disc ports) in an attempt to prevent overlap across tests (ie Test1: api=8000(tcp), disc=9000(udp), and Test2: api=9000(tcp), disc=10000(udp))
- print stderr when exitcode == 1 if there's < 3 lines of stdout
- Logging:
- Always write test manager harness chronicles logs to file, ie testmanager.chronicles.log in the root of the `integration/logs/<run name>` dir
- Always write individual test stdout to file, ie `<test file name>.stdout.log` in the root of the `integration/logs/<run name>/<test file name>` dir
- On error, print stderr to screen and write stderr to file. Or on failure, if stdout is sufficiently short, write stderr to screen and file in `integration/logs/<run name>/<test file name>/<test file name>.stderr.log`
- When debugging, ie DebugCodexNodes == true
- Removes DebugTestHarness from controlling anything other than printing chronicles output from the testmanager to the terminal
- Now, if DebugCodexNodes is set to true:
- Codex node (chronicles) output for multinodesuite tests is logged to file, eg `integration/logs/<run name>/<test file name>/<test name>/<role>_<idx>.log`
- Codex chronicles output is logged to stdout, which also written to file (see above)
2025-03-06 15:49:08 +11:00
|
|
|
var sinks = @["textlines[nocolors,file]"]
|
2025-01-22 19:23:08 +11:00
|
|
|
for i in 2 ..< paramCount():
|
|
|
|
|
if "DebugTestHarness" in paramStr(i) and truthy paramStr(i).split('=')[1]:
|
refactor: debug and logging
- Move test manager values to config object
- Increase codex port separation (between api and disc ports) in an attempt to prevent overlap across tests (ie Test1: api=8000(tcp), disc=9000(udp), and Test2: api=9000(tcp), disc=10000(udp))
- print stderr when exitcode == 1 if there's < 3 lines of stdout
- Logging:
- Always write test manager harness chronicles logs to file, ie testmanager.chronicles.log in the root of the `integration/logs/<run name>` dir
- Always write individual test stdout to file, ie `<test file name>.stdout.log` in the root of the `integration/logs/<run name>/<test file name>` dir
- On error, print stderr to screen and write stderr to file. Or on failure, if stdout is sufficiently short, write stderr to screen and file in `integration/logs/<run name>/<test file name>/<test file name>.stderr.log`
- When debugging, ie DebugCodexNodes == true
- Removes DebugTestHarness from controlling anything other than printing chronicles output from the testmanager to the terminal
- Now, if DebugCodexNodes is set to true:
- Codex node (chronicles) output for multinodesuite tests is logged to file, eg `integration/logs/<run name>/<test file name>/<test name>/<role>_<idx>.log`
- Codex chronicles output is logged to stdout, which also written to file (see above)
2025-03-06 15:49:08 +11:00
|
|
|
sinks.add "textlines[stdout]"
|
2025-02-03 19:04:24 +11:00
|
|
|
break
|
refactor: debug and logging
- Move test manager values to config object
- Increase codex port separation (between api and disc ports) in an attempt to prevent overlap across tests (ie Test1: api=8000(tcp), disc=9000(udp), and Test2: api=9000(tcp), disc=10000(udp))
- print stderr when exitcode == 1 if there's < 3 lines of stdout
- Logging:
- Always write test manager harness chronicles logs to file, ie testmanager.chronicles.log in the root of the `integration/logs/<run name>` dir
- Always write individual test stdout to file, ie `<test file name>.stdout.log` in the root of the `integration/logs/<run name>/<test file name>` dir
- On error, print stderr to screen and write stderr to file. Or on failure, if stdout is sufficiently short, write stderr to screen and file in `integration/logs/<run name>/<test file name>/<test file name>.stderr.log`
- When debugging, ie DebugCodexNodes == true
- Removes DebugTestHarness from controlling anything other than printing chronicles output from the testmanager to the terminal
- Now, if DebugCodexNodes is set to true:
- Codex node (chronicles) output for multinodesuite tests is logged to file, eg `integration/logs/<run name>/<test file name>/<test name>/<role>_<idx>.log`
- Codex chronicles output is logged to stdout, which also written to file (see above)
2025-03-06 15:49:08 +11:00
|
|
|
var testParams =
|
|
|
|
|
"-d:chronicles_log_level=TRACE -d:chronicles_sinks=\"" & sinks.join(",") & "\""
|
2025-01-22 19:23:08 +11:00
|
|
|
test "testIntegration", params = testParams
|
2024-12-14 06:07:55 +01:00
|
|
|
# use params to enable logging from the integration test executable
|
|
|
|
|
# test "testIntegration", params = "-d:chronicles_sinks=textlines[notimestamps,stdout],textlines[dynamic] " &
|
2025-03-04 13:05:03 +11:00
|
|
|
# "-d:chronicles_enabled_topics:integration:TRACE"
|
2023-08-01 16:47:57 -07:00
|
|
|
|
|
|
|
|
task build, "build codex binary":
|
|
|
|
|
codexTask()
|
|
|
|
|
|
|
|
|
|
task test, "Run tests":
|
|
|
|
|
testCodexTask()
|
|
|
|
|
|
2024-09-23 16:37:17 +02:00
|
|
|
task testTools, "Run Tools tests":
|
|
|
|
|
toolsCirdlTask()
|
|
|
|
|
test "testTools"
|
|
|
|
|
|
2023-09-13 16:17:56 +02:00
|
|
|
task testAll, "Run all tests (except for Taiko L2 tests)":
|
2023-08-01 16:47:57 -07:00
|
|
|
testCodexTask()
|
|
|
|
|
testContractsTask()
|
|
|
|
|
testIntegrationTask()
|
2024-09-23 16:37:17 +02:00
|
|
|
testToolsTask()
|
2023-08-01 16:47:57 -07:00
|
|
|
|
2023-09-13 16:17:56 +02:00
|
|
|
task testTaiko, "Run Taiko L2 tests":
|
|
|
|
|
codexTask()
|
|
|
|
|
test "testTaiko"
|
|
|
|
|
|
2023-08-01 16:47:57 -07:00
|
|
|
import strutils
|
|
|
|
|
import os
|
|
|
|
|
|
|
|
|
|
task coverage, "generates code coverage report":
|
|
|
|
|
var (output, exitCode) = gorgeEx("which lcov")
|
|
|
|
|
if exitCode != 0:
|
|
|
|
|
echo " ************************** ⛔️ ERROR ⛔️ **************************"
|
|
|
|
|
echo " ** ERROR: lcov not found, it must be installed to run code **"
|
|
|
|
|
echo " ** coverage locally **"
|
|
|
|
|
echo " *****************************************************************"
|
|
|
|
|
quit 1
|
|
|
|
|
|
|
|
|
|
(output, exitCode) = gorgeEx("gcov --version")
|
|
|
|
|
if output.contains("Apple LLVM"):
|
|
|
|
|
echo " ************************* ⚠️ WARNING ⚠️ *************************"
|
|
|
|
|
echo " ** WARNING: Using Apple's llvm-cov in place of gcov, which **"
|
|
|
|
|
echo " ** emulates an old version of gcov (4.2.0) and therefore **"
|
|
|
|
|
echo " ** coverage results will differ than those on CI (which **"
|
|
|
|
|
echo " ** uses a much newer version of gcov). **"
|
|
|
|
|
echo " *****************************************************************"
|
|
|
|
|
|
|
|
|
|
var nimSrcs = " "
|
|
|
|
|
for f in walkDirRec("codex", {pcFile}):
|
2025-02-11 16:00:05 -03:00
|
|
|
if f.endswith(".nim"):
|
|
|
|
|
nimSrcs.add " " & f.absolutePath.quoteShell()
|
2023-08-01 16:47:57 -07:00
|
|
|
|
|
|
|
|
echo "======== Running Tests ======== "
|
2025-02-11 16:00:05 -03:00
|
|
|
test "coverage",
|
|
|
|
|
srcDir = "tests/",
|
|
|
|
|
params =
|
|
|
|
|
" --nimcache:nimcache/coverage -d:release -d:codex_enable_proof_failures=true"
|
2023-08-01 16:47:57 -07:00
|
|
|
exec("rm nimcache/coverage/*.c")
|
2025-02-11 16:00:05 -03:00
|
|
|
rmDir("coverage")
|
|
|
|
|
mkDir("coverage")
|
2023-08-01 16:47:57 -07:00
|
|
|
echo " ======== Running LCOV ======== "
|
2025-02-11 16:00:05 -03:00
|
|
|
exec(
|
2025-04-02 12:09:43 +03:00
|
|
|
"lcov --capture --keep-going --directory nimcache/coverage --output-file coverage/coverage.info"
|
2025-02-11 16:00:05 -03:00
|
|
|
)
|
|
|
|
|
exec(
|
2025-04-02 12:09:43 +03:00
|
|
|
"lcov --extract coverage/coverage.info --keep-going --output-file coverage/coverage.f.info " &
|
2025-02-11 16:00:05 -03:00
|
|
|
nimSrcs
|
|
|
|
|
)
|
2023-08-01 16:47:57 -07:00
|
|
|
echo " ======== Generating HTML coverage report ======== "
|
2025-04-02 12:09:43 +03:00
|
|
|
exec("genhtml coverage/coverage.f.info --keep-going --output-directory coverage/report ")
|
2023-08-01 16:47:57 -07:00
|
|
|
echo " ======== Coverage report Done ======== "
|
|
|
|
|
|
|
|
|
|
task showCoverage, "open coverage html":
|
|
|
|
|
echo " ======== Opening HTML coverage report in browser... ======== "
|
|
|
|
|
if findExe("open") != "":
|
|
|
|
|
exec("open coverage/report/index.html")
|