1483 lines
45 KiB
Nim
1483 lines
45 KiB
Nim
# unittest2
|
|
#
|
|
# (c) Copyright 2015 Nim Contributors
|
|
# (c) Copyright 2019-2021 Ștefan Talpalaru
|
|
# (c) Copyright 2021-Onwards Status Research and Development
|
|
#
|
|
|
|
{.push raises: [].}
|
|
|
|
## :Authors: Zahary Karadjov, Ștefan Talpalaru, Status Research and Development
|
|
##
|
|
## This module makes unit testing easy.
|
|
##
|
|
## .. code::
|
|
## nim c -r testfile.nim
|
|
##
|
|
## exits with 0 or 1.
|
|
##
|
|
## Running individual tests
|
|
## ========================
|
|
##
|
|
## Specify the test names as command line arguments.
|
|
##
|
|
## .. code::
|
|
##
|
|
## nim c -r test "my test name" "another test"
|
|
##
|
|
## Multiple arguments can be used.
|
|
##
|
|
## Running a single test suite
|
|
## ===========================
|
|
##
|
|
## Specify the suite name delimited by ``"::"``.
|
|
##
|
|
## .. code::
|
|
##
|
|
## nim c -r test "my suite name::"
|
|
##
|
|
## Selecting tests by pattern
|
|
## ==========================
|
|
##
|
|
## A single ``"*"`` can be used for globbing.
|
|
##
|
|
## Delimit the end of a suite name with ``"::"``.
|
|
##
|
|
## Tests matching **any** of the arguments are executed.
|
|
##
|
|
## .. code::
|
|
##
|
|
## nim c -r test fast_suite::mytest1 fast_suite::mytest2
|
|
## nim c -r test "fast_suite::mytest*"
|
|
## nim c -r test "auth*::" "crypto::hashing*"
|
|
## # Run suites starting with 'bug #' and standalone tests starting with '#'
|
|
## nim c -r test 'bug #*::' '::#*'
|
|
##
|
|
## Command line arguments
|
|
## ======================
|
|
##
|
|
## The unit test runner recognises serveral parameters that can be specified
|
|
## either via environment or command line, the latter taking precedence.
|
|
##
|
|
## Several options also have defaults that can be controlled at compile-time.
|
|
##
|
|
## --help Print short help and quit
|
|
## --xml:file Write JUnit-compatible XML report to `file`
|
|
## --console Write report to the console (default, when no other output
|
|
## is selected)
|
|
## --output-lvl:level Verbosity of output [COMPACT, VERBOSE, FAILURES, NONE] (env: UNITTEST2_OUTPUT_LVL)
|
|
## --verbose, -v Shorthand for --output-lvl:VERBOSE
|
|
##
|
|
## Command line parsing can be disabled with `-d:unittest2DisableParamFiltering`.
|
|
##
|
|
## Running tests in parallel
|
|
## =========================
|
|
##
|
|
## Early versions of this library had rudimentary support for running tests in
|
|
## parallel - this has since been removed due to safety issues in the
|
|
## implementation and may be reintroduced at a future date.
|
|
##
|
|
## Example
|
|
## -------
|
|
##
|
|
## .. code:: nim
|
|
##
|
|
## suite "description for this stuff":
|
|
## echo "suite setup: run once before the tests"
|
|
##
|
|
## setup:
|
|
## echo "run before each test"
|
|
##
|
|
## teardown:
|
|
## echo "run after each test"
|
|
##
|
|
## test "essential truths":
|
|
## # give up and stop if this fails
|
|
## require(true)
|
|
##
|
|
## test "slightly less obvious stuff":
|
|
## # print a nasty message and move on, skipping
|
|
## # the remainder of this block
|
|
## check(1 != 1)
|
|
## check("asd"[2] == 'd')
|
|
##
|
|
## test "out of bounds error is thrown on bad access":
|
|
## let v = @[1, 2, 3] # you can do initialization here
|
|
## expect(IndexError):
|
|
## discard v[4]
|
|
##
|
|
## suiteTeardown:
|
|
## echo "suite teardown: run once after the tests"
|
|
|
|
import std/[
|
|
macros, sequtils, sets, strutils, streams, tables, times, monotimes]
|
|
|
|
when defined(nimHasWarnBareExcept):
|
|
# In unit tests, we want to at least attempt to catch Exception no matter its
|
|
# UB
|
|
{.warning[BareExcept]: off.}
|
|
|
|
{.warning[LockLevel]: off.}
|
|
|
|
when declared(stdout):
|
|
import std/os
|
|
|
|
const useTerminal = declared(stdout) and not defined(js)
|
|
|
|
type
|
|
OutputLevel* = enum ## The output verbosity of the tests.
|
|
VERBOSE, ## Print as much as possible.
|
|
COMPACT ## Print failures and compact success information
|
|
FAILURES, ## Print only failures
|
|
NONE ## Print nothing.
|
|
|
|
const
|
|
outputLevelDefault = COMPACT
|
|
slowThreshold = initDuration(seconds = 5)
|
|
|
|
# `unittest` compatibility
|
|
nimUnittestOutputLevel {.strdefine.} = $outputLevelDefault
|
|
nimUnittestColor {.strdefine.} = "auto" ## auto|on|off
|
|
nimUnittestAbortOnError {.booldefine.} = false
|
|
|
|
# `unittest2` compile-time configuration options
|
|
unittest2DisableParamFiltering {.booldefine.} = false
|
|
## Disables automatic command line argument parsing - parsing is available
|
|
## via the `parseParameters` function instead
|
|
unittest2Compat {.booldefine.} = true # This will be disabled in the future
|
|
## Compatibility mode for `unittest` for easier porting and improved
|
|
## backwards compatibility - no stability guarantees
|
|
unittest2NoCollect {.booldefine.} = false
|
|
## Disable test collection mode where tests are enumerated before they are
|
|
## run - in particular, this affects the order in which tests and suites
|
|
## have their bodies evaluated and disables several features that require
|
|
## knowing how many tests will be executed - experimental feature
|
|
unittest2PreviewIsolate {.booldefine.} = false
|
|
## Preview isolation mode where each test is run in a separate process - may
|
|
## be removed in the future
|
|
unittest2Static* {.booldefine.} = false
|
|
## Run tests at compile time as well - only a subset of functionality is
|
|
## enabled at compile-time meaning that tests must be written
|
|
## conservatively. `suite` features (`setup` etc) in particular are not
|
|
## supported.
|
|
|
|
when useTerminal:
|
|
import std/terminal
|
|
|
|
const
|
|
collect = (not unittest2NoCollect and not unittest2Compat) or unittest2PreviewIsolate
|
|
autoParseArgs = not unittest2DisableParamFiltering
|
|
isolate = unittest2PreviewIsolate
|
|
|
|
when isolate:
|
|
let
|
|
isolated = getEnv("UNITTEST2_ISOLATED") == "1"
|
|
## Test is running in the isolated environment
|
|
|
|
from std/exitprocs import nil
|
|
template addExitProc(p: proc) =
|
|
try:
|
|
exitprocs.addExitProc(p)
|
|
except Exception as e:
|
|
echo "Can't add exit proc", e.msg
|
|
quit(1)
|
|
|
|
type
|
|
Test = object
|
|
suiteName: string
|
|
testName: string
|
|
impl: proc(suite, name: string): TestStatus
|
|
|
|
TestStatus* = enum ## The status of a test when it is done.
|
|
OK,
|
|
FAILED,
|
|
SKIPPED
|
|
|
|
TestResult* = object
|
|
suiteName*: string
|
|
## Name of the test suite that contains this test case.
|
|
testName*: string
|
|
## Name of the test case
|
|
status*: TestStatus
|
|
duration*: Duration # How long the test took, in seconds
|
|
output*: string
|
|
errors*: string
|
|
|
|
OutputFormatter* = ref object of RootObj
|
|
|
|
ConsoleOutputFormatter* = ref object of OutputFormatter
|
|
colorOutput: bool
|
|
## Have test results printed in color.
|
|
## Default is `auto` depending on `isatty(stdout)`, or override it with
|
|
## `-d:nimUnittestColor:auto|on|off`.
|
|
##
|
|
## Deprecated: Setting the environment variable `NIMTEST_COLOR` to `always`
|
|
## or `never` changes the default for the non-js target to true or false respectively.
|
|
## Deprecated: the environment variable `NIMTEST_NO_COLOR`, when set, changes the
|
|
## default to true, if `NIMTEST_COLOR` is undefined.
|
|
outputLevel: OutputLevel
|
|
## Set the verbosity of test results.
|
|
## Default is `VERBOSE`, or override with:
|
|
## `-d:nimUnittestOutputLevel:VERBOSE|FAILURES|NONE`.
|
|
##
|
|
## Deprecated: the `NIMTEST_OUTPUT_LVL` environment variable is set for the non-js target.
|
|
|
|
when collect:
|
|
tests: Table[string, int]
|
|
|
|
curSuiteName: string
|
|
curSuite: int
|
|
curTestName: string
|
|
curTest: int
|
|
|
|
statuses: array[TestStatus, int]
|
|
|
|
totalDuration: Duration
|
|
|
|
results: seq[TestResult]
|
|
|
|
failures: seq[TestResult]
|
|
|
|
errors: string
|
|
|
|
JUnitTest = object
|
|
name: string
|
|
result: TestResult
|
|
error: (seq[string], string)
|
|
failures: seq[seq[string]]
|
|
|
|
JUnitSuite = object
|
|
name: string
|
|
tests: seq[JUnitTest]
|
|
|
|
JUnitOutputFormatter* = ref object of OutputFormatter
|
|
stream: Stream
|
|
defaultSuite: JUnitSuite
|
|
suites: seq[JUnitSuite]
|
|
currentSuite: int
|
|
|
|
# TODO these globals are threadvar so as to avoid gc-safety-issues - this should
|
|
# probably be resolved in a better way down the line specially since we
|
|
# don't support threads _really_
|
|
|
|
var
|
|
abortOnError* {.threadvar.}: bool
|
|
## Set to true in order to quit
|
|
## immediately on fail. Default is false,
|
|
## or override with `-d:nimUnittestAbortOnError:on|off`.
|
|
|
|
checkpoints {.threadvar.}: seq[string]
|
|
formatters {.threadvar.}: seq[OutputFormatter]
|
|
testsFilters {.threadvar.}: HashSet[string]
|
|
|
|
currentSuite {.threadvar.}: string
|
|
|
|
when collect:
|
|
var
|
|
tests {.threadvar.}: OrderedTable[string, seq[Test]]
|
|
|
|
abortOnError = nimUnittestAbortOnError
|
|
|
|
when declared(stdout):
|
|
if existsEnv("UNITTEST2_ABORT_ON_ERROR") or existsEnv("NIMTEST_ABORT_ON_ERROR"):
|
|
abortOnError = true
|
|
|
|
when collect:
|
|
method suiteRunStarted*(
|
|
formatter: OutputFormatter, tests: OrderedTable[string, seq[Test]]) {.base, gcsafe.} =
|
|
# Run when a round of running discovered suites starts - these may result
|
|
# in subsequent tests being added meaning subsequent suite runs
|
|
discard
|
|
method suiteStarted*(formatter: OutputFormatter, suiteName: string) {.base, gcsafe.} =
|
|
discard
|
|
method testStarted*(formatter: OutputFormatter, testName: string) {.base, gcsafe.} =
|
|
discard
|
|
method failureOccurred*(formatter: OutputFormatter, checkpoints: seq[string],
|
|
stackTrace: string) {.base, gcsafe.} =
|
|
## ``stackTrace`` is provided only if the failure occurred due to an exception.
|
|
## ``checkpoints`` is never ``nil``.
|
|
discard
|
|
method testEnded*(formatter: OutputFormatter, testResult: TestResult) {.base, gcsafe.} =
|
|
discard
|
|
method suiteEnded*(formatter: OutputFormatter) {.base, gcsafe.} =
|
|
discard
|
|
when collect:
|
|
method suiteRunEnded*(
|
|
formatter: OutputFormatter) {.base, gcsafe.} =
|
|
discard
|
|
|
|
method testRunEnded*(formatter: OutputFormatter) {.base, gcsafe.} =
|
|
# Runs when the test executable is about to end, which is implemented using
|
|
# addExitProc, a best-effort kind of place to do cleanups
|
|
discard
|
|
|
|
when collect:
|
|
proc suiteRunStarted(tests: OrderedTable[string, seq[Test]]) =
|
|
for formatter in formatters:
|
|
formatter.suiteRunStarted(tests)
|
|
|
|
proc suiteStarted(name: string) =
|
|
for formatter in formatters:
|
|
formatter.suiteStarted(name)
|
|
|
|
proc testStarted(name: string) =
|
|
for formatter in formatters:
|
|
formatter.testStarted(name)
|
|
|
|
proc testEnded(testResult: TestResult) =
|
|
for formatter in formatters:
|
|
formatter.testEnded(testResult)
|
|
|
|
proc suiteEnded() =
|
|
for formatter in formatters:
|
|
formatter.suiteEnded()
|
|
|
|
when collect:
|
|
proc suiteRunEnded() =
|
|
for formatter in formatters:
|
|
formatter.suiteRunEnded()
|
|
|
|
proc testRunEnded() =
|
|
when not collect:
|
|
if currentSuite.len > 0:
|
|
suiteEnded()
|
|
currentSuite.reset()
|
|
|
|
for formatter in formatters:
|
|
testRunEnded(formatter)
|
|
|
|
proc addOutputFormatter*(formatter: OutputFormatter) =
|
|
formatters.add(formatter)
|
|
|
|
proc resetOutputFormatters*() =
|
|
formatters.reset()
|
|
|
|
proc newConsoleOutputFormatter*(outputLevel: OutputLevel = outputLevelDefault,
|
|
colorOutput = true): ConsoleOutputFormatter =
|
|
ConsoleOutputFormatter(
|
|
outputLevel: outputLevel,
|
|
colorOutput: colorOutput,
|
|
)
|
|
|
|
proc defaultColorOutput(): bool =
|
|
let color = nimUnittestColor
|
|
case color
|
|
of "auto":
|
|
when declared(stdout): result = isatty(stdout)
|
|
else: result = false
|
|
of "on": result = true
|
|
of "off": result = false
|
|
else: raiseAssert "Unrecognised nimUnittestColor setting: " & color
|
|
|
|
when declared(stdout):
|
|
# TODO unittest2-equivalent color parsing
|
|
if existsEnv("NIMTEST_COLOR"):
|
|
let colorEnv = getEnv("NIMTEST_COLOR")
|
|
if colorEnv == "never":
|
|
result = false
|
|
elif colorEnv == "always":
|
|
result = true
|
|
elif existsEnv("NIMTEST_NO_COLOR"):
|
|
result = false
|
|
|
|
proc defaultOutputLevel(): OutputLevel =
|
|
when declared(stdout):
|
|
const levelEnv = "UNITTEST2_OUTPUT_LVL"
|
|
const nimtestEnv = "NIMTEST_OUTPUT_LVL"
|
|
if existsEnv(levelEnv):
|
|
try:
|
|
parseEnum[OutputLevel](getEnv(levelEnv))
|
|
except ValueError:
|
|
echo "Cannot parse UNITTEST2_OUTPUT_LVL: ", getEnv(levelEnv)
|
|
quit 1
|
|
elif existsEnv(nimtestEnv):
|
|
# std-compatible parsing and translation
|
|
case toUpper(getEnv(nimtestEnv))
|
|
of "PRINT_ALL": OutputLevel.VERBOSE
|
|
of "PRINT_FAILURES": OutputLevel.FAILURES
|
|
of "PRINT_NONE": OutputLevel.NONE
|
|
else:
|
|
echo "Cannot parse NIMTEST_OUTPUT_LVL: ", getEnv(nimtestEnv)
|
|
quit 1
|
|
else:
|
|
const defaultLevel = static: nimUnittestOutputLevel.parseEnum[:OutputLevel]
|
|
defaultLevel
|
|
|
|
proc defaultConsoleFormatter*(): ConsoleOutputFormatter =
|
|
newConsoleOutputFormatter(defaultOutputLevel(), defaultColorOutput())
|
|
|
|
const
|
|
maxStatusLen = 7
|
|
maxDurationLen = 6
|
|
|
|
func formatStatus(status: string): string =
|
|
"[" & alignLeft(status, maxStatusLen) & "]"
|
|
|
|
func formatStatus(status: TestStatus): string =
|
|
formatStatus($status)
|
|
|
|
proc formatDuration(dur: Duration, aligned = true): string =
|
|
let
|
|
seconds = dur.inMilliseconds.float / 1000.0
|
|
precision = max(3 - ($seconds.int).len, 1)
|
|
str = formatFloat(seconds, ffDecimal, precision)
|
|
|
|
if aligned:
|
|
"(" & align(str, maxDurationLen) & "s)"
|
|
else:
|
|
"(" & str & "s)"
|
|
|
|
when collect:
|
|
proc formatFraction(cur, total: int): string =
|
|
let
|
|
cur = $cur
|
|
total = $total
|
|
"[" & align(cur, max(0, maxStatusLen - total.len - 1)) & "/" & total & "]"
|
|
|
|
template write(
|
|
formatter: ConsoleOutputFormatter, styled: untyped, unstyled: untyped) =
|
|
template ignoreExceptions(body: untyped) =
|
|
# We ignore exceptions throughout assuming there's no way to
|
|
try: body except CatchableError: discard
|
|
|
|
when useTerminal:
|
|
if formatter.colorOutput:
|
|
ignoreExceptions: styled
|
|
else: ignoreExceptions: unstyled
|
|
else: ignoreExceptions: unstyled
|
|
|
|
when collect:
|
|
method suiteRunStarted*(
|
|
formatter: ConsoleOutputFormatter, tests: OrderedTable[string, seq[Test]]) =
|
|
for k, v in tests:
|
|
formatter.tests[k] = v.len
|
|
|
|
when collect:
|
|
method suiteRunEnded*(formatter: ConsoleOutputFormatter) =
|
|
formatter.tests.reset()
|
|
|
|
method suiteStarted*(formatter: ConsoleOutputFormatter, suiteName: string) =
|
|
formatter.curSuiteName = suiteName
|
|
formatter.curSuite += 1
|
|
|
|
formatter.curTest.reset()
|
|
|
|
if formatter.outputLevel in {OutputLevel.FAILURES, OutputLevel.NONE}:
|
|
return
|
|
|
|
let
|
|
counter =
|
|
when collect: formatFraction(formatter.curSuite, formatter.tests.len) & " "
|
|
else:
|
|
if formatter.outputLevel == VERBOSE: formatStatus("Suite") & " " else: ""
|
|
maxNameLen = when collect: max(toSeq(formatter.tests.keys()).mapIt(it.len)) else: 0
|
|
eol = if formatter.outputLevel == VERBOSE: "\n" else: " "
|
|
formatter.write do:
|
|
stdout.styledWrite(styleBright, fgBlue, counter, alignLeft(suiteName, maxNameLen), eol)
|
|
do:
|
|
stdout.write(counter, alignLeft(suiteName, maxNameLen), eol)
|
|
stdout.flushFile()
|
|
|
|
proc writeTestName(formatter: ConsoleOutputFormatter, testName: string) =
|
|
formatter.write do:
|
|
stdout.styledWrite fgBlue, testName
|
|
do:
|
|
stdout.write(testName)
|
|
|
|
method testStarted*(formatter: ConsoleOutputFormatter, testName: string) =
|
|
formatter.curTestName = testName
|
|
formatter.curTest += 1
|
|
|
|
if formatter.outputLevel != VERBOSE:
|
|
return
|
|
|
|
# In verbose mode, print a line when the test starts so that output can be
|
|
# correlated with the test that's currently running rather than misleadingly
|
|
# being printed just below the test that just finished running.
|
|
let
|
|
counter =
|
|
when collect:
|
|
try: formatFraction(formatter.curTest, formatter.tests[formatter.curSuiteName]) & " "
|
|
except CatchableError: ""
|
|
else:
|
|
formatStatus("Test")
|
|
|
|
formatter.write do:
|
|
stdout.styledWrite " ", fgBlue, alignLeft(counter, maxStatusLen + maxDurationLen + 7)
|
|
do:
|
|
stdout.write " ", alignLeft(counter, maxStatusLen + maxDurationLen + 7)
|
|
|
|
writeTestName(formatter, testName)
|
|
echo ""
|
|
|
|
method failureOccurred*(formatter: ConsoleOutputFormatter,
|
|
checkpoints: seq[string], stackTrace: string) =
|
|
if stackTrace.len > 0:
|
|
formatter.errors.add(stackTrace)
|
|
formatter.errors.add("\n")
|
|
for msg in items(checkpoints):
|
|
formatter.errors.add(" ")
|
|
formatter.errors.add(msg)
|
|
formatter.errors.add("\n")
|
|
|
|
proc color(status: TestStatus): ForegroundColor =
|
|
case status
|
|
of TestStatus.OK: fgGreen
|
|
of TestStatus.FAILED: fgRed
|
|
of TestStatus.SKIPPED: fgYellow
|
|
proc marker(status: TestStatus): string =
|
|
case status
|
|
of TestStatus.OK: "."
|
|
of TestStatus.FAILED: "F"
|
|
of TestStatus.SKIPPED: "s"
|
|
|
|
proc getAppFilename2(): string =
|
|
# TODO https://github.com/nim-lang/Nim/pull/22544
|
|
try:
|
|
getAppFilename()
|
|
except OSError:
|
|
""
|
|
|
|
proc printFailureInfo(formatter: ConsoleOutputFormatter, testResult: TestResult) =
|
|
# Show how to re-run this test case
|
|
echo repeat('=', testResult.testName.len)
|
|
echo " ", getAppFilename2(), " ", quoteShell(testResult.suiteName & "::" & testResult.testName)
|
|
echo repeat('-', testResult.testName.len)
|
|
|
|
# Show the output
|
|
if testResult.output.len > 0:
|
|
echo testResult.output
|
|
if testResult.errors.len > 0:
|
|
echo testResult.errors
|
|
|
|
proc printTestResultStatus(formatter: ConsoleOutputFormatter, testResult: TestResult) =
|
|
let
|
|
status = formatStatus(testResult.status)
|
|
duration = formatDuration(testResult.duration)
|
|
|
|
formatter.write do:
|
|
stdout.styledWrite(
|
|
" ", styleBright, testResult.status.color, status, " ")
|
|
if testResult.duration > slowThreshold:
|
|
stdout.styledWrite styleBright, duration
|
|
else:
|
|
stdout.write(duration)
|
|
stdout.write " ", testResult.testName
|
|
do:
|
|
stdout.styledWrite " ", status, " ", duration, " ", testResult.testName
|
|
echo ""
|
|
|
|
method testEnded*(formatter: ConsoleOutputFormatter, testResult: TestResult) =
|
|
formatter.statuses[testResult.status] += 1
|
|
formatter.totalDuration += testResult.duration
|
|
|
|
if formatter.outputLevel == NONE:
|
|
return
|
|
|
|
var testResult = testResult
|
|
testResult.errors = move(formatter.errors)
|
|
|
|
formatter.results.add(testResult)
|
|
|
|
if formatter.outputLevel == VERBOSE and testResult.status == TestStatus.FAILED:
|
|
# We'll print it again when all tests have completed
|
|
formatter.failures.add testResult
|
|
|
|
if formatter.outputLevel in {VERBOSE, FAILURES}:
|
|
if testResult.status == TestStatus.FAILED:
|
|
printFailureInfo(formatter, testResult)
|
|
if formatter.outputLevel == VERBOSE or testResult.status == TestStatus.FAILED:
|
|
printTestResultStatus(formatter, testResult)
|
|
else:
|
|
# In compact mode, we use a small marker to mark progress within the suite -
|
|
# we have to be careful about line breaks and flushing so that the marker
|
|
# really ends up on the screen where it's supposed to
|
|
# TODO if the test writes to stdout, the display with be disrupted
|
|
# capturing / redirecting stdout with `dup2` or process isolation could
|
|
# fix this
|
|
|
|
let
|
|
marker = testResult.status.marker()
|
|
color = testResult.status.color()
|
|
formatter.write do:
|
|
stdout.styledWrite styleBright, color, marker
|
|
do:
|
|
stdout.write marker
|
|
stdout.flushFile()
|
|
|
|
method suiteEnded*(formatter: ConsoleOutputFormatter) =
|
|
if formatter.outputLevel == OutputLevel.NONE:
|
|
return
|
|
|
|
let
|
|
totalDur = formatter.results.foldl(a + b.duration, DurationZero)
|
|
totalDurStr = formatDuration(totalDur, false)
|
|
|
|
if formatter.outputLevel == OutputLevel.COMPACT:
|
|
# Complete the line with timing information
|
|
formatter.write do:
|
|
if totalDur > slowThreshold:
|
|
stdout.styledWrite(" ", styleBright, totalDurStr)
|
|
else:
|
|
stdout.write(" ", totalDurStr)
|
|
echo ""
|
|
do:
|
|
echo(" ", totalDurStr)
|
|
|
|
var failed = false
|
|
if formatter.outputLevel notin {VERBOSE, FAILURES}:
|
|
for testResult in formatter.results:
|
|
if testResult.status == TestStatus.FAILED:
|
|
failed = true
|
|
formatter.printFailureInfo(testResult)
|
|
formatter.printTestResultStatus(testResult)
|
|
echo ""
|
|
|
|
formatter.results.reset()
|
|
|
|
if failed or formatter.outputLevel == VERBOSE:
|
|
formatter.write do:
|
|
if totalDur > slowThreshold:
|
|
stdout.styledWrite styleBright, align(totalDurStr, maxStatusLen)
|
|
else:
|
|
stdout.write(align(totalDurStr, maxStatusLen))
|
|
do:
|
|
stdout.write(align(totalDurStr, maxStatusLen))
|
|
|
|
echo(" ", formatter.curSuiteName)
|
|
echo("")
|
|
|
|
method testRunEnded*(formatter: ConsoleOutputFormatter) =
|
|
if formatter.outputLevel notin {VERBOSE, COMPACT} or
|
|
(formatter.outputLevel == FAILURES and
|
|
formatter.statuses[TestStatus.FAILED] > 0):
|
|
return
|
|
|
|
let totalDurStr = formatDuration(formatter.totalDuration, false)
|
|
|
|
try:
|
|
let total = foldl(formatter.statuses, a + b, 0)
|
|
stdout.write("[Summary] ", $total, " tests run ", totalDurStr, ": ")
|
|
|
|
var first = true
|
|
for s, c in formatter.statuses:
|
|
if first:
|
|
first = false
|
|
else:
|
|
stdout.write(", ")
|
|
if c > 0:
|
|
formatter.write do: stdout.styledWrite(s.color, $c, " ", $s)
|
|
do: stdout.write($c, " ", $s)
|
|
else:
|
|
stdout.write($c, " ", $s)
|
|
echo ""
|
|
except CatchableError: discard
|
|
|
|
# In verbose mode, it's likely failures got spammed away - print the specifics
|
|
# so that they can more easily be looked up:
|
|
for testResult in formatter.failures:
|
|
formatter.printTestResultStatus(testResult)
|
|
|
|
proc xmlEscape(s: string): string =
|
|
result = newStringOfCap(s.len)
|
|
for c in items(s):
|
|
case c:
|
|
of '<': result.add("<")
|
|
of '>': result.add(">")
|
|
of '&': result.add("&")
|
|
of '"': result.add(""")
|
|
of '\'': result.add("'")
|
|
else:
|
|
if ord(c) < 32:
|
|
result.add("&#" & $ord(c) & ';')
|
|
else:
|
|
result.add(c)
|
|
|
|
proc newJUnitOutputFormatter*(stream: Stream): JUnitOutputFormatter =
|
|
## Creates a formatter that writes report to the specified stream in
|
|
## JUnit format.
|
|
## The ``stream`` is NOT closed automatically when the test are finished,
|
|
## because the formatter has no way to know when all tests are finished.
|
|
## You should invoke formatter.close() to finalize the report.
|
|
result = JUnitOutputFormatter(
|
|
stream: stream,
|
|
defaultSuite: JUnitSuite(name: "default"),
|
|
currentSuite: -1,
|
|
)
|
|
try:
|
|
stream.writeLine("<?xml version=\"1.0\" encoding=\"UTF-8\"?>")
|
|
except CatchableError as exc:
|
|
echo "Cannot write JUnit: ", exc.msg
|
|
quit 1
|
|
|
|
template suite(formatter: JUnitOutputFormatter): untyped =
|
|
if formatter.currentSuite == -1:
|
|
addr formatter.defaultSuite
|
|
else:
|
|
addr formatter.suites[formatter.currentSuite]
|
|
|
|
method suiteStarted*(formatter: JUnitOutputFormatter, suiteName: string) =
|
|
formatter.currentSuite = formatter.suites.len()
|
|
formatter.suites.add(JUnitSuite(name: suiteName))
|
|
|
|
method testStarted*(formatter: JUnitOutputFormatter, testName: string) =
|
|
formatter.suite().tests.add(JUnitTest(name: testName))
|
|
|
|
method failureOccurred*(formatter: JUnitOutputFormatter,
|
|
checkpoints: seq[string], stackTrace: string) =
|
|
## ``stackTrace`` is provided only if the failure occurred due to an exception.
|
|
## ``checkpoints`` is never ``nil``.
|
|
if stackTrace.len > 0:
|
|
formatter.suite().tests[^1].error = (checkpoints, stackTrace)
|
|
else:
|
|
formatter.suite().tests[^1].failures.add(checkpoints)
|
|
|
|
method testEnded*(formatter: JUnitOutputFormatter, testResult: TestResult) =
|
|
formatter.suite().tests[^1].result = testResult
|
|
|
|
method suiteEnded*(formatter: JUnitOutputFormatter) =
|
|
formatter.currentSuite = -1
|
|
|
|
func toFloatSeconds(duration: Duration): float64 =
|
|
duration.inNanoseconds().float64 / 1_000_000_000.0
|
|
|
|
proc writeTest(s: Stream, test: JUnitTest) {.raises: [CatchableError].} =
|
|
let
|
|
time = test.result.duration.toFloatSeconds()
|
|
timeStr = time.formatFloat(ffDecimal, precision = 6)
|
|
|
|
s.writeLine("\t\t<testcase name=\"$#\" time=\"$#\">" % [
|
|
xmlEscape(test.name), timeStr])
|
|
case test.result.status
|
|
of TestStatus.OK:
|
|
discard
|
|
of TestStatus.SKIPPED:
|
|
s.writeLine("\t\t\t<skipped />")
|
|
of TestStatus.FAILED:
|
|
if test.error[0].len > 0:
|
|
s.writeLine("\t\t\t<error message=\"$#\">$#</error>" % [
|
|
xmlEscape(join(test.error[0], "\n")), xmlEscape(test.error[1])])
|
|
|
|
for failure in test.failures:
|
|
s.writeLine("\t\t\t<failure message=\"$#\">$#</failure>" %
|
|
[xmlEscape(failure[^1]), xmlEscape(join(failure[0..^2], "\n"))])
|
|
|
|
s.writeLine("\t\t</testcase>")
|
|
|
|
proc countTests(counts: var (int, int, int, int, float), suite: JUnitSuite) =
|
|
counts[0] += suite.tests.len()
|
|
for test in suite.tests:
|
|
counts[4] += test.result.duration.toFloatSeconds()
|
|
case test.result.status
|
|
of TestStatus.OK:
|
|
discard
|
|
of TestStatus.SKIPPED:
|
|
counts[3] += 1
|
|
of TestStatus.FAILED:
|
|
if test.error[0].len > 0:
|
|
counts[2] += 1
|
|
else:
|
|
counts[1] += 1
|
|
|
|
proc writeSuite(s: Stream, suite: JUnitSuite) {.raises: [CatchableError].} =
|
|
var counts: (int, int, int, int, float)
|
|
countTests(counts, suite)
|
|
|
|
let timeStr = counts[4].formatFloat(ffDecimal, precision = 6)
|
|
|
|
s.writeLine("\t" & """<testsuite name="$1" tests="$2" failures="$3" errors="$4" skipped="$5" time="$6">""" % [
|
|
xmlEscape(suite.name), $counts[0], $counts[1], $counts[2], $counts[3], timeStr])
|
|
|
|
for test in suite.tests.items():
|
|
s.writeTest(test)
|
|
|
|
s.writeLine("\t</testsuite>")
|
|
|
|
method testRunEnded*(formatter: JUnitOutputFormatter) =
|
|
## Completes the report and closes the underlying stream.
|
|
let s = formatter.stream
|
|
|
|
when defined(nimHasWarnBareExcept):
|
|
{.warning[BareExcept]:off.}
|
|
try:
|
|
s.writeLine("<testsuites>")
|
|
|
|
for suite in formatter.suites.mitems():
|
|
s.writeSuite(suite)
|
|
|
|
if formatter.defaultSuite.tests.len() > 0:
|
|
s.writeSuite(formatter.defaultSuite)
|
|
|
|
s.writeLine("</testsuites>")
|
|
s.close()
|
|
except Exception as exc: # Work around Exception raised in stream
|
|
echo "Cannot write JUnit: ", exc.msg
|
|
quit 1
|
|
|
|
when defined(nimHasWarnBareExcept):
|
|
{.warning[BareExcept]:on.}
|
|
|
|
proc glob(matcher, filter: string): bool =
|
|
## Globbing using a single `*`. Empty `filter` matches everything.
|
|
if filter.len == 0:
|
|
return true
|
|
|
|
if not filter.contains('*'):
|
|
return matcher == filter
|
|
|
|
let beforeAndAfter = filter.split('*', maxsplit=1)
|
|
if beforeAndAfter.len == 1:
|
|
# "foo*"
|
|
return matcher.startsWith(beforeAndAfter[0])
|
|
|
|
if matcher.len < filter.len - 1:
|
|
return false # "12345" should not match "123*345"
|
|
|
|
return matcher.startsWith(beforeAndAfter[0]) and matcher.endsWith(
|
|
beforeAndAfter[1])
|
|
|
|
proc matchFilter(suiteName, testName, filter: string): bool =
|
|
if filter == "":
|
|
return true
|
|
if testName == filter:
|
|
# corner case for tests containing "::" in their name
|
|
return true
|
|
let suiteAndTestFilters = filter.split("::", maxsplit=1)
|
|
|
|
if suiteAndTestFilters.len == 1:
|
|
# no suite specified
|
|
let testFilter = suiteAndTestFilters[0]
|
|
return glob(testName, testFilter)
|
|
|
|
return glob(suiteName, suiteAndTestFilters[0]) and
|
|
glob(testName, suiteAndTestFilters[1])
|
|
|
|
when defined(testing): export matchFilter
|
|
|
|
proc shouldRun(currentSuiteName, testName: string): bool =
|
|
## Check if a test should be run by matching suiteName and testName against
|
|
## test filters.
|
|
when nimvm:
|
|
true
|
|
else:
|
|
if testsFilters.len == 0:
|
|
return true
|
|
|
|
for f in testsFilters:
|
|
if matchFilter(currentSuiteName, testName, f):
|
|
return true
|
|
|
|
return false
|
|
|
|
proc parseParameters*(args: openArray[string]) =
|
|
var
|
|
hasConsole = false
|
|
hasXml: string
|
|
hasVerbose = false
|
|
hasLevel = defaultOutputLevel()
|
|
|
|
# Read tests to run from the command line.
|
|
for str in args:
|
|
if str.startsWith("--help"):
|
|
echo "Usage: [--xml=file.xml] [--console] [--output-level=[VERBOSE,COMPACT,FAILURES,NONE]] [test-name-glob]"
|
|
quit 0
|
|
elif str.startsWith("--xml:") or str.startsWith("--xml="):
|
|
hasXml = str[("--xml".len + 1)..^1] # skip separator char as well
|
|
elif str.startsWith("--console"):
|
|
hasConsole = true
|
|
elif str.startsWith("--output-level:") or str.startsWith("--output-level="):
|
|
hasLevel = try: parseEnum[OutputLevel](str[("--output-level".len + 1)..^1])
|
|
except ValueError:
|
|
echo "Unknown output level ", str[("--output-level".len + 1)..^1]
|
|
quit 1
|
|
elif str.startsWith("--verbose") or str == "-v":
|
|
hasVerbose = true
|
|
else:
|
|
testsFilters.incl(str)
|
|
if hasXml.len > 0:
|
|
try:
|
|
formatters.add(newJUnitOutputFormatter(newFileStream(hasXml, fmWrite)))
|
|
except CatchableError as exc:
|
|
echo "Cannot open ", hasXml, " for writing: ", exc.msg
|
|
quit 1
|
|
|
|
if hasConsole or hasXml.len == 0:
|
|
let level =
|
|
if hasVerbose: OutputLevel.VERBOSE
|
|
else: hasLevel
|
|
formatters.add(newConsoleOutputFormatter(level, defaultColorOutput()))
|
|
|
|
proc ensureInitialized() =
|
|
if autoParseArgs and declared(paramCount):
|
|
parseParameters(commandLineParams())
|
|
|
|
if formatters.len == 0:
|
|
formatters = @[OutputFormatter(defaultConsoleFormatter())]
|
|
|
|
ensureInitialized() # Run once!
|
|
|
|
template suite*(nameParam: string, body: untyped) {.dirty.} =
|
|
## Declare a test suite identified by `name` with optional ``setup``
|
|
## and/or ``teardown`` section.
|
|
##
|
|
## A test suite is a series of one or more related tests sharing a
|
|
## common fixture (``setup``, ``teardown``). The fixture is executed
|
|
## for EACH test.
|
|
##
|
|
## .. code-block:: nim
|
|
## suite "test suite for addition":
|
|
## setup:
|
|
## let result = 4
|
|
##
|
|
## test "2 + 2 = 4":
|
|
## check(2+2 == result)
|
|
##
|
|
## test "(2 + -2) != 4":
|
|
## check(2 + -2 != result)
|
|
##
|
|
## # No teardown needed
|
|
##
|
|
## The suite will run the individual test cases in the order in which
|
|
## they were listed. With default global settings the above code prints:
|
|
##
|
|
## .. code-block::
|
|
##
|
|
## [Suite] test suite for addition
|
|
## [OK] 2 + 2 = 4
|
|
## [OK] (2 + -2) != 4
|
|
bind collect, currentSuite, suiteStarted, suiteEnded
|
|
|
|
block:
|
|
template setup(setupBody: untyped) {.dirty, used.} =
|
|
var testSetupIMPLFlag {.used.} = true
|
|
template testSetupIMPL: untyped {.dirty.} = setupBody
|
|
|
|
template teardown(teardownBody: untyped) {.dirty, used.} =
|
|
var testTeardownIMPLFlag {.used.} = true
|
|
template testTeardownIMPL: untyped {.dirty.} = teardownBody
|
|
|
|
template suiteTeardown(suiteTeardownBody: untyped) {.dirty, used.} =
|
|
var testSuiteTeardownIMPLFlag {.used.} = true
|
|
template testSuiteTeardownIMPL: untyped {.dirty.} = suiteTeardownBody
|
|
|
|
when nimvm:
|
|
discard
|
|
else:
|
|
let suiteName {.inject.} = nameParam
|
|
when not collect:
|
|
# TODO deal with suite nesting
|
|
if currentSuite.len > 0:
|
|
suiteEnded()
|
|
currentSuite.reset()
|
|
currentSuite = suiteName
|
|
|
|
suiteStarted(suiteName)
|
|
|
|
# TODO what about exceptions in the suite itself?
|
|
body
|
|
|
|
when declared(testSuiteTeardownIMPLFlag):
|
|
testSuiteTeardownIMPL()
|
|
|
|
when nimvm:
|
|
discard
|
|
else:
|
|
when not collect:
|
|
suiteEnded()
|
|
currentSuite.reset()
|
|
|
|
template checkpoint*(msg: string) =
|
|
## Set a checkpoint identified by `msg`. Upon test failure all
|
|
## checkpoints encountered so far are printed out. Example:
|
|
##
|
|
## .. code-block:: nim
|
|
##
|
|
## checkpoint("Checkpoint A")
|
|
## check((42, "the Answer to life and everything") == (1, "a"))
|
|
## checkpoint("Checkpoint B")
|
|
##
|
|
## outputs "Checkpoint A" once it fails.
|
|
when nimvm:
|
|
when compiles(testName):
|
|
echo testName
|
|
|
|
echo msg
|
|
else:
|
|
bind checkpoints
|
|
|
|
checkpoints.add(msg)
|
|
# TODO: add support for something like SCOPED_TRACE from Google Test
|
|
|
|
template fail* =
|
|
## Print out the checkpoints encountered so far and quit if ``abortOnError``
|
|
## is true. Otherwise, erase the checkpoints and indicate the test has
|
|
## failed (change exit code and test status). This template is useful
|
|
## for debugging, but is otherwise mostly used internally. Example:
|
|
##
|
|
## .. code-block:: nim
|
|
##
|
|
## checkpoint("Checkpoint A")
|
|
## complicatedProcInThread()
|
|
## fail()
|
|
##
|
|
## outputs "Checkpoint A" before quitting.
|
|
when nimvm:
|
|
echo "Tests failed"
|
|
quit 1
|
|
else:
|
|
when declared(testStatusIMPL):
|
|
testStatusIMPL = TestStatus.FAILED
|
|
|
|
exitProcs.setProgramResult(1)
|
|
|
|
for formatter in formatters:
|
|
let formatter = formatter # avoid lent iterator
|
|
when declared(stackTrace):
|
|
when stackTrace is string:
|
|
formatter.failureOccurred(checkpoints, stackTrace)
|
|
else:
|
|
formatter.failureOccurred(checkpoints, "")
|
|
else:
|
|
formatter.failureOccurred(checkpoints, "")
|
|
|
|
if abortOnError: quit(1)
|
|
|
|
checkpoints.reset()
|
|
|
|
template skip* =
|
|
## Mark the test as skipped. Should be used directly
|
|
## in case when it is not possible to perform test
|
|
## for reasons depending on outer environment,
|
|
## or certain application logic conditions or configurations.
|
|
## The test code is still executed.
|
|
##
|
|
## .. code-block:: nim
|
|
##
|
|
## if not isGLContextCreated():
|
|
## skip()
|
|
when nimvm:
|
|
discard
|
|
else:
|
|
bind checkpoints
|
|
|
|
testStatusIMPL = TestStatus.SKIPPED
|
|
checkpoints = @[]
|
|
|
|
proc runDirect(test: Test) =
|
|
when not collect:
|
|
# In collection mode, we implicitly create a suite based on the module name
|
|
# and start it based on the test list but in non-collect mode, we have to
|
|
# emulate this with this hack
|
|
if currentSuite != test.suiteName:
|
|
if currentSuite.len > 0:
|
|
suiteEnded()
|
|
suiteStarted(test.suiteName)
|
|
currentSuite = test.suiteName
|
|
|
|
let startTime = getMonoTime()
|
|
testStarted(test.testName)
|
|
|
|
# TODO this annotation works around a limitation where we know that we only
|
|
# call the callback from the main thread but the compiler doesn't -
|
|
# when / if testing becomes multithreaded, this will need a proper
|
|
# solution
|
|
{.gcsafe.}:
|
|
let
|
|
status = test.impl(test.suiteName, test.testName)
|
|
duration = getMonoTime() - startTime
|
|
|
|
testEnded(TestResult(
|
|
suiteName: test.suiteName,
|
|
testName: test.testName,
|
|
status: status,
|
|
duration: duration
|
|
))
|
|
|
|
template runtimeTest*(nameParam: string, body: untyped) =
|
|
## Similar to `test` but runs only at run time, no matter the `unittest2Static`
|
|
## setting
|
|
bind collect, runDirect, shouldRun, checkpoints
|
|
|
|
proc runTest(suiteName, testName: string): TestStatus {.raises: [], gensym.} =
|
|
var testStatusIMPL {.inject.} = TestStatus.OK
|
|
let suiteName {.inject, used.} = suiteName
|
|
let testName {.inject, used.} = testName
|
|
|
|
template fail(prefix: string, eClass: string, e: auto): untyped =
|
|
let eName = "[" & $e.name & "]"
|
|
checkpoint(prefix & "Unhandled " & eClass & ": " & e.msg & " " & eName)
|
|
var stackTrace {.inject.} = e.getStackTrace()
|
|
fail()
|
|
|
|
template failingOnExceptions(prefix: string, code: untyped): untyped =
|
|
when NimMajor>=2:
|
|
{.push warning[UnnamedBreak]:off.}
|
|
try:
|
|
block:
|
|
code
|
|
except CatchableError as e:
|
|
prefix.fail("error", e)
|
|
except Defect as e: # This may or may not work dependings on --panics
|
|
prefix.fail("defect", e)
|
|
except Exception as e:
|
|
prefix.fail("exception that may cause undefined behavior", e)
|
|
when NimMajor>=2:
|
|
{.pop.}
|
|
|
|
failingOnExceptions("[setup] "):
|
|
when declared(testSetupIMPLFlag): testSetupIMPL()
|
|
defer: failingOnExceptions("[teardown] "):
|
|
when declared(testTeardownIMPLFlag): testTeardownIMPL()
|
|
failingOnExceptions(""):
|
|
body
|
|
|
|
checkpoints = @[]
|
|
|
|
testStatusIMPL
|
|
|
|
let
|
|
localSuiteName =
|
|
when declared(suiteName):
|
|
suiteName
|
|
else: instantiationInfo().filename
|
|
localTestName = nameParam
|
|
if shouldRun(localSuiteName, localTestName):
|
|
let
|
|
instance =
|
|
Test(testName: localTestName, suiteName: localSuiteName, impl: runTest)
|
|
when collect:
|
|
tests.mgetOrPut(localSuiteName, default(seq[Test])).add(instance)
|
|
else:
|
|
runDirect(instance)
|
|
|
|
template staticTest*(nameParam: string, body: untyped) =
|
|
## Similar to `test` but runs only at compiletime, no matter the
|
|
## `unittest2Static` flag
|
|
static:
|
|
block:
|
|
echo "[Test ] ", nameParam
|
|
body
|
|
echo "[", TestStatus.OK, " ] ", nameParam
|
|
|
|
template dualTest*(nameParam: string, body: untyped) =
|
|
## Similar to `test` but run the test both compuletime and run time, no
|
|
## matter the `unittest2Static` flag
|
|
staticTest nameParam:
|
|
body
|
|
runtimeTest nameParam:
|
|
body
|
|
|
|
template test*(nameParam: string, body: untyped) =
|
|
## Define a single test case identified by `name`.
|
|
##
|
|
## .. code-block:: nim
|
|
##
|
|
## test "roses are red":
|
|
## let roses = "red"
|
|
## check(roses == "red")
|
|
##
|
|
## The above code outputs:
|
|
##
|
|
## .. code-block::
|
|
##
|
|
## [OK] roses are red
|
|
when nimvm:
|
|
when unittest2Static:
|
|
staticTest nameParam:
|
|
body
|
|
runtimeTest nameParam:
|
|
body
|
|
|
|
{.pop.} # raises: []
|
|
|
|
macro check*(conditions: untyped): untyped =
|
|
## Verify if a statement or a list of statements is true.
|
|
## A helpful error message and set checkpoints are printed out on
|
|
## failure (if ``outputLevel`` is not ``NONE``).
|
|
runnableExamples:
|
|
import std/strutils
|
|
|
|
check("AKB48".toLowerAscii() == "akb48")
|
|
|
|
let teams = {'A', 'K', 'B', '4', '8'}
|
|
|
|
check:
|
|
"AKB48".toLowerAscii() == "akb48"
|
|
'C' notin teams
|
|
|
|
{.warning[Deprecated]:off.}
|
|
let checked = callsite()[1]
|
|
{.warning[Deprecated]:on.}
|
|
|
|
template asgn(a: untyped, value: typed) =
|
|
var a = value # XXX: we need "var: var" here in order to
|
|
# preserve the semantics of var params
|
|
|
|
template print(name: untyped, value: typed) =
|
|
when compiles(string($value)):
|
|
checkpoint(name & " was " & $value)
|
|
|
|
proc inspectArgs(exp: NimNode): tuple[assigns, check, printOuts: NimNode] =
|
|
result.check = copyNimTree(exp)
|
|
result.assigns = newNimNode(nnkStmtList)
|
|
result.printOuts = newNimNode(nnkStmtList)
|
|
|
|
var counter = 0
|
|
|
|
if exp[0].kind in {nnkIdent, nnkOpenSymChoice, nnkClosedSymChoice, nnkSym} and
|
|
$exp[0] in ["not", "in", "notin", "==", "<=",
|
|
">=", "<", ">", "!=", "is", "isnot"]:
|
|
|
|
for i in 1 ..< exp.len:
|
|
if exp[i].kind notin nnkLiterals:
|
|
inc counter
|
|
let argStr = exp[i].toStrLit
|
|
let paramAst = exp[i]
|
|
if exp[i].kind == nnkIdent:
|
|
result.printOuts.add getAst(print(argStr, paramAst))
|
|
if exp[i].kind in nnkCallKinds + {nnkDotExpr, nnkBracketExpr, nnkPar} and
|
|
(exp[i].typeKind notin {ntyTypeDesc} or $exp[0] notin ["is", "isnot"]):
|
|
let callVar = newIdentNode(":c" & $counter)
|
|
result.assigns.add getAst(asgn(callVar, paramAst))
|
|
result.check[i] = callVar
|
|
result.printOuts.add getAst(print(argStr, callVar))
|
|
if exp[i].kind == nnkExprEqExpr:
|
|
# ExprEqExpr
|
|
# Ident "v"
|
|
# IntLit 2
|
|
result.check[i] = exp[i][1]
|
|
if exp[i].typeKind notin {ntyTypeDesc}:
|
|
let arg = newIdentNode(":p" & $counter)
|
|
result.assigns.add getAst(asgn(arg, paramAst))
|
|
result.printOuts.add getAst(print(argStr, arg))
|
|
if exp[i].kind != nnkExprEqExpr:
|
|
result.check[i] = arg
|
|
else:
|
|
result.check[i][1] = arg
|
|
|
|
proc buildCheck(lineinfo, callLit, assigns, check, printOuts: NimNode): NimNode =
|
|
let
|
|
checkpointSym = bindSym("checkpoint")
|
|
failSym = bindSym("fail")
|
|
|
|
nnkBlockStmt.newTree(
|
|
newEmptyNode(),
|
|
nnkStmtList.newTree(
|
|
assigns,
|
|
nnkIfStmt.newTree(
|
|
nnkElifBranch.newTree(
|
|
nnkCall.newTree(ident("not"), check),
|
|
nnkStmtList.newTree(
|
|
nnkCall.newTree(
|
|
checkpointSym,
|
|
nnkInfix.newTree(
|
|
ident("&"),
|
|
nnkInfix.newTree(
|
|
ident("&"),
|
|
lineinfo,
|
|
newLit(": Check failed: ")
|
|
),
|
|
callLit
|
|
)
|
|
),
|
|
printOuts,
|
|
nnkCall.newTree(failSym)
|
|
)
|
|
)
|
|
)
|
|
)
|
|
)
|
|
|
|
let
|
|
checkSym = bindSym("check")
|
|
|
|
case checked.kind
|
|
of nnkCallKinds:
|
|
let
|
|
(assigns, check, printOuts) = inspectArgs(checked)
|
|
lineinfo = newStrLitNode(checked.lineInfo)
|
|
callLit = checked.toStrLit
|
|
result = buildCheck(lineinfo, callLit, assigns, check, printOuts)
|
|
|
|
of nnkStmtList:
|
|
result = newNimNode(nnkStmtList)
|
|
for node in checked:
|
|
if node.kind != nnkCommentStmt:
|
|
result.add(newCall(checkSym, node))
|
|
|
|
else:
|
|
let
|
|
lineinfo = newStrLitNode(checked.lineInfo)
|
|
callLit = checked.toStrLit
|
|
|
|
result = buildCheck(
|
|
lineinfo, callLit, newEmptyNode(), checked, newEmptyNode())
|
|
|
|
template require*(conditions: untyped) =
|
|
## Same as `check` except any failed test causes the program to quit
|
|
## immediately. Any teardown statements are not executed and the failed
|
|
## test output is not generated.
|
|
when nimvm:
|
|
check conditions
|
|
else:
|
|
let savedAbortOnError = abortOnError
|
|
block:
|
|
abortOnError = true
|
|
check conditions
|
|
abortOnError = savedAbortOnError
|
|
|
|
macro expect*(exceptions: varargs[typed], body: untyped): untyped =
|
|
## Test if `body` raises an exception found in the passed `exceptions`.
|
|
## The test passes if the raised exception is part of the acceptable
|
|
## exceptions. Otherwise, it fails.
|
|
runnableExamples:
|
|
import std/[math, random, strutils]
|
|
proc defectiveRobot() =
|
|
randomize()
|
|
case rand(1..4)
|
|
of 1: raise newException(OSError, "CANNOT COMPUTE!")
|
|
of 2: discard parseInt("Hello World!")
|
|
of 3: raise newException(IOError, "I can't do that Dave.")
|
|
else: assert 2 + 2 == 5
|
|
|
|
expect IOError, OSError, ValueError, AssertionDefect:
|
|
defectiveRobot()
|
|
|
|
template expectBody(errorTypes, lineInfoLit, body): NimNode {.dirty.} =
|
|
try:
|
|
try:
|
|
body
|
|
checkpoint(lineInfoLit & ": Expect Failed, no exception was thrown.")
|
|
fail()
|
|
except errorTypes:
|
|
discard
|
|
except CatchableError as e:
|
|
checkpoint(lineInfoLit & ": Expect Failed, unexpected " & $e.name &
|
|
" (" & e.msg & ") was thrown.\n" & e.getStackTrace())
|
|
fail()
|
|
except Defect as e:
|
|
checkpoint(lineInfoLit & ": Expect Failed, unexpected " & $e.name &
|
|
" (" & e.msg & ") was thrown.\n" & e.getStackTrace())
|
|
fail()
|
|
|
|
var errorTypes = newNimNode(nnkBracket)
|
|
for exp in exceptions:
|
|
errorTypes.add(exp)
|
|
|
|
result = getAst(expectBody(errorTypes, errorTypes.lineInfo, body))
|
|
|
|
proc disableParamFiltering* {.deprecated:
|
|
"Compile with -d:unittest2DisableParamFiltering instead".} =
|
|
discard
|
|
|
|
when unittest2PreviewIsolate:
|
|
import std/[osproc, strtabs]
|
|
proc runIsolated(test: Test) =
|
|
# Run test in an isolated process - this has the advantage that we can
|
|
# trivially capture stdout but has a number of problems:
|
|
# * suite and other global stuff gets executed for each test
|
|
# * on unix, `fork` could work around this but not on windows
|
|
# * there's no good way to separate errors from stdout
|
|
# * there's process overhead
|
|
#
|
|
# There are advantages too:
|
|
# * reduced cross-test pollution
|
|
# * simple to parallelise
|
|
# * we can abort long-running tests after a timeout
|
|
|
|
let startTime = getMonoTime()
|
|
testStarted(test.testName)
|
|
|
|
let runner = startProcess(
|
|
getAppFilename2(),
|
|
args = [test.suiteName & "::" & test.testName],
|
|
env = newStringTable(
|
|
"UNITTEST2_ISOLATED", "1",
|
|
StringTableMode.modeCaseSensitive),
|
|
options = {poStdErrToStdOut})
|
|
|
|
close(runner.inputStream) # EOF so the test doesn't think it'll get input
|
|
|
|
var output: string
|
|
|
|
while true:
|
|
let pos = output.len
|
|
output.setLen(pos + 4096)
|
|
|
|
let bytes = runner.outputStream.readData(addr output[pos], 4096)
|
|
if bytes >= 0:
|
|
output.setLen(pos + bytes)
|
|
|
|
if bytes <= 0:
|
|
break
|
|
|
|
let status = runner.waitForExit()
|
|
|
|
runner.close()
|
|
|
|
testEnded(TestResult(
|
|
suiteName: test.suiteName,
|
|
testName: test.testName,
|
|
status: if status == 0: TestStatus.OK else: TestStatus.FAILED,
|
|
duration: getMonoTime() - startTime,
|
|
output: output
|
|
))
|
|
|
|
type
|
|
IsolatedFormatter* = ref object of OutputFormatter
|
|
## Formatter suitable for using the process-isolated environment
|
|
##
|
|
## This is a work in progress with several open issues
|
|
## * we could use stderr for "unittest" traffic but it would be
|
|
## compromised by application output (typically ok in nim) and makes
|
|
## reading messy
|
|
## * we could print all errors after test providing some sort of
|
|
## separator - has escape issues
|
|
## * we could redirect stdout/stderr to a file and use stdout for errors
|
|
## * as an addon to the above, we could read back the file then print
|
|
## a structured test format to stdout which the parent process can
|
|
## capture easily
|
|
|
|
if isolated:
|
|
formatters.add(IsolatedFormatter())
|
|
|
|
method failureOccurred*(formatter: IsolatedFormatter,
|
|
checkpoints: seq[string], stackTrace: string) =
|
|
if stackTrace.len > 0:
|
|
echo(stackTrace)
|
|
echo("\n")
|
|
for msg in items(checkpoints):
|
|
echo(" ")
|
|
echo(msg)
|
|
echo("\n")
|
|
|
|
when collect:
|
|
proc runScheduledTests() {.noconv.} =
|
|
# Tests can be added inside tests - this is weird and only partially
|
|
# supported
|
|
while tests.len > 0:
|
|
var tmp = move(tests)
|
|
suiteRunStarted(tmp)
|
|
for suiteName, suite in tmp:
|
|
if suite.len == 0: continue
|
|
|
|
suiteStarted(suiteName)
|
|
for test in suite:
|
|
when isolate:
|
|
if not isolated:
|
|
runIsolated(test)
|
|
else:
|
|
runDirect(test)
|
|
else:
|
|
runDirect(test)
|
|
|
|
suiteEnded()
|
|
|
|
suiteRunEnded()
|
|
testRunEnded()
|
|
|
|
addExitProc(runScheduledTests)
|
|
|
|
else:
|
|
addExitProc(proc() {.noconv.} = testRunEnded())
|