remove testutils from chronicles
This commit is contained in:
Andy Davidoff
zah
parent
b9315ced4e
commit
1ac38c5a42
|
|
@ -7,9 +7,10 @@ description = "A crafty implementation of structured logging for Nim"
|
|||
license = "Apache License 2.0"
|
||||
skipDirs = @["tests"]
|
||||
|
||||
requires "nim >= 0.18.1", "json_serialization"
|
||||
requires "nim >= 0.18.1"
|
||||
requires "json_serialization"
|
||||
requires "https://github.com/disruptek/testutils < 2.0.0"
|
||||
|
||||
task test, "run CPU tests":
|
||||
cd "tests"
|
||||
exec "nim c -r testrunner ."
|
||||
exec "testrunner tests"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,128 +0,0 @@
|
|||
import os, parsecfg, parseopt, strutils, streams
|
||||
|
||||
const
|
||||
Usage = """Usage:
|
||||
testrunner [options] path
|
||||
Run the test(s) specified at path. Will search recursively for test files
|
||||
provided path is a directory.
|
||||
Options:
|
||||
--targets:"c c++ js objc" [Not implemented] Run tests for specified targets
|
||||
--include:"test1 test2" Run only listed tests (space/comma seperated)
|
||||
--exclude:"test1 test2" Skip listed tests (space/comma seperated)
|
||||
--help Display this help and exit"""
|
||||
|
||||
type
|
||||
TestConfig* = object
|
||||
path*: string
|
||||
includedTests*: seq[string]
|
||||
excludedTests*: seq[string]
|
||||
releaseBuild*: bool
|
||||
noThreads*: bool
|
||||
|
||||
TestSpec* = object
|
||||
name*: string
|
||||
skip*: bool
|
||||
program*: string
|
||||
flags*: string
|
||||
outputs*: seq[tuple[name: string, expectedOutput: string]]
|
||||
timestampPeg*: string
|
||||
errorMsg*: string
|
||||
maxSize*: int64
|
||||
compileError*: string
|
||||
errorFile*: string
|
||||
errorLine*: int
|
||||
errorColumn*: int
|
||||
os*: seq[string]
|
||||
|
||||
proc processArguments*(): TestConfig =
|
||||
var opt = initOptParser()
|
||||
var length = 0
|
||||
for kind, key, value in opt.getopt():
|
||||
case kind
|
||||
of cmdArgument:
|
||||
if result.path == "":
|
||||
result.path = key
|
||||
of cmdLongOption, cmdShortOption:
|
||||
inc(length)
|
||||
case key.toLowerAscii()
|
||||
of "help", "h": quit(Usage, QuitSuccess)
|
||||
of "release":
|
||||
result.releaseBuild = true
|
||||
of "nothreads":
|
||||
result.noThreads = true
|
||||
of "targets", "t": discard # not implemented
|
||||
of "include":
|
||||
result.includedTests.add value.split(Whitespace + {','})
|
||||
of "exclude":
|
||||
result.excludedTests.add value.split(Whitespace + {','})
|
||||
else: quit(Usage)
|
||||
of cmdEnd:
|
||||
quit(Usage)
|
||||
|
||||
if result.path == "":
|
||||
quit(Usage)
|
||||
|
||||
proc defaults(result: var TestSpec) =
|
||||
result.os = @["linux", "macosx", "windows"]
|
||||
|
||||
proc parseTestFile*(filePath: string): TestSpec =
|
||||
result.defaults()
|
||||
result.name = splitFile(filePath).name
|
||||
var f = newFileStream(filePath, fmRead)
|
||||
var outputSection = false
|
||||
if f != nil:
|
||||
var p: CfgParser
|
||||
open(p, f, filePath)
|
||||
while true:
|
||||
var e = next(p)
|
||||
case e.kind
|
||||
of cfgEof:
|
||||
break
|
||||
of cfgSectionStart:
|
||||
if e.section.cmpIgnoreCase("Output") == 0:
|
||||
outputSection = true
|
||||
of cfgKeyValuePair:
|
||||
if outputSection:
|
||||
result.outputs.add((e.key, e.value))
|
||||
else:
|
||||
case e.key
|
||||
of "program":
|
||||
result.program = e.value
|
||||
of "timestamp_peg":
|
||||
result.timestampPeg = e.value
|
||||
of "max_size":
|
||||
try:
|
||||
result.maxSize = parseInt(e.value)
|
||||
except ValueError:
|
||||
echo("Parsing warning: value of " & e.key &
|
||||
" is not a number (value = " & e.value & ").")
|
||||
of "compile_error":
|
||||
result.compileError = e.value
|
||||
of "error_file":
|
||||
result.errorFile = e.value
|
||||
of "os":
|
||||
result.os = e.value.normalize.split({','} + Whitespace)
|
||||
else:
|
||||
result.flags &= ("-d:$#:$#" % [e.key, e.value]).quoteShell & " "
|
||||
of cfgOption:
|
||||
case e.key
|
||||
of "skip":
|
||||
result.skip = true
|
||||
else:
|
||||
result.flags &= ("--$#:$#" % [e.key, e.value]).quoteShell & " "
|
||||
of cfgError:
|
||||
echo("Parsing warning:" & e.msg)
|
||||
close(p)
|
||||
if result.program == "":
|
||||
echo("Parsing error: no program value")
|
||||
else:
|
||||
echo("Parsing error: cannot open " & filePath)
|
||||
|
||||
func shouldSkip*(c: TestConfig, testName: string): bool =
|
||||
if testName in c.excludedTests:
|
||||
return true
|
||||
|
||||
if c.includedTests.len > 0:
|
||||
return testName notin c.includedTests
|
||||
|
||||
return false
|
||||
|
|
@ -1,97 +0,0 @@
|
|||
import os, osproc, strutils, streams, pegs
|
||||
|
||||
type
|
||||
CompileInfo* = object
|
||||
templFile*: string
|
||||
errorFile*: string
|
||||
errorLine*, errorColumn*: int
|
||||
templLine*, templColumn*: int
|
||||
msg*: string
|
||||
fullMsg*: string
|
||||
compileTime*: float
|
||||
exitCode*: int
|
||||
|
||||
let
|
||||
# Error pegs, taken from testament tester
|
||||
pegLineTemplate =
|
||||
peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' 'template/generic instantiation from here'.*"
|
||||
pegLineError =
|
||||
peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' ('Error') ':' \s* {.*}"
|
||||
pegOtherError = peg"'Error:' \s* {.*}"
|
||||
pegError = pegLineError / pegOtherError
|
||||
pegSuccess = peg"'Hint: operation successful' {[^;]*} '; ' {\d+} '.' {\d+} .*"
|
||||
|
||||
# Timestamp pegs
|
||||
# peg for unix timestamp, basically any float with 6 digits after the decimal
|
||||
# Not ideal - could also improve by checking for the location in the line
|
||||
pegUnixTimestamp = peg"{\d+} '.' {\d\d\d\d\d\d} \s"
|
||||
# peg for timestamp with format yyyy-MM-dd HH:mm:sszzz
|
||||
pegRfcTimestamp = peg"{\d\d\d\d} '-' {\d\d} '-' {\d\d} ' ' {\d\d} ':' {\d\d} ':' {\d\d} {'+' / '-'} {\d\d} ':' {\d\d} \s"
|
||||
# Thread/process id is unpredictable..
|
||||
pegXid* = peg"""'tid' (('=') / ('":') / (': [1m') / (': ') / ('[0m=[94m') / ('>')) \d+"""
|
||||
|
||||
proc cmpIgnorePegs*(a, b: string, pegs: varargs[Peg]): bool =
|
||||
var
|
||||
aa = a
|
||||
bb = b
|
||||
for peg in pegs:
|
||||
aa = aa.replace(peg, "dummy")
|
||||
bb = bb.replace(peg, "dummy")
|
||||
|
||||
return aa == bb
|
||||
|
||||
proc cmpIgnoreTimestamp*(a, b: string, timestamp = ""): bool =
|
||||
if timestamp.len == 0:
|
||||
return cmpIgnorePegs(a, b, pegXid)
|
||||
else:
|
||||
if timestamp == "RfcTime":
|
||||
return cmpIgnorePegs(a, b, pegRfcTimestamp, pegXid)
|
||||
elif timestamp == "UnixTime":
|
||||
return cmpIgnorePegs(a, b, pegUnixTimestamp, pegXid)
|
||||
|
||||
proc cmpIgnoreDefaultTimestamps*(a, b: string): bool =
|
||||
if cmpIgnorePegs(a, b, pegRfcTimestamp, pegXid):
|
||||
return true
|
||||
elif cmpIgnorePegs(a, b, pegUnixTimestamp, pegXid):
|
||||
return true
|
||||
else: return false
|
||||
|
||||
# parsing based on testament tester
|
||||
proc parseCompileStream*(p: Process, output: Stream): CompileInfo =
|
||||
result.exitCode = -1
|
||||
var line = newStringOfCap(120).TaintedString
|
||||
var suc, err, tmpl = ""
|
||||
|
||||
while true:
|
||||
if output.readLine(line):
|
||||
if line =~ pegError:
|
||||
# `err` should contain the last error/warning message
|
||||
err = line
|
||||
elif line =~ pegLineTemplate and err == "":
|
||||
# `tmpl` contains the last template expansion before the error
|
||||
tmpl = line
|
||||
elif line =~ pegSuccess:
|
||||
suc = line
|
||||
|
||||
if err != "":
|
||||
result.fullMsg.add(line.string & "\p")
|
||||
else:
|
||||
result.exitCode = peekExitCode(p)
|
||||
if result.exitCode != -1: break
|
||||
|
||||
if tmpl =~ pegLineTemplate:
|
||||
result.templFile = extractFilename(matches[0])
|
||||
result.templLine = parseInt(matches[1])
|
||||
result.templColumn = parseInt(matches[2])
|
||||
if err =~ pegLineError:
|
||||
result.errorFile = extractFilename(matches[0])
|
||||
result.errorLine = parseInt(matches[1])
|
||||
result.errorColumn = parseInt(matches[2])
|
||||
result.msg = matches[3]
|
||||
elif err =~ pegOtherError:
|
||||
result.msg = matches[0]
|
||||
elif suc =~ pegSuccess:
|
||||
result.msg = suc
|
||||
result.compileTime = parseFloat(matches[1] & "." & matches[2])
|
||||
|
||||
proc parseExecuteOutput*() = discard
|
||||
|
|
@ -1,234 +0,0 @@
|
|||
import os, osproc, strutils, terminal, times, pegs
|
||||
import test_config, test_helpers
|
||||
|
||||
# Testrunner to run tests for chronicles library
|
||||
# However, could in theory be used to run any test that needs to check
|
||||
# compile time or runtime output
|
||||
#
|
||||
# The runner will look recursively for all *.test files at given path.
|
||||
# A test file should have at minimum a program name. This is the name of the nim
|
||||
# source minus the .nim extension)
|
||||
#
|
||||
# Code is here and there influenced by nim testament tester and unittest module.
|
||||
|
||||
const
|
||||
# defaultOptions = "--verbosity:1 --warnings:off --hint[Processing]:off " &
|
||||
# "--hint[Conf]:off --hint[XDeclaredButNotUsed]:off " &
|
||||
# "--hint[Link]:off --hint[Pattern]:off"
|
||||
defaultOptions = "--verbosity:1 --warnings:off "
|
||||
|
||||
type
|
||||
TestStatus* = enum
|
||||
OK,
|
||||
FAILED,
|
||||
SKIPPED,
|
||||
INVALID
|
||||
|
||||
# If needed pass more info to the logresult via a TestResult object
|
||||
# TestResult = object
|
||||
# status: TestStatus
|
||||
# compileTime: float
|
||||
# fileSize: uint
|
||||
|
||||
TestError* = enum
|
||||
SourceFileNotFound,
|
||||
ExeFileNotFound,
|
||||
OutputFileNotFound,
|
||||
CompileError,
|
||||
RuntimeError,
|
||||
OutputsDiffer,
|
||||
FileSizeTooLarge,
|
||||
CompileErrorDiffers
|
||||
|
||||
proc logFailure(test: TestSpec, error: TestError, data: varargs[string] = [""]) =
|
||||
case error
|
||||
of SourceFileNotFound:
|
||||
styledEcho(fgYellow, styleBright, "source file not found: ",
|
||||
resetStyle, test.program.addFileExt(".nim"))
|
||||
of ExeFileNotFound:
|
||||
styledEcho(fgYellow, styleBright, "file not found: ",
|
||||
resetStyle, test.program.addFileExt(ExeExt))
|
||||
of OutputFileNotFound:
|
||||
styledEcho(fgYellow, styleBright, "file not found: ",
|
||||
resetStyle, data[0])
|
||||
of CompileError:
|
||||
styledEcho(fgYellow, styleBright, "compile error:\p",
|
||||
resetStyle, data[0])
|
||||
of RuntimeError:
|
||||
styledEcho(fgYellow, styleBright, "runtime error:\p",
|
||||
resetStyle, data[0])
|
||||
of OutputsDiffer:
|
||||
styledEcho(fgYellow, styleBright, "outputs are different:\p",
|
||||
resetStyle,"Expected output to $#:\p$#" % [data[0], data[1]],
|
||||
"Resulted output to $#:\p$#" % [data[0], data[2]])
|
||||
of FileSizeTooLarge:
|
||||
styledEcho(fgYellow, styleBright, "file size is too large: ",
|
||||
resetStyle, data[0] & " > " & $test.maxSize)
|
||||
of CompileErrorDiffers:
|
||||
styledEcho(fgYellow, styleBright, "compile error is different:\p",
|
||||
resetStyle, data[0])
|
||||
|
||||
styledEcho(fgCyan, styleBright, "command: ", resetStyle,
|
||||
"nim c $#$#$#" % [defaultOptions, test.flags,
|
||||
test.program.addFileExt(".nim")])
|
||||
|
||||
proc logResult(testName: string, status: TestStatus, time: float) =
|
||||
var color = case status
|
||||
of OK: fgGreen
|
||||
of FAILED: fgRed
|
||||
of SKIPPED: fgYellow
|
||||
of INVALID: fgRed
|
||||
styledEcho(styleBright, color, "[", $status, "] ",
|
||||
resetStyle, testName,
|
||||
fgYellow, " ", time.formatFloat(ffDecimal, 3), " s")
|
||||
|
||||
template time(duration, body): untyped =
|
||||
let t0 = epochTime()
|
||||
block:
|
||||
body
|
||||
duration = epochTime() - t0
|
||||
|
||||
proc cmpOutputs(test: TestSpec, stdout: string): TestStatus =
|
||||
result = OK
|
||||
for output in test.outputs:
|
||||
var testOutput: string
|
||||
if output.name == "stdout":
|
||||
testOutput = stdout
|
||||
else:
|
||||
if not existsFile(output.name):
|
||||
logFailure(test, OutputFileNotFound, output.name)
|
||||
result = FAILED
|
||||
continue
|
||||
|
||||
testOutput = readFile(output.name)
|
||||
|
||||
# Would be nice to do a real diff here instead of simple compare
|
||||
if test.timestampPeg.len > 0:
|
||||
if not cmpIgnorePegs(testOutput, output.expectedOutput, peg(test.timestampPeg), pegXid):
|
||||
logFailure(test, OutputsDiffer, output.name, output.expectedOutput, testOutput)
|
||||
result = FAILED
|
||||
else:
|
||||
if not cmpIgnoreDefaultTimestamps(testOutput, output.expectedOutput):
|
||||
logFailure(test, OutputsDiffer, output.name, output.expectedOutput, testOutput)
|
||||
result = FAILED
|
||||
|
||||
if output.name != "stdout":
|
||||
removeFile(output.name)
|
||||
|
||||
proc compile(test: TestSpec): TestStatus =
|
||||
let source = test.program.addFileExt(".nim")
|
||||
if not existsFile(source):
|
||||
logFailure(test, SourceFileNotFound)
|
||||
return FAILED
|
||||
|
||||
let cmd = "nim c $#$#$#" % [defaultOptions, test.flags, source.quoteShell]
|
||||
let c = parseCmdLine(cmd)
|
||||
var p = startProcess(command=c[0], args=c[1.. ^1],
|
||||
options={poStdErrToStdOut, poUsePath})
|
||||
let compileInfo = parseCompileStream(p, p.outputStream)
|
||||
close(p)
|
||||
|
||||
if compileInfo.exitCode != 0:
|
||||
if test.compileError.len == 0:
|
||||
logFailure(test, CompileError, compileInfo.fullMsg)
|
||||
return FAILED
|
||||
else:
|
||||
if test.compileError == compileInfo.msg and
|
||||
(test.errorFile.len == 0 or test.errorFile == compileInfo.errorFile) and
|
||||
(test.errorLine == 0 or test.errorLine == compileInfo.errorLine) and
|
||||
(test.errorColumn == 0 or test.errorColumn == compileInfo.errorColumn):
|
||||
return OK
|
||||
else:
|
||||
logFailure(test, CompileErrorDiffers, compileInfo.fullMsg)
|
||||
return FAILED
|
||||
|
||||
# Lets also check file size here as it kinda belongs to the compilation result
|
||||
if test.maxSize != 0:
|
||||
var size = getFileSize(test.program.addFileExt(ExeExt))
|
||||
if size > test.maxSize:
|
||||
logFailure(test, FileSizeTooLarge, $size)
|
||||
return FAILED
|
||||
|
||||
return OK
|
||||
|
||||
proc execute(test: TestSpec): TestStatus =
|
||||
let program = test.program.addFileExt(ExeExt)
|
||||
if not existsFile(program):
|
||||
logFailure(test, ExeFileNotFound)
|
||||
return FAILED
|
||||
|
||||
let (output, exitCode) = execCmdEx(CurDir & DirSep & program.quoteShell)
|
||||
|
||||
if exitCode != 0:
|
||||
# parseExecuteOutput() # Need to parse the run time failures?
|
||||
logFailure(test, RuntimeError, output)
|
||||
return FAILED
|
||||
else:
|
||||
return test.cmpOutputs(output)
|
||||
|
||||
proc scanTestPath(path: string): seq[string] =
|
||||
result = @[]
|
||||
|
||||
if fileExists(path):
|
||||
result.add(path)
|
||||
else:
|
||||
for file in walkDirRec path:
|
||||
if file.endsWith(".test"):
|
||||
result.add(file)
|
||||
|
||||
proc test(config: TestConfig, testPath: string): TestStatus =
|
||||
var test: TestSpec
|
||||
var duration: float
|
||||
|
||||
time duration:
|
||||
test = parseTestFile(testPath)
|
||||
test.flags &= (if config.releaseBuild: "-d:release " else: "-d:debug ")
|
||||
if not config.noThreads:
|
||||
test.flags &= "--threads:on "
|
||||
if test.program.len == 0: # a program name is bare minimum of a test file
|
||||
result = INVALID
|
||||
break
|
||||
if test.skip or hostOS notin test.os or config.shouldSkip(test.name):
|
||||
result = SKIPPED
|
||||
break
|
||||
|
||||
result = test.compile()
|
||||
if result != OK or test.compileError.len > 0:
|
||||
break
|
||||
|
||||
result = test.execute()
|
||||
try:
|
||||
# this may fail in 64-bit AppVeyor images with "The process cannot access the file because it is being used by another process. [OSError]"
|
||||
removeFile(test.program.addFileExt(ExeExt))
|
||||
except CatchableError as e:
|
||||
echo e.msg
|
||||
|
||||
logResult(test.name, result, duration)
|
||||
|
||||
proc main() =
|
||||
let config = processArguments()
|
||||
let testFiles = scanTestPath(config.path)
|
||||
var successful, skipped = 0
|
||||
|
||||
if testFiles.len == 0:
|
||||
styledEcho(styleBright, "No test files found")
|
||||
program_result = 1
|
||||
return
|
||||
|
||||
for testFile in testFiles:
|
||||
# Here we could do multithread or multiprocess
|
||||
# but we will have to work with different nim caches per test
|
||||
# and also the executables have to be in a unique location as several tests
|
||||
# can use the same source
|
||||
var result = test(config, testFile)
|
||||
if result == OK:
|
||||
successful += 1
|
||||
elif result == SKIPPED:
|
||||
skipped += 1
|
||||
|
||||
styledEcho(styleBright, "Finished run: $#/$# tests successful" %
|
||||
[$successful, $(testFiles.len - skipped)])
|
||||
program_result = testFiles.len - successful - skipped
|
||||
|
||||
when isMainModule:
|
||||
main()
|
||||
Loading…
Reference in New Issue