commit d41ed048244586c03ffbf44bd59a8db54f40c31c Author: Andy Davidoff Date: Fri Feb 21 16:14:14 2020 -0500 initial diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 0000000..880b8e1 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,39 @@ +version: '{build}' + +image: Visual Studio 2015 + +cache: + - NimBinaries + +matrix: + # We always want 32 and 64-bit compilation + fast_finish: false + +platform: + - x86 + - x64 + +# when multiple CI builds are queued, the tested commit needs to be in the last X commits cloned with "--depth X" +clone_depth: 10 + +install: + # use the newest versions documented here: https://www.appveyor.com/docs/windows-images-software/#mingw-msys-cygwin + - IF "%PLATFORM%" == "x86" SET PATH=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin;%PATH% + - IF "%PLATFORM%" == "x64" SET PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% + + # build nim from our own branch - this to avoid the day-to-day churn and + # regressions of the fast-paced Nim development while maintaining the + # flexibility to apply patches + - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh + - env MAKE="mingw32-make -j2" ARCH_OVERRIDE=%PLATFORM% bash build_nim.sh Nim csources dist/nimble NimBinaries + - SET PATH=%CD%\Nim\bin;%PATH% + +build_script: + - cd C:\projects\%APPVEYOR_PROJECT_SLUG% + - nimble install -y + +test_script: + - nimble test + +deploy: off + diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..6cc19ff --- /dev/null +++ b/.editorconfig @@ -0,0 +1,5 @@ +[*] +indent_style = space +insert_final_newline = true +indent_size = 2 +trim_trailing_whitespace = true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cc935b3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +# ignore all executable files +* +!*.* +!*/ +*.exe + +nimcache/ + diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..03fab11 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,26 @@ +language: c + +# https://docs.travis-ci.com/user/caching/ +cache: + directories: + - NimBinaries + +git: + # when multiple CI builds are queued, the tested commit needs to be in the last X commits cloned with "--depth X" + depth: 10 + +os: + - linux + - osx + +install: + # build nim from our own branch - this to avoid the day-to-day churn and + # regressions of the fast-paced Nim development while maintaining the + # flexibility to apply patches + - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh + - env MAKE="make -j2" bash build_nim.sh Nim csources dist/nimble NimBinaries + - export PATH=$PWD/Nim/bin:$PATH + +script: + - nimble install -y + - nimble test diff --git a/README.md b/README.md new file mode 100644 index 0000000..10eb105 --- /dev/null +++ b/README.md @@ -0,0 +1,64 @@ +# Testrunner +## Usage +Command syntax: +``` +testrunner [options] path +Run the test(s) specified at path. Will search recursively for test files +provided path is a directory. +Options: +--targets:"c c++ js objc" [Not implemented] Run tests for specified targets +--include:"test1 test2" Run only listed tests (space/comma seperated) +--exclude:"test1 test2" Skip listed tests (space/comma seperated) +--help Display this help and exit +``` + +The runner will look recursively for all `*.test` files at given path. + +## Test file options +The test files follow the configuration file syntax (similar as `.ini`), see also +[nim parsecfg module](https://nim-lang.org/docs/parsecfg.html). + +### Required +- **program**: A test file should have at minimum a program name. This is the name +of the nim source minus the `.nim` extension. + +### Optional +- **max_size**: To check the maximum size of the binary, in bytes. +- **timestamp_peg**: If you don't want to use the default timestamps, you can define +your own timestamp peg here. +- **compile_error**: When expecting a compilation failure, the error message that +should be expected. +- **error_file**: When expecting a compilation failure, the source file where the +error should occur. +- **os**: Space and/or comma separated list of operating systems for which the +test should be run. Defaults to `"linux, macosx, windows"`. Tests meant for a +different OS than the host will be marked as `SKIPPED`. +- **--skip**: This will simply skip the test (will not be marked as failure). + +### Forwarded Options +Any other options or key-value pairs will be forwarded to the nim compiler. + +A **key-value** pair will become a conditional symbol + value (`-d:SYMBOL(:VAL)`) +for the nim compiler, e.g. for `-d:chronicles_timestamps="UnixTime"` the test +file requires: +``` +chronicles_timestamps="UnixTime" +``` +If only a key is given, an empty value will be forwarded. + +An **option** will be forwarded as is to the nim compiler, e.g. this can be +added in a test file: +``` +--opt:size +``` + +### Outputs +For outputs to be compared, the output string should be set to the output +name (stdout or filename) from within the "Output" section, e.g.: +``` +[Output] +stdout="""expected stdout output""" +file.log="""expected file output""" +``` + +Triple quotes can be used for multiple lines. diff --git a/testrunner.nim b/testrunner.nim new file mode 100644 index 0000000..a89cc33 --- /dev/null +++ b/testrunner.nim @@ -0,0 +1,248 @@ +import std/os +import std/osproc +import std/strutils +import std/terminal +import std/times +import std/pegs + +import testutils/config +import testutils/spec + +##[ + +The runner will look recursively for all *.test files at given path. A +test file should have at minimum a program name. This is the name of the +nim source minus the .nim extension) + +]## + + +# Code is here and there influenced by nim testament tester and unittest +# module. + +const + # defaultOptions = "--verbosity:1 --warnings:off --hint[Processing]:off " & + # "--hint[Conf]:off --hint[XDeclaredButNotUsed]:off " & + # "--hint[Link]:off --hint[Pattern]:off" + defaultOptions = "--verbosity:1 --warnings:off " + +type + TestStatus* = enum + OK + FAILED + SKIPPED + INVALID + + #[ + If needed, pass more info to the logresult via a TestResult object + TestResult = object + status: TestStatus + compileTime: float + fileSize: uint + ]# + + TestError* = enum + SourceFileNotFound + ExeFileNotFound + OutputFileNotFound + CompileError + RuntimeError + OutputsDiffer + FileSizeTooLarge + CompileErrorDiffers + +proc logFailure(test: TestSpec, error: TestError, data: varargs[string] = [""]) = + case error + of SourceFileNotFound: + styledEcho(fgYellow, styleBright, "source file not found: ", + resetStyle, test.program.addFileExt(".nim")) + of ExeFileNotFound: + styledEcho(fgYellow, styleBright, "file not found: ", + resetStyle, test.program.addFileExt(ExeExt)) + of OutputFileNotFound: + styledEcho(fgYellow, styleBright, "file not found: ", + resetStyle, data[0]) + of CompileError: + styledEcho(fgYellow, styleBright, "compile error:\p", + resetStyle, data[0]) + of RuntimeError: + styledEcho(fgYellow, styleBright, "runtime error:\p", + resetStyle, data[0]) + of OutputsDiffer: + styledEcho(fgYellow, styleBright, "outputs are different:\p", + resetStyle,"Expected output to $#:\p$#" % [data[0], data[1]], + "Resulted output to $#:\p$#" % [data[0], data[2]]) + of FileSizeTooLarge: + styledEcho(fgYellow, styleBright, "file size is too large: ", + resetStyle, data[0] & " > " & $test.maxSize) + of CompileErrorDiffers: + styledEcho(fgYellow, styleBright, "compile error is different:\p", + resetStyle, data[0]) + + styledEcho(fgCyan, styleBright, "command: ", resetStyle, + "nim c $#$#$#" % [defaultOptions, test.flags, + test.program.addFileExt(".nim")]) + +proc logResult(testName: string, status: TestStatus, time: float) = + var color = case status + of OK: fgGreen + of FAILED: fgRed + of SKIPPED: fgYellow + of INVALID: fgRed + styledEcho(styleBright, color, "[", $status, "] ", + resetStyle, testName, + fgYellow, " ", time.formatFloat(ffDecimal, 3), " s") + +template time(duration, body): untyped = + let t0 = epochTime() + block: + body + duration = epochTime() - t0 + +proc cmpOutputs(test: TestSpec, stdout: string): TestStatus = + result = OK + for output in test.outputs: + var testOutput: string + if output.name == "stdout": + testOutput = stdout + else: + if not existsFile(output.name): + logFailure(test, OutputFileNotFound, output.name) + result = FAILED + continue + + testOutput = readFile(output.name) + + # Would be nice to do a real diff here instead of simple compare + if test.timestampPeg.len > 0: + if not cmpIgnorePegs(testOutput, output.expectedOutput, peg(test.timestampPeg), pegXid): + logFailure(test, OutputsDiffer, output.name, output.expectedOutput, testOutput) + result = FAILED + else: + if not cmpIgnoreDefaultTimestamps(testOutput, output.expectedOutput): + logFailure(test, OutputsDiffer, output.name, output.expectedOutput, testOutput) + result = FAILED + + if output.name != "stdout": + removeFile(output.name) + +proc compile(test: TestSpec): TestStatus = + let + source = test.program.addFileExt(".nim") + cmd = "nim c $#$#$#" % [defaultOptions, test.flags, source.quoteShell] + c = parseCmdLine(cmd) + if not existsFile(source): + logFailure(test, SourceFileNotFound) + return FAILED + + var + p = startProcess(command=c[0], args=c[1.. ^1], + options={poStdErrToStdOut, poUsePath}) + defer: + close(p) + let + compileInfo = parseCompileStream(p, p.outputStream) + + if compileInfo.exitCode != 0: + if test.compileError.len == 0: + logFailure(test, CompileError, compileInfo.fullMsg) + return FAILED + else: + if test.compileError == compileInfo.msg and + (test.errorFile.len == 0 or test.errorFile == compileInfo.errorFile) and + (test.errorLine == 0 or test.errorLine == compileInfo.errorLine) and + (test.errorColumn == 0 or test.errorColumn == compileInfo.errorColumn): + return OK + else: + logFailure(test, CompileErrorDiffers, compileInfo.fullMsg) + return FAILED + + # Lets also check file size here as it kinda belongs to the compilation result + if test.maxSize != 0: + var size = getFileSize(test.program.addFileExt(ExeExt)) + if size > test.maxSize: + logFailure(test, FileSizeTooLarge, $size) + return FAILED + + return OK + +proc execute(test: TestSpec): TestStatus = + let program = test.program.addFileExt(ExeExt) + if not existsFile(program): + logFailure(test, ExeFileNotFound) + return FAILED + + let (output, exitCode) = execCmdEx(CurDir & DirSep & program.quoteShell) + + if exitCode != 0: + # parseExecuteOutput() # Need to parse the run time failures? + logFailure(test, RuntimeError, output) + return FAILED + else: + return test.cmpOutputs(output) + +proc scanTestPath(path: string): seq[string] = + if fileExists(path): + result.add path + else: + for file in walkDirRec path: + if file.endsWith ".test": + result.add file + +proc test(config: TestConfig, testPath: string): TestStatus = + var test: TestSpec + var duration: float + + time duration: + test = parseTestFile(testPath) + test.flags &= (if config.releaseBuild: "-d:release " else: "-d:debug ") + if not config.noThreads: + test.flags &= "--threads:on " + if test.program.len == 0: # a program name is bare minimum of a test file + result = INVALID + break + if test.skip or hostOS notin test.os or config.shouldSkip(test.name): + result = SKIPPED + break + + result = test.compile() + if result != OK or test.compileError.len > 0: + break + + result = test.execute() + try: + # this may fail in 64-bit AppVeyor images with "The process cannot access the file because it is being used by another process. [OSError]" + removeFile(test.program.addFileExt(ExeExt)) + except CatchableError as e: + echo e.msg + + logResult(test.name, result, duration) + +proc main() = + let + config = processArguments() + testFiles = scanTestPath(config.path) + var successful, skipped = 0 + + if testFiles.len == 0: + styledEcho(styleBright, "No test files found") + program_result = 1 + return + + for testFile in testFiles: + # Here we could do multithread or multiprocess + # but we will have to work with different nim caches per test + # and also the executables have to be in a unique location as several tests + # can use the same source + var result = test(config, testFile) + if result == OK: + successful += 1 + elif result == SKIPPED: + skipped += 1 + + styledEcho(styleBright, "Finished run: $#/$# tests successful" % + [$successful, $(testFiles.len - skipped)]) + program_result = testFiles.len - successful - skipped + +when isMainModule: + main() diff --git a/testutils.nimble b/testutils.nimble new file mode 100644 index 0000000..202521c --- /dev/null +++ b/testutils.nimble @@ -0,0 +1,16 @@ +mode = ScriptMode.Verbose + +packageName = "testutils" +version = "0.0.1" +author = "Status Research & Development GmbH" +description = "A unittest framework" +license = "Apache License 2.0" +skipDirs = @["tests"] + +requires "nim >= 1.0.2" +#requires "json_serialization" + +task test, "run CPU tests": + cd "tests" + exec "nim c -r testrunner ." + diff --git a/testutils/config.nim b/testutils/config.nim new file mode 100644 index 0000000..952229b --- /dev/null +++ b/testutils/config.nim @@ -0,0 +1,66 @@ +import std/parseopt +import std/strutils + +const + Usage = """ + + Usage: + testrunner [options] path + Run the test(s) specified at path. Will search recursively for test files + provided path is a directory. +Options: + --targets:"c c++ js objc" [Not implemented] Run tests for specified targets + --include:"test1 test2" Run only listed tests (space/comma seperated) + --exclude:"test1 test2" Skip listed tests (space/comma seperated) + --help Display this help and exit + + """.unindent.strip + +type + TestConfig* = object + path*: string + includedTests*: seq[string] + excludedTests*: seq[string] + releaseBuild*: bool + noThreads*: bool + +proc processArguments*(): TestConfig = + ## consume the arguments supplied to testrunner and yield a computed + ## configuration object + var + opt = initOptParser() + + for kind, key, value in opt.getOpt: + case kind + of cmdArgument: + if result.path == "": + result.path = key + of cmdLongOption, cmdShortOption: + case key.toLowerAscii + of "help", "h": + quit(Usage, QuitSuccess) + of "release": + result.releaseBuild = true + of "nothreads": + result.noThreads = true + of "targets", "t": + discard # not implemented + of "include": + result.includedTests.add value.split(Whitespace + {','}) + of "exclude": + result.excludedTests.add value.split(Whitespace + {','}) + else: + quit(Usage) + of cmdEnd: + quit(Usage) + + if result.path == "": + quit(Usage) + +func shouldSkip*(config: TestConfig, name: string): bool = + ## true if the named test should be skipped + if name in config.excludedTests: + result = true + elif config.includedTests.len > 0: + if name notin config.includedTests: + result = true diff --git a/testutils/helpers.nim b/testutils/helpers.nim new file mode 100644 index 0000000..6ef9eeb --- /dev/null +++ b/testutils/helpers.nim @@ -0,0 +1,103 @@ +import std/os +import std/osproc +import std/strutils +import std/streams +import std/pegs + +type + CompileInfo* = object + templFile*: string + errorFile*: string + errorLine*, errorColumn*: int + templLine*, templColumn*: int + msg*: string + fullMsg*: string + compileTime*: float + exitCode*: int + +let + # Error pegs, taken from testament tester + pegLineTemplate = + peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' 'template/generic instantiation from here'.*" + pegLineError = + peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' ('Error') ':' \s* {.*}" + pegOtherError = peg"'Error:' \s* {.*}" + pegError = pegLineError / pegOtherError + pegSuccess = peg"'Hint: operation successful' {[^;]*} '; ' {\d+} '.' {\d+} .*" + + # Timestamp pegs + # peg for unix timestamp, basically any float with 6 digits after the decimal + # Not ideal - could also improve by checking for the location in the line + pegUnixTimestamp = peg"{\d+} '.' {\d\d\d\d\d\d} \s" + # peg for timestamp with format yyyy-MM-dd HH:mm:sszzz + pegRfcTimestamp = peg"{\d\d\d\d} '-' {\d\d} '-' {\d\d} ' ' {\d\d} ':' {\d\d} ':' {\d\d} {'+' / '-'} {\d\d} ':' {\d\d} \s" + # Thread/process id is unpredictable.. + pegXid* = peg"""'tid' (('=') / ('":') / (': ') / (': ') / ('=') / ('>')) \d+""" + +proc cmpIgnorePegs*(a, b: string, pegs: varargs[Peg]): bool = + ## true when input strings are equal without regard to supplied pegs + var + aa = a + bb = b + for peg in pegs: + aa = aa.replace(peg, "dummy") + bb = bb.replace(peg, "dummy") + result = aa == bb + +proc cmpIgnoreTimestamp*(a, b: string, timestamp = ""): bool = + ## true when input strings are equal without regard to supplied timestamp form + if timestamp.len == 0: + result = cmpIgnorePegs(a, b, pegXid) + elif timestamp == "RfcTime": + result = cmpIgnorePegs(a, b, pegRfcTimestamp, pegXid) + elif timestamp == "UnixTime": + result = cmpIgnorePegs(a, b, pegUnixTimestamp, pegXid) + +proc cmpIgnoreDefaultTimestamps*(a, b: string): bool = + ## true when input strings are equal without regard to timestamp + if cmpIgnorePegs(a, b, pegRfcTimestamp, pegXid): + result = true + elif cmpIgnorePegs(a, b, pegUnixTimestamp, pegXid): + result = true + +proc parseCompileStream*(p: Process, output: Stream): CompileInfo = + ## parsing compiler output (based on testament tester) + result.exitCode = -1 + var + line = newStringOfCap(120).TaintedString + suc, err, tmpl = "" + + while true: + if output.readLine(line): + if line =~ pegError: + # `err` should contain the last error/warning message + err = line + elif line =~ pegLineTemplate and err == "": + # `tmpl` contains the last template expansion before the error + tmpl = line + elif line =~ pegSuccess: + suc = line + + if err != "": + result.fullMsg.add(line.string & "\p") + else: + result.exitCode = peekExitCode(p) + if result.exitCode != -1: + break + + if tmpl =~ pegLineTemplate: + result.templFile = extractFilename(matches[0]) + result.templLine = parseInt(matches[1]) + result.templColumn = parseInt(matches[2]) + if err =~ pegLineError: + result.errorFile = extractFilename(matches[0]) + result.errorLine = parseInt(matches[1]) + result.errorColumn = parseInt(matches[2]) + result.msg = matches[3] + elif err =~ pegOtherError: + result.msg = matches[0] + elif suc =~ pegSuccess: + result.msg = suc + result.compileTime = parseFloat(matches[1] & "." & matches[2]) + +proc parseExecuteOutput*() = discard diff --git a/testutils/spec.nim b/testutils/spec.nim new file mode 100644 index 0000000..0a4e8a9 --- /dev/null +++ b/testutils/spec.nim @@ -0,0 +1,97 @@ +import std/os +import std/parsecfg +import std/strutils +import std/streams + +const + DefaultOses = @["linux", "macosx", "windows"] + +type + TestSpec* = object + name*: string + skip*: bool + program*: string + flags*: string + outputs*: seq[tuple[name: string, expectedOutput: string]] + timestampPeg*: string + errorMsg*: string + maxSize*: int64 + compileError*: string + errorFile*: string + errorLine*: int + errorColumn*: int + os*: seq[string] + +proc defaults(spec: var TestSpec) = + ## assert some default values for a given spec + spec.os = DefaultOses + +proc consumeConfigEvent(spec: var TestSpec; event: CfgEvent) = + ## parse a specification supplied prior to any sections + case event.key + of "program": + spec.program = event.value + of "timestamp_peg": + spec.timestampPeg = event.value + of "max_size": + if event.value[0].isDigit: + spec.maxSize = parseInt(event.value) + else: + # XXX crash? + echo "Parsing warning: value of " & event.key & + " is not a number (value = " & event.value & ")." + of "compile_error": + spec.compileError = event.value + of "error_file": + spec.errorFile = event.value + of "os": + spec.os = event.value.normalize.split({','} + Whitespace) + else: + let + flag = "--define:$#:$#" % [event.key, event.value] + spec.flags.add flag.quoteShell & " " + +proc parseTestFile*(filePath: string): TestSpec = + ## parse a test input file into a spec + result.defaults + result.name = splitFile(filePath).name + block: + var + f = newFileStream(filePath, fmRead) + if f == nil: + # XXX crash? + echo "Parsing error: cannot open " & filePath + break + + var + outputSection = false + p: CfgParser + p.open(f, filePath) + try: + while true: + var e = next(p) + case e.kind + of cfgEof: + break + of cfgError: + # XXX crash? + echo "Parsing warning:" & e.msg + of cfgSectionStart: + if e.section.cmpIgnoreCase("Output") == 0: + outputSection = true + of cfgKeyValuePair: + if outputSection: + result.outputs.add((e.key, e.value)) + else: + result.consumeConfigEvent(e) + of cfgOption: + case e.key + of "skip": + result.skip = true + else: + result.flags &= ("--$#:$#" % [e.key, e.value]).quoteShell & " " + finally: + close p + if result.program == "": + # XXX crash? + echo "Parsing error: no program value"