From d41ed048244586c03ffbf44bd59a8db54f40c31c Mon Sep 17 00:00:00 2001 From: Andy Davidoff Date: Fri, 21 Feb 2020 16:14:14 -0500 Subject: [PATCH] initial --- .appveyor.yml | 39 +++++++ .editorconfig | 5 + .gitignore | 8 ++ .travis.yml | 26 +++++ README.md | 64 +++++++++++ testrunner.nim | 248 ++++++++++++++++++++++++++++++++++++++++++ testutils.nimble | 16 +++ testutils/config.nim | 66 +++++++++++ testutils/helpers.nim | 103 ++++++++++++++++++ testutils/spec.nim | 97 +++++++++++++++++ 10 files changed, 672 insertions(+) create mode 100644 .appveyor.yml create mode 100644 .editorconfig create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 README.md create mode 100644 testrunner.nim create mode 100644 testutils.nimble create mode 100644 testutils/config.nim create mode 100644 testutils/helpers.nim create mode 100644 testutils/spec.nim diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 0000000..880b8e1 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,39 @@ +version: '{build}' + +image: Visual Studio 2015 + +cache: + - NimBinaries + +matrix: + # We always want 32 and 64-bit compilation + fast_finish: false + +platform: + - x86 + - x64 + +# when multiple CI builds are queued, the tested commit needs to be in the last X commits cloned with "--depth X" +clone_depth: 10 + +install: + # use the newest versions documented here: https://www.appveyor.com/docs/windows-images-software/#mingw-msys-cygwin + - IF "%PLATFORM%" == "x86" SET PATH=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin;%PATH% + - IF "%PLATFORM%" == "x64" SET PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% + + # build nim from our own branch - this to avoid the day-to-day churn and + # regressions of the fast-paced Nim development while maintaining the + # flexibility to apply patches + - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh + - env MAKE="mingw32-make -j2" ARCH_OVERRIDE=%PLATFORM% bash build_nim.sh Nim csources dist/nimble NimBinaries + - SET PATH=%CD%\Nim\bin;%PATH% + +build_script: + - cd C:\projects\%APPVEYOR_PROJECT_SLUG% + - nimble install -y + +test_script: + - nimble test + +deploy: off + diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..6cc19ff --- /dev/null +++ b/.editorconfig @@ -0,0 +1,5 @@ +[*] +indent_style = space +insert_final_newline = true +indent_size = 2 +trim_trailing_whitespace = true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cc935b3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +# ignore all executable files +* +!*.* +!*/ +*.exe + +nimcache/ + diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..03fab11 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,26 @@ +language: c + +# https://docs.travis-ci.com/user/caching/ +cache: + directories: + - NimBinaries + +git: + # when multiple CI builds are queued, the tested commit needs to be in the last X commits cloned with "--depth X" + depth: 10 + +os: + - linux + - osx + +install: + # build nim from our own branch - this to avoid the day-to-day churn and + # regressions of the fast-paced Nim development while maintaining the + # flexibility to apply patches + - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh + - env MAKE="make -j2" bash build_nim.sh Nim csources dist/nimble NimBinaries + - export PATH=$PWD/Nim/bin:$PATH + +script: + - nimble install -y + - nimble test diff --git a/README.md b/README.md new file mode 100644 index 0000000..10eb105 --- /dev/null +++ b/README.md @@ -0,0 +1,64 @@ +# Testrunner +## Usage +Command syntax: +``` +testrunner [options] path +Run the test(s) specified at path. Will search recursively for test files +provided path is a directory. +Options: +--targets:"c c++ js objc" [Not implemented] Run tests for specified targets +--include:"test1 test2" Run only listed tests (space/comma seperated) +--exclude:"test1 test2" Skip listed tests (space/comma seperated) +--help Display this help and exit +``` + +The runner will look recursively for all `*.test` files at given path. + +## Test file options +The test files follow the configuration file syntax (similar as `.ini`), see also +[nim parsecfg module](https://nim-lang.org/docs/parsecfg.html). + +### Required +- **program**: A test file should have at minimum a program name. This is the name +of the nim source minus the `.nim` extension. + +### Optional +- **max_size**: To check the maximum size of the binary, in bytes. +- **timestamp_peg**: If you don't want to use the default timestamps, you can define +your own timestamp peg here. +- **compile_error**: When expecting a compilation failure, the error message that +should be expected. +- **error_file**: When expecting a compilation failure, the source file where the +error should occur. +- **os**: Space and/or comma separated list of operating systems for which the +test should be run. Defaults to `"linux, macosx, windows"`. Tests meant for a +different OS than the host will be marked as `SKIPPED`. +- **--skip**: This will simply skip the test (will not be marked as failure). + +### Forwarded Options +Any other options or key-value pairs will be forwarded to the nim compiler. + +A **key-value** pair will become a conditional symbol + value (`-d:SYMBOL(:VAL)`) +for the nim compiler, e.g. for `-d:chronicles_timestamps="UnixTime"` the test +file requires: +``` +chronicles_timestamps="UnixTime" +``` +If only a key is given, an empty value will be forwarded. + +An **option** will be forwarded as is to the nim compiler, e.g. this can be +added in a test file: +``` +--opt:size +``` + +### Outputs +For outputs to be compared, the output string should be set to the output +name (stdout or filename) from within the "Output" section, e.g.: +``` +[Output] +stdout="""expected stdout output""" +file.log="""expected file output""" +``` + +Triple quotes can be used for multiple lines. diff --git a/testrunner.nim b/testrunner.nim new file mode 100644 index 0000000..a89cc33 --- /dev/null +++ b/testrunner.nim @@ -0,0 +1,248 @@ +import std/os +import std/osproc +import std/strutils +import std/terminal +import std/times +import std/pegs + +import testutils/config +import testutils/spec + +##[ + +The runner will look recursively for all *.test files at given path. A +test file should have at minimum a program name. This is the name of the +nim source minus the .nim extension) + +]## + + +# Code is here and there influenced by nim testament tester and unittest +# module. + +const + # defaultOptions = "--verbosity:1 --warnings:off --hint[Processing]:off " & + # "--hint[Conf]:off --hint[XDeclaredButNotUsed]:off " & + # "--hint[Link]:off --hint[Pattern]:off" + defaultOptions = "--verbosity:1 --warnings:off " + +type + TestStatus* = enum + OK + FAILED + SKIPPED + INVALID + + #[ + If needed, pass more info to the logresult via a TestResult object + TestResult = object + status: TestStatus + compileTime: float + fileSize: uint + ]# + + TestError* = enum + SourceFileNotFound + ExeFileNotFound + OutputFileNotFound + CompileError + RuntimeError + OutputsDiffer + FileSizeTooLarge + CompileErrorDiffers + +proc logFailure(test: TestSpec, error: TestError, data: varargs[string] = [""]) = + case error + of SourceFileNotFound: + styledEcho(fgYellow, styleBright, "source file not found: ", + resetStyle, test.program.addFileExt(".nim")) + of ExeFileNotFound: + styledEcho(fgYellow, styleBright, "file not found: ", + resetStyle, test.program.addFileExt(ExeExt)) + of OutputFileNotFound: + styledEcho(fgYellow, styleBright, "file not found: ", + resetStyle, data[0]) + of CompileError: + styledEcho(fgYellow, styleBright, "compile error:\p", + resetStyle, data[0]) + of RuntimeError: + styledEcho(fgYellow, styleBright, "runtime error:\p", + resetStyle, data[0]) + of OutputsDiffer: + styledEcho(fgYellow, styleBright, "outputs are different:\p", + resetStyle,"Expected output to $#:\p$#" % [data[0], data[1]], + "Resulted output to $#:\p$#" % [data[0], data[2]]) + of FileSizeTooLarge: + styledEcho(fgYellow, styleBright, "file size is too large: ", + resetStyle, data[0] & " > " & $test.maxSize) + of CompileErrorDiffers: + styledEcho(fgYellow, styleBright, "compile error is different:\p", + resetStyle, data[0]) + + styledEcho(fgCyan, styleBright, "command: ", resetStyle, + "nim c $#$#$#" % [defaultOptions, test.flags, + test.program.addFileExt(".nim")]) + +proc logResult(testName: string, status: TestStatus, time: float) = + var color = case status + of OK: fgGreen + of FAILED: fgRed + of SKIPPED: fgYellow + of INVALID: fgRed + styledEcho(styleBright, color, "[", $status, "] ", + resetStyle, testName, + fgYellow, " ", time.formatFloat(ffDecimal, 3), " s") + +template time(duration, body): untyped = + let t0 = epochTime() + block: + body + duration = epochTime() - t0 + +proc cmpOutputs(test: TestSpec, stdout: string): TestStatus = + result = OK + for output in test.outputs: + var testOutput: string + if output.name == "stdout": + testOutput = stdout + else: + if not existsFile(output.name): + logFailure(test, OutputFileNotFound, output.name) + result = FAILED + continue + + testOutput = readFile(output.name) + + # Would be nice to do a real diff here instead of simple compare + if test.timestampPeg.len > 0: + if not cmpIgnorePegs(testOutput, output.expectedOutput, peg(test.timestampPeg), pegXid): + logFailure(test, OutputsDiffer, output.name, output.expectedOutput, testOutput) + result = FAILED + else: + if not cmpIgnoreDefaultTimestamps(testOutput, output.expectedOutput): + logFailure(test, OutputsDiffer, output.name, output.expectedOutput, testOutput) + result = FAILED + + if output.name != "stdout": + removeFile(output.name) + +proc compile(test: TestSpec): TestStatus = + let + source = test.program.addFileExt(".nim") + cmd = "nim c $#$#$#" % [defaultOptions, test.flags, source.quoteShell] + c = parseCmdLine(cmd) + if not existsFile(source): + logFailure(test, SourceFileNotFound) + return FAILED + + var + p = startProcess(command=c[0], args=c[1.. ^1], + options={poStdErrToStdOut, poUsePath}) + defer: + close(p) + let + compileInfo = parseCompileStream(p, p.outputStream) + + if compileInfo.exitCode != 0: + if test.compileError.len == 0: + logFailure(test, CompileError, compileInfo.fullMsg) + return FAILED + else: + if test.compileError == compileInfo.msg and + (test.errorFile.len == 0 or test.errorFile == compileInfo.errorFile) and + (test.errorLine == 0 or test.errorLine == compileInfo.errorLine) and + (test.errorColumn == 0 or test.errorColumn == compileInfo.errorColumn): + return OK + else: + logFailure(test, CompileErrorDiffers, compileInfo.fullMsg) + return FAILED + + # Lets also check file size here as it kinda belongs to the compilation result + if test.maxSize != 0: + var size = getFileSize(test.program.addFileExt(ExeExt)) + if size > test.maxSize: + logFailure(test, FileSizeTooLarge, $size) + return FAILED + + return OK + +proc execute(test: TestSpec): TestStatus = + let program = test.program.addFileExt(ExeExt) + if not existsFile(program): + logFailure(test, ExeFileNotFound) + return FAILED + + let (output, exitCode) = execCmdEx(CurDir & DirSep & program.quoteShell) + + if exitCode != 0: + # parseExecuteOutput() # Need to parse the run time failures? + logFailure(test, RuntimeError, output) + return FAILED + else: + return test.cmpOutputs(output) + +proc scanTestPath(path: string): seq[string] = + if fileExists(path): + result.add path + else: + for file in walkDirRec path: + if file.endsWith ".test": + result.add file + +proc test(config: TestConfig, testPath: string): TestStatus = + var test: TestSpec + var duration: float + + time duration: + test = parseTestFile(testPath) + test.flags &= (if config.releaseBuild: "-d:release " else: "-d:debug ") + if not config.noThreads: + test.flags &= "--threads:on " + if test.program.len == 0: # a program name is bare minimum of a test file + result = INVALID + break + if test.skip or hostOS notin test.os or config.shouldSkip(test.name): + result = SKIPPED + break + + result = test.compile() + if result != OK or test.compileError.len > 0: + break + + result = test.execute() + try: + # this may fail in 64-bit AppVeyor images with "The process cannot access the file because it is being used by another process. [OSError]" + removeFile(test.program.addFileExt(ExeExt)) + except CatchableError as e: + echo e.msg + + logResult(test.name, result, duration) + +proc main() = + let + config = processArguments() + testFiles = scanTestPath(config.path) + var successful, skipped = 0 + + if testFiles.len == 0: + styledEcho(styleBright, "No test files found") + program_result = 1 + return + + for testFile in testFiles: + # Here we could do multithread or multiprocess + # but we will have to work with different nim caches per test + # and also the executables have to be in a unique location as several tests + # can use the same source + var result = test(config, testFile) + if result == OK: + successful += 1 + elif result == SKIPPED: + skipped += 1 + + styledEcho(styleBright, "Finished run: $#/$# tests successful" % + [$successful, $(testFiles.len - skipped)]) + program_result = testFiles.len - successful - skipped + +when isMainModule: + main() diff --git a/testutils.nimble b/testutils.nimble new file mode 100644 index 0000000..202521c --- /dev/null +++ b/testutils.nimble @@ -0,0 +1,16 @@ +mode = ScriptMode.Verbose + +packageName = "testutils" +version = "0.0.1" +author = "Status Research & Development GmbH" +description = "A unittest framework" +license = "Apache License 2.0" +skipDirs = @["tests"] + +requires "nim >= 1.0.2" +#requires "json_serialization" + +task test, "run CPU tests": + cd "tests" + exec "nim c -r testrunner ." + diff --git a/testutils/config.nim b/testutils/config.nim new file mode 100644 index 0000000..952229b --- /dev/null +++ b/testutils/config.nim @@ -0,0 +1,66 @@ +import std/parseopt +import std/strutils + +const + Usage = """ + + Usage: + testrunner [options] path + Run the test(s) specified at path. Will search recursively for test files + provided path is a directory. +Options: + --targets:"c c++ js objc" [Not implemented] Run tests for specified targets + --include:"test1 test2" Run only listed tests (space/comma seperated) + --exclude:"test1 test2" Skip listed tests (space/comma seperated) + --help Display this help and exit + + """.unindent.strip + +type + TestConfig* = object + path*: string + includedTests*: seq[string] + excludedTests*: seq[string] + releaseBuild*: bool + noThreads*: bool + +proc processArguments*(): TestConfig = + ## consume the arguments supplied to testrunner and yield a computed + ## configuration object + var + opt = initOptParser() + + for kind, key, value in opt.getOpt: + case kind + of cmdArgument: + if result.path == "": + result.path = key + of cmdLongOption, cmdShortOption: + case key.toLowerAscii + of "help", "h": + quit(Usage, QuitSuccess) + of "release": + result.releaseBuild = true + of "nothreads": + result.noThreads = true + of "targets", "t": + discard # not implemented + of "include": + result.includedTests.add value.split(Whitespace + {','}) + of "exclude": + result.excludedTests.add value.split(Whitespace + {','}) + else: + quit(Usage) + of cmdEnd: + quit(Usage) + + if result.path == "": + quit(Usage) + +func shouldSkip*(config: TestConfig, name: string): bool = + ## true if the named test should be skipped + if name in config.excludedTests: + result = true + elif config.includedTests.len > 0: + if name notin config.includedTests: + result = true diff --git a/testutils/helpers.nim b/testutils/helpers.nim new file mode 100644 index 0000000..6ef9eeb --- /dev/null +++ b/testutils/helpers.nim @@ -0,0 +1,103 @@ +import std/os +import std/osproc +import std/strutils +import std/streams +import std/pegs + +type + CompileInfo* = object + templFile*: string + errorFile*: string + errorLine*, errorColumn*: int + templLine*, templColumn*: int + msg*: string + fullMsg*: string + compileTime*: float + exitCode*: int + +let + # Error pegs, taken from testament tester + pegLineTemplate = + peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' 'template/generic instantiation from here'.*" + pegLineError = + peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' ('Error') ':' \s* {.*}" + pegOtherError = peg"'Error:' \s* {.*}" + pegError = pegLineError / pegOtherError + pegSuccess = peg"'Hint: operation successful' {[^;]*} '; ' {\d+} '.' {\d+} .*" + + # Timestamp pegs + # peg for unix timestamp, basically any float with 6 digits after the decimal + # Not ideal - could also improve by checking for the location in the line + pegUnixTimestamp = peg"{\d+} '.' {\d\d\d\d\d\d} \s" + # peg for timestamp with format yyyy-MM-dd HH:mm:sszzz + pegRfcTimestamp = peg"{\d\d\d\d} '-' {\d\d} '-' {\d\d} ' ' {\d\d} ':' {\d\d} ':' {\d\d} {'+' / '-'} {\d\d} ':' {\d\d} \s" + # Thread/process id is unpredictable.. + pegXid* = peg"""'tid' (('=') / ('":') / (': ') / (': ') / ('=') / ('>')) \d+""" + +proc cmpIgnorePegs*(a, b: string, pegs: varargs[Peg]): bool = + ## true when input strings are equal without regard to supplied pegs + var + aa = a + bb = b + for peg in pegs: + aa = aa.replace(peg, "dummy") + bb = bb.replace(peg, "dummy") + result = aa == bb + +proc cmpIgnoreTimestamp*(a, b: string, timestamp = ""): bool = + ## true when input strings are equal without regard to supplied timestamp form + if timestamp.len == 0: + result = cmpIgnorePegs(a, b, pegXid) + elif timestamp == "RfcTime": + result = cmpIgnorePegs(a, b, pegRfcTimestamp, pegXid) + elif timestamp == "UnixTime": + result = cmpIgnorePegs(a, b, pegUnixTimestamp, pegXid) + +proc cmpIgnoreDefaultTimestamps*(a, b: string): bool = + ## true when input strings are equal without regard to timestamp + if cmpIgnorePegs(a, b, pegRfcTimestamp, pegXid): + result = true + elif cmpIgnorePegs(a, b, pegUnixTimestamp, pegXid): + result = true + +proc parseCompileStream*(p: Process, output: Stream): CompileInfo = + ## parsing compiler output (based on testament tester) + result.exitCode = -1 + var + line = newStringOfCap(120).TaintedString + suc, err, tmpl = "" + + while true: + if output.readLine(line): + if line =~ pegError: + # `err` should contain the last error/warning message + err = line + elif line =~ pegLineTemplate and err == "": + # `tmpl` contains the last template expansion before the error + tmpl = line + elif line =~ pegSuccess: + suc = line + + if err != "": + result.fullMsg.add(line.string & "\p") + else: + result.exitCode = peekExitCode(p) + if result.exitCode != -1: + break + + if tmpl =~ pegLineTemplate: + result.templFile = extractFilename(matches[0]) + result.templLine = parseInt(matches[1]) + result.templColumn = parseInt(matches[2]) + if err =~ pegLineError: + result.errorFile = extractFilename(matches[0]) + result.errorLine = parseInt(matches[1]) + result.errorColumn = parseInt(matches[2]) + result.msg = matches[3] + elif err =~ pegOtherError: + result.msg = matches[0] + elif suc =~ pegSuccess: + result.msg = suc + result.compileTime = parseFloat(matches[1] & "." & matches[2]) + +proc parseExecuteOutput*() = discard diff --git a/testutils/spec.nim b/testutils/spec.nim new file mode 100644 index 0000000..0a4e8a9 --- /dev/null +++ b/testutils/spec.nim @@ -0,0 +1,97 @@ +import std/os +import std/parsecfg +import std/strutils +import std/streams + +const + DefaultOses = @["linux", "macosx", "windows"] + +type + TestSpec* = object + name*: string + skip*: bool + program*: string + flags*: string + outputs*: seq[tuple[name: string, expectedOutput: string]] + timestampPeg*: string + errorMsg*: string + maxSize*: int64 + compileError*: string + errorFile*: string + errorLine*: int + errorColumn*: int + os*: seq[string] + +proc defaults(spec: var TestSpec) = + ## assert some default values for a given spec + spec.os = DefaultOses + +proc consumeConfigEvent(spec: var TestSpec; event: CfgEvent) = + ## parse a specification supplied prior to any sections + case event.key + of "program": + spec.program = event.value + of "timestamp_peg": + spec.timestampPeg = event.value + of "max_size": + if event.value[0].isDigit: + spec.maxSize = parseInt(event.value) + else: + # XXX crash? + echo "Parsing warning: value of " & event.key & + " is not a number (value = " & event.value & ")." + of "compile_error": + spec.compileError = event.value + of "error_file": + spec.errorFile = event.value + of "os": + spec.os = event.value.normalize.split({','} + Whitespace) + else: + let + flag = "--define:$#:$#" % [event.key, event.value] + spec.flags.add flag.quoteShell & " " + +proc parseTestFile*(filePath: string): TestSpec = + ## parse a test input file into a spec + result.defaults + result.name = splitFile(filePath).name + block: + var + f = newFileStream(filePath, fmRead) + if f == nil: + # XXX crash? + echo "Parsing error: cannot open " & filePath + break + + var + outputSection = false + p: CfgParser + p.open(f, filePath) + try: + while true: + var e = next(p) + case e.kind + of cfgEof: + break + of cfgError: + # XXX crash? + echo "Parsing warning:" & e.msg + of cfgSectionStart: + if e.section.cmpIgnoreCase("Output") == 0: + outputSection = true + of cfgKeyValuePair: + if outputSection: + result.outputs.add((e.key, e.value)) + else: + result.consumeConfigEvent(e) + of cfgOption: + case e.key + of "skip": + result.skip = true + else: + result.flags &= ("--$#:$#" % [e.key, e.value]).quoteShell & " " + finally: + close p + if result.program == "": + # XXX crash? + echo "Parsing error: no program value"