initial

.appveyor.yml

@@ -0,0 +1,39 @@
+version: '{build}'
+
+image: Visual Studio 2015
+
+cache:
+  - NimBinaries
+
+matrix:
+  # We always want 32 and 64-bit compilation
+  fast_finish: false
+
+platform:
+  - x86
+  - x64
+
+# when multiple CI builds are queued, the tested commit needs to be in the last X commits cloned with "--depth X"
+clone_depth: 10
+
+install:
+  # use the newest versions documented here: https://www.appveyor.com/docs/windows-images-software/#mingw-msys-cygwin
+  - IF "%PLATFORM%" == "x86" SET PATH=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin;%PATH%
+  - IF "%PLATFORM%" == "x64" SET PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH%
+
+  # build nim from our own branch - this to avoid the day-to-day churn and
+  # regressions of the fast-paced Nim development while maintaining the
+  # flexibility to apply patches
+  - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
+  - env MAKE="mingw32-make -j2" ARCH_OVERRIDE=%PLATFORM% bash build_nim.sh Nim csources dist/nimble NimBinaries
+  - SET PATH=%CD%\Nim\bin;%PATH%
+
+build_script:
+  - cd C:\projects\%APPVEYOR_PROJECT_SLUG%
+  - nimble install -y
+
+test_script:
+  - nimble test
+
+deploy: off
+

.editorconfig

@@ -0,0 +1,5 @@
+[*]
+indent_style = space
+insert_final_newline = true
+indent_size = 2
+trim_trailing_whitespace = true

.gitignore

@@ -0,0 +1,8 @@
+# ignore all executable files
+*
+!*.*
+!*/
+*.exe
+
+nimcache/
+

.travis.yml

@@ -0,0 +1,26 @@
+language: c
+
+# https://docs.travis-ci.com/user/caching/
+cache:
+  directories:
+    - NimBinaries
+
+git:
+  # when multiple CI builds are queued, the tested commit needs to be in the last X commits cloned with "--depth X"
+  depth: 10
+
+os:
+  - linux
+  - osx
+
+install:
+  # build nim from our own branch - this to avoid the day-to-day churn and
+  # regressions of the fast-paced Nim development while maintaining the
+  # flexibility to apply patches
+  - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
+  - env MAKE="make -j2" bash build_nim.sh Nim csources dist/nimble NimBinaries
+  - export PATH=$PWD/Nim/bin:$PATH
+
+script:
+  - nimble install -y
+  - nimble test

README.md

@@ -0,0 +1,64 @@
+# Testrunner
+## Usage
+Command syntax:
+```
+testrunner [options] path
+Run the test(s) specified at path. Will search recursively for test files
+provided path is a directory.
+Options:
+--targets:"c c++ js objc" [Not implemented] Run tests for specified targets
+--include:"test1 test2"   Run only listed tests (space/comma seperated)
+--exclude:"test1 test2"   Skip listed tests (space/comma seperated)
+--help                    Display this help and exit
+```
+
+The runner will look recursively for all `*.test` files at given path.
+
+## Test file options
+The test files follow the configuration file syntax (similar as `.ini`), see also
+[nim parsecfg module](https://nim-lang.org/docs/parsecfg.html).
+
+### Required
+- **program**: A test file should have at minimum a program name. This is the name
+of the nim source minus the `.nim` extension.
+
+### Optional
+- **max_size**: To check the maximum size of the binary, in bytes.
+- **timestamp_peg**: If you don't want to use the default timestamps, you can define
+your own timestamp peg here.
+- **compile_error**: When expecting a compilation failure, the error message that
+should be expected.
+- **error_file**: When expecting a compilation failure, the source file where the
+error should occur.
+- **os**: Space and/or comma separated list of operating systems for which the
+test should be run. Defaults to `"linux, macosx, windows"`. Tests meant for a
+different OS than the host will be marked as `SKIPPED`.
+- **--skip**: This will simply skip the test (will not be marked as failure).
+
+### Forwarded Options
+Any other options or key-value pairs will be forwarded to the nim compiler.
+
+A **key-value** pair will become a conditional symbol + value (`-d:SYMBOL(:VAL)`)
+for the nim compiler, e.g. for `-d:chronicles_timestamps="UnixTime"` the test
+file requires:
+```
+chronicles_timestamps="UnixTime"
+```
+If only a key is given, an empty value will be forwarded.
+
+An **option** will be forwarded as is to the nim compiler, e.g. this can be
+added in a test file:
+```
+--opt:size
+```
+
+### Outputs
+For outputs to be compared, the output string should be set to the output
+name (stdout or filename) from within the "Output" section, e.g.:
+```
+[Output]
+stdout="""expected stdout output"""
+file.log="""expected file output"""
+```
+
+Triple quotes can be used for multiple lines.

testrunner.nim

@@ -0,0 +1,248 @@
+import std/os
+import std/osproc
+import std/strutils
+import std/terminal
+import std/times
+import std/pegs
+
+import testutils/config
+import testutils/spec
+
+##[
+
+The runner will look recursively for all *.test files at given path. A
+test file should have at minimum a program name. This is the name of the
+nim source minus the .nim extension)
+
+]##
+
+
+# Code is here and there influenced by nim testament tester and unittest
+# module.
+
+const
+  # defaultOptions = "--verbosity:1 --warnings:off --hint[Processing]:off " &
+  #                  "--hint[Conf]:off --hint[XDeclaredButNotUsed]:off " &
+  #                  "--hint[Link]:off --hint[Pattern]:off"
+  defaultOptions = "--verbosity:1 --warnings:off "
+
+type
+  TestStatus* = enum
+    OK
+    FAILED
+    SKIPPED
+    INVALID
+
+  #[
+   If needed, pass more info to the logresult via a TestResult object
+   TestResult = object
+     status: TestStatus
+     compileTime: float
+     fileSize: uint
+  ]#
+
+  TestError* = enum
+    SourceFileNotFound
+    ExeFileNotFound
+    OutputFileNotFound
+    CompileError
+    RuntimeError
+    OutputsDiffer
+    FileSizeTooLarge
+    CompileErrorDiffers
+
+proc logFailure(test: TestSpec, error: TestError, data: varargs[string] = [""]) =
+  case error
+  of SourceFileNotFound:
+    styledEcho(fgYellow, styleBright, "source file not found: ",
+               resetStyle, test.program.addFileExt(".nim"))
+  of ExeFileNotFound:
+    styledEcho(fgYellow, styleBright, "file not found: ",
+               resetStyle, test.program.addFileExt(ExeExt))
+  of OutputFileNotFound:
+    styledEcho(fgYellow, styleBright, "file not found: ",
+               resetStyle, data[0])
+  of CompileError:
+    styledEcho(fgYellow, styleBright, "compile error:\p",
+               resetStyle, data[0])
+  of RuntimeError:
+    styledEcho(fgYellow, styleBright, "runtime error:\p",
+               resetStyle, data[0])
+  of OutputsDiffer:
+    styledEcho(fgYellow, styleBright, "outputs are different:\p",
+               resetStyle,"Expected output to $#:\p$#" % [data[0], data[1]],
+                          "Resulted output to $#:\p$#" % [data[0], data[2]])
+  of FileSizeTooLarge:
+    styledEcho(fgYellow, styleBright, "file size is too large: ",
+               resetStyle, data[0] & " > " & $test.maxSize)
+  of CompileErrorDiffers:
+    styledEcho(fgYellow, styleBright, "compile error is different:\p",
+               resetStyle, data[0])
+
+  styledEcho(fgCyan, styleBright, "command: ", resetStyle,
+             "nim c $#$#$#" % [defaultOptions, test.flags,
+                                 test.program.addFileExt(".nim")])
+
+proc logResult(testName: string, status: TestStatus, time: float) =
+  var color = case status
+              of OK: fgGreen
+              of FAILED: fgRed
+              of SKIPPED: fgYellow
+              of INVALID: fgRed
+  styledEcho(styleBright, color, "[", $status, "] ",
+             resetStyle, testName,
+             fgYellow, " ", time.formatFloat(ffDecimal, 3), " s")
+
+template time(duration, body): untyped =
+  let t0 = epochTime()
+  block:
+    body
+  duration =  epochTime() - t0
+
+proc cmpOutputs(test: TestSpec, stdout: string): TestStatus =
+  result = OK
+  for output in test.outputs:
+    var testOutput: string
+    if output.name == "stdout":
+      testOutput = stdout
+    else:
+      if not existsFile(output.name):
+        logFailure(test, OutputFileNotFound, output.name)
+        result = FAILED
+        continue
+
+      testOutput = readFile(output.name)
+
+    # Would be nice to do a real diff here instead of simple compare
+    if test.timestampPeg.len > 0:
+      if not cmpIgnorePegs(testOutput, output.expectedOutput, peg(test.timestampPeg), pegXid):
+        logFailure(test, OutputsDiffer, output.name, output.expectedOutput, testOutput)
+        result = FAILED
+    else:
+      if not cmpIgnoreDefaultTimestamps(testOutput, output.expectedOutput):
+        logFailure(test, OutputsDiffer, output.name, output.expectedOutput, testOutput)
+        result = FAILED
+
+    if output.name != "stdout":
+      removeFile(output.name)
+
+proc compile(test: TestSpec): TestStatus =
+  let
+    source = test.program.addFileExt(".nim")
+    cmd = "nim c $#$#$#" % [defaultOptions, test.flags, source.quoteShell]
+    c = parseCmdLine(cmd)
+  if not existsFile(source):
+    logFailure(test, SourceFileNotFound)
+    return FAILED
+
+  var
+    p = startProcess(command=c[0], args=c[1.. ^1],
+                     options={poStdErrToStdOut, poUsePath})
+  defer:
+    close(p)
+  let
+    compileInfo = parseCompileStream(p, p.outputStream)
+
+  if compileInfo.exitCode != 0:
+    if test.compileError.len == 0:
+      logFailure(test, CompileError, compileInfo.fullMsg)
+      return FAILED
+    else:
+      if test.compileError == compileInfo.msg and
+         (test.errorFile.len == 0 or test.errorFile == compileInfo.errorFile) and
+         (test.errorLine == 0 or test.errorLine == compileInfo.errorLine) and
+         (test.errorColumn == 0 or test.errorColumn == compileInfo.errorColumn):
+        return OK
+      else:
+        logFailure(test, CompileErrorDiffers, compileInfo.fullMsg)
+        return FAILED
+
+  # Lets also check file size here as it kinda belongs to the compilation result
+  if test.maxSize != 0:
+    var size = getFileSize(test.program.addFileExt(ExeExt))
+    if size > test.maxSize:
+      logFailure(test, FileSizeTooLarge, $size)
+      return FAILED
+
+  return OK
+
+proc execute(test: TestSpec): TestStatus =
+  let program = test.program.addFileExt(ExeExt)
+  if not existsFile(program):
+    logFailure(test, ExeFileNotFound)
+    return FAILED
+
+  let (output, exitCode) = execCmdEx(CurDir & DirSep & program.quoteShell)
+
+  if exitCode != 0:
+    # parseExecuteOutput() # Need to parse the run time failures?
+    logFailure(test, RuntimeError, output)
+    return FAILED
+  else:
+    return test.cmpOutputs(output)
+
+proc scanTestPath(path: string): seq[string] =
+  if fileExists(path):
+    result.add path
+  else:
+    for file in walkDirRec path:
+      if file.endsWith ".test":
+        result.add file
+
+proc test(config: TestConfig, testPath: string): TestStatus =
+  var test: TestSpec
+  var duration: float
+
+  time duration:
+    test = parseTestFile(testPath)
+    test.flags &= (if config.releaseBuild: "-d:release " else: "-d:debug ")
+    if not config.noThreads:
+      test.flags &= "--threads:on "
+    if test.program.len == 0: # a program name is bare minimum of a test file
+      result = INVALID
+      break
+    if test.skip or hostOS notin test.os or config.shouldSkip(test.name):
+      result = SKIPPED
+      break
+
+    result = test.compile()
+    if result != OK or test.compileError.len > 0:
+      break
+
+    result = test.execute()
+    try:
+      # this may fail in 64-bit AppVeyor images with "The process cannot access the file because it is being used by another process. [OSError]"
+      removeFile(test.program.addFileExt(ExeExt))
+    except CatchableError as e:
+      echo e.msg
+
+  logResult(test.name, result, duration)
+
+proc main() =
+  let
+    config = processArguments()
+    testFiles = scanTestPath(config.path)
+  var successful, skipped = 0
+
+  if testFiles.len == 0:
+    styledEcho(styleBright, "No test files found")
+    program_result = 1
+    return
+
+  for testFile in testFiles:
+    # Here we could do multithread or multiprocess
+    # but we will have to work with different nim caches per test
+    # and also the executables have to be in a unique location as several tests
+    # can use the same source
+    var result = test(config, testFile)
+    if result == OK:
+      successful += 1
+    elif result == SKIPPED:
+      skipped += 1
+
+  styledEcho(styleBright, "Finished run: $#/$# tests successful" %
+                          [$successful, $(testFiles.len - skipped)])
+  program_result = testFiles.len - successful - skipped
+
+when isMainModule:
+  main()

testutils.nimble

@@ -0,0 +1,16 @@
+mode = ScriptMode.Verbose
+
+packageName   = "testutils"
+version       = "0.0.1"
+author        = "Status Research & Development GmbH"
+description   = "A unittest framework"
+license       = "Apache License 2.0"
+skipDirs      = @["tests"]
+
+requires "nim >= 1.0.2"
+#requires "json_serialization"
+
+task test, "run CPU tests":
+  cd "tests"
+  exec "nim c -r testrunner ."
+

testutils/config.nim

@@ -0,0 +1,66 @@
+import std/parseopt
+import std/strutils
+
+const
+  Usage = """
+
+  Usage:
+  testrunner [options] path
+  Run the test(s) specified at path. Will search recursively for test files
+  provided path is a directory.
+Options:
+  --targets:"c c++ js objc" [Not implemented] Run tests for specified targets
+  --include:"test1 test2"   Run only listed tests (space/comma seperated)
+  --exclude:"test1 test2"   Skip listed tests (space/comma seperated)
+  --help                    Display this help and exit
+
+  """.unindent.strip
+
+type
+  TestConfig* = object
+    path*: string
+    includedTests*: seq[string]
+    excludedTests*: seq[string]
+    releaseBuild*: bool
+    noThreads*: bool
+
+proc processArguments*(): TestConfig =
+  ## consume the arguments supplied to testrunner and yield a computed
+  ## configuration object
+  var
+    opt = initOptParser()
+
+  for kind, key, value in opt.getOpt:
+    case kind
+    of cmdArgument:
+      if result.path == "":
+        result.path = key
+    of cmdLongOption, cmdShortOption:
+      case key.toLowerAscii
+      of "help", "h":
+        quit(Usage, QuitSuccess)
+      of "release":
+        result.releaseBuild = true
+      of "nothreads":
+        result.noThreads = true
+      of "targets", "t":
+        discard # not implemented
+      of "include":
+        result.includedTests.add value.split(Whitespace + {','})
+      of "exclude":
+        result.excludedTests.add value.split(Whitespace + {','})
+      else:
+        quit(Usage)
+    of cmdEnd:
+      quit(Usage)
+
+  if result.path == "":
+    quit(Usage)
+
+func shouldSkip*(config: TestConfig, name: string): bool =
+  ## true if the named test should be skipped
+  if name in config.excludedTests:
+    result = true
+  elif config.includedTests.len > 0:
+    if name notin config.includedTests:
+      result = true

testutils/helpers.nim

@@ -0,0 +1,103 @@
+import std/os
+import std/osproc
+import std/strutils
+import std/streams
+import std/pegs
+
+type
+  CompileInfo* = object
+    templFile*: string
+    errorFile*: string
+    errorLine*, errorColumn*: int
+    templLine*, templColumn*: int
+    msg*: string
+    fullMsg*: string
+    compileTime*: float
+    exitCode*: int
+
+let
+  # Error pegs, taken from testament tester
+  pegLineTemplate =
+    peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' 'template/generic instantiation from here'.*"
+  pegLineError =
+    peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' ('Error') ':' \s* {.*}"
+  pegOtherError = peg"'Error:' \s* {.*}"
+  pegError = pegLineError / pegOtherError
+  pegSuccess = peg"'Hint: operation successful' {[^;]*} '; '  {\d+} '.' {\d+} .*"
+
+  # Timestamp pegs
+  # peg for unix timestamp, basically any float with 6 digits after the decimal
+  # Not ideal - could also improve by checking for the location in the line
+  pegUnixTimestamp = peg"{\d+} '.' {\d\d\d\d\d\d} \s"
+  # peg for timestamp with format yyyy-MM-dd HH:mm:sszzz
+  pegRfcTimestamp = peg"{\d\d\d\d} '-' {\d\d} '-' {\d\d} ' ' {\d\d} ':' {\d\d} ':' {\d\d} {'+' / '-'} {\d\d} ':' {\d\d} \s"
+  # Thread/process id is unpredictable..
+  pegXid* = peg"""'tid' (('=') / ('":') / (': ') / (': ') / ('=') / ('>')) \d+"""
+
+proc cmpIgnorePegs*(a, b: string, pegs: varargs[Peg]): bool =
+  ## true when input strings are equal without regard to supplied pegs
+  var
+    aa = a
+    bb = b
+  for peg in pegs:
+    aa = aa.replace(peg, "dummy")
+    bb = bb.replace(peg, "dummy")
+  result = aa == bb
+
+proc cmpIgnoreTimestamp*(a, b: string, timestamp = ""): bool =
+  ## true when input strings are equal without regard to supplied timestamp form
+  if timestamp.len == 0:
+    result = cmpIgnorePegs(a, b, pegXid)
+  elif timestamp == "RfcTime":
+    result = cmpIgnorePegs(a, b, pegRfcTimestamp, pegXid)
+  elif timestamp == "UnixTime":
+    result = cmpIgnorePegs(a, b, pegUnixTimestamp, pegXid)
+
+proc cmpIgnoreDefaultTimestamps*(a, b: string): bool =
+  ## true when input strings are equal without regard to timestamp
+  if cmpIgnorePegs(a, b, pegRfcTimestamp, pegXid):
+    result = true
+  elif cmpIgnorePegs(a, b, pegUnixTimestamp, pegXid):
+    result = true
+
+proc parseCompileStream*(p: Process, output: Stream): CompileInfo =
+  ## parsing compiler output (based on testament tester)
+  result.exitCode = -1
+  var
+    line = newStringOfCap(120).TaintedString
+    suc, err, tmpl = ""
+
+  while true:
+    if output.readLine(line):
+      if line =~ pegError:
+       # `err` should contain the last error/warning message
+       err = line
+      elif line =~ pegLineTemplate and err == "":
+       # `tmpl` contains the last template expansion before the error
+       tmpl = line
+      elif line =~ pegSuccess:
+       suc = line
+
+      if err != "":
+       result.fullMsg.add(line.string & "\p")
+    else:
+     result.exitCode = peekExitCode(p)
+     if result.exitCode != -1:
+       break
+
+  if tmpl =~ pegLineTemplate:
+    result.templFile = extractFilename(matches[0])
+    result.templLine = parseInt(matches[1])
+    result.templColumn = parseInt(matches[2])
+  if err =~ pegLineError:
+    result.errorFile = extractFilename(matches[0])
+    result.errorLine = parseInt(matches[1])
+    result.errorColumn = parseInt(matches[2])
+    result.msg = matches[3]
+  elif err =~ pegOtherError:
+    result.msg = matches[0]
+  elif suc =~ pegSuccess:
+    result.msg = suc
+    result.compileTime = parseFloat(matches[1] & "." & matches[2])
+
+proc parseExecuteOutput*() = discard

testutils/spec.nim

@@ -0,0 +1,97 @@
+import std/os
+import std/parsecfg
+import std/strutils
+import std/streams
+
+const
+  DefaultOses = @["linux", "macosx", "windows"]
+
+type
+  TestSpec* = object
+    name*: string
+    skip*: bool
+    program*: string
+    flags*: string
+    outputs*: seq[tuple[name: string, expectedOutput: string]]
+    timestampPeg*: string
+    errorMsg*: string
+    maxSize*: int64
+    compileError*: string
+    errorFile*: string
+    errorLine*: int
+    errorColumn*: int
+    os*: seq[string]
+
+proc defaults(spec: var TestSpec) =
+  ## assert some default values for a given spec
+  spec.os = DefaultOses
+
+proc consumeConfigEvent(spec: var TestSpec; event: CfgEvent) =
+  ## parse a specification supplied prior to any sections
+  case event.key
+  of "program":
+    spec.program = event.value
+  of "timestamp_peg":
+    spec.timestampPeg = event.value
+  of "max_size":
+    if event.value[0].isDigit:
+      spec.maxSize = parseInt(event.value)
+    else:
+      # XXX crash?
+      echo "Parsing warning: value of " & event.key &
+           " is not a number (value = " & event.value & ")."
+  of "compile_error":
+    spec.compileError = event.value
+  of "error_file":
+    spec.errorFile = event.value
+  of "os":
+    spec.os = event.value.normalize.split({','} + Whitespace)
+  else:
+    let
+      flag = "--define:$#:$#" % [event.key, event.value]
+    spec.flags.add flag.quoteShell & " "
+
+proc parseTestFile*(filePath: string): TestSpec =
+  ## parse a test input file into a spec
+  result.defaults
+  result.name = splitFile(filePath).name
+  block:
+    var
+      f = newFileStream(filePath, fmRead)
+    if f == nil:
+      # XXX crash?
+      echo "Parsing error: cannot open " & filePath
+      break
+
+    var
+      outputSection = false
+      p: CfgParser
+    p.open(f, filePath)
+    try:
+      while true:
+        var e = next(p)
+        case e.kind
+        of cfgEof:
+          break
+        of cfgError:
+          # XXX crash?
+          echo "Parsing warning:" & e.msg
+        of cfgSectionStart:
+          if e.section.cmpIgnoreCase("Output") == 0:
+            outputSection = true
+        of cfgKeyValuePair:
+          if outputSection:
+            result.outputs.add((e.key, e.value))
+          else:
+            result.consumeConfigEvent(e)
+        of cfgOption:
+          case e.key
+          of "skip":
+            result.skip = true
+          else:
+            result.flags &= ("--$#:$#" % [e.key, e.value]).quoteShell & " "
+    finally:
+      close p
+    if result.program == "":
+      # XXX crash?
+      echo "Parsing error: no program value"