From 45f2d5d84a9ffd1bf7e62f49e7a7c5fe042aa4fe Mon Sep 17 00:00:00 2001
From: Jacek Sieka <jacek@status.im>
Date: Thu, 30 Dec 2021 13:33:27 +0100
Subject: [PATCH 1/2] bump cppsnappy, split out benchmark

* benchmark both compression and decompression
* bump C++-snappy: it's gotten better over the years

```
fastStreams,       openArrays,       nimStreams,           cppLib,      Samples,         Size,         Test
  0.281 /  0.139,   0.295 /  0.220,   0.528 /  0.238,   0.140 /  0.047,          100,       102400, html
  2.697 /  1.457,   3.263 /  1.993,   6.106 /  2.026,   1.655 /  0.530,          100,       702087, urls.10K
  0.021 /  0.011,   0.017 /  0.011,   0.031 /  0.011,   0.014 /  0.009,          100,       123093, fireworks.jpeg
  0.057 /  0.019,   0.051 /  0.036,   0.092 /  0.042,   0.021 /  0.011,          100,       102400, paper-100k.pdf
  1.153 /  0.565,   1.210 /  0.904,   2.172 /  0.985,   0.575 /  0.184,          100,       409600, html_x_4
  0.887 /  0.582,   0.976 /  0.743,   1.923 /  0.712,   0.536 /  0.202,          100,       152089, alice29.txt
  0.771 /  0.504,   0.850 /  0.639,   1.678 /  0.606,   0.482 /  0.184,          100,       129301, asyoulik.txt
  2.349 /  1.518,   2.553 /  1.946,   5.057 /  1.913,   1.391 /  0.522,          100,       426754, lcet10.txt
  2.983 /  1.951,   3.241 /  2.408,   6.680 /  2.342,   1.882 /  0.735,          100,       481861, plrabn12.txt
  0.293 /  0.120,   0.306 /  0.210,   0.542 /  0.241,   0.131 /  0.042,          100,       118588, geo.protodata
  0.743 /  0.501,   0.738 /  0.620,   1.597 /  0.640,   0.414 /  0.191,          100,       184320, kppkn.gtb
  0.087 /  0.054,   0.102 /  0.067,   0.193 /  0.061,   0.052 /  0.022,          100,        14564, Mark.Twain-Tom.Sawyer.txt
 66.886 / 20.880, 105.393 / 37.907, 210.197 / 38.867,  34.273 / 10.193,           10,     38942424, state-2560000-114a593d-0d5e08e8.ssz
```

In general, we're consistently about 2x slower than C++ right now.
---
 tests/benchmark.nim      | 112 +++++++++++++++++++
 tests/build_snappycpp.sh |   9 ++
 tests/cpp_snappy.nim     |  46 ++++++++
 tests/snappycpp          |   2 +-
 tests/test_codec.nim     | 228 +++++++++------------------------------
 5 files changed, 221 insertions(+), 176 deletions(-)
 create mode 100644 tests/benchmark.nim
 create mode 100755 tests/build_snappycpp.sh
 create mode 100644 tests/cpp_snappy.nim

diff --git a/tests/benchmark.nim b/tests/benchmark.nim
new file mode 100644
index 0000000..7c7d861
--- /dev/null
+++ b/tests/benchmark.nim
@@ -0,0 +1,112 @@
+{.used.}
+
+import
+  os, strformat, stats, times,
+  snappy, openarrays_snappy, nimstreams_snappy, cpp_snappy
+
+const
+  currentDir = currentSourcePath.parentDir
+  dataDir = currentDir & DirSep & "data" & DirSep
+
+type
+  TestTimes = object
+    fastStreams: array[2, RunningStat]
+    openArrays: array[2, RunningStat]
+    nimStreams: array[2, RunningStat]
+    cppLib: array[2, RunningStat]
+    size: int
+
+template timeit(timerVar: var RunningStat, code: untyped) =
+  let t0 = cpuTime()
+  code
+  timerVar.push cpuTime() - t0
+
+var printedHeader = false
+
+proc printTimes(t: TestTimes, name: string) =
+  func f(t: array[2, RunningStat]): string =
+    &"{t[0].mean * 1000 :>7.3f} /{t[1].mean * 1000 :>7.3f}, "
+
+  if not printedHeader:
+    printedHeader = true
+    echo &"{\"fastStreams\" :>16}, {\"openArrays\" :>16}, {\"nimStreams\" :>16}, " &
+      &"{\"cppLib\" :>16}, {\"Samples\" :>12}, {\"Size\" :>12}, {\"Test\" :>12}"
+
+  echo f(t.fastStreams),
+    f(t.openArrays),
+    f(t.nimStreams),
+    f(t.cppLib),
+    &"{t.fastStreams[0].n :>12}, ",
+    &"{t.size :>12}, ",
+    name
+
+proc readSource(sourceName: string): seq[byte] =
+  var f = open(sourceName, fmRead)
+  if f.isNil: return
+  let size = f.getFileSize()
+  result = newSeq[byte](size)
+  doAssert(size == f.readBytes(result, 0, size))
+  f.close()
+
+proc timedRoundTrip(msg: string, source: openarray[byte], iterations = 100) =
+  when declared(GC_fullCollect):
+    GC_fullCollect()
+
+  var timers: TestTimes
+  timers.size = source.len()
+
+  for i in 0..<iterations:
+    timeit(timers.fastStreams[0]):
+      let encodedWithFastStreams = snappy.encode(source)
+
+    timeit(timers.fastStreams[1]):
+      var decodedWithFastStreams = snappy.decode(encodedWithFastStreams)
+
+    timeit(timers.nimStreams[0]):
+      var encodedWithNimStreams = nimstreams_snappy.encode(source)
+
+    timeit(timers.nimStreams[1]):
+      var decodedWithNimStreams = nimstreams_snappy.decode(encodedWithNimStreams)
+
+    timeit(timers.openArrays[0]):
+      var encodedWithOpenArrays = openarrays_snappy.encode(source)
+
+    timeit(timers.openArrays[1]):
+      var decodedWithOpenArrays = openarrays_snappy.decode(encodedWithOpenArrays)
+
+    timeit(timers.cppLib[0]):
+      var encodedWithCpp = cpp_snappy.encode(source)
+
+    timeit(timers.cppLib[1]):
+      var decodedWithCpp = cpp_snappy.decode(encodedWithCpp)
+
+    doAssert decodedWithFastStreams == source
+    doAssert decodedWithNimStreams == source
+    doAssert decodedWithOpenArrays == source
+    doAssert decodedWithCpp == source
+
+  printTimes(timers, msg)
+
+proc roundTrip(msg: string, source: openArray[byte], iterations = 100) =
+  timedRoundTrip(msg, source, iterations)
+
+proc roundTrip(sourceName: string, iterations = 100) =
+  var src = readSource(sourceName)
+  roundTrip(extractFilename(sourceName), src, iterations)
+
+roundTrip(dataDir & "html")
+roundTrip(dataDir & "urls.10K")
+roundTrip(dataDir & "fireworks.jpeg")
+roundTrip(dataDir & "paper-100k.pdf")
+roundTrip(dataDir & "html_x_4")
+roundTrip(dataDir & "alice29.txt")
+roundTrip(dataDir & "asyoulik.txt")
+roundTrip(dataDir & "lcet10.txt")
+roundTrip(dataDir & "plrabn12.txt")
+roundTrip(dataDir & "geo.protodata")
+roundTrip(dataDir & "kppkn.gtb")
+roundTrip(dataDir & "Mark.Twain-Tom.Sawyer.txt")
+
+# ncli_db --db:db dumpState 0x114a593d248af2ad05580299b803657d4b78a3b6578f47425cc396c9644e800e 2560000
+if fileExists(dataDir & "state-2560000-114a593d-0d5e08e8.ssz"):
+  roundTrip(dataDir & "state-2560000-114a593d-0d5e08e8.ssz", 10)
diff --git a/tests/build_snappycpp.sh b/tests/build_snappycpp.sh
new file mode 100755
index 0000000..b087c14
--- /dev/null
+++ b/tests/build_snappycpp.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+cd snappycpp
+mkdir -p build
+cd build
+cmake -DCMAKE_BUILD_TYPE=Release ../
+
+make snappy
+cp libsnappy.a ../..
diff --git a/tests/cpp_snappy.nim b/tests/cpp_snappy.nim
new file mode 100644
index 0000000..68b42e3
--- /dev/null
+++ b/tests/cpp_snappy.nim
@@ -0,0 +1,46 @@
+import std/os
+
+const
+  currentDir = currentSourcePath.parentDir
+
+{.passl: "-lsnappy -L\"" & currentDir & "\" -lstdc++".}
+
+proc snappy_compress*(input: cstring, input_length: csize_t, compressed: ptr cchar, compressed_length: var csize_t): cint {.importc, cdecl.}
+proc snappy_uncompress*(compressed: cstring, compressed_length: csize_t, uncompressed: ptr cchar, uncompressed_length: var csize_t): cint {.importc, cdecl.}
+proc snappy_max_compressed_length*(source_length: csize_t): csize_t {.importc, cdecl.}
+proc snappy_uncompressed_length*(compressed: cstring, compressed_length: csize_t, res: var csize_t): cint {.importc, cdecl.}
+
+proc encode*(input: openArray[byte]): seq[byte] =
+  result = newSeqUninitialized[byte](
+    snappy_max_compressed_length(input.len.csize_t))
+  var bytes = result.len.csize_t
+  let res = if input.len() == 0:
+    snappy_compress(nil, 0, cast[ptr cchar](result[0].addr), bytes)
+  else:
+    snappy_compress(
+      cast[cstring](unsafeAddr input[0]), input.len().csize_t,
+      cast[ptr cchar](result[0].addr), bytes)
+  if res != 0:
+    raise (ref ValueError)(msg: "Cannot compress")
+
+  result.setLen(bytes.int)
+
+proc decode*(input: openArray[byte]): seq[byte] =
+  if input.len() == 0:
+    raise (ref ValueError)(msg: "empty input")
+
+  var bytes: csize_t
+  if snappy_uncompressed_length(
+    cast[cstring](input[0].unsafeAddr), input.len.csize_t, bytes) != 0:
+    raise (ref ValueError)(msg: "Cannot get length")
+
+  if bytes == 0:
+    return
+
+  result = newSeqUninitialized[byte](bytes.int)
+
+
+  if snappy_uncompress(
+      cast[cstring](unsafeAddr input[0]), input.len().csize_t,
+      cast[ptr cchar](result[0].addr), bytes) != 0:
+    raise (ref ValueError)(msg: "Cannot compress")
diff --git a/tests/snappycpp b/tests/snappycpp
index 837f38b..65dc7b3 160000
--- a/tests/snappycpp
+++ b/tests/snappycpp
@@ -1 +1 @@
-Subproject commit 837f38b3e0cae01ccd5021b176509cb72eb25d81
+Subproject commit 65dc7b383985eb4f63cd3e752136db8d9b4be8c0
diff --git a/tests/test_codec.nim b/tests/test_codec.nim
index 91e1e15..419cea1 100644
--- a/tests/test_codec.nim
+++ b/tests/test_codec.nim
@@ -3,41 +3,13 @@
 import
   os, unittest, terminal, strutils,
   faststreams,
-  snappy, randgen, openarrays_snappy, nimstreams_snappy
+  snappy, randgen, openarrays_snappy, nimstreams_snappy, cpp_snappy
 
 include system/timers
 
 const
   currentDir = currentSourcePath.parentDir
-
-{.passl: "-lsnappy -L\"" & currentDir & "\" -lstdc++".}
-
-type
-  TestTimes = object
-    fastStreams: int
-    appendSnappyBytes: int
-    openArrays: int
-    nimStreams: int
-    cppLib: int
-
-template timeit(timerVar: var Nanos, code: untyped) =
-  let t0 = getTicks()
-  code
-  timerVar = int(getTicks() - t0) div 1000000
-
-proc printTimes(t: TestTimes) =
-  styledEcho "  cpu time [OpenArrays]: ", styleBright, $t.openArrays, "ms"
-  styledEcho "  cpu time [FastStream]: ", styleBright, $t.fastStreams, "ms"
-  styledEcho "  cpu time [NimStreams]: ", styleBright, $t.nimStreams, "ms"
-  styledEcho "  cpu time [C++ Snappy]: ", styleBright, $t.cppLib, "ms"
-
-proc snappy_compress(input: cstring, input_length: csize_t, compressed: cstring, compressed_length: var csize_t): cint {.importc, cdecl.}
-proc snappy_uncompress(compressed: cstring, compressed_length: csize_t, uncompressed: cstring, uncompressed_length: var csize_t): cint {.importc, cdecl.}
-proc snappy_max_compressed_length(source_length: csize_t): csize_t {.importc, cdecl.}
-proc snappy_uncompressed_length(compressed: cstring, compressed_length: csize_t, res: var csize_t): cint {.importc, cdecl.}
-
-const
-  testDataDir = "data" & DirSep
+  dataDir = currentDir & DirSep & "data" & DirSep
 
 let
   empty: seq[byte] = @[]
@@ -51,122 +23,52 @@ proc readSource(sourceName: string): seq[byte] =
   doAssert(size == f.readBytes(result, 0, size))
   f.close()
 
-proc timedRoundTrip(msg: string, source: openarray[byte]): (bool, TestTimes) =
-  var timers: TestTimes
-  timeit(timers.fastStreams):
-    var encodedWithFastStreams = snappy.encode(source)
+proc roundTrip(msg: string, source: openarray[byte]) =
+  var encodedWithFastStreams = snappy.encode(source)
+  var encodedWithAppendSnappyBytes = block:
+    let output = memoryOutput()
+    snappy.appendSnappyBytes(output, source)
+    output.getOutput
 
-  timeit(timers.appendSnappyBytes):
-    var encodedWithAppendSnappyBytes = block:
-      let output = memoryOutput()
-      snappy.appendSnappyBytes(output, source)
-      output.getOutput
+  var encodedWithNimStreams = nimstreams_snappy.encode(source)
+  var encodedWithOpenArrays = openarrays_snappy.encode(source)
 
-  timeit(timers.nimStreams):
-    var encodedWithNimStreams = nimstreams_snappy.encode(source)
+  # Test that everything can decode with C++ snappy - there may be minor
+  # differences in encoding however!
+  check:
+    snappy.decode(encodedWithFastStreams) == source
+    cpp_snappy.decode(encodedWithFastStreams) == source
 
-  timeit(timers.openArrays):
-    var encodedWithOpenArrays = openarrays_snappy.encode(source)
+    snappy.decode(encodedWithAppendSnappyBytes) == source
+    cpp_snappy.decode(encodedWithAppendSnappyBytes) == source
 
-  var
-    encodedWithCpp = newString(snappy_max_compressed_length(source.len.csize_t))
-    outputSize = csize_t encodedWithCpp.len
+    nimstreams_snappy.decode(encodedWithNimStreams) == source
+    cpp_snappy.decode(encodedWithNimStreams) == source
 
-  timeit(timers.cppLib):
-    var success = if source.len > 0:
-      snappy_compress(cast[cstring](source[0].unsafeAddr), source.len.csize_t, encodedWithCpp[0].addr, outputSize)
-    else:
-      snappy_compress(cast[cstring](0), source.len.csize_t, encodedWithCpp[0].addr, outputSize)
+    openarrays_snappy.decode(encodedWithOpenArrays) == source
+    cpp_snappy.decode(encodedWithOpenArrays) == source
 
-  var ok = success == 0
-  if not ok: echo "cpp_compress failed"
-
-  ok = outputSize == encodedWithOpenArrays.len.csize_t
-  if not ok: echo "cpp output size and nim output size differ"
-
-  if ok:
-    ok = equalMem(encodedWithOpenArrays[0].addr, encodedWithCpp[0].addr, outputSize.int)
-    if not ok: echo "cpp output and nim output differ"
-
-  if ok:
-    ok = encodedWithOpenArrays == encodedWithFastStreams
-    if not ok:
-      echo "OpenArray and FastStreams implementations disagree"
-
-  if ok:
-    ok = encodedWithOpenArrays == encodedWithAppendSnappyBytes
-    if not ok:
-      echo "OpenArray and AppendSnappyBytes implementations disagree"
-
-  if ok:
-    ok = encodedWithOpenArrays == encodedWithNimStreams
-    if not ok:
-      echo "OpenArray and NimStreams implementations disagree"
-
-  if ok:
-    ok = snappy.decode(encodedWithOpenArrays) == source
-    if not ok: echo "roundtrip failure"
-
-  if ok:
-    stdout.styledWriteLine("  ", msg, "...", fgGreen, "[PASS]")
-  else:
-    stdout.styledWriteLine("  ", msg, "...", fgRed, "[FAILED]")
-
-  (ok, timers)
-
-proc roundTrip(msg: string, source: openArray[byte]): bool =
-  timedRoundTrip(msg, source)[0]
-
-proc roundTrip(msg: string, sourceName: string): bool =
-  var src = readSource(sourceName)
-  roundTrip(msg, src)
-
-proc timedRoundTrip(msg: string, sourceName: string): auto =
-  var src = readSource(sourceName)
-  timedRoundTrip(msg, src)
-
-proc roundTripRev(msg: string, source: openArray[byte]): bool =
+proc roundTripRev(msg: string, source: openArray[byte]) =
   var
     decoded = snappy.decode(source)
     outputSize: csize_t = 0
     ok = snappy_uncompressed_length(cast[cstring](source[0].unsafeAddr), source.len.csize_t, outputSize) == 0
-    cpp_decoded: string
+    cpp_decoded = cpp_snappy.decode(source)
 
-  if not ok: echo "maybe a bad data"
+  check:
+    decoded == cpp_decoded
 
-  if ok:
-    cpp_decoded = newString(outputSize)
-    ok = snappy_uncompress(cast[cstring](source[0].unsafeAddr), source.len.csize_t, cpp_decoded, outputSize) == 0
-    if not ok: echo "cpp failed to uncompress"
-
-  if ok:
-    ok = equalMem(decoded[0].addr, cpp_decoded[0].addr, outputSize.int)
-    if not ok: echo "cpp output and nim output differ"
-
-  if ok:
-    ok = snappy.encode(decoded) == source
-    if not ok: echo "rev roundtrip failure"
-
-  if ok:
-    stdout.styledWriteLine("  ", msg, "...", fgGreen, "[PASS]")
-  else:
-    stdout.styledWriteLine("  ", msg, "...", fgRed, "[FAILED]")
-
-  result = ok
-
-proc roundTripRev(msg: string, sourceName: string): bool =
+proc roundTripRev(msg: string, sourceName: string) =
   var src = readSource(sourceName)
   roundTripRev(msg, src)
 
+proc roundTrip(msg: string, sourceName: string) =
+  var src = readSource(sourceName)
+  roundTrip(msg, src)
+
 template toBytes(s: string): auto =
   toOpenArrayByte(s, 0, s.len-1)
 
-proc compressFileWithFaststreams(src, dst: string) =
-  var input = memFileInput(src)
-  var output = fileOutput(dst)
-  output.appendSnappyBytes input.read(input.len.get)
-  output.flush()
-
 when false:
   # TODO: This is not tested yet
   import streams
@@ -177,59 +79,35 @@ when false:
     output.appendSnappyBytes input, getFileSize(src).int
 
 suite "snappy":
-  let
-    dataDir = getAppDir() & DirSep & testDataDir
-    largeFile = dataDir / "largefile.bin"
-
-  if fileExists(largeFile):
-    test "test large file performance":
-      let (success, times) = timedRoundTrip("empty", largeFile)
-      printTimes times
-      check success and float64(times.fastStreams) < float64(times.openArrays) * 1.1
-
-    let
-      largeFileCopy1 = dataDir / "largefile.bin.copy.1"
-      largeFileCopy2 = dataDir / "largefile.bin.copy.2"
-
-    var time = 0
-    timeit(time): compressFileWithFaststreams(largeFile, largeFileCopy1)
-    styledEcho "  compress file [Faststreams]: ", styleBright, $time, "ms"
-
-    timeit(time): compressFileWithFaststreams(largeFile, largeFileCopy2)
-    styledEcho "  compress file [Faststreams]: ", styleBright, $time, "ms"
-
-    removeFile largeFileCopy1
-    removeFile largeFileCopy2
-
   test "basic roundtrip test":
-    check roundTrip("empty", empty)
-    check roundTrip("oneZero", oneZero)
-    check roundTrip("data_html",    dataDir & "html")
-    check roundTrip("data_urls",    dataDir & "urls.10K")
-    check roundTrip("data_jpg",     dataDir & "fireworks.jpeg")
-    check roundTrip("data_pdf",     dataDir & "paper-100k.pdf")
-    check roundTrip("data_html4",   dataDir & "html_x_4")
-    check roundTrip("data_txt1",    dataDir & "alice29.txt")
-    check roundTrip("data_txt2",    dataDir & "asyoulik.txt")
-    check roundTrip("data_txt3",    dataDir & "lcet10.txt")
-    check roundTrip("data_txt4",    dataDir & "plrabn12.txt")
-    check roundTrip("data_pb",      dataDir & "geo.protodata")
-    check roundTrip("data_gaviota", dataDir & "kppkn.gtb")
-    check roundTrip("data_golden",  dataDir & "Mark.Twain-Tom.Sawyer.txt")
-    check roundTripRev("data_golden_rev", dataDir & "Mark.Twain-Tom.Sawyer.txt.rawsnappy")
+    roundTrip("empty", empty)
+    roundTrip("oneZero", oneZero)
+    roundTrip("data_html",    dataDir & "html")
+    roundTrip("data_urls",    dataDir & "urls.10K")
+    roundTrip("data_jpg",     dataDir & "fireworks.jpeg")
+    roundTrip("data_pdf",     dataDir & "paper-100k.pdf")
+    roundTrip("data_html4",   dataDir & "html_x_4")
+    roundTrip("data_txt1",    dataDir & "alice29.txt")
+    roundTrip("data_txt2",    dataDir & "asyoulik.txt")
+    roundTrip("data_txt3",    dataDir & "lcet10.txt")
+    roundTrip("data_txt4",    dataDir & "plrabn12.txt")
+    roundTrip("data_pb",      dataDir & "geo.protodata")
+    roundTrip("data_gaviota", dataDir & "kppkn.gtb")
+    roundTrip("data_golden",  dataDir & "Mark.Twain-Tom.Sawyer.txt")
+    roundTripRev("data_golden_rev", dataDir & "Mark.Twain-Tom.Sawyer.txt.rawsnappy")
 
   test "misc test":
     for i in 1..32:
       let x = repeat("b", i)
       let y = "aaaa$1aaaabbbb" % [x]
-      check roundTrip("repeat " & $i, toBytes(y))
+      roundTrip("repeat " & $i, toBytes(y))
 
     var i = 1
     while i < 20_000:
       var buf = newSeq[byte](i)
       for j in 0..<buf.len:
         buf[j] = byte((j mod 10) + int('a'))
-      check roundTrip("buf " & $buf.len, buf)
+      roundTrip("buf " & $buf.len, buf)
       inc(i, 23)
 
     for m in 1 .. 5:
@@ -237,7 +115,7 @@ suite "snappy":
         var buf = newSeq[byte](i)
         for j in 0..<buf.len:
           buf[j] = byte((j mod 10) + int('a'))
-        check roundTrip("buf " & $buf.len, buf)
+        roundTrip("buf " & $buf.len, buf)
 
     block:
       let encoded = [27'u8, 0b000010_00, 1, 2, 3, 0b000_000_10, 3, 0,
@@ -339,10 +217,10 @@ suite "snappy":
       0, 0, 1, 2, 0, 0, 1, 3, 0, 0, 1, 4, 0, 0, 2, 1, 0, 0, 0, 4, 0, 1, 0, 0, 0,
       0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
 
-    check roundTrip("random1", random1)
-    check roundTrip("random2", random2)
-    check roundTrip("random3", random3)
-    check roundTrip("random4", random4)
+    roundTrip("random1", random1)
+    roundTrip("random2", random2)
+    roundTrip("random3", random3)
+    roundTrip("random4", random4)
 
     const
       listLen = 100
@@ -350,4 +228,4 @@ suite "snappy":
       maxStringSize = 10000
 
     for x in randList(string, randGen(minStringSize, maxStringSize), randGen(listLen, listLen)):
-      check roundTrip("random " & $x.len, toBytes(x))
+      roundTrip("random " & $x.len, toBytes(x))

From 2256d6efb266dd1f65fc24077fb148377edb5b8c Mon Sep 17 00:00:00 2001
From: Jacek Sieka <jacek@status.im>
Date: Thu, 30 Dec 2021 13:49:33 +0100
Subject: [PATCH 2/2] ensure benchmarks are built on test

---
 snappy.nimble | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/snappy.nimble b/snappy.nimble
index 6bfefba..5016a9f 100644
--- a/snappy.nimble
+++ b/snappy.nimble
@@ -28,9 +28,10 @@ proc test(env, path: string) =
     mkDir "build"
 
   exec "nim " & lang & " " & env &
-    " -r --hints:off --skipParentCfg " & path
+    " --hints:off --skipParentCfg " & path
 
 task test, "Run all tests":
-  test "-d:debug", "tests/all_tests"
-  test "-d:release", "tests/all_tests"
-  test "--threads:on -d:release", "tests/all_tests"
+  test "-d:debug -r", "tests/all_tests"
+  test "-d:release -r", "tests/all_tests"
+  test "--threads:on -d:release -r", "tests/all_tests"
+  test "", "tests/benchmark" # don't run