random helpers (#64)

* random helpers

* arrayops as a home for small array/openArray utilities
* assign2 - a replacement for genericAssign and assignment operators in
general which in nim are very slow
* use assign2 in a few places to speed things up

* fixes

* fixes

README.md

@@ -21,6 +21,8 @@ broken out into separate repositories.
 Libraries are documented either in-module or on a separate README in their
 respective folders
 
+- `arrayops` - small helpers and operations on `array`/`openArray`
+- `assign2` - fast assignments (unlike the `=` operator in nim which is very slow)
 - `bitops2` - an updated version of `bitops.nim`, filling in gaps in original code
 - `byteutils` - utilities that make working with the Nim `byte` type convenient
 - `endians2` - utilities for converting to and from little / big endian integers
@@ -28,6 +30,8 @@ respective folders
 - `ptrops` - pointer arithmetic utilities
 - `result` - friendly, exception-free value-or-error returns, similar to `Option[T]`, from [nim-result](https://github.com/arnetheduck/nim-result/)
 - `shims` - backports of nim `devel` code to the stable version that Status is using
+- `sequtils2` - extensions to the `sequtils` module for working conveniently with `seq`
+- `varints` - helpers for working with variable length integers
 
 ## Layout
 

stew/arrayops.nim

@@ -0,0 +1,80 @@
+# Copyright (c) 2020 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.push raises: [Defect].}
+
+## Operations on array and openArray
+
+import
+  ./assign2
+
+export assign2
+
+template eachElement(x, res, op: untyped) =
+  for i in 0..<res.len:
+    res[i] = op(x[i])
+
+template eachElement(x, y, res, op: untyped) =
+  for i in 0..<res.len:
+    res[i] = op(x[i], y[i])
+
+func `and`*[N: static int; T](x, y: array[N, T]): array[N, T] =
+  eachElement(x, y, result, `and`)
+
+func `or`*[N: static int; T](x, y: array[N, T]): array[N, T] =
+  eachElement(x, y, result, `or`)
+
+func `xor`*[N: static int; T](x, y: array[N, T]): array[N, T] =
+  eachElement(x, y, result, `xor`)
+
+func `not`*[N: static int; T](x: array[N, T]): array[N, T] =
+  eachElement(x, result, `not`)
+
+func mand*[N: static int; T](x: var array[N, T], y: array[N, T]) =
+  eachElement(x, y, x, `and`)
+
+func mor*[N: static int; T](x: var array[N, T], y: array[N, T]) =
+  eachElement(x, y, x, `or`)
+
+func mxor*[N: static int; T](x: var array[N, T], y: array[N, T]) =
+  eachElement(x, y, x, `xor`)
+
+func mnot*[N: static int; T](x: var array[N, T], y: array[N, T]) =
+  eachElement(x, x, `not`)
+
+func copyFrom*[T](
+    v: var openArray[T], src: openArray[T]): Natural {.raises: [Defect].} =
+  ## Copy `src` contents into `v` - this is a permissive assignment where `src`
+  ## may contain both fewer and more elements than `v`. Returns the number of
+  ## elements copied which may be less than N when `src` is shorter than v
+  let elems = min(v.len, src.len)
+  assign(v.toOpenArray(0, elems - 1), src.toOpenArray(0, elems - 1))
+  elems
+
+func initArrayWith*[N: static[int], T](value: T): array[N, T] {.noInit, inline.}=
+  result.fill(value)
+
+func `&`*[N1, N2: static[int], T](
+    a: array[N1, T],
+    b: array[N2, T]
+    ): array[N1 + N2, T] {.inline, noInit.}=
+  ## Array concatenation
+  assign(result.toOpenArray(0, N1 - 1), a)
+  assign(result.toOpenArray(N1, result.high), b)
+
+template `^^`(s, i: untyped): untyped =
+  (when i is BackwardsIndex: s.len - int(i) else: int(i))
+
+func `[]=`*[T, U, V](r: var openArray[T], s: HSlice[U, V], v: openArray[T]) =
+  ## openArray slice assignment:
+  ## v[0..<2] = [0, 1]
+  let a = r ^^ s.a
+  let b = r ^^ s.b
+  let L = b - a + 1
+  if L == v.len:
+    assign(r.toOpenArray(a, a + L - 1), v)
+  else:
+    raiseAssert "different lengths for slice assignment: " & $L & " vs " & $v

stew/assign2.nim

@@ -0,0 +1,58 @@
+import
+  std/typetraits,
+  ./shims/macros
+
+{.push raises: [Defect].}
+
+func assignImpl[T](tgt: var openArray[T], src: openArray[T]) =
+  mixin assign
+  when supportsCopyMem(T):
+    if tgt.len > 0:
+      copyMem(addr tgt[0], unsafeAddr src[0], sizeof(tgt[0]) * tgt.len)
+  else:
+    for i in 0..<tgt.len:
+      assign(tgt[i], src[i])
+
+func assign*[T](tgt: var openArray[T], src: openArray[T]) =
+  mixin assign
+
+  if tgt.len != src.len:
+    raiseAssert "Target and source lengths don't match: " &
+      $tgt.len & " vs " & $src.len
+
+  assignImpl(tgt, src)
+
+func assign*[T](tgt: var seq[T], src: openArray[T]) =
+  mixin assign
+
+  tgt.setLen(src.len)
+  assignImpl(tgt.toOpenArray(0, tgt.high), src)
+
+macro unsupported(T: typed): untyped =
+  error "Assignment of the type " & humaneTypeName(T) & " is not supported"
+
+func assign*[T](tgt: var T, src: T) =
+  # The default `genericAssignAux` that gets generated for assignments in nim
+  # is ridiculously slow. When syncing, the application was spending 50%+ CPU
+  # time in it - `assign`, in the same test, doesn't even show in the perf trace
+  mixin assign
+
+  when supportsCopyMem(T):
+    when sizeof(src) <= sizeof(int):
+      tgt = src
+    else:
+      copyMem(addr tgt, unsafeAddr src, sizeof(tgt))
+  elif T is object|tuple:
+    for t, s in fields(tgt, src):
+      when supportsCopyMem(type s) and sizeof(s) <= sizeof(int) * 2:
+        t = s # Shortcut
+      else:
+        assign(t, s)
+  elif T is seq:
+    assign(tgt, src.toOpenArray(0, src.high))
+  elif T is ref:
+    tgt = src
+  elif compiles(distinctBase(tgt)):
+    assign(distinctBase tgt, distinctBase src)
+  else:
+    unsupported T

stew/byteutils.nim

@@ -9,35 +9,15 @@
 ########################################################################################################
 ####################################   Array utilities   ###############################################
 
-import algorithm
+import
+  std/[algorithm, typetraits],
+  ./assign2, ./arrayops
+
+# backwards compat
+export arrayops.`&`, arrayops.initArrayWith, arrayops.`[]=`
 
 {.push raises: [Defect].}
 
-func initArrayWith*[N: static[int], T](value: T): array[N, T] {.noInit, inline.}=
-  result.fill(value)
-
-func `&`*[N1, N2: static[int], T](
-    a: array[N1, T],
-    b: array[N2, T]
-    ): array[N1 + N2, T] {.inline, noInit.}=
-  ## Array concatenation
-  result[0 ..< N1] = a
-  result[N1 ..< result.len] = b
-
-template `^^`(s, i: untyped): untyped =
-  (when i is BackwardsIndex: s.len - int(i) else: int(i))
-
-func `[]=`*[T, U, V](r: var openArray[T], s: HSlice[U, V], v: openArray[T]) =
-  ## openArray slice assignment:
-  ## v[0..<2] = [0, 1]
-  let a = r ^^ s.a
-  let b = r ^^ s.b
-  let L = b - a + 1
-  if L == v.len:
-    for i in 0..<L: r[i + a] = v[i]
-  else:
-    raise newException(RangeError, "different lengths for slice assignment")
-
 ########################################################################################################
 #####################################   Hex utilities   ################################################
 

stew/sequtils2.nim

@@ -0,0 +1,22 @@
+import
+  std/typetraits,
+  ./assign2
+
+func write*[T](s: var seq[T], v: openArray[T]) =
+  # The Nim standard library is inefficient when copying simple types
+  # into a seq: it will first zero-init the new memory then copy the items
+  # one by one, when a copyMem would be sufficient - semantically, this
+  # function performs the same thing as `add`, but similar to faststreams, from
+  # where the `write` name comes from, it is much faster. Unfortunately, there's
+  # no easy way to avoid the zero-init, but a smart compiler might be able
+  # to elide it.
+  when nimvm:
+    s.add(v)
+  else:
+    if v.len > 0:
+      let start = s.len
+      s.setLen(start + v.len)
+      when supportsCopyMem(T): # shortcut
+        copyMem(addr s[start], unsafeAddr v[0], v.len * sizeof(T))
+      else:
+        assign(s.toOpenArray(start, s.high), v)

tests/all_tests.nim

@@ -9,18 +9,21 @@
 
 import
   ranges/all,
+  test_assign2,
+  test_arrayops,
   test_base32,
   test_base58,
   test_base64,
   test_bitops2,
   test_bitseqs,
   test_byteutils,
+  test_ctops,
   test_endians2,
+  test_io2,
   test_macros,
   test_objects,
   test_ptrops,
+  test_sequtils2,
   test_results,
   test_varints,
-  test_ctops,
+  test_winacl
-  test_io2,
-  test_winacl

tests/test_arrayops.nim

@@ -0,0 +1,26 @@
+# stew
+# Copyright 2018-2019 Status Research & Development GmbH
+# Licensed under either of
+#
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
+#
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.used.}
+
+import
+  std/unittest,
+  ../stew/arrayops
+
+suite "arrayops":
+  test "basic":
+    let
+      a = [byte 0, 1]
+      b = [byte 4, 5]
+
+    check:
+      (a and b) == [a[0] and b[0], a[1] and b[1]]
+      (a or b) == [a[0] or b[0], a[1] or b[1]]
+      (a xor b) == [a[0] xor b[0], a[1] xor b[1]]
+      (not a) == [not a[0], not a[1]]

tests/test_assign2.nim

@@ -0,0 +1,29 @@
+import
+  std/unittest,
+  ../stew/assign2
+
+suite "assign2":
+  test "basic":
+    type X = distinct int
+    var
+      a = 5
+      b = [2, 3]
+      c = @[5, 6]
+
+    assign(c, b)
+    check: c == b
+    assign(b, [4, 5])
+    check: b == [4, 5]
+
+    assign(a, 6)
+    check: a == 6
+
+    assign(c.toOpenArray(0, 1), [2, 2])
+    check: c == [2, 2]
+
+    var
+      dis = X(53)
+
+    assign(dis, X(55))
+
+    check: int(dis) == 55

tests/test_byteutils.nim

@@ -5,8 +5,9 @@
 #   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
-import  unittest,
+import
-        ../stew/byteutils
+  std/unittest,
+  ../stew/byteutils
 
 proc compilationTest {.exportc: "compilationTest".} =
   var bytes = @[1.byte, 2, 3, 4]

tests/test_sequtils2.nim

@@ -0,0 +1,39 @@
+# byteutils
+# Copyright (c) 2018 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import
+  std/unittest,
+  ../stew/sequtils2
+
+suite "sequtils2":
+  test "write":
+    block:
+      var a: seq[int]
+
+      a.write([0, 1, 2, 3])
+
+      check:
+        a == @[0, 1, 2, 3]
+
+      a.write([])
+      a.write([4])
+
+      check:
+        a == @[0, 1, 2, 3, 4]
+    block:
+      var a: seq[byte]
+
+      a.write([byte 0, 1, 2, 3])
+
+      check:
+        a == [byte 0, 1, 2, 3]
+
+      a.write([])
+      a.write([byte 4])
+
+      check:
+        a == [byte 0, 1, 2, 3, 4]