allow non-power-of-2 limits in hashlist

fixes "make eth2_network_simulation"

a bit sad: no test coverage except against our own tests

beacon_chain/ssz/merkleization.nim

@@ -21,6 +21,9 @@ import
 export
   spec_types, types
 
+when hasSerializationTracing:
+  import stew/byteutils, typetraits
+
 const
   zero64 = default array[64, byte]
   bitsPerChunk = bytesPerChunk * 8
@@ -375,7 +378,6 @@ func mergedDataHash(x: HashList|HashArray, chunkIdx: int64): Eth2Digest =
 
 template mergedHash(x: HashList|HashArray, vIdxParam: int64): Eth2Digest =
   # The merged hash of the data at `vIdx` and `vIdx + 1`
-
   let vIdx = vIdxParam
   if vIdx >= x.maxChunks:
     let dataIdx = vIdx - x.maxChunks
@@ -393,8 +395,9 @@ func hashTreeRootCached*(x: HashList, vIdx: int64): Eth2Digest =
     idxInLayer = vIdx - (1'i64 shl layer)
     layerIdx = idxInlayer + x.indices[layer]
 
-  doAssert layer < x.maxDepth
   trs "GETTING ", vIdx, " ", layerIdx, " ", layer, " ", x.indices.len
+
+  doAssert layer < x.maxDepth
   if layerIdx >= x.indices[layer + 1]:
     trs "ZERO ", x.indices[layer], " ", x.indices[layer + 1]
     zeroHashes[x.maxDepth - layer]

beacon_chain/ssz/types.nim

@@ -25,6 +25,25 @@ type
 # * vIdx - virtual index in merkle tree - the root is found at index 1, its
 #          two children at 2, 3 then 4, 5, 6, 7 etc
 
+func nextPow2Int64(x: int64): int64 =
+  # TODO the nextPow2 in bitops2 works with uint64 - there's a bug in the nim
+  #      compiler preventing it to be used - it seems that a conversion to
+  #      uint64 cannot be done with the static maxLen :(
+  var v = x - 1
+
+  # round down, make sure all bits are 1 below the threshold, then add 1
+  v = v or v shr 1
+  v = v or v shr 2
+  v = v or v shr 4
+  when bitsof(x) > 8:
+    v = v or v shr 8
+  when bitsof(x) > 16:
+    v = v or v shr 16
+  when bitsof(x) > 32:
+    v = v or v shr 32
+
+  v + 1
+
 template dataPerChunk(T: type): int =
   # How many data items fit in a chunk
   when T is BasicType:
@@ -38,7 +57,10 @@ template chunkIdx*(T: type, dataIdx: int64): int64 =
 
 template maxChunkIdx*(T: type, maxLen: int64): int64 =
   # Given a number of data items, how many chunks are needed?
-  chunkIdx(T, maxLen + dataPerChunk(T) - 1)
+  # TODO compiler bug:
+  # beacon_chain/ssz/types.nim(75, 53) Error: cannot generate code for: maxLen
+  # nextPow2(chunkIdx(T, maxLen + dataPerChunk(T) - 1).uint64).int64
+  nextPow2Int64(chunkIdx(T, maxLen + dataPerChunk(T) - 1))
 
 template layer*(vIdx: int64): int =
   ## Layer 0 = layer at which the root hash is
@@ -57,7 +79,7 @@ type
   HashList*[T; maxLen: static Limit] = object
     data*: List[T, maxLen]
     hashes* {.dontSerialize.}: seq[Eth2Digest]
-    indices* {.dontSerialize.}: array[layer(maxChunkIdx(T, maxLen)) + 1, int64]
+    indices* {.dontSerialize.}: array[int(layer(maxChunkIdx(T, maxLen))) + 1, int64]
 
   # Note for readers:
   # We use `array` for `Vector` and
@@ -130,11 +152,11 @@ template clearCache*(v: var Eth2Digest) =
 
 template maxChunks*(a: HashList|HashArray): int64 =
   ## Layer where data is
-  chunkIdx(a.T, a.maxLen)
+  maxChunkIdx(a.T, a.maxLen)
 
 template maxDepth*(a: HashList|HashArray): int =
   ## Layer where data is
-  layer(a.maxChunks)
+  layer(nextPow2(a.maxChunks.uint64).int64)
 
 template chunkIdx(a: HashList|HashArray, dataIdx: int64): int64 =
   chunkIdx(a.T, dataIdx)

beacon_chain/validator_duties.nim

@@ -10,7 +10,7 @@ import
   os, tables, strutils,
 
   # Nimble packages
-  stew/[objects], stew/shims/macros,
+  stew/[byteutils, objects], stew/shims/macros,
   chronos, metrics, json_rpc/[rpcserver, jsonmarshal],
   chronicles,
   json_serialization/std/[options, sets, net], serialization/errors,
@@ -215,8 +215,10 @@ proc proposeBlock(node: BeaconNode,
       cat = "fastforward"
     return head
 
+  var graffiti: Eth2Digest
+  graffiti.data[0..<5] = toBytes("quack")
   let valInfo = ValidatorInfoForMakeBeaconBlock(kind: viValidator, validator: validator)
-  let beaconBlockTuple = makeBeaconBlockForHeadAndSlot(node, valInfo, validator_index, Eth2Digest(), head, slot)
+  let beaconBlockTuple = makeBeaconBlockForHeadAndSlot(node, valInfo, validator_index, graffiti, head, slot)
 
   if not beaconBlockTuple.message.isSome():
     return head # already logged elsewhere!

tests/test_ssz.nim

@@ -127,6 +127,13 @@ suiteReport "SSZ navigator":
     leaves2.add c
     check hash_tree_root(leaves2) == hash_tree_root(leaves2.data)
 
+    var leaves3 = HashList[Eth2Digest, 7]() # Non-power-of-2
+    check hash_tree_root(leaves3) == hash_tree_root(leaves3.data)
+    leaves3.add a
+    leaves3.add b
+    leaves3.add c
+    check hash_tree_root(leaves3) == hash_tree_root(leaves3.data)
+
   timedTest "basictype":
     var leaves = HashList[uint64, 1'i64 shl 3]()
     while leaves.len < leaves.maxLen: