Remove SSZ files forgotten in move to ssz repo (#3091)
This commit is contained in:
Kim De Mey
GitHub
parent
00bbc8e0fe
commit
2da9309d2a
|
|
@ -1,254 +0,0 @@
|
||||||
# beacon_chain
|
|
||||||
# Copyright (c) 2018-2021 Status Research & Development GmbH
|
|
||||||
# Licensed and distributed under either of
|
|
||||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
|
||||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
|
||||||
# at your option. This file may not be copied, modified, or distributed except according to those terms.
|
|
||||||
|
|
||||||
{.push raises: [Defect].}
|
|
||||||
{.pragma: raisesssz, raises: [Defect, MalformedSszError, SszSizeMismatchError].}
|
|
||||||
|
|
||||||
# Coding and decoding of primitive SSZ types - every "simple" type passed to
|
|
||||||
# and from the SSZ library must have a `fromSssBytes` and `toSszType` overload.
|
|
||||||
|
|
||||||
import
|
|
||||||
std/typetraits,
|
|
||||||
stew/[endians2, objects],
|
|
||||||
../spec/digest, ./types
|
|
||||||
|
|
||||||
export
|
|
||||||
digest, types
|
|
||||||
|
|
||||||
template raiseIncorrectSize*(T: type) =
|
|
||||||
const typeName = name(T)
|
|
||||||
raise newException(MalformedSszError,
|
|
||||||
"SSZ " & typeName & " input of incorrect size")
|
|
||||||
|
|
||||||
template setOutputSize[R, T](a: var array[R, T], length: int) =
|
|
||||||
if length != a.len:
|
|
||||||
raiseIncorrectSize a.type
|
|
||||||
|
|
||||||
proc setOutputSize(list: var List, length: int) {.raisesssz.} =
|
|
||||||
if not list.setLen length:
|
|
||||||
raise newException(MalformedSszError, "SSZ list maximum size exceeded")
|
|
||||||
|
|
||||||
# fromSszBytes copies the wire representation to a Nim variable,
|
|
||||||
# assuming there's enough data in the buffer
|
|
||||||
func fromSszBytes*(T: type UintN, data: openArray[byte]): T {.raisesssz.} =
|
|
||||||
## Convert directly to bytes the size of the int. (e.g. ``uint16 = 2 bytes``)
|
|
||||||
## All integers are serialized as **little endian**.
|
|
||||||
if data.len != sizeof(result):
|
|
||||||
raiseIncorrectSize T
|
|
||||||
|
|
||||||
T.fromBytesLE(data)
|
|
||||||
|
|
||||||
func fromSszBytes*(T: type bool, data: openArray[byte]): T {.raisesssz.} =
|
|
||||||
# Strict: only allow 0 or 1
|
|
||||||
if data.len != 1 or byte(data[0]) > byte(1):
|
|
||||||
raise newException(MalformedSszError, "invalid boolean value")
|
|
||||||
data[0] == 1
|
|
||||||
|
|
||||||
func fromSszBytes*(T: type Eth2Digest, data: openArray[byte]): T {.raisesssz.} =
|
|
||||||
if data.len != sizeof(result.data):
|
|
||||||
raiseIncorrectSize T
|
|
||||||
copyMem(result.data.addr, unsafeAddr data[0], sizeof(result.data))
|
|
||||||
|
|
||||||
template fromSszBytes*(T: type BitSeq, bytes: openArray[byte]): auto =
|
|
||||||
BitSeq @bytes
|
|
||||||
|
|
||||||
proc `[]`[T, U, V](s: openArray[T], x: HSlice[U, V]) {.error:
|
|
||||||
"Please don't use openArray's [] as it allocates a result sequence".}
|
|
||||||
|
|
||||||
template checkForForbiddenBits(ResulType: type,
|
|
||||||
input: openArray[byte],
|
|
||||||
expectedBits: static int64) =
|
|
||||||
## This checks if the input contains any bits set above the maximum
|
|
||||||
## sized allowed. We only need to check the last byte to verify this:
|
|
||||||
const bitsInLastByte = (expectedBits mod 8)
|
|
||||||
when bitsInLastByte != 0:
|
|
||||||
# As an example, if there are 3 bits expected in the last byte,
|
|
||||||
# we calculate a bitmask equal to 11111000. If the input has any
|
|
||||||
# raised bits in range of the bitmask, this would be a violation
|
|
||||||
# of the size of the BitArray:
|
|
||||||
const forbiddenBitsMask = byte(byte(0xff) shl bitsInLastByte)
|
|
||||||
|
|
||||||
if (input[^1] and forbiddenBitsMask) != 0:
|
|
||||||
raiseIncorrectSize ResulType
|
|
||||||
|
|
||||||
func readSszValue*[T](input: openArray[byte],
|
|
||||||
val: var T) {.raisesssz.} =
|
|
||||||
mixin fromSszBytes, toSszType
|
|
||||||
|
|
||||||
template readOffsetUnchecked(n: int): uint32 {.used.}=
|
|
||||||
fromSszBytes(uint32, input.toOpenArray(n, n + offsetSize - 1))
|
|
||||||
|
|
||||||
template readOffset(n: int): int {.used.} =
|
|
||||||
let offset = readOffsetUnchecked(n)
|
|
||||||
if offset > input.len.uint32:
|
|
||||||
raise newException(MalformedSszError, "SSZ list element offset points past the end of the input")
|
|
||||||
int(offset)
|
|
||||||
|
|
||||||
when val is BitList:
|
|
||||||
if input.len == 0:
|
|
||||||
raise newException(MalformedSszError, "Invalid empty SSZ BitList value")
|
|
||||||
|
|
||||||
# Since our BitLists have an in-memory representation that precisely
|
|
||||||
# matches their SSZ encoding, we can deserialize them as regular Lists:
|
|
||||||
const maxExpectedSize = (val.maxLen div 8) + 1
|
|
||||||
type MatchingListType = List[byte, maxExpectedSize]
|
|
||||||
|
|
||||||
when false:
|
|
||||||
# TODO: Nim doesn't like this simple type coercion,
|
|
||||||
# we'll rely on `cast` for now (see below)
|
|
||||||
readSszValue(input, MatchingListType val)
|
|
||||||
else:
|
|
||||||
static:
|
|
||||||
# As a sanity check, we verify that the coercion is accepted by the compiler:
|
|
||||||
doAssert MatchingListType(val) is MatchingListType
|
|
||||||
readSszValue(input, cast[ptr MatchingListType](addr val)[])
|
|
||||||
|
|
||||||
let resultBytesCount = len bytes(val)
|
|
||||||
|
|
||||||
if bytes(val)[resultBytesCount - 1] == 0:
|
|
||||||
raise newException(MalformedSszError, "SSZ BitList is not properly terminated")
|
|
||||||
|
|
||||||
if resultBytesCount == maxExpectedSize:
|
|
||||||
checkForForbiddenBits(T, input, val.maxLen + 1)
|
|
||||||
|
|
||||||
elif val is HashList | HashArray:
|
|
||||||
readSszValue(input, val.data)
|
|
||||||
val.resetCache()
|
|
||||||
|
|
||||||
elif val is List|array:
|
|
||||||
type E = type val[0]
|
|
||||||
|
|
||||||
when E is byte:
|
|
||||||
val.setOutputSize input.len
|
|
||||||
if input.len > 0:
|
|
||||||
copyMem(addr val[0], unsafeAddr input[0], input.len)
|
|
||||||
|
|
||||||
elif isFixedSize(E):
|
|
||||||
const elemSize = fixedPortionSize(E)
|
|
||||||
if input.len mod elemSize != 0:
|
|
||||||
var ex = new SszSizeMismatchError
|
|
||||||
ex.deserializedType = cstring typetraits.name(T)
|
|
||||||
ex.actualSszSize = input.len
|
|
||||||
ex.elementSize = elemSize
|
|
||||||
raise ex
|
|
||||||
val.setOutputSize input.len div elemSize
|
|
||||||
for i in 0 ..< val.len:
|
|
||||||
let offset = i * elemSize
|
|
||||||
readSszValue(input.toOpenArray(offset, offset + elemSize - 1), val[i])
|
|
||||||
|
|
||||||
else:
|
|
||||||
if input.len == 0:
|
|
||||||
# This is an empty list.
|
|
||||||
# The default initialization of the return value is fine.
|
|
||||||
val.setOutputSize 0
|
|
||||||
return
|
|
||||||
elif input.len < offsetSize:
|
|
||||||
raise newException(MalformedSszError, "SSZ input of insufficient size")
|
|
||||||
|
|
||||||
var offset = readOffset 0
|
|
||||||
let resultLen = offset div offsetSize
|
|
||||||
|
|
||||||
if resultLen == 0:
|
|
||||||
# If there are too many elements, other constraints detect problems
|
|
||||||
# (not monotonically increasing, past end of input, or last element
|
|
||||||
# not matching up with its nextOffset properly)
|
|
||||||
raise newException(MalformedSszError, "SSZ list incorrectly encoded of zero length")
|
|
||||||
|
|
||||||
val.setOutputSize resultLen
|
|
||||||
for i in 1 ..< resultLen:
|
|
||||||
let nextOffset = readOffset(i * offsetSize)
|
|
||||||
if nextOffset < offset:
|
|
||||||
raise newException(MalformedSszError, "SSZ list element offsets are decreasing")
|
|
||||||
else:
|
|
||||||
readSszValue(input.toOpenArray(offset, nextOffset - 1), val[i - 1])
|
|
||||||
offset = nextOffset
|
|
||||||
|
|
||||||
readSszValue(input.toOpenArray(offset, input.len - 1), val[resultLen - 1])
|
|
||||||
|
|
||||||
elif val is SingleMemberUnion:
|
|
||||||
readSszValue(input.toOpenArray(0, 0), val.selector)
|
|
||||||
if val.selector != 0'u8:
|
|
||||||
raise newException(MalformedSszError, "SingleMemberUnion selector must be 0")
|
|
||||||
readSszValue(input.toOpenArray(1, input.len - 1), val.value)
|
|
||||||
|
|
||||||
elif val is UintN|bool:
|
|
||||||
val = fromSszBytes(T, input)
|
|
||||||
|
|
||||||
elif val is BitArray:
|
|
||||||
if sizeof(val) != input.len:
|
|
||||||
raiseIncorrectSize(T)
|
|
||||||
checkForForbiddenBits(T, input, val.bits)
|
|
||||||
copyMem(addr val.bytes[0], unsafeAddr input[0], input.len)
|
|
||||||
|
|
||||||
elif val is object|tuple:
|
|
||||||
let inputLen = uint32 input.len
|
|
||||||
const minimallyExpectedSize = uint32 fixedPortionSize(T)
|
|
||||||
|
|
||||||
if inputLen < minimallyExpectedSize:
|
|
||||||
raise newException(MalformedSszError, "SSZ input of insufficient size")
|
|
||||||
|
|
||||||
enumInstanceSerializedFields(val, fieldName, field):
|
|
||||||
const boundingOffsets = getFieldBoundingOffsets(T, fieldName)
|
|
||||||
|
|
||||||
# type FieldType = type field # buggy
|
|
||||||
# For some reason, Nim gets confused about the alias here. This could be a
|
|
||||||
# generics caching issue caused by the use of distinct types. Such an
|
|
||||||
# issue is very scary in general.
|
|
||||||
# The bug can be seen with the two List[uint64, N] types that exist in
|
|
||||||
# the spec, with different N.
|
|
||||||
|
|
||||||
type SszType = type toSszType(declval type(field))
|
|
||||||
|
|
||||||
when isFixedSize(SszType):
|
|
||||||
const
|
|
||||||
startOffset = boundingOffsets[0]
|
|
||||||
endOffset = boundingOffsets[1]
|
|
||||||
else:
|
|
||||||
let
|
|
||||||
startOffset = readOffsetUnchecked(boundingOffsets[0])
|
|
||||||
endOffset = if boundingOffsets[1] == -1: inputLen
|
|
||||||
else: readOffsetUnchecked(boundingOffsets[1])
|
|
||||||
|
|
||||||
when boundingOffsets.isFirstOffset:
|
|
||||||
if startOffset != minimallyExpectedSize:
|
|
||||||
raise newException(MalformedSszError, "SSZ object dynamic portion starts at invalid offset")
|
|
||||||
|
|
||||||
if startOffset > endOffset:
|
|
||||||
raise newException(MalformedSszError, "SSZ field offsets are not monotonically increasing")
|
|
||||||
elif endOffset > inputLen:
|
|
||||||
raise newException(MalformedSszError, "SSZ field offset points past the end of the input")
|
|
||||||
elif startOffset < minimallyExpectedSize:
|
|
||||||
raise newException(MalformedSszError, "SSZ field offset points outside bounding offsets")
|
|
||||||
|
|
||||||
# TODO The extra type escaping here is a work-around for a Nim issue:
|
|
||||||
when type(field) is type(SszType):
|
|
||||||
readSszValue(
|
|
||||||
input.toOpenArray(int(startOffset), int(endOffset - 1)),
|
|
||||||
field)
|
|
||||||
else:
|
|
||||||
field = fromSszBytes(
|
|
||||||
type(field),
|
|
||||||
input.toOpenArray(int(startOffset), int(endOffset - 1)))
|
|
||||||
|
|
||||||
else:
|
|
||||||
unsupported T
|
|
||||||
|
|
||||||
# Identity conversions for core SSZ types
|
|
||||||
|
|
||||||
template toSszType*(v: auto): auto =
|
|
||||||
## toSszType converts a given value into one of the primitive types supported
|
|
||||||
## by SSZ - to add support for a custom type (for example a `distinct` type),
|
|
||||||
## add an overload for `toSszType` which converts it to one of the `SszType`
|
|
||||||
## types, as well as a `fromSszBytes`.
|
|
||||||
type T = type(v)
|
|
||||||
when T is SszType:
|
|
||||||
when T is Eth2Digest:
|
|
||||||
v.data
|
|
||||||
else:
|
|
||||||
v
|
|
||||||
else:
|
|
||||||
unsupported T
|
|
||||||
|
|
@ -1,570 +0,0 @@
|
||||||
# beacon_chain
|
|
||||||
# Copyright (c) 2018-2021 Status Research & Development GmbH
|
|
||||||
# Licensed and distributed under either of
|
|
||||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
|
||||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
|
||||||
# at your option. This file may not be copied, modified, or distributed except according to those terms.
|
|
||||||
|
|
||||||
{.push raises: [Defect].}
|
|
||||||
|
|
||||||
import
|
|
||||||
std/[tables, typetraits, strformat],
|
|
||||||
stew/shims/macros, stew/[byteutils, bitops2, objects], stint,
|
|
||||||
serialization/[object_serialization, errors],
|
|
||||||
json_serialization,
|
|
||||||
"."/[bitseqs],
|
|
||||||
../spec/digest
|
|
||||||
|
|
||||||
export stint, bitseqs, json_serialization
|
|
||||||
|
|
||||||
const
|
|
||||||
offsetSize* = 4
|
|
||||||
bytesPerChunk* = 32
|
|
||||||
|
|
||||||
type
|
|
||||||
UintN* = SomeUnsignedInt|UInt128|UInt256
|
|
||||||
BasicType* = bool|UintN
|
|
||||||
|
|
||||||
Limit* = int64
|
|
||||||
|
|
||||||
# A few index types from here onwards:
|
|
||||||
# * dataIdx - leaf index starting from 0 to maximum length of collection
|
|
||||||
# * chunkIdx - leaf data index after chunking starting from 0
|
|
||||||
# * vIdx - virtual index in merkle tree - the root is found at index 1, its
|
|
||||||
# two children at 2, 3 then 4, 5, 6, 7 etc
|
|
||||||
|
|
||||||
func nextPow2Int64(x: int64): int64 =
|
|
||||||
# TODO the nextPow2 in bitops2 works with uint64 - there's a bug in the nim
|
|
||||||
# compiler preventing it to be used - it seems that a conversion to
|
|
||||||
# uint64 cannot be done with the static maxLen :(
|
|
||||||
var v = x - 1
|
|
||||||
|
|
||||||
# round down, make sure all bits are 1 below the threshold, then add 1
|
|
||||||
v = v or v shr 1
|
|
||||||
v = v or v shr 2
|
|
||||||
v = v or v shr 4
|
|
||||||
when bitsof(x) > 8:
|
|
||||||
v = v or v shr 8
|
|
||||||
when bitsof(x) > 16:
|
|
||||||
v = v or v shr 16
|
|
||||||
when bitsof(x) > 32:
|
|
||||||
v = v or v shr 32
|
|
||||||
|
|
||||||
v + 1
|
|
||||||
|
|
||||||
template dataPerChunk(T: type): int =
|
|
||||||
# How many data items fit in a chunk
|
|
||||||
when T is BasicType:
|
|
||||||
bytesPerChunk div sizeof(T)
|
|
||||||
else:
|
|
||||||
1
|
|
||||||
|
|
||||||
template chunkIdx*(T: type, dataIdx: int64): int64 =
|
|
||||||
# Given a data index, which chunk does it belong to?
|
|
||||||
dataIdx div dataPerChunk(T)
|
|
||||||
|
|
||||||
template maxChunkIdx*(T: type, maxLen: Limit): int64 =
|
|
||||||
# Given a number of data items, how many chunks are needed?
|
|
||||||
# TODO compiler bug:
|
|
||||||
# beacon_chain/ssz/types.nim(75, 53) Error: cannot generate code for: maxLen
|
|
||||||
# nextPow2(chunkIdx(T, maxLen + dataPerChunk(T) - 1).uint64).int64
|
|
||||||
nextPow2Int64(chunkIdx(T, maxLen.int64 + dataPerChunk(T) - 1))
|
|
||||||
|
|
||||||
template layer*(vIdx: int64): int =
|
|
||||||
## Layer 0 = layer at which the root hash is
|
|
||||||
## We place the root hash at index 1 which simplifies the math and leaves
|
|
||||||
## index 0 for the mixed-in-length
|
|
||||||
log2trunc(vIdx.uint64).int
|
|
||||||
|
|
||||||
func hashListIndicesLen(maxChunkIdx: int64): int =
|
|
||||||
# TODO: This exists only to work-around a compilation issue when the complex
|
|
||||||
# expression is used directly in the HastList array size definition below
|
|
||||||
int(layer(maxChunkIdx)) + 1
|
|
||||||
|
|
||||||
type
|
|
||||||
List*[T; maxLen: static Limit] = distinct seq[T]
|
|
||||||
BitList*[maxLen: static Limit] = distinct BitSeq
|
|
||||||
|
|
||||||
SingleMemberUnion*[T] = object
|
|
||||||
selector*: uint8
|
|
||||||
value*: T
|
|
||||||
|
|
||||||
HashArray*[maxLen: static Limit; T] = object
|
|
||||||
## Array implementation that caches the hash of each chunk of data - see
|
|
||||||
## also HashList for more details.
|
|
||||||
data*: array[maxLen, T]
|
|
||||||
hashes* {.dontSerialize.}: array[maxChunkIdx(T, maxLen), Eth2Digest]
|
|
||||||
|
|
||||||
HashList*[T; maxLen: static Limit] = object
|
|
||||||
## List implementation that caches the hash of each chunk of data as well
|
|
||||||
## as the combined hash of each level of the merkle tree using a flattened
|
|
||||||
## list of hashes.
|
|
||||||
##
|
|
||||||
## The merkle tree of a list is formed by imagining a virtual buffer of
|
|
||||||
## `maxLen` length which is zero-filled where there is no data. Then,
|
|
||||||
## a merkle tree of hashes is formed as usual - at each level of the tree,
|
|
||||||
## iff the hash is combined from two zero-filled chunks, the hash is not
|
|
||||||
## stored in the `hashes` list - instead, `indices` keeps track of where in
|
|
||||||
## the list each level starts. When the length of `data` changes, the
|
|
||||||
## `hashes` and `indices` structures must be updated accordingly using
|
|
||||||
## `growHashes`.
|
|
||||||
##
|
|
||||||
## All mutating operators (those that take `var HashList`) will
|
|
||||||
## automatically invalidate the cache for the relevant chunks - the leaf and
|
|
||||||
## all intermediate chunk hashes up to the root. When large changes are made
|
|
||||||
## to `data`, it might be more efficient to batch the updates then reset
|
|
||||||
## the cache using resetCache` instead.
|
|
||||||
|
|
||||||
data*: List[T, maxLen]
|
|
||||||
hashes* {.dontSerialize.}: seq[Eth2Digest] ## \
|
|
||||||
## Flattened tree store that skips "empty" branches of the tree - the
|
|
||||||
## starting index in this sequence of each "level" in the tree is found
|
|
||||||
## in `indices`.
|
|
||||||
indices* {.dontSerialize.}: array[hashListIndicesLen(maxChunkIdx(T, maxLen)), int64] ##\
|
|
||||||
## Holds the starting index in the hashes list for each level of the tree
|
|
||||||
|
|
||||||
# Note for readers:
|
|
||||||
# We use `array` for `Vector` and
|
|
||||||
# `BitArray` for `BitVector`
|
|
||||||
|
|
||||||
SszError* = object of SerializationError
|
|
||||||
|
|
||||||
MalformedSszError* = object of SszError
|
|
||||||
|
|
||||||
SszSizeMismatchError* = object of SszError
|
|
||||||
deserializedType*: cstring
|
|
||||||
actualSszSize*: int
|
|
||||||
elementSize*: int
|
|
||||||
|
|
||||||
# These are supported by the SSZ library - anything that's not covered here
|
|
||||||
# needs to overload toSszType and fromSszBytes
|
|
||||||
SszType* =
|
|
||||||
BasicType | array | HashArray | List | HashList | BitArray | BitList |
|
|
||||||
object | tuple
|
|
||||||
|
|
||||||
template asSeq*(x: List): auto = distinctBase(x)
|
|
||||||
|
|
||||||
template init*[T, N](L: type List[T, N], x: seq[T]): auto =
|
|
||||||
List[T, N](x)
|
|
||||||
|
|
||||||
template `$`*(x: List): auto = $(distinctBase x)
|
|
||||||
template len*(x: List): auto = len(distinctBase x)
|
|
||||||
template low*(x: List): auto = low(distinctBase x)
|
|
||||||
template high*(x: List): auto = high(distinctBase x)
|
|
||||||
template `[]`*(x: List, idx: auto): untyped = distinctBase(x)[idx]
|
|
||||||
template `[]=`*(x: var List, idx: auto, val: auto) = distinctBase(x)[idx] = val
|
|
||||||
template `==`*(a, b: List): bool = distinctBase(a) == distinctBase(b)
|
|
||||||
|
|
||||||
template `&`*(a, b: List): auto = (type(a)(distinctBase(a) & distinctBase(b)))
|
|
||||||
|
|
||||||
template items* (x: List): untyped = items(distinctBase x)
|
|
||||||
template pairs* (x: List): untyped = pairs(distinctBase x)
|
|
||||||
template mitems*(x: var List): untyped = mitems(distinctBase x)
|
|
||||||
template mpairs*(x: var List): untyped = mpairs(distinctBase x)
|
|
||||||
|
|
||||||
proc add*(x: var List, val: auto): bool =
|
|
||||||
if x.len < x.maxLen:
|
|
||||||
add(distinctBase x, val)
|
|
||||||
true
|
|
||||||
else:
|
|
||||||
false
|
|
||||||
|
|
||||||
proc setLen*(x: var List, newLen: int): bool =
|
|
||||||
if newLen <= x.maxLen:
|
|
||||||
setLen(distinctBase x, newLen)
|
|
||||||
true
|
|
||||||
else:
|
|
||||||
false
|
|
||||||
|
|
||||||
template init*(L: type BitList, x: seq[byte], N: static Limit): auto =
|
|
||||||
BitList[N](data: x)
|
|
||||||
|
|
||||||
template init*[N](L: type BitList[N], x: seq[byte]): auto =
|
|
||||||
L(data: x)
|
|
||||||
|
|
||||||
template init*(T: type BitList, len: int): auto = T init(BitSeq, len)
|
|
||||||
template len*(x: BitList): auto = len(BitSeq(x))
|
|
||||||
template bytes*(x: BitList): auto = seq[byte](x)
|
|
||||||
template `[]`*(x: BitList, idx: auto): auto = BitSeq(x)[idx]
|
|
||||||
template `[]=`*(x: var BitList, idx: auto, val: bool) = BitSeq(x)[idx] = val
|
|
||||||
template `==`*(a, b: BitList): bool = BitSeq(a) == BitSeq(b)
|
|
||||||
template getBit*(x: var BitList, idx: Natural): bool = getBit(BitSeq(x), idx)
|
|
||||||
template setBit*(x: var BitList, idx: Natural) = setBit(BitSeq(x), idx)
|
|
||||||
template clearBit*(x: var BitList, idx: Natural) = clearBit(BitSeq(x), idx)
|
|
||||||
template overlaps*(a, b: BitList): bool = overlaps(BitSeq(a), BitSeq(b))
|
|
||||||
template incl*(a: var BitList, b: BitList) = incl(BitSeq(a), BitSeq(b))
|
|
||||||
template isSubsetOf*(a, b: BitList): bool = isSubsetOf(BitSeq(a), BitSeq(b))
|
|
||||||
template isZeros*(x: BitList): bool = isZeros(BitSeq(x))
|
|
||||||
template countOnes*(x: BitList): int = countOnes(BitSeq(x))
|
|
||||||
template countZeros*(x: BitList): int = countZeros(BitSeq(x))
|
|
||||||
template countOverlap*(x, y: BitList): int = countOverlap(BitSeq(x), BitSeq(y))
|
|
||||||
template `$`*(a: BitList): string = $(BitSeq(a))
|
|
||||||
|
|
||||||
iterator items*(x: BitList): bool =
|
|
||||||
for i in 0 ..< x.len:
|
|
||||||
yield x[i]
|
|
||||||
|
|
||||||
template isCached*(v: Eth2Digest): bool =
|
|
||||||
## An entry is "in the cache" if the first 8 bytes are zero - conveniently,
|
|
||||||
## Nim initializes values this way, and while there may be false positives,
|
|
||||||
## that's fine.
|
|
||||||
|
|
||||||
# Checking and resetting the cache status are hotspots - profile before
|
|
||||||
# touching!
|
|
||||||
cast[ptr uint64](unsafeAddr v.data[0])[] != 0 # endian safe
|
|
||||||
|
|
||||||
template clearCache*(v: var Eth2Digest) =
|
|
||||||
cast[ptr uint64](addr v.data[0])[] = 0 # endian safe
|
|
||||||
|
|
||||||
template maxChunks*(a: HashList|HashArray): int64 =
|
|
||||||
## Layer where data is
|
|
||||||
maxChunkIdx(a.T, a.maxLen)
|
|
||||||
|
|
||||||
template maxDepth*(a: HashList|HashArray): int =
|
|
||||||
## Layer where data is
|
|
||||||
static: doAssert a.maxChunks <= high(int64) div 2
|
|
||||||
layer(nextPow2(a.maxChunks.uint64).int64)
|
|
||||||
|
|
||||||
template chunkIdx(a: HashList|HashArray, dataIdx: int64): int64 =
|
|
||||||
chunkIdx(a.T, dataIdx)
|
|
||||||
|
|
||||||
proc clearCaches*(a: var HashArray, dataIdx: auto) =
|
|
||||||
## Clear all cache entries after data at dataIdx has been modified
|
|
||||||
var idx = 1 shl (a.maxDepth - 1) + (chunkIdx(a, dataIdx) shr 1)
|
|
||||||
while idx != 0:
|
|
||||||
clearCache(a.hashes[idx])
|
|
||||||
idx = idx shr 1
|
|
||||||
|
|
||||||
func nodesAtLayer*(layer, depth, leaves: int): int =
|
|
||||||
## Given a number of leaves, how many nodes do you need at a given layer
|
|
||||||
## in a binary tree structure?
|
|
||||||
let leavesPerNode = 1'i64 shl (depth - layer)
|
|
||||||
int((leaves + leavesPerNode - 1) div leavesPerNode)
|
|
||||||
|
|
||||||
func cacheNodes*(depth, leaves: int): int =
|
|
||||||
## Total number of nodes needed to cache a tree of a given depth with
|
|
||||||
## `leaves` items in it - chunks that are zero-filled have well-known hash
|
|
||||||
## trees and don't need to be stored in the tree.
|
|
||||||
var res = 0
|
|
||||||
for i in 0..<depth:
|
|
||||||
res += nodesAtLayer(i, depth, leaves)
|
|
||||||
res
|
|
||||||
|
|
||||||
proc clearCaches*(a: var HashList, dataIdx: int64) =
|
|
||||||
## Clear each level of the merkle tree up to the root affected by a data
|
|
||||||
## change at `dataIdx`.
|
|
||||||
if a.hashes.len == 0:
|
|
||||||
return
|
|
||||||
|
|
||||||
var
|
|
||||||
idx = 1'i64 shl (a.maxDepth - 1) + (chunkIdx(a, dataIdx) shr 1)
|
|
||||||
layer = a.maxDepth - 1
|
|
||||||
while idx > 0:
|
|
||||||
let
|
|
||||||
idxInLayer = idx - (1'i64 shl layer)
|
|
||||||
layerIdx = idxInlayer + a.indices[layer]
|
|
||||||
if layerIdx < a.indices[layer + 1]:
|
|
||||||
# Only clear cache when we're actually storing it - ie it hasn't been
|
|
||||||
# skipped by the "combined zero hash" optimization
|
|
||||||
clearCache(a.hashes[layerIdx])
|
|
||||||
|
|
||||||
idx = idx shr 1
|
|
||||||
layer = layer - 1
|
|
||||||
|
|
||||||
clearCache(a.hashes[0])
|
|
||||||
|
|
||||||
proc clearCache*(a: var HashList) =
|
|
||||||
# Clear the full merkle tree, in anticipation of a complete rewrite of the
|
|
||||||
# contents
|
|
||||||
for c in a.hashes.mitems(): clearCache(c)
|
|
||||||
|
|
||||||
proc growHashes*(a: var HashList) =
|
|
||||||
## Ensure that the hash cache is big enough for the data in the list - must
|
|
||||||
## be called whenever `data` grows.
|
|
||||||
let
|
|
||||||
leaves = int(
|
|
||||||
chunkIdx(a, a.data.len() + dataPerChunk(a.T) - 1))
|
|
||||||
newSize = 1 + cacheNodes(a.maxDepth, leaves)
|
|
||||||
|
|
||||||
if a.hashes.len >= newSize:
|
|
||||||
return
|
|
||||||
|
|
||||||
var
|
|
||||||
newHashes = newSeq[Eth2Digest](newSize)
|
|
||||||
newIndices = default(type a.indices)
|
|
||||||
|
|
||||||
if a.hashes.len != newSize:
|
|
||||||
newIndices[0] = nodesAtLayer(0, a.maxDepth, leaves)
|
|
||||||
for i in 1..a.maxDepth:
|
|
||||||
newIndices[i] = newIndices[i - 1] + nodesAtLayer(i - 1, a.maxDepth, leaves)
|
|
||||||
|
|
||||||
for i in 1..<a.maxDepth:
|
|
||||||
for j in 0..<(a.indices[i] - a.indices[i-1]):
|
|
||||||
newHashes[newIndices[i - 1] + j] = a.hashes[a.indices[i - 1] + j]
|
|
||||||
|
|
||||||
swap(a.hashes, newHashes)
|
|
||||||
a.indices = newIndices
|
|
||||||
|
|
||||||
proc resetCache*(a: var HashList) =
|
|
||||||
## Perform a full reset of the hash cache, for example after data has been
|
|
||||||
## rewritten "manually" without going through the exported operators
|
|
||||||
a.hashes.setLen(0)
|
|
||||||
a.indices = default(type a.indices)
|
|
||||||
a.growHashes()
|
|
||||||
|
|
||||||
proc resetCache*(a: var HashArray) =
|
|
||||||
for h in a.hashes.mitems():
|
|
||||||
clearCache(h)
|
|
||||||
|
|
||||||
template len*(a: type HashArray): auto = int(a.maxLen)
|
|
||||||
|
|
||||||
proc add*(x: var HashList, val: auto): bool =
|
|
||||||
if add(x.data, val):
|
|
||||||
x.growHashes()
|
|
||||||
clearCaches(x, x.data.len() - 1)
|
|
||||||
true
|
|
||||||
else:
|
|
||||||
false
|
|
||||||
|
|
||||||
proc addDefault*(x: var HashList): ptr x.T =
|
|
||||||
if x.data.len >= x.maxLen:
|
|
||||||
return nil
|
|
||||||
|
|
||||||
distinctBase(x.data).setLen(x.data.len + 1)
|
|
||||||
x.growHashes()
|
|
||||||
clearCaches(x, x.data.len() - 1)
|
|
||||||
addr x.data[^1]
|
|
||||||
|
|
||||||
template init*[T, N](L: type HashList[T, N], x: seq[T]): auto =
|
|
||||||
var tmp = HashList[T, N](data: List[T, N].init(x))
|
|
||||||
tmp.growHashes()
|
|
||||||
tmp
|
|
||||||
|
|
||||||
template len*(x: HashList|HashArray): auto = len(x.data)
|
|
||||||
template low*(x: HashList|HashArray): auto = low(x.data)
|
|
||||||
template high*(x: HashList|HashArray): auto = high(x.data)
|
|
||||||
template `[]`*(x: HashList|HashArray, idx: auto): auto = x.data[idx]
|
|
||||||
|
|
||||||
proc `[]`*(a: var HashArray, b: auto): var a.T =
|
|
||||||
# Access item and clear cache - use asSeq when only reading!
|
|
||||||
clearCaches(a, b.Limit)
|
|
||||||
a.data[b]
|
|
||||||
|
|
||||||
proc `[]=`*(a: var HashArray, b: auto, c: auto) =
|
|
||||||
clearCaches(a, b.Limit)
|
|
||||||
a.data[b] = c
|
|
||||||
|
|
||||||
proc `[]`*(x: var HashList, idx: auto): var x.T =
|
|
||||||
# Access item and clear cache - use asSeq when only reading!
|
|
||||||
clearCaches(x, idx.int64)
|
|
||||||
x.data[idx]
|
|
||||||
|
|
||||||
proc `[]=`*(x: var HashList, idx: auto, val: auto) =
|
|
||||||
clearCaches(x, idx.int64)
|
|
||||||
x.data[idx] = val
|
|
||||||
|
|
||||||
template `==`*(a, b: HashList|HashArray): bool = a.data == b.data
|
|
||||||
template asSeq*(x: HashList): auto = asSeq(x.data)
|
|
||||||
template `$`*(x: HashList): auto = $(x.data)
|
|
||||||
|
|
||||||
template items* (x: HashList|HashArray): untyped = items(x.data)
|
|
||||||
template pairs* (x: HashList|HashArray): untyped = pairs(x.data)
|
|
||||||
|
|
||||||
template swap*(a, b: var HashList) =
|
|
||||||
swap(a.data, b.data)
|
|
||||||
swap(a.hashes, b.hashes)
|
|
||||||
swap(a.indices, b.indices)
|
|
||||||
|
|
||||||
template clear*(a: var HashList) =
|
|
||||||
if not a.data.setLen(0):
|
|
||||||
raiseAssert "length 0 should always succeed"
|
|
||||||
a.hashes.setLen(0)
|
|
||||||
a.indices = default(type a.indices)
|
|
||||||
|
|
||||||
template fill*(a: var HashArray, c: auto) =
|
|
||||||
mixin fill
|
|
||||||
fill(a.data, c)
|
|
||||||
template sum*[maxLen; T](a: var HashArray[maxLen, T]): T =
|
|
||||||
mixin sum
|
|
||||||
sum(a.data)
|
|
||||||
|
|
||||||
macro unsupported*(T: typed): untyped =
|
|
||||||
# TODO: {.fatal.} breaks compilation even in `compiles()` context,
|
|
||||||
# so we use this macro instead. It's also much better at figuring
|
|
||||||
# out the actual type that was used in the instantiation.
|
|
||||||
# File both problems as issues.
|
|
||||||
when T is enum:
|
|
||||||
error "Nim `enum` types map poorly to SSZ and make it easy to introduce security issues because of spurious Defect's"
|
|
||||||
else:
|
|
||||||
error "SSZ serialization of the type " & humaneTypeName(T) & " is not supported, overload toSszType and fromSszBytes"
|
|
||||||
|
|
||||||
template ElemType*(T0: type HashArray): untyped =
|
|
||||||
T0.T
|
|
||||||
|
|
||||||
template ElemType*(T0: type HashList): untyped =
|
|
||||||
T0.T
|
|
||||||
|
|
||||||
template ElemType*(T: type array): untyped =
|
|
||||||
type(default(T)[low(T)])
|
|
||||||
|
|
||||||
template ElemType*(T: type seq): untyped =
|
|
||||||
type(default(T)[0])
|
|
||||||
|
|
||||||
template ElemType*(T0: type List): untyped =
|
|
||||||
T0.T
|
|
||||||
|
|
||||||
func isFixedSize*(T0: type): bool {.compileTime.} =
|
|
||||||
mixin toSszType, enumAllSerializedFields
|
|
||||||
|
|
||||||
type T = type toSszType(declval T0)
|
|
||||||
|
|
||||||
when T is BasicType:
|
|
||||||
return true
|
|
||||||
elif T is array|HashArray:
|
|
||||||
return isFixedSize(ElemType(T))
|
|
||||||
elif T is object|tuple:
|
|
||||||
enumAllSerializedFields(T):
|
|
||||||
when not isFixedSize(FieldType):
|
|
||||||
return false
|
|
||||||
return true
|
|
||||||
|
|
||||||
func fixedPortionSize*(T0: type): int {.compileTime.} =
|
|
||||||
mixin enumAllSerializedFields, toSszType
|
|
||||||
|
|
||||||
type T = type toSszType(declval T0)
|
|
||||||
|
|
||||||
when T is BasicType: sizeof(T)
|
|
||||||
elif T is array|HashArray:
|
|
||||||
type E = ElemType(T)
|
|
||||||
when isFixedSize(E): int(len(T)) * fixedPortionSize(E)
|
|
||||||
else: int(len(T)) * offsetSize
|
|
||||||
elif T is object|tuple:
|
|
||||||
enumAllSerializedFields(T):
|
|
||||||
when isFixedSize(FieldType):
|
|
||||||
result += fixedPortionSize(FieldType)
|
|
||||||
else:
|
|
||||||
result += offsetSize
|
|
||||||
else:
|
|
||||||
unsupported T0
|
|
||||||
|
|
||||||
# TODO This should have been an iterator, but the VM can't compile the
|
|
||||||
# code due to "too many registers required".
|
|
||||||
proc fieldInfos*(RecordType: type): seq[tuple[name: string,
|
|
||||||
offset: int,
|
|
||||||
fixedSize: int,
|
|
||||||
branchKey: string]] =
|
|
||||||
mixin enumAllSerializedFields
|
|
||||||
|
|
||||||
var
|
|
||||||
offsetInBranch = {"": 0}.toTable
|
|
||||||
nestedUnder = initTable[string, string]()
|
|
||||||
|
|
||||||
enumAllSerializedFields(RecordType):
|
|
||||||
const
|
|
||||||
isFixed = isFixedSize(FieldType)
|
|
||||||
fixedSize = when isFixed: fixedPortionSize(FieldType)
|
|
||||||
else: 0
|
|
||||||
branchKey = when fieldCaseDiscriminator.len == 0: ""
|
|
||||||
else: fieldCaseDiscriminator & ":" & $fieldCaseBranches
|
|
||||||
fieldSize = when isFixed: fixedSize
|
|
||||||
else: offsetSize
|
|
||||||
|
|
||||||
nestedUnder[fieldName] = branchKey
|
|
||||||
|
|
||||||
var fieldOffset: int
|
|
||||||
offsetInBranch.withValue(branchKey, val):
|
|
||||||
fieldOffset = val[]
|
|
||||||
val[] += fieldSize
|
|
||||||
do:
|
|
||||||
try:
|
|
||||||
let parentBranch = nestedUnder.getOrDefault(fieldCaseDiscriminator, "")
|
|
||||||
fieldOffset = offsetInBranch[parentBranch]
|
|
||||||
offsetInBranch[branchKey] = fieldOffset + fieldSize
|
|
||||||
except KeyError as e:
|
|
||||||
raiseAssert e.msg
|
|
||||||
|
|
||||||
result.add((fieldName, fieldOffset, fixedSize, branchKey))
|
|
||||||
|
|
||||||
func getFieldBoundingOffsetsImpl(RecordType: type,
|
|
||||||
fieldName: static string):
|
|
||||||
tuple[fieldOffset, nextFieldOffset: int, isFirstOffset: bool] {.compileTime.} =
|
|
||||||
result = (-1, -1, false)
|
|
||||||
var fieldBranchKey: string
|
|
||||||
var isFirstOffset = true
|
|
||||||
|
|
||||||
for f in fieldInfos(RecordType):
|
|
||||||
if fieldName == f.name:
|
|
||||||
result[0] = f.offset
|
|
||||||
if f.fixedSize > 0:
|
|
||||||
result[1] = result[0] + f.fixedSize
|
|
||||||
return
|
|
||||||
else:
|
|
||||||
fieldBranchKey = f.branchKey
|
|
||||||
result.isFirstOffset = isFirstOffset
|
|
||||||
|
|
||||||
elif result[0] != -1 and
|
|
||||||
f.fixedSize == 0 and
|
|
||||||
f.branchKey == fieldBranchKey:
|
|
||||||
# We have found the next variable sized field
|
|
||||||
result[1] = f.offset
|
|
||||||
return
|
|
||||||
|
|
||||||
if f.fixedSize == 0:
|
|
||||||
isFirstOffset = false
|
|
||||||
|
|
||||||
func getFieldBoundingOffsets*(RecordType: type,
|
|
||||||
fieldName: static string):
|
|
||||||
tuple[fieldOffset, nextFieldOffset: int, isFirstOffset: bool] {.compileTime.} =
|
|
||||||
## Returns the start and end offsets of a field.
|
|
||||||
##
|
|
||||||
## For fixed-size fields, the start offset points to the first
|
|
||||||
## byte of the field and the end offset points to 1 byte past the
|
|
||||||
## end of the field.
|
|
||||||
##
|
|
||||||
## For variable-size fields, the returned offsets point to the
|
|
||||||
## statically known positions of the 32-bit offset values written
|
|
||||||
## within the SSZ object. You must read the 32-bit values stored
|
|
||||||
## at the these locations in order to obtain the actual offsets.
|
|
||||||
##
|
|
||||||
## For variable-size fields, the end offset may be -1 when the
|
|
||||||
## designated field is the last variable sized field within the
|
|
||||||
## object. Then the SSZ object boundary known at run-time marks
|
|
||||||
## the end of the variable-size field.
|
|
||||||
type T = RecordType
|
|
||||||
anonConst getFieldBoundingOffsetsImpl(T, fieldName)
|
|
||||||
|
|
||||||
template enumerateSubFields*(holder, fieldVar, body: untyped) =
|
|
||||||
when holder is array|HashArray:
|
|
||||||
for fieldVar in holder: body
|
|
||||||
else:
|
|
||||||
enumInstanceSerializedFields(holder, _{.used.}, fieldVar): body
|
|
||||||
|
|
||||||
method formatMsg*(
|
|
||||||
err: ref SszSizeMismatchError,
|
|
||||||
filename: string): string {.gcsafe, raises: [Defect].} =
|
|
||||||
try:
|
|
||||||
&"SSZ size mismatch, element {err.elementSize}, actual {err.actualSszSize}, type {err.deserializedType}, file {filename}"
|
|
||||||
except CatchableError:
|
|
||||||
"SSZ size mismatch"
|
|
||||||
|
|
||||||
template readValue*(reader: var JsonReader, value: var List) =
|
|
||||||
value = type(value)(readValue(reader, seq[type value[0]]))
|
|
||||||
|
|
||||||
template writeValue*(writer: var JsonWriter, value: List) =
|
|
||||||
writeValue(writer, asSeq value)
|
|
||||||
|
|
||||||
proc writeValue*(writer: var JsonWriter, value: HashList)
|
|
||||||
{.raises: [IOError, SerializationError, Defect].} =
|
|
||||||
writeValue(writer, value.data)
|
|
||||||
|
|
||||||
proc readValue*(reader: var JsonReader, value: var HashList)
|
|
||||||
{.raises: [IOError, SerializationError, Defect].} =
|
|
||||||
value.resetCache()
|
|
||||||
readValue(reader, value.data)
|
|
||||||
|
|
||||||
template readValue*(reader: var JsonReader, value: var BitList) =
|
|
||||||
type T = type(value)
|
|
||||||
value = T readValue(reader, BitSeq)
|
|
||||||
|
|
||||||
template writeValue*(writer: var JsonWriter, value: BitList) =
|
|
||||||
writeValue(writer, BitSeq value)
|
|
||||||
Loading…
Reference in New Issue