status-im
/
nimbus-eth2
mirror of https://github.com/status-im/nimbus-eth2.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity
nimbus-eth2/beacon_chain/ssz.nim

598 lines
20 KiB
Nim
Raw Blame History

# beacon_chain
# Copyright (c) 2018 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
# SSZ Serialization (simple serialize)
# See https://github.com/ethereum/eth2.0-specs/blob/master/specs/simple-serialize.md
# TODO Cannot override push, even though the function is annotated
# nim-beacon-chain/beacon_chain/ssz.nim(212, 18) Error: can raise an unlisted exception: IOError
#{.push raises: [Defect].}
import
stew/shims/macros, options, algorithm, options,
stew/[bitops2, bitseqs, endians2, objects, varints, ptrops, ranges/ptr_arith], stint,
faststreams/input_stream, serialization, serialization/testing/tracing,
./spec/[crypto, datatypes, digest],
./ssz/[types, bytes_reader]
# ################### Helper functions ###################################
export
serialization, types, bytes_reader
when defined(serialization_tracing):
import
typetraits, stew/ranges/ptr_arith
const
bytesPerChunk = 32
bitsPerChunk = bytesPerChunk * 8
maxChunkTreeDepth = 25
defaultMaxObjectSize = 1 * 1024 * 1024
type
SszReader* = object
stream: InputStream
maxObjectSize: int
SszWriter* = object
stream: OutputStream
BasicType = char|bool|SomeUnsignedInt|StUint|ValidatorIndex
SszChunksMerkleizer = ref object
combinedChunks: array[maxChunkTreeDepth, Eth2Digest]
totalChunks: uint64
limit: uint64
SszHashingStream = ref object of OutputStream
merkleizer: SszChunksMerkleizer
TypeWithMaxLen*[T; maxLen: static int64] = distinct T
SizePrefixed*[T] = distinct T
SszMaxSizeExceeded* = object of SerializationError
VarSizedWriterCtx = object
fixedParts: WriteCursor
offset: int
FixedSizedWriterCtx = object
ByteList = seq[byte]
serializationFormat SSZ,
Reader = SszReader,
Writer = SszWriter,
PreferedOutput = seq[byte]
template sizePrefixed*[TT](x: TT): untyped =
type T = TT
SizePrefixed[T](x)
proc init*(T: type SszReader,
stream: InputStream,
maxObjectSize = defaultMaxObjectSize): T {.raises: [Defect].} =
T(stream: stream, maxObjectSize: maxObjectSize)
proc mount*(F: type SSZ, stream: InputStream, T: type): T {.raises: [Defect].} =
mixin readValue
var reader = init(SszReader, stream)
reader.readValue(T)
method formatMsg*(err: ref SszSizeMismatchError, filename: string): string {.gcsafe, raises: [Defect].} =
# TODO: implement proper error string
"Serialisation error while processing " & filename
when false:
# TODO: Nim can't handle yet this simpler definition. File an issue.
template valueOf[T; N](x: TypeWithMaxLen[T, N]): auto = T(x)
else:
proc unwrapImpl[T; N: static int64](x: ptr TypeWithMaxLen[T, N]): ptr T =
cast[ptr T](x)
template valueOf(x: TypeWithMaxLen): auto =
let xaddr = unsafeAddr x
unwrapImpl(xaddr)[]
template sszList*(x: seq|array, maxLen: static int64): auto =
TypeWithMaxLen[type(x), maxLen](x)
template toSszType*(x: auto): auto =
mixin toSszType
when x is Slot|Epoch|ValidatorIndex|enum: uint64(x)
elif x is Eth2Digest: x.data
elif x is BlsCurveType: toRaw(x)
elif x is BitSeq|BitList: ByteList(x)
elif x is ref|ptr: toSszType x[]
elif x is Option: toSszType x.get
elif x is TypeWithMaxLen: toSszType valueOf(x)
elif useListType and x is List: seq[x.T](x)
else: x
proc writeFixedSized(c: var WriteCursor, x: auto) {.raises: [Defect, IOError].} =
mixin toSszType
when x is byte:
c.append x
elif x is bool|char:
c.append byte(ord(x))
elif x is SomeUnsignedInt:
let value = x.toBytesLE()
trs "APPENDING INT ", x, " = ", value
c.appendMemCopy value
elif x is StUint:
c.appendMemCopy x # TODO: Is this always correct?
elif x is array|string|seq|openarray:
when x[0] is byte:
trs "APPENDING FIXED SIZE BYTES", x
c.append x
else:
for elem in x:
trs "WRITING FIXED SIZE ARRAY ELEMENT"
c.writeFixedSized toSszType(elem)
elif x is tuple|object:
enumInstanceSerializedFields(x, fieldName, field):
trs "WRITING FIXED SIZE FIELD", fieldName
c.writeFixedSized toSszType(field)
else:
unsupported x.type
template writeFixedSized(s: OutputStream, x: auto) =
writeFixedSized(s.cursor, x)
template supports*(_: type SSZ, T: type): bool =
mixin toSszType
anonConst compiles(fixedPortionSize toSszType(declval T))
func init*(T: type SszWriter, stream: OutputStream): T {.raises: [Defect].} =
result.stream = stream
template enumerateSubFields(holder, fieldVar, body: untyped) =
when holder is array|string|seq|openarray:
for fieldVar in holder: body
else:
enumInstanceSerializedFields(holder, _, fieldVar): body
proc writeVarSizeType(w: var SszWriter, value: auto) {.gcsafe.}
proc beginRecord*(w: var SszWriter, TT: type): auto {.raises: [Defect].} =
type T = TT
when isFixedSize(T):
FixedSizedWriterCtx()
else:
const offset = when T is array: len(T) * offsetSize
else: fixedPortionSize(T)
VarSizedWriterCtx(offset: offset,
fixedParts: w.stream.delayFixedSizeWrite(offset))
template writeField*(w: var SszWriter,
ctx: var auto,
fieldName: string,
field: auto) =
mixin toSszType
when ctx is FixedSizedWriterCtx:
writeFixedSized(w.stream, toSszType(field))
else:
type FieldType = type toSszType(field)
when isFixedSize(FieldType):
ctx.fixedParts.writeFixedSized toSszType(field)
else:
trs "WRITING OFFSET ", ctx.offset, " FOR ", fieldName
ctx.fixedParts.writeFixedSized uint32(ctx.offset)
let initPos = w.stream.pos
trs "WRITING VAR SIZE VALUE OF TYPE ", name(FieldType)
when FieldType is BitSeq:
trs "BIT SEQ ", ByteList(field)
writeVarSizeType(w, toSszType(field))
ctx.offset += w.stream.pos - initPos
template endRecord*(w: var SszWriter, ctx: var auto) =
when ctx is VarSizedWriterCtx:
finalize ctx.fixedParts
proc writeVarSizeType(w: var SszWriter, value: auto) {.raises: [Defect, IOError].} =
trs "STARTING VAR SIZE TYPE"
mixin toSszType
type T = type toSszType(value)
when T is seq|string|openarray:
type E = ElemType(T)
const isFixed = when E is Option: false
else: isFixedSize(E)
when isFixed:
trs "WRITING LIST WITH FIXED SIZE ELEMENTS"
for elem in value:
w.stream.writeFixedSized toSszType(elem)
trs "DONE"
else:
trs "WRITING LIST WITH VAR SIZE ELEMENTS"
var offset = value.len * offsetSize
var cursor = w.stream.delayFixedSizeWrite offset
for elem in value:
cursor.writeFixedSized uint32(offset)
when elem is Option:
if not isSome(elem): continue
elif elem is ptr|ref:
if isNil(elem): continue
let initPos = w.stream.pos
w.writeVarSizeType toSszType(elem)
offset += w.stream.pos - initPos
finalize cursor
trs "DONE"
elif T is object|tuple|array:
trs "WRITING OBJECT OR ARRAY"
var ctx = beginRecord(w, T)
enumerateSubFields(value, field):
writeField w, ctx, astToStr(field), field
endRecord w, ctx
proc writeValue*(w: var SszWriter, x: auto) {.gcsafe, raises: [Defect, IOError].} =
mixin toSszType
type T = type toSszType(x)
when isFixedSize(T):
w.stream.writeFixedSized toSszType(x)
elif T is array|seq|openarray|string|object|tuple:
w.writeVarSizeType toSszType(x)
else:
unsupported type(x)
proc writeValue*[T](w: var SszWriter, x: SizePrefixed[T]) {.raises: [Defect, IOError].} =
var cursor = w.stream.delayVarSizeWrite(10)
let initPos = w.stream.pos
w.writeValue T(x)
let length = uint64(w.stream.pos - initPos)
when false:
discard
# TODO varintBytes is sub-optimal at the moment
# cursor.writeAndFinalize length.varintBytes
else:
var buf: VarintBuffer
buf.appendVarint length
cursor.writeAndFinalize buf.writtenBytes
template fromSszBytes*[T; N](_: type TypeWithMaxLen[T, N],
bytes: openarray[byte]): auto =
mixin fromSszBytes
fromSszBytes(T, bytes)
proc readValue*[T](r: var SszReader, val: var T) {.raises: [Defect, MalformedSszError, SszSizeMismatchError].} =
when isFixedSize(T):
const minimalSize = fixedPortionSize(T)
if r.stream.readable(minimalSize):
val = readSszValue(r.stream.read(minimalSize), T)
else:
raise newException(MalformedSszError, "SSZ input of insufficient size")
else:
# TODO Read the fixed portion first and precisely measure the size of
# the dynamic portion to consume the right number of bytes.
val = readSszValue(r.stream.read(r.stream.endPos), T)
proc readValue*[T](r: var SszReader, val: var SizePrefixed[T]) {.raises: [Defect].} =
let length = r.stream.readVarint(uint64)
if length > r.maxObjectSize:
raise newException(SszMaxSizeExceeded,
"Maximum SSZ object size exceeded: " & $length)
val = readSszValue(r.stream.read(length), T)
const
zeroChunk = default array[32, byte]
func hash(a, b: openArray[byte]): Eth2Digest =
result = withEth2Hash:
trs "MERGING BRANCHES "
trs a
trs b
h.update a
h.update b
trs "HASH RESULT ", result
func mergeBranches(existing: Eth2Digest, newData: openarray[byte]): Eth2Digest =
result = withEth2Hash:
trs "MERGING BRANCHES OPEN ARRAY"
trs existing.data
trs newData
h.update existing.data
h.update newData
let paddingBytes = bytesPerChunk - newData.len
if paddingBytes > 0:
trs "USING ", paddingBytes, " PADDING BYTES"
h.update zeroChunk[0 ..< paddingBytes]
trs "HASH RESULT ", result
template mergeBranches(a, b: Eth2Digest): Eth2Digest =
hash(a.data, b.data)
func computeZeroHashes: array[100, Eth2Digest] =
result[0] = Eth2Digest(data: zeroChunk)
for i in 1 .. result.high:
result[i] = mergeBranches(result[i - 1], result[i - 1])
let zeroHashes = computeZeroHashes()
func getZeroHashWithoutSideEffect(idx: int): Eth2Digest =
# TODO this is a work-around for the somewhat broken side
# effects analysis of Nim - reading from global let variables
# is considered a side-effect.
{.noSideEffect.}:
zeroHashes[idx]
func addChunk(merkleizer: SszChunksMerkleizer, data: openarray[byte]) =
doAssert data.len > 0 and data.len <= bytesPerChunk
if not getBitLE(merkleizer.totalChunks, 0):
let chunkStartAddr = addr merkleizer.combinedChunks[0].data[0]
copyMem(chunkStartAddr, unsafeAddr data[0], data.len)
zeroMem(chunkStartAddr.offset(data.len), bytesPerChunk - data.len)
trs "WROTE BASE CHUNK ", merkleizer.combinedChunks[0]
else:
var hash = mergeBranches(merkleizer.combinedChunks[0], data)
for i in 1 .. high(merkleizer.combinedChunks):
trs "ITERATING"
if getBitLE(merkleizer.totalChunks, i):
trs "CALLING MERGE BRANCHES"
hash = mergeBranches(merkleizer.combinedChunks[i], hash)
else:
trs "WRITING FRESH CHUNK AT ", i, " = ", hash
merkleizer.combinedChunks[i] = hash
break
inc merkleizer.totalChunks
func getFinalHash(merkleizer: SszChunksMerkleizer): Eth2Digest =
let limit = merkleizer.limit
if merkleizer.totalChunks == 0:
let limitHeight = if limit != 0: bitWidth(limit - 1) else: 0
return getZeroHashWithoutSideEffect(limitHeight)
let
bottomHashIdx = firstOne(merkleizer.totalChunks) - 1
submittedChunksHeight = bitWidth(merkleizer.totalChunks - 1)
topHashIdx = if limit <= 1: submittedChunksHeight
else: max(submittedChunksHeight, bitWidth(limit - 1))
trs "BOTTOM HASH ", bottomHashIdx
trs "SUBMITTED HEIGHT ", submittedChunksHeight
trs "LIMIT ", limit
if bottomHashIdx != submittedChunksHeight:
# Our tree is not finished. We must complete the work in progress
# branches and then extend the tree to the right height.
result = mergeBranches(merkleizer.combinedChunks[bottomHashIdx],
getZeroHashWithoutSideEffect(bottomHashIdx))
for i in bottomHashIdx + 1 ..< topHashIdx:
if getBitLE(merkleizer.totalChunks, i):
result = mergeBranches(merkleizer.combinedChunks[i], result)
trs "COMBINED"
else:
result = mergeBranches(result, getZeroHashWithoutSideEffect(i))
trs "COMBINED WITH ZERO"
elif bottomHashIdx == topHashIdx:
# We have a perfect tree (chunks == 2**n) at just the right height!
result = merkleizer.combinedChunks[bottomHashIdx]
else:
# We have a perfect tree of user chunks, but we have more work to
# do - we must extend it to reach the desired height
result = mergeBranches(merkleizer.combinedChunks[bottomHashIdx],
getZeroHashWithoutSideEffect(bottomHashIdx))
for i in bottomHashIdx + 1 ..< topHashIdx:
result = mergeBranches(result, getZeroHashWithoutSideEffect(i))
let SszHashingStreamVTable = OutputStreamVTable(
writePageSync: proc (s: OutputStream, data: openarray[byte])
{.nimcall, gcsafe, raises: [Defect, IOError].} =
trs "ADDING STREAM CHUNK ", data
SszHashingStream(s).merkleizer.addChunk(data)
,
flushSync: proc (s: OutputStream) {.nimcall, gcsafe.} =
discard
)
func newSszHashingStream(merkleizer: SszChunksMerkleizer): OutputStream =
result = SszHashingStream(vtable: vtableAddr SszHashingStreamVTable,
pageSize: bytesPerChunk,
maxWriteSize: bytesPerChunk,
minWriteSize: bytesPerChunk,
merkleizer: merkleizer)
result.initWithSinglePage()
func mixInLength(root: Eth2Digest, length: int): Eth2Digest =
var dataLen: array[32, byte]
dataLen[0..<8] = uint64(length).toBytesLE()
hash(root.data, dataLen)
func merkleizeSerializedChunks(merkleizer: SszChunksMerkleizer,
obj: auto): Eth2Digest =
try:
var hashingStream = newSszHashingStream merkleizer
{.noSideEffect.}:
# We assume there are no side-effects here, because the
# SszHashingStream is keeping all of its output in memory.
hashingStream.writeFixedSized obj
hashingStream.flush
merkleizer.getFinalHash
except IOError as e:
# Hashing shouldn't raise in theory but because of abstraction
# tax in the faststreams library, we have to do this at runtime
raiseAssert($e.msg)
func merkleizeSerializedChunks(obj: auto): Eth2Digest =
merkleizeSerializedChunks(SszChunksMerkleizer(), obj)
func hash_tree_root*(x: auto): Eth2Digest {.gcsafe, raises: [Defect].}
template merkleizeFields(body: untyped): Eth2Digest {.dirty.} =
var merkleizer {.inject.} = SszChunksMerkleizer()
template addField(field) =
let hash = hash_tree_root(field)
trs "MERKLEIZING FIELD ", astToStr(field), " = ", hash
addChunk(merkleizer, hash.data)
trs "CHUNK ADDED"
template addField2(field) {.used.}=
const maxLen = fieldMaxLen(field)
when maxLen > 0:
type FieldType = type field
addField TypeWithMaxLen[FieldType, maxLen](field)
else:
addField field
body
merkleizer.getFinalHash
func bitlistHashTreeRoot(merkleizer: SszChunksMerkleizer, x: BitSeq): Eth2Digest =
trs "CHUNKIFYING BIT SEQ WITH LIMIT ", merkleizer.limit
var
totalBytes = ByteList(x).len
lastCorrectedByte = ByteList(x)[^1]
if lastCorrectedByte == byte(1):
if totalBytes == 1:
# This is an empty bit list.
# It should be hashed as a tree containing all zeros:
let treeHeight = if merkleizer.limit == 0: 0
else: log2trunc(merkleizer.limit)
return mergeBranches(getZeroHashWithoutSideEffect(treeHeight),
getZeroHashWithoutSideEffect(0)) # this is the mixed length
totalBytes -= 1
lastCorrectedByte = ByteList(x)[^2]
else:
let markerPos = log2trunc(lastCorrectedByte)
lastCorrectedByte.clearBit(markerPos)
var
bytesInLastChunk = totalBytes mod bytesPerChunk
fullChunks = totalBytes div bytesPerChunk
if bytesInLastChunk == 0:
fullChunks -= 1
bytesInLastChunk = 32
for i in 0 ..< fullChunks:
let
chunkStartPos = i * bytesPerChunk
chunkEndPos = chunkStartPos + bytesPerChunk - 1
merkleizer.addChunk ByteList(x).toOpenArray(chunkEndPos, chunkEndPos)
var
lastChunk: array[bytesPerChunk, byte]
chunkStartPos = fullChunks * bytesPerChunk
for i in 0 .. bytesInLastChunk - 2:
lastChunk[i] = ByteList(x)[chunkStartPos + i]
lastChunk[bytesInLastChunk - 1] = lastCorrectedByte
merkleizer.addChunk lastChunk.toOpenArray(0, bytesInLastChunk - 1)
let contentsHash = merkleizer.getFinalHash
mixInLength contentsHash, x.len
func hashTreeRootImpl[T](x: T): Eth2Digest =
when T is uint64:
trs "UINT64; LITTLE-ENDIAN IDENTITY MAPPING"
result.data[0..<8] = x.toBytesLE()
elif (when T is array: ElemType(T) is byte and
sizeof(T) == sizeof(Eth2Digest) else: false):
# TODO is this sizeof comparison guranteed? it's whole structure vs field
trs "ETH2DIGEST; IDENTITY MAPPING"
Eth2Digest(data: x)
elif (T is BasicType) or (when T is array: ElemType(T) is BasicType else: false):
trs "FIXED TYPE; USE CHUNK STREAM"
merkleizeSerializedChunks x
elif T is string or (when T is (seq|openarray): ElemType(T) is BasicType else: false):
trs "TYPE WITH LENGTH"
mixInLength merkleizeSerializedChunks(x), x.len
elif T is array|object|tuple:
trs "MERKLEIZING FIELDS"
merkleizeFields:
x.enumerateSubFields(f):
const maxLen = fieldMaxLen(f)
when maxLen > 0:
type FieldType = type f
addField TypeWithMaxLen[FieldType, maxLen](f)
else:
addField f
elif T is seq:
trs "SEQ WITH VAR SIZE"
let hash = merkleizeFields(for e in x: addField e)
mixInLength hash, x.len
#elif isCaseObject(T):
# # TODO implement this
else:
unsupported T
func maxChunksCount(T: type, maxLen: static int64): int64 {.compileTime.} =
when T is BitList:
(maxLen + bitsPerChunk - 1) div bitsPerChunk
elif T is seq:
type E = ElemType(T)
when E is BasicType:
(maxLen * sizeof(E) + bytesPerChunk - 1) div bytesPerChunk
else:
maxLen
else:
unsupported T # This should never happen
func hash_tree_root*(x: auto): Eth2Digest {.raises: [Defect].} =
trs "STARTING HASH TREE ROOT FOR TYPE ", name(type(x))
mixin toSszType
when x is SignedBeaconBlock:
doassert false
when x is TypeWithMaxLen:
const maxLen = x.maxLen
type T = type valueOf(x)
const limit = maxChunksCount(T, maxLen)
var merkleizer = SszChunksMerkleizer(limit: uint64(limit))
when T is BitList:
result = merkleizer.bitlistHashTreeRoot(BitSeq valueOf(x))
elif T is seq:
type E = ElemType(T)
let contentsHash = when E is BasicType:
merkleizeSerializedChunks(merkleizer, valueOf(x))
else:
for elem in valueOf(x):
let elemHash = hash_tree_root(elem)
merkleizer.addChunk(elemHash.data)
merkleizer.getFinalHash()
result = mixInLength(contentsHash, valueOf(x).len)
else:
unsupported T # This should never happen
else:
result = hashTreeRootImpl toSszType(x)
trs "HASH TREE ROOT FOR ", name(type x), " = ", "0x", $result
iterator hash_tree_roots_prefix*[T](lst: openarray[T], limit: auto):
Eth2Digest =
# This is a particular type's instantiation of a general fold, reduce,
# accumulation, prefix sums, etc family of operations. As long as that
# Eth1 deposit case is the only notable example -- the usual uses of a
# list involve, at some point, tree-hashing it -- finalized hashes are
# the only abstraction that escapes from this module this way.
var merkleizer = SszChunksMerkleizer(limit: uint64(limit))
for i, elem in lst:
merkleizer.addChunk(hash_tree_root(elem).data)
yield mixInLength(merkleizer.getFinalHash(), i + 1)
Reference in New Issue View Git Blame Copy Permalink