status-im
/
nimbus-eth1
mirror of https://github.com/status-im/nimbus-eth1.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

append instead of reallocate in blobify (#2277) 

...otherwise, we get lots and lots of temporary allocations of seq's
This commit is contained in:
Jacek Sieka 2024-06-01 17:13:24 +02:00 committed by GitHub
parent f9765e617b
commit 9f879406f3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 52 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
nimbus/db/aristo/aristo_blobify.nim
View File

@ -39,39 +39,41 @@ proc load256(data: openArray[byte]; start: var int): Result[UInt256,AristoError]
# Public functions # Public functions
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
proc blobify*(pyl: PayloadRef): Blob = proc blobifyTo*(pyl: PayloadRef, data: var Blob) =
if pyl.isNil: if pyl.isNil:
return return
case pyl.pType case pyl.pType
of RawData: of RawData:
result = pyl.rawBlob & @[0x6b.byte] data &= pyl.rawBlob
data &= [0x6b.byte]
of RlpData: of RlpData:
result = pyl.rlpBlob & @[0x6a.byte] data &= pyl.rlpBlob
data &= @[0x6a.byte]
of AccountData: of AccountData:
var mask: byte var mask: byte
if 0 < pyl.account.nonce: if 0 < pyl.account.nonce:
mask = mask or 0x01 mask = mask or 0x01
result &= pyl.account.nonce.uint64.toBytesBE.toSeq data &= pyl.account.nonce.uint64.toBytesBE
if high(uint64).u256 < pyl.account.balance: if high(uint64).u256 < pyl.account.balance:
mask = mask or 0x08 mask = mask or 0x08
result &= pyl.account.balance.toBytesBE.toSeq data &= pyl.account.balance.toBytesBE
elif 0 < pyl.account.balance: elif 0 < pyl.account.balance:
mask = mask or 0x04 mask = mask or 0x04
result &= pyl.account.balance.truncate(uint64).uint64.toBytesBE.toSeq data &= pyl.account.balance.truncate(uint64).uint64.toBytesBE
if VertexID(0) < pyl.account.storageID: if VertexID(0) < pyl.account.storageID:
mask = mask or 0x10 mask = mask or 0x10
result &= pyl.account.storageID.uint64.toBytesBE.toSeq data &= pyl.account.storageID.uint64.toBytesBE
if pyl.account.codeHash != VOID_CODE_HASH: if pyl.account.codeHash != VOID_CODE_HASH:
mask = mask or 0x80 mask = mask or 0x80
result &= pyl.account.codeHash.data.toSeq data &= pyl.account.codeHash.data
result &= @[mask] data &= [mask]
proc blobify*(vtx: VertexRef; data: var Blob): Result[void,AristoError] = proc blobifyTo*(vtx: VertexRef; data: var Blob): Result[void,AristoError] =
## This function serialises the vertex argument to a database record. ## This function serialises the vertex argument to a database record.
## Contrary to RLP based serialisation, these records aim to align on ## Contrary to RLP based serialisation, these records aim to align on
## fixed byte boundaries. ## fixed byte boundaries.
@ -102,14 +104,15 @@ proc blobify*(vtx: VertexRef; data: var Blob): Result[void,AristoError] =
of Branch: of Branch:
var var
access = 0u16 access = 0u16
refs: Blob pos = data.len
for n in 0..15: for n in 0..15:
if vtx.bVid[n].isValid: if vtx.bVid[n].isValid:
access = access or (1u16 shl n) access = access or (1u16 shl n)
refs &= vtx.bVid[n].uint64.toBytesBE.toSeq data &= vtx.bVid[n].uint64.toBytesBE
if refs.len < 16: if data.len - pos < 16:
return err(BlobifyBranchMissingRefs) return err(BlobifyBranchMissingRefs)
data = refs & access.toBytesBE.toSeq & @[0x08u8] data &= access.toBytesBE
data &= [0x08u8]
of Extension: of Extension:
let let
pSegm = vtx.ePfx.hexPrefixEncode(isleaf = false) pSegm = vtx.ePfx.hexPrefixEncode(isleaf = false)
@ -118,39 +121,42 @@ proc blobify*(vtx: VertexRef; data: var Blob): Result[void,AristoError] =
return err(BlobifyExtPathOverflow) return err(BlobifyExtPathOverflow)
if not vtx.eVid.isValid: if not vtx.eVid.isValid:
return err(BlobifyExtMissingRefs) return err(BlobifyExtMissingRefs)
data = vtx.eVid.uint64.toBytesBE.toSeq & pSegm & @[0x80u8 or psLen] data &= vtx.eVid.uint64.toBytesBE
data &= pSegm
data &= [0x80u8 or psLen]
of Leaf: of Leaf:
let let
pSegm = vtx.lPfx.hexPrefixEncode(isleaf = true) pSegm = vtx.lPfx.hexPrefixEncode(isleaf = true)
psLen = pSegm.len.byte psLen = pSegm.len.byte
if psLen == 0 or 33 < psLen: if psLen == 0 or 33 < psLen:
return err(BlobifyLeafPathOverflow) return err(BlobifyLeafPathOverflow)
data = vtx.lData.blobify & pSegm & @[0xC0u8 or psLen] vtx.lData.blobifyTo(data)
data &= pSegm
data &= [0xC0u8 or psLen]
ok() ok()
proc blobify*(vtx: VertexRef): Result[Blob, AristoError] = proc blobify*(vtx: VertexRef): Result[Blob, AristoError] =
## Variant of `blobify()` ## Variant of `blobify()`
var data: Blob var data: Blob
? vtx.blobify data ? vtx.blobifyTo data
ok(move(data)) ok(move(data))
proc blobify*(vGen: openArray[VertexID]; data: var Blob) = proc blobifyTo*(vGen: openArray[VertexID]; data: var Blob) =
## This function serialises a list of vertex IDs. ## This function serialises a list of vertex IDs.
## :: ## ::
## uint64, ... -- list of IDs ## uint64, ... -- list of IDs
## 0x7c -- marker(8) ## 0x7c -- marker(8)
## ##
data.setLen(0)
for w in vGen: for w in vGen:
data &= w.uint64.toBytesBE.toSeq data &= w.uint64.toBytesBE
data.add 0x7Cu8 data.add 0x7Cu8
proc blobify*(vGen: openArray[VertexID]): Blob = proc blobify*(vGen: openArray[VertexID]): Blob =
## Variant of `blobify()` ## Variant of `blobify()`
vGen.blobify result vGen.blobifyTo result
proc blobify*(lSst: SavedState; data: var Blob) = proc blobifyTo*(lSst: SavedState; data: var Blob) =
## Serialise a last saved state record ## Serialise a last saved state record
data.setLen(73) data.setLen(73)
(addr data[0]).copyMem(unsafeAddr lSst.src.data[0], 32) (addr data[0]).copyMem(unsafeAddr lSst.src.data[0], 32)
@ -161,10 +167,10 @@ proc blobify*(lSst: SavedState; data: var Blob) =
proc blobify*(lSst: SavedState): Blob = proc blobify*(lSst: SavedState): Blob =
## Variant of `blobify()` ## Variant of `blobify()`
lSst.blobify result lSst.blobifyTo result
proc blobify*(filter: FilterRef; data: var Blob): Result[void,AristoError] = proc blobifyTo*(filter: FilterRef; data: var Blob): Result[void,AristoError] =
## This function serialises an Aristo DB filter object ## This function serialises an Aristo DB filter object
## :: ## ::
## uint64 -- filter ID ## uint64 -- filter ID
@ -189,17 +195,17 @@ proc blobify*(filter: FilterRef; data: var Blob): Result[void,AristoError] =
if not filter.isValid: if not filter.isValid:
return err(BlobifyNilFilter) return err(BlobifyNilFilter)
data.setLen(0)
data &= filter.fid.uint64.toBytesBE.toSeq
data &= @(filter.src.data)
data &= @(filter.trg.data)
data &= filter.vGen.len.uint32.toBytesBE.toSeq data &= filter.fid.uint64.toBytesBE
data &= newSeq[byte](4) # place holder data &= filter.src.data
data &= filter.trg.data
data &= filter.vGen.len.uint32.toBytesBE
data &= default(array[4, byte]) # place holder
# Store vertex ID generator state # Store vertex ID generator state
for w in filter.vGen: for w in filter.vGen:
data &= w.uint64.toBytesBE.toSeq data &= w.uint64.toBytesBE
var var
n = 0 n = 0
@ -224,18 +230,17 @@ proc blobify*(filter: FilterRef; data: var Blob): Result[void,AristoError] =
keyMode = 0x4000_0000u # void hash key => considered deleted keyMode = 0x4000_0000u # void hash key => considered deleted
if vtx.isValid: if vtx.isValid:
? vtx.blobify vtxBlob ? vtx.blobifyTo vtxBlob
vtxLen = vtxBlob.len.uint vtxLen = vtxBlob.len.uint
if 0x3fff_ffff <= vtxLen: if 0x3fff_ffff <= vtxLen:
return err(BlobifyFilterRecordOverflow) return err(BlobifyFilterRecordOverflow)
else: else:
vtxLen = 0x3fff_ffff # nil vertex => considered deleted vtxLen = 0x3fff_ffff # nil vertex => considered deleted
data &= data &= (keyMode or vtxLen).uint32.toBytesBE
(keyMode or vtxLen).uint32.toBytesBE.toSeq & data &= vid.uint64.toBytesBE
vid.uint64.toBytesBE.toSeq & data &= keyBlob
keyBlob & data &= vtxBlob
vtxBlob
# Loop over remaining data from key table # Loop over remaining data from key table
for vid in leftOver: for vid in leftOver:
@ -251,36 +256,34 @@ proc blobify*(filter: FilterRef; data: var Blob): Result[void,AristoError] =
else: else:
keyMode = 0x4000_0000u # void hash key => considered deleted keyMode = 0x4000_0000u # void hash key => considered deleted
data &= data &= keyMode.uint32.toBytesBE
keyMode.uint32.toBytesBE.toSeq & data &= vid.uint64.toBytesBE
vid.uint64.toBytesBE.toSeq & data &= keyBlob
keyBlob
data[76 ..< 80] = n.uint32.toBytesBE.toSeq data[76 ..< 80] = n.uint32.toBytesBE
data.add 0x7Du8 data.add 0x7Du8
ok() ok()
proc blobify*(filter: FilterRef): Result[Blob, AristoError] = proc blobify*(filter: FilterRef): Result[Blob, AristoError] =
## ... ## ...
var data: Blob var data: Blob
? filter.blobify data ? filter.blobifyTo data
ok move(data) ok move(data)
proc blobify*(vFqs: openArray[(QueueID,QueueID)]; data: var Blob) = proc blobifyTo*(vFqs: openArray[(QueueID,QueueID)]; data: var Blob) =
## This function serialises a list of filter queue IDs. ## This function serialises a list of filter queue IDs.
## :: ## ::
## uint64, ... -- list of IDs ## uint64, ... -- list of IDs
## 0x7e -- marker(8) ## 0x7e -- marker(8)
## ##
data.setLen(0)
for w in vFqs: for w in vFqs:
data &= w[0].uint64.toBytesBE.toSeq data &= w[0].uint64.toBytesBE
data &= w[1].uint64.toBytesBE.toSeq data &= w[1].uint64.toBytesBE
data.add 0x7Eu8 data.add 0x7Eu8
proc blobify*(vFqs: openArray[(QueueID,QueueID)]): Blob = proc blobify*(vFqs: openArray[(QueueID,QueueID)]): Blob =
## Variant of `blobify()` ## Variant of `blobify()`
vFqs.blobify result vFqs.blobifyTo result
# ------------- # -------------

2
nimbus/db/aristo/aristo_init/rocks_db/rdb_walk.nim
View File

@ -88,7 +88,7 @@ iterator walk*(
let val = rit.value() let val = rit.value()
if val.len != 0: if val.len != 0:
yield (uint64.fromBytesBE key[1..^1], val) yield (uint64.fromBytesBE key.toOpenArray(1, key.high()), val)
# Update Iterator # Update Iterator
rit.next() rit.next()