2023-06-20 14:26:25 +01:00
|
|
|
# nimbus-eth1
|
2024-03-05 12:54:42 +08:00
|
|
|
# Copyright (c) 2023-2024 Status Research & Development GmbH
|
2023-06-20 14:26:25 +01:00
|
|
|
# Licensed under either of
|
|
|
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
|
|
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
|
|
|
# http://opensource.org/licenses/MIT)
|
|
|
|
# at your option. This file may not be copied, modified, or distributed
|
|
|
|
# except according to those terms.
|
|
|
|
|
|
|
|
## Rocks DB store data record
|
|
|
|
## ==========================
|
|
|
|
|
|
|
|
{.push raises: [].}
|
|
|
|
|
|
|
|
import
|
|
|
|
eth/common,
|
|
|
|
rocksdb,
|
2023-09-12 19:45:12 +01:00
|
|
|
results,
|
2024-06-10 12:04:22 +00:00
|
|
|
../../[aristo_blobify, aristo_desc],
|
2023-08-25 23:53:59 +01:00
|
|
|
../init_common,
|
2023-06-20 14:26:25 +01:00
|
|
|
./rdb_desc
|
|
|
|
|
|
|
|
const
|
2024-04-16 20:39:11 +00:00
|
|
|
extraTraceMessages = false
|
2023-06-20 14:26:25 +01:00
|
|
|
## Enable additional logging noise
|
|
|
|
|
2024-04-16 20:39:11 +00:00
|
|
|
when extraTraceMessages:
|
|
|
|
import chronicles
|
2023-06-20 14:26:25 +01:00
|
|
|
|
2024-04-16 20:39:11 +00:00
|
|
|
logScope:
|
|
|
|
topics = "aristo-rocksdb"
|
2023-06-20 14:26:25 +01:00
|
|
|
|
|
|
|
# ------------------------------------------------------------------------------
|
2024-04-16 20:39:11 +00:00
|
|
|
# Private helpers
|
2023-06-20 14:26:25 +01:00
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
|
2024-04-16 20:39:11 +00:00
|
|
|
proc disposeSession(rdb: var RdbInst) =
|
|
|
|
rdb.session.close()
|
|
|
|
rdb.session = WriteBatchRef(nil)
|
2023-06-20 14:26:25 +01:00
|
|
|
|
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
# Public functions
|
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
|
2024-04-16 20:39:11 +00:00
|
|
|
proc begin*(rdb: var RdbInst) =
|
|
|
|
if rdb.session.isNil:
|
2024-06-10 12:04:22 +00:00
|
|
|
rdb.session = rdb.baseDb.openWriteBatch()
|
2024-04-16 20:39:11 +00:00
|
|
|
|
|
|
|
proc rollback*(rdb: var RdbInst) =
|
|
|
|
if not rdb.session.isClosed():
|
2024-09-13 15:47:50 +02:00
|
|
|
rdb.rdKeyLru = typeof(rdb.rdKeyLru).init(rdb.rdKeySize)
|
|
|
|
rdb.rdVtxLru = typeof(rdb.rdVtxLru).init(rdb.rdVtxSize)
|
2024-12-11 11:53:26 +01:00
|
|
|
rdb.rdBranchLru = typeof(rdb.rdBranchLru).init(rdb.rdBranchSize)
|
2024-04-16 20:39:11 +00:00
|
|
|
rdb.disposeSession()
|
|
|
|
|
|
|
|
proc commit*(rdb: var RdbInst): Result[void,(AristoError,string)] =
|
|
|
|
if not rdb.session.isClosed():
|
|
|
|
defer: rdb.disposeSession()
|
2024-06-10 12:04:22 +00:00
|
|
|
rdb.baseDb.write(rdb.session).isOkOr:
|
2024-04-16 20:39:11 +00:00
|
|
|
const errSym = RdbBeDriverWriteError
|
|
|
|
when extraTraceMessages:
|
|
|
|
trace logTxt "commit", error=errSym, info=error
|
|
|
|
return err((errSym,error))
|
|
|
|
ok()
|
2023-06-20 14:26:25 +01:00
|
|
|
|
2024-06-10 12:04:22 +00:00
|
|
|
|
|
|
|
proc putAdm*(
|
2024-04-22 19:02:22 +00:00
|
|
|
rdb: var RdbInst;
|
2024-06-10 12:04:22 +00:00
|
|
|
xid: AdminTabID;
|
2024-07-02 20:25:06 +02:00
|
|
|
data: openArray[byte];
|
2024-06-10 12:04:22 +00:00
|
|
|
): Result[void,(AdminTabID,AristoError,string)] =
|
|
|
|
let dsc = rdb.session
|
|
|
|
if data.len == 0:
|
2024-06-27 16:51:43 +08:00
|
|
|
dsc.delete(xid.toOpenArray, rdb.admCol.handle()).isOkOr:
|
2024-06-10 12:04:22 +00:00
|
|
|
const errSym = RdbBeDriverDelAdmError
|
|
|
|
when extraTraceMessages:
|
|
|
|
trace logTxt "putAdm()", xid, error=errSym, info=error
|
|
|
|
return err((xid,errSym,error))
|
|
|
|
else:
|
2024-06-27 16:51:43 +08:00
|
|
|
dsc.put(xid.toOpenArray, data, rdb.admCol.handle()).isOkOr:
|
2024-06-10 12:04:22 +00:00
|
|
|
const errSym = RdbBeDriverPutAdmError
|
|
|
|
when extraTraceMessages:
|
|
|
|
trace logTxt "putAdm()", xid, error=errSym, info=error
|
|
|
|
return err((xid,errSym,error))
|
2024-04-22 19:02:22 +00:00
|
|
|
ok()
|
|
|
|
|
|
|
|
proc putVtx*(
|
|
|
|
rdb: var RdbInst;
|
Store keys together with node data (#2849)
Currently, computed hash keys are stored in a separate column family
with respect to the MPT data they're generated from - this has several
disadvantages:
* A lot of space is wasted because the lookup key (`RootedVertexID`) is
repeated in both tables - this is 30% of the `AriKey` content!
* rocksdb must maintain in-memory bloom filters and LRU caches for said
keys, doubling its "minimal efficient cache size"
* An extra disk traversal must be made to check for existence of cached
hash key
* Doubles the amount of files on disk due to each column family being
its own set of files
Here, the two CFs are joined such that both key and data is stored in
`AriVtx`. This means:
* we save ~30% disk space on repeated lookup keys
* we save ~2gb of memory overhead that can be used to cache data instead
of indices
* we can skip storing hash keys for MPT leaf nodes - these are trivial
to compute and waste a lot of space - previously they had to present in
the `AriKey` CF to avoid having to look in two tables on the happy path.
* There is a small increase in write amplification because when a hash
value is updated for a branch node, we must write both key and branch
data - previously we would write only the key
* There's a small shift in CPU usage - instead of performing lookups in
the database, hashes for leaf nodes are (re)-computed on the fly
* We can return to slightly smaller on-disk SST files since there's
fewer of them, which should reduce disk traffic a bit
Internally, there are also other advantages:
* when clearing keys, we no longer have to store a zero hash in memory -
instead, we deduce staleness of the cached key from the presence of an
updated VertexRef - this saves ~1gb of mem overhead during import
* hash key cache becomes dedicated to branch keys since leaf keys are no
longer stored in memory, reducing churn
* key computation is a lot faster thanks to the skipped second disk
traversal - a key computation for mainnet can be completed in 11 hours
instead of ~2 days (!) thanks to better cache usage and less read
amplification - with additional improvements to the on-disk format, we
can probably get rid of the initial full traversal method of seeding the
key cache on first start after import
All in all, this PR reduces the size of a mainnet database from 160gb to
110gb and the peak memory footprint during import by ~1-2gb.
2024-11-20 09:56:27 +01:00
|
|
|
rvid: RootedVertexID; vtx: VertexRef, key: HashKey
|
2024-06-10 12:04:22 +00:00
|
|
|
): Result[void,(VertexID,AristoError,string)] =
|
|
|
|
let dsc = rdb.session
|
2024-06-25 13:39:53 +02:00
|
|
|
if vtx.isValid:
|
Store keys together with node data (#2849)
Currently, computed hash keys are stored in a separate column family
with respect to the MPT data they're generated from - this has several
disadvantages:
* A lot of space is wasted because the lookup key (`RootedVertexID`) is
repeated in both tables - this is 30% of the `AriKey` content!
* rocksdb must maintain in-memory bloom filters and LRU caches for said
keys, doubling its "minimal efficient cache size"
* An extra disk traversal must be made to check for existence of cached
hash key
* Doubles the amount of files on disk due to each column family being
its own set of files
Here, the two CFs are joined such that both key and data is stored in
`AriVtx`. This means:
* we save ~30% disk space on repeated lookup keys
* we save ~2gb of memory overhead that can be used to cache data instead
of indices
* we can skip storing hash keys for MPT leaf nodes - these are trivial
to compute and waste a lot of space - previously they had to present in
the `AriKey` CF to avoid having to look in two tables on the happy path.
* There is a small increase in write amplification because when a hash
value is updated for a branch node, we must write both key and branch
data - previously we would write only the key
* There's a small shift in CPU usage - instead of performing lookups in
the database, hashes for leaf nodes are (re)-computed on the fly
* We can return to slightly smaller on-disk SST files since there's
fewer of them, which should reduce disk traffic a bit
Internally, there are also other advantages:
* when clearing keys, we no longer have to store a zero hash in memory -
instead, we deduce staleness of the cached key from the presence of an
updated VertexRef - this saves ~1gb of mem overhead during import
* hash key cache becomes dedicated to branch keys since leaf keys are no
longer stored in memory, reducing churn
* key computation is a lot faster thanks to the skipped second disk
traversal - a key computation for mainnet can be completed in 11 hours
instead of ~2 days (!) thanks to better cache usage and less read
amplification - with additional improvements to the on-disk format, we
can probably get rid of the initial full traversal method of seeding the
key cache on first start after import
All in all, this PR reduces the size of a mainnet database from 160gb to
110gb and the peak memory footprint during import by ~1-2gb.
2024-11-20 09:56:27 +01:00
|
|
|
dsc.put(rvid.blobify().data(), vtx.blobify(key), rdb.vtxCol.handle()).isOkOr:
|
2024-06-25 13:39:53 +02:00
|
|
|
# Caller must `rollback()` which will flush the `rdVtxLru` cache
|
|
|
|
const errSym = RdbBeDriverPutVtxError
|
|
|
|
when extraTraceMessages:
|
|
|
|
trace logTxt "putVtx()", vid, error=errSym, info=error
|
2024-07-04 15:46:52 +02:00
|
|
|
return err((rvid.vid,errSym,error))
|
2024-06-25 13:39:53 +02:00
|
|
|
|
2024-09-13 15:47:50 +02:00
|
|
|
# Update existing cached items but don't add new ones since doing so is
|
|
|
|
# likely to evict more useful items (when putting many items, we might even
|
|
|
|
# evict those that were just added)
|
2024-12-11 11:53:26 +01:00
|
|
|
|
|
|
|
if vtx.vType == Branch and vtx.pfx.len == 0:
|
|
|
|
rdb.rdVtxLru.del(rvid.vid)
|
|
|
|
if rdb.rdBranchLru.len < rdb.rdBranchLru.capacity:
|
|
|
|
rdb.rdBranchLru.put(rvid.vid, (vtx.startVid, vtx.used))
|
|
|
|
else:
|
|
|
|
discard rdb.rdBranchLru.update(rvid.vid, (vtx.startVid, vtx.used))
|
|
|
|
else:
|
|
|
|
rdb.rdBranchLru.del(rvid.vid)
|
|
|
|
if rdb.rdVtxLru.len < rdb.rdVtxLru.capacity:
|
|
|
|
rdb.rdVtxLru.put(rvid.vid, vtx)
|
|
|
|
else:
|
|
|
|
discard rdb.rdVtxLru.update(rvid.vid, vtx)
|
|
|
|
|
Store keys together with node data (#2849)
Currently, computed hash keys are stored in a separate column family
with respect to the MPT data they're generated from - this has several
disadvantages:
* A lot of space is wasted because the lookup key (`RootedVertexID`) is
repeated in both tables - this is 30% of the `AriKey` content!
* rocksdb must maintain in-memory bloom filters and LRU caches for said
keys, doubling its "minimal efficient cache size"
* An extra disk traversal must be made to check for existence of cached
hash key
* Doubles the amount of files on disk due to each column family being
its own set of files
Here, the two CFs are joined such that both key and data is stored in
`AriVtx`. This means:
* we save ~30% disk space on repeated lookup keys
* we save ~2gb of memory overhead that can be used to cache data instead
of indices
* we can skip storing hash keys for MPT leaf nodes - these are trivial
to compute and waste a lot of space - previously they had to present in
the `AriKey` CF to avoid having to look in two tables on the happy path.
* There is a small increase in write amplification because when a hash
value is updated for a branch node, we must write both key and branch
data - previously we would write only the key
* There's a small shift in CPU usage - instead of performing lookups in
the database, hashes for leaf nodes are (re)-computed on the fly
* We can return to slightly smaller on-disk SST files since there's
fewer of them, which should reduce disk traffic a bit
Internally, there are also other advantages:
* when clearing keys, we no longer have to store a zero hash in memory -
instead, we deduce staleness of the cached key from the presence of an
updated VertexRef - this saves ~1gb of mem overhead during import
* hash key cache becomes dedicated to branch keys since leaf keys are no
longer stored in memory, reducing churn
* key computation is a lot faster thanks to the skipped second disk
traversal - a key computation for mainnet can be completed in 11 hours
instead of ~2 days (!) thanks to better cache usage and less read
amplification - with additional improvements to the on-disk format, we
can probably get rid of the initial full traversal method of seeding the
key cache on first start after import
All in all, this PR reduces the size of a mainnet database from 160gb to
110gb and the peak memory footprint during import by ~1-2gb.
2024-11-20 09:56:27 +01:00
|
|
|
if key.isValid:
|
2024-12-11 11:53:26 +01:00
|
|
|
if rdb.rdKeyLru.len < rdb.rdKeyLru.capacity:
|
|
|
|
rdb.rdKeyLru.put(rvid.vid, key)
|
|
|
|
else:
|
|
|
|
discard rdb.rdKeyLru.update(rvid.vid, key)
|
Store keys together with node data (#2849)
Currently, computed hash keys are stored in a separate column family
with respect to the MPT data they're generated from - this has several
disadvantages:
* A lot of space is wasted because the lookup key (`RootedVertexID`) is
repeated in both tables - this is 30% of the `AriKey` content!
* rocksdb must maintain in-memory bloom filters and LRU caches for said
keys, doubling its "minimal efficient cache size"
* An extra disk traversal must be made to check for existence of cached
hash key
* Doubles the amount of files on disk due to each column family being
its own set of files
Here, the two CFs are joined such that both key and data is stored in
`AriVtx`. This means:
* we save ~30% disk space on repeated lookup keys
* we save ~2gb of memory overhead that can be used to cache data instead
of indices
* we can skip storing hash keys for MPT leaf nodes - these are trivial
to compute and waste a lot of space - previously they had to present in
the `AriKey` CF to avoid having to look in two tables on the happy path.
* There is a small increase in write amplification because when a hash
value is updated for a branch node, we must write both key and branch
data - previously we would write only the key
* There's a small shift in CPU usage - instead of performing lookups in
the database, hashes for leaf nodes are (re)-computed on the fly
* We can return to slightly smaller on-disk SST files since there's
fewer of them, which should reduce disk traffic a bit
Internally, there are also other advantages:
* when clearing keys, we no longer have to store a zero hash in memory -
instead, we deduce staleness of the cached key from the presence of an
updated VertexRef - this saves ~1gb of mem overhead during import
* hash key cache becomes dedicated to branch keys since leaf keys are no
longer stored in memory, reducing churn
* key computation is a lot faster thanks to the skipped second disk
traversal - a key computation for mainnet can be completed in 11 hours
instead of ~2 days (!) thanks to better cache usage and less read
amplification - with additional improvements to the on-disk format, we
can probably get rid of the initial full traversal method of seeding the
key cache on first start after import
All in all, this PR reduces the size of a mainnet database from 160gb to
110gb and the peak memory footprint during import by ~1-2gb.
2024-11-20 09:56:27 +01:00
|
|
|
else:
|
|
|
|
rdb.rdKeyLru.del rvid.vid
|
2024-06-25 13:39:53 +02:00
|
|
|
|
|
|
|
else:
|
2024-07-04 15:46:52 +02:00
|
|
|
dsc.delete(rvid.blobify().data(), rdb.vtxCol.handle()).isOkOr:
|
2024-06-25 13:39:53 +02:00
|
|
|
# Caller must `rollback()` which will flush the `rdVtxLru` cache
|
|
|
|
const errSym = RdbBeDriverDelVtxError
|
|
|
|
when extraTraceMessages:
|
|
|
|
trace logTxt "putVtx()", vid, error=errSym, info=error
|
2024-07-04 15:46:52 +02:00
|
|
|
return err((rvid.vid,errSym,error))
|
2024-06-25 13:39:53 +02:00
|
|
|
|
|
|
|
# Update cache, vertex will most probably never be visited anymore
|
2024-12-11 11:53:26 +01:00
|
|
|
rdb.rdBranchLru.del rvid.vid
|
2024-07-04 15:46:52 +02:00
|
|
|
rdb.rdVtxLru.del rvid.vid
|
Store keys together with node data (#2849)
Currently, computed hash keys are stored in a separate column family
with respect to the MPT data they're generated from - this has several
disadvantages:
* A lot of space is wasted because the lookup key (`RootedVertexID`) is
repeated in both tables - this is 30% of the `AriKey` content!
* rocksdb must maintain in-memory bloom filters and LRU caches for said
keys, doubling its "minimal efficient cache size"
* An extra disk traversal must be made to check for existence of cached
hash key
* Doubles the amount of files on disk due to each column family being
its own set of files
Here, the two CFs are joined such that both key and data is stored in
`AriVtx`. This means:
* we save ~30% disk space on repeated lookup keys
* we save ~2gb of memory overhead that can be used to cache data instead
of indices
* we can skip storing hash keys for MPT leaf nodes - these are trivial
to compute and waste a lot of space - previously they had to present in
the `AriKey` CF to avoid having to look in two tables on the happy path.
* There is a small increase in write amplification because when a hash
value is updated for a branch node, we must write both key and branch
data - previously we would write only the key
* There's a small shift in CPU usage - instead of performing lookups in
the database, hashes for leaf nodes are (re)-computed on the fly
* We can return to slightly smaller on-disk SST files since there's
fewer of them, which should reduce disk traffic a bit
Internally, there are also other advantages:
* when clearing keys, we no longer have to store a zero hash in memory -
instead, we deduce staleness of the cached key from the presence of an
updated VertexRef - this saves ~1gb of mem overhead during import
* hash key cache becomes dedicated to branch keys since leaf keys are no
longer stored in memory, reducing churn
* key computation is a lot faster thanks to the skipped second disk
traversal - a key computation for mainnet can be completed in 11 hours
instead of ~2 days (!) thanks to better cache usage and less read
amplification - with additional improvements to the on-disk format, we
can probably get rid of the initial full traversal method of seeding the
key cache on first start after import
All in all, this PR reduces the size of a mainnet database from 160gb to
110gb and the peak memory footprint during import by ~1-2gb.
2024-11-20 09:56:27 +01:00
|
|
|
rdb.rdKeyLru.del rvid.vid
|
2024-06-10 12:04:22 +00:00
|
|
|
|
2023-06-20 14:26:25 +01:00
|
|
|
ok()
|
|
|
|
|
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
# End
|
|
|
|
# ------------------------------------------------------------------------------
|