nimbus-eth1/nimbus/db/aristo/aristo_delete.nim

302 lines
9.1 KiB
Nim
Raw Normal View History

2023-06-02 19:21:46 +00:00
# nimbus-eth1
# Copyright (c) 2023-2024 Status Research & Development GmbH
2023-06-02 19:21:46 +00:00
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed
# except according to those terms.
## Aristo DB -- Patricia Trie delete funcionality
## ==============================================
##
{.push raises: [].}
import
std/typetraits,
eth/common,
results,
./aristo_delete/[delete_helpers, delete_subtree],
"."/[aristo_desc, aristo_fetch, aristo_get, aristo_hike, aristo_layers]
2023-06-02 19:21:46 +00:00
# ------------------------------------------------------------------------------
# Private heplers
2023-06-02 19:21:46 +00:00
# ------------------------------------------------------------------------------
proc branchStillNeeded(vtx: VertexRef, removed: int): Result[int,void] =
## Returns the nibble if there is only one reference left.
var nibble = -1
2023-06-02 19:21:46 +00:00
for n in 0 .. 15:
if n == removed:
continue
if vtx.bVid[n].isValid:
if 0 <= nibble:
return ok(-1)
nibble = n
if 0 <= nibble:
return ok(nibble)
# Oops, degenerated branch node
err()
2023-06-02 19:21:46 +00:00
# ------------------------------------------------------------------------------
# Private functions
# ------------------------------------------------------------------------------
2023-06-02 19:21:46 +00:00
proc deleteImpl(
db: AristoDbRef; # Database, top layer
2023-06-02 19:21:46 +00:00
hike: Hike; # Fully expanded path
): Result[VertexRef,AristoError] =
## Removes the last node in the hike and returns the updated leaf in case
## a branch collapsed
Core db aristo hasher profiling and timing improvement (#1938) * Explicitly use shared `Kvt` table on `Ledger` and `Clique` lookup. why: Speeds up lookup time with `Aristo` backend. For writing `Clique` data, the `Companion` model allows to write `Clique` data past the database locked by evm transactions. * Implement `CoreDb` profiling with API tracking why: Chasing time spent per APT procs ... * Implement `Ledger` profiling with API tracking why: Chasing time spent per APT procs ... * Always hashify when commiting or storing why: A dirty cache makes no sense when committing * Make sure that a zero key is created when adding/updating vertices why: This is an error fix mainly for edge cases. A typical error was that the root key got deleted when there were only a few vertices left on the DB. * Need all created and changed vertices zero-keyed on the cache why: A zero key (i.e. empty Merkle hash) indicates that a vertex key needs to be updated. This would not be needed immediately after a merge as there is an actual leaf path on the cache layer. But after subsequent merge and delete operations this information might get blurred. * Re-org hashing algorithm why: Apart from errors, the previous implementation was too slow for two reasons: + some control hashes were calculated for debugging (now all verification is done in `aristo_check` module) + the leaf paths stored on the cache are used to build the labelling (aka hashing) schedule; there paths were accumulated over successive hash sessions although it is clear that all keys were generated, already
2023-12-12 17:47:41 +00:00
# Remove leaf entry
let lf = hike.legs[^1].wp
if lf.vtx.vType != Leaf:
return err(DelLeafExpexted)
db.disposeOfVtx((hike.root, lf.vid))
if hike.legs.len == 1:
# This was the last node in the trie, meaning we don't have any branches or
# leaves to update
return ok(nil)
if hike.legs[^2].wp.vtx.vType != Branch:
return err(DelBranchExpexted)
# Get current `Branch` vertex `br`
let
br = hike.legs[^2].wp
nbl = br.vtx.branchStillNeeded(hike.legs[^2].nibble).valueOr:
return err(DelBranchWithoutRefs)
# Clear all Merkle hash keys up to the root key
for n in 0 .. hike.legs.len - 2:
let vid = hike.legs[n].wp.vid
db.layersResKey((hike.root, vid))
if 0 <= nbl:
# Branch has only one entry - move that entry to where the branch was and
# update its path
# Get child vertex (there must be one after a `Branch` node)
let
vid = br.vtx.bVid[nbl]
nxt = db.getVtx (hike.root, vid)
if not nxt.isValid:
return err(DelVidStaleVtx)
db.disposeOfVtx((hike.root, vid))
let vtx =
case nxt.vType
of Leaf:
VertexRef(
vType: Leaf,
pfx: br.vtx.pfx & NibblesBuf.nibble(nbl.byte) & nxt.pfx,
lData: nxt.lData)
of Branch:
VertexRef(
vType: Branch,
pfx: br.vtx.pfx & NibblesBuf.nibble(nbl.byte) & nxt.pfx,
bVid: nxt.bVid)
# Put the new vertex at the id of the obsolete branch
db.layersPutVtx((hike.root, br.vid), vtx)
if vtx.vType == Leaf:
ok(vtx)
else:
ok(nil)
else:
# Clear the removed leaf from the branch (that still contains other children)
let brDup = br.vtx.dup
brDup.bVid[hike.legs[^2].nibble] = VertexID(0)
db.layersPutVtx((hike.root, br.vid), brDup)
ok(nil)
2023-06-02 19:21:46 +00:00
# ------------------------------------------------------------------------------
# Public functions
# ------------------------------------------------------------------------------
Update storage tree admin (#2419) * Tighten `CoreDb` API for accounts why: Apart from cruft, the way to fetch the accounts state root via a `CoreDbColRef` record was unnecessarily complicated. * Extend `CoreDb` API for accounts to cover storage tries why: In future, this will make the notion of column objects obsolete. Storage trees will then be indexed by the account address rather than the vertex ID equivalent like a `CoreDbColRef`. * Apply new/extended accounts API to ledger and tests details: This makes the `distinct_ledger` module obsolete * Remove column object constructors why: They were needed as an abstraction of MPT sub-trees including storage trees. Now, storage trees are handled by the account (e.g. via address) they belong to and all other trees can be identified by a constant well known vertex ID. So there is no need for column objects anymore. Still there are some left-over column object methods wnich will be removed next. * Remove `serialise()` and `PayloadRef` from default Aristo API why: Not needed. `PayloadRef` was used for unstructured/unknown payload formats (account or blob) and `serialise()` was used for decodng `PayloadRef`. Now it is known in advance what the payload looks like. * Added query function `hasStorageData()` whether a storage area exists why: Useful for supporting `slotStateEmpty()` of the `CoreDb` API * In the `Ledger` replace `storage.stateEmpty()` by `slotStateEmpty()` * On Aristo, hide the storage root/vertex ID in the `PayloadRef` why: The storage vertex ID is fully controlled by Aristo while the `AristoAccount` object is controlled by the application. With the storage root part of the `AristoAccount` object, there was a useless administrative burden to keep that storage root field up to date. * Remove cruft, update comments etc. * Update changed MPT access paradigms why: Fixes verified proxy tests * Fluffy cosmetics
2024-06-27 09:01:26 +00:00
proc deleteAccountRecord*(
db: AristoDbRef;
accPath: Hash256;
): Result[void,AristoError] =
## Delete the account leaf entry addressed by the argument `path`. If this
## leaf entry referres to a storage tree, this one will be deleted as well.
##
var accHike: Hike
db.fetchAccountHike(accPath, accHike).isOkOr:
if error == FetchAccInaccessible:
return err(DelPathNotFound)
return err(error)
let
stoID = accHike.legs[^1].wp.vtx.lData.stoID
# Delete storage tree if present
if stoID.isValid:
? db.delStoTreeImpl((stoID.vid, stoID.vid), accPath)
let otherLeaf = ?db.deleteImpl(accHike)
db.layersPutAccLeaf(accPath, nil)
if otherLeaf.isValid:
db.layersPutAccLeaf(
Hash32(getBytes(NibblesBuf.fromBytes(accPath.data).replaceSuffix(otherLeaf.pfx))),
otherLeaf)
ok()
proc deleteGenericData*(
db: AristoDbRef;
root: VertexID;
path: openArray[byte];
): Result[bool,AristoError] =
## Delete the leaf data entry addressed by the argument `path`. The MPT
## sub-tree the leaf data entry is subsumed under is passed as argument
## `root` which must be greater than `VertexID(1)` and smaller than
## `LEAST_FREE_VID`.
Core db and aristo maintenance update (#2014) * Aristo: Update error return code why: Failing of `Aristo` function `delete()` might fail because there is no such data item on the db. This must return a single error code as is done with `fetch()`. * Ledger: Better error handling why: The `expect()` clauses have been replaced by raising asserts indicating the error from the database backend. Also, `delete()` failures are legitimate if the item to delete does not exist. * Aristo: Delete function must always leave a label on DB for `hashify()` why: The `hashify()` uses the labels left bu `merge()` and `delete()` to compile (and optimise) a scheduler for subsequent hashing. Originally, the labels were not used for deleted entries and `delete()` still had some edge case where the deletion label was not properly handled. * Aristo: Update `hashify()` scheduler, remove buggy optimisation why: Was left over from version without virtual state roots which did not know about account payload leaf vertices referring to storage roots. * Aristo: Label storage trie account in `delete()` similar to `merge()` details; The `delete()` function applied to a non-static state root (assumed to be a storage root) will check the payload of an accounts leaf and mark its Merkle keys to be re-checked when runninh `hashify()` * Aristo: Clean up and re-org recycled vertex IDs in `hashify()` why: Re-organising the recycled vertex IDs list intends to reduce the size of the list. This list is organised as a LIFO (or stack.) By reorganising it in a way so that the least vertex ID numbers are on top, the list will be kept smaller as observed on some examples (less than 30%.) * CoreDb: Accept storage trie deletion requests in non-initialised state why: Due to lazy initialisation, the root vertex ID might not yet exist. So the `Aristo` database handlers would reject this call with an error and this condition needs to be handled by the API (which realises the lazy feature.) * Cosmetics & code massage, prettify logging * fix missing import
2024-02-08 16:32:16 +00:00
##
## The return value is `true` if the argument `path` deleted was the last
## one and the tree does not exist anymore.
Aristo db api extensions for use as core db backend (#1754) * Update docu * Update Aristo/Kvt constructor prototype why: Previous version used an `enum` value to indicate what backend is to be used. This was replaced by using the backend object type. * Rewrite `hikeUp()` return code into `Result[Hike,(Hike,AristoError)]` why: Better code maintenance. Previously, the `Hike` object was returned. It had an internal error field so partial success was also available on a failure. This error field has been removed. * Use `openArray[byte]` rather than `Blob` in functions prototypes * Provide synchronised multi instance transactions why: The `CoreDB` object was geared towards the legacy DB which used a single transaction for the key-value backend DB. Different state roots are provided by the backend database, so all instances work directly on the same backend. Aristo db instances have different in-memory mappings (aka different state roots) and the transactions are on top of there mappings. So each instance might run different transactions. Multi instance transactions are a compromise to converge towards the legacy behaviour. The synchronised transactions span over all instances available at the time when base transaction was opened. Instances created later are unaffected. * Provide key-value pair database iterator why: Needed in `CoreDB` for `replicate()` emulation also: Some update of internal code * Extend API (i.e. prototype variants) why: Needed for `CoreDB` geared towards the legacy backend which has a more basic API than Aristo.
2023-09-15 15:23:53 +00:00
##
# Verify that `root` is neither an accounts tree nor a strorage tree.
if not root.isValid:
return err(DelRootVidMissing)
elif root == VertexID(1):
return err(DelAccRootNotAccepted)
elif LEAST_FREE_VID <= root.distinctBase:
return err(DelStoRootNotAccepted)
var hike: Hike
path.hikeUp(root, db, Opt.none(VertexRef), hike).isOkOr:
if error[1] in HikeAcceptableStopsNotFound:
return err(DelPathNotFound)
return err(error[1])
2023-06-02 19:21:46 +00:00
discard ?db.deleteImpl(hike)
ok(not db.getVtx((root, root)).isValid)
proc deleteGenericTree*(
db: AristoDbRef; # Database, top layer
root: VertexID; # Root vertex
): Result[void,AristoError] =
## Variant of `deleteGenericData()` for purging the whole MPT sub-tree.
Aristo db api extensions for use as core db backend (#1754) * Update docu * Update Aristo/Kvt constructor prototype why: Previous version used an `enum` value to indicate what backend is to be used. This was replaced by using the backend object type. * Rewrite `hikeUp()` return code into `Result[Hike,(Hike,AristoError)]` why: Better code maintenance. Previously, the `Hike` object was returned. It had an internal error field so partial success was also available on a failure. This error field has been removed. * Use `openArray[byte]` rather than `Blob` in functions prototypes * Provide synchronised multi instance transactions why: The `CoreDB` object was geared towards the legacy DB which used a single transaction for the key-value backend DB. Different state roots are provided by the backend database, so all instances work directly on the same backend. Aristo db instances have different in-memory mappings (aka different state roots) and the transactions are on top of there mappings. So each instance might run different transactions. Multi instance transactions are a compromise to converge towards the legacy behaviour. The synchronised transactions span over all instances available at the time when base transaction was opened. Instances created later are unaffected. * Provide key-value pair database iterator why: Needed in `CoreDB` for `replicate()` emulation also: Some update of internal code * Extend API (i.e. prototype variants) why: Needed for `CoreDB` geared towards the legacy backend which has a more basic API than Aristo.
2023-09-15 15:23:53 +00:00
##
# Verify that `root` is neither an accounts tree nor a strorage tree.
if not root.isValid:
return err(DelRootVidMissing)
elif root == VertexID(1):
return err(DelAccRootNotAccepted)
elif LEAST_FREE_VID <= root.distinctBase:
return err(DelStoRootNotAccepted)
db.delSubTreeImpl root
proc deleteStorageData*(
Aristo db api extensions for use as core db backend (#1754) * Update docu * Update Aristo/Kvt constructor prototype why: Previous version used an `enum` value to indicate what backend is to be used. This was replaced by using the backend object type. * Rewrite `hikeUp()` return code into `Result[Hike,(Hike,AristoError)]` why: Better code maintenance. Previously, the `Hike` object was returned. It had an internal error field so partial success was also available on a failure. This error field has been removed. * Use `openArray[byte]` rather than `Blob` in functions prototypes * Provide synchronised multi instance transactions why: The `CoreDB` object was geared towards the legacy DB which used a single transaction for the key-value backend DB. Different state roots are provided by the backend database, so all instances work directly on the same backend. Aristo db instances have different in-memory mappings (aka different state roots) and the transactions are on top of there mappings. So each instance might run different transactions. Multi instance transactions are a compromise to converge towards the legacy behaviour. The synchronised transactions span over all instances available at the time when base transaction was opened. Instances created later are unaffected. * Provide key-value pair database iterator why: Needed in `CoreDB` for `replicate()` emulation also: Some update of internal code * Extend API (i.e. prototype variants) why: Needed for `CoreDB` geared towards the legacy backend which has a more basic API than Aristo.
2023-09-15 15:23:53 +00:00
db: AristoDbRef;
accPath: Hash256; # Implies storage data tree
stoPath: Hash256;
): Result[bool,AristoError] =
## For a given account argument `accPath`, this function deletes the
## argument `stoPath` from the associated storage tree (if any, at all.) If
## the if the argument `stoPath` deleted was the last one on the storage tree,
## account leaf referred to by `accPath` will be updated so that it will
## not refer to a storage tree anymore. In the latter case only the function
## will return `true`.
Aristo db api extensions for use as core db backend (#1754) * Update docu * Update Aristo/Kvt constructor prototype why: Previous version used an `enum` value to indicate what backend is to be used. This was replaced by using the backend object type. * Rewrite `hikeUp()` return code into `Result[Hike,(Hike,AristoError)]` why: Better code maintenance. Previously, the `Hike` object was returned. It had an internal error field so partial success was also available on a failure. This error field has been removed. * Use `openArray[byte]` rather than `Blob` in functions prototypes * Provide synchronised multi instance transactions why: The `CoreDB` object was geared towards the legacy DB which used a single transaction for the key-value backend DB. Different state roots are provided by the backend database, so all instances work directly on the same backend. Aristo db instances have different in-memory mappings (aka different state roots) and the transactions are on top of there mappings. So each instance might run different transactions. Multi instance transactions are a compromise to converge towards the legacy behaviour. The synchronised transactions span over all instances available at the time when base transaction was opened. Instances created later are unaffected. * Provide key-value pair database iterator why: Needed in `CoreDB` for `replicate()` emulation also: Some update of internal code * Extend API (i.e. prototype variants) why: Needed for `CoreDB` geared towards the legacy backend which has a more basic API than Aristo.
2023-09-15 15:23:53 +00:00
##
let
mixPath = mixUp(accPath, stoPath)
stoLeaf = db.cachedStoLeaf(mixPath)
if stoLeaf == Opt.some(nil):
return err(DelPathNotFound)
var accHike: Hike
db.fetchAccountHike(accPath, accHike).isOkOr:
if error == FetchAccInaccessible:
return err(DelStoAccMissing)
return err(error)
let
wpAcc = accHike.legs[^1].wp
Update storage tree admin (#2419) * Tighten `CoreDb` API for accounts why: Apart from cruft, the way to fetch the accounts state root via a `CoreDbColRef` record was unnecessarily complicated. * Extend `CoreDb` API for accounts to cover storage tries why: In future, this will make the notion of column objects obsolete. Storage trees will then be indexed by the account address rather than the vertex ID equivalent like a `CoreDbColRef`. * Apply new/extended accounts API to ledger and tests details: This makes the `distinct_ledger` module obsolete * Remove column object constructors why: They were needed as an abstraction of MPT sub-trees including storage trees. Now, storage trees are handled by the account (e.g. via address) they belong to and all other trees can be identified by a constant well known vertex ID. So there is no need for column objects anymore. Still there are some left-over column object methods wnich will be removed next. * Remove `serialise()` and `PayloadRef` from default Aristo API why: Not needed. `PayloadRef` was used for unstructured/unknown payload formats (account or blob) and `serialise()` was used for decodng `PayloadRef`. Now it is known in advance what the payload looks like. * Added query function `hasStorageData()` whether a storage area exists why: Useful for supporting `slotStateEmpty()` of the `CoreDb` API * In the `Ledger` replace `storage.stateEmpty()` by `slotStateEmpty()` * On Aristo, hide the storage root/vertex ID in the `PayloadRef` why: The storage vertex ID is fully controlled by Aristo while the `AristoAccount` object is controlled by the application. With the storage root part of the `AristoAccount` object, there was a useless administrative burden to keep that storage root field up to date. * Remove cruft, update comments etc. * Update changed MPT access paradigms why: Fixes verified proxy tests * Fluffy cosmetics
2024-06-27 09:01:26 +00:00
stoID = wpAcc.vtx.lData.stoID
if not stoID.isValid:
return err(DelStoRootMissing)
let stoNibbles = NibblesBuf.fromBytes(stoPath.data)
var stoHike: Hike
stoNibbles.hikeUp(stoID.vid, db, stoLeaf, stoHike).isOkOr:
if error[1] in HikeAcceptableStopsNotFound:
return err(DelPathNotFound)
return err(error[1])
# Mark account path Merkle keys for update
db.layersResKeys accHike
let otherLeaf = ?db.deleteImpl(stoHike)
db.layersPutStoLeaf(mixPath, nil)
if otherLeaf.isValid:
let leafMixPath = mixUp(
accPath,
Hash32(getBytes(stoNibbles.replaceSuffix(otherLeaf.pfx))))
db.layersPutStoLeaf(leafMixPath, otherLeaf)
# If there was only one item (that got deleted), update the account as well
if stoHike.legs.len > 1:
return ok(false)
# De-register the deleted storage tree from the account record
let leaf = wpAcc.vtx.dup # Dup on modify
leaf.lData.stoID.isValid = false
db.layersPutAccLeaf(accPath, leaf)
db.layersPutVtx((accHike.root, wpAcc.vid), leaf)
ok(true)
proc deleteStorageTree*(
db: AristoDbRef; # Database, top layer
accPath: Hash256; # Implies storage data tree
): Result[void,AristoError] =
## Variant of `deleteStorageData()` for purging the whole storage tree
## associated to the account argument `accPath`.
##
var accHike: Hike
db.fetchAccountHike(accPath, accHike).isOkOr:
if error == FetchAccInaccessible:
return err(DelStoAccMissing)
return err(error)
let
wpAcc = accHike.legs[^1].wp
Update storage tree admin (#2419) * Tighten `CoreDb` API for accounts why: Apart from cruft, the way to fetch the accounts state root via a `CoreDbColRef` record was unnecessarily complicated. * Extend `CoreDb` API for accounts to cover storage tries why: In future, this will make the notion of column objects obsolete. Storage trees will then be indexed by the account address rather than the vertex ID equivalent like a `CoreDbColRef`. * Apply new/extended accounts API to ledger and tests details: This makes the `distinct_ledger` module obsolete * Remove column object constructors why: They were needed as an abstraction of MPT sub-trees including storage trees. Now, storage trees are handled by the account (e.g. via address) they belong to and all other trees can be identified by a constant well known vertex ID. So there is no need for column objects anymore. Still there are some left-over column object methods wnich will be removed next. * Remove `serialise()` and `PayloadRef` from default Aristo API why: Not needed. `PayloadRef` was used for unstructured/unknown payload formats (account or blob) and `serialise()` was used for decodng `PayloadRef`. Now it is known in advance what the payload looks like. * Added query function `hasStorageData()` whether a storage area exists why: Useful for supporting `slotStateEmpty()` of the `CoreDb` API * In the `Ledger` replace `storage.stateEmpty()` by `slotStateEmpty()` * On Aristo, hide the storage root/vertex ID in the `PayloadRef` why: The storage vertex ID is fully controlled by Aristo while the `AristoAccount` object is controlled by the application. With the storage root part of the `AristoAccount` object, there was a useless administrative burden to keep that storage root field up to date. * Remove cruft, update comments etc. * Update changed MPT access paradigms why: Fixes verified proxy tests * Fluffy cosmetics
2024-06-27 09:01:26 +00:00
stoID = wpAcc.vtx.lData.stoID
if not stoID.isValid:
return err(DelStoRootMissing)
# Mark account path Merkle keys for update
db.layersResKeys accHike
? db.delStoTreeImpl((stoID.vid, stoID.vid), accPath)
# De-register the deleted storage tree from the accounts record
let leaf = wpAcc.vtx.dup # Dup on modify
leaf.lData.stoID.isValid = false
db.layersPutAccLeaf(accPath, leaf)
db.layersPutVtx((accHike.root, wpAcc.vid), leaf)
ok()
2023-06-02 19:21:46 +00:00
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------