diff --git a/nimbus/db/aristo/aristo_delete.nim b/nimbus/db/aristo/aristo_delete.nim index ecaf79c91..9fad02515 100644 --- a/nimbus/db/aristo/aristo_delete.nim +++ b/nimbus/db/aristo/aristo_delete.nim @@ -58,7 +58,7 @@ proc collapseBranch( db: AristoDbRef; # Database, top layer hike: Hike; # Fully expanded path nibble: byte; # Applicable link for `Branch` vertex - ): Result[void,(VertexID,AristoError)] = + ): Result[void,AristoError] = ## Convert/merge vertices: ## :: ## current | becomes | condition @@ -95,7 +95,7 @@ proc collapseBranch( xt.vtx.ePfx = par.vtx.ePfx & xt.vtx.ePfx of Leaf: - return err((par.vid,DelLeafUnexpected)) + return err(DelLeafUnexpected) else: # (3) # Replace `br` (use `xt` as-is) @@ -110,7 +110,7 @@ proc collapseExt( hike: Hike; # Fully expanded path nibble: byte; # Link for `Branch` vertex `^2` vtx: VertexRef; # Follow up extension vertex (nibble) - ): Result[void,(VertexID,AristoError)] = + ): Result[void,AristoError] = ## Convert/merge vertices: ## :: ## ^3 ^2 `vtx` | ^3 ^2 | @@ -145,7 +145,7 @@ proc collapseExt( xt.vtx.ePfx = par.vtx.ePfx & xt.vtx.ePfx of Leaf: - return err((par.vid,DelLeafUnexpected)) + return err(DelLeafUnexpected) else: # (3) # Replace ^2 by `^2 & vtx` (use `xt` as-is) @@ -160,7 +160,7 @@ proc collapseLeaf( hike: Hike; # Fully expanded path nibble: byte; # Link for `Branch` vertex `^2` vtx: VertexRef; # Follow up leaf vertex (from nibble) - ): Result[void,(VertexID,AristoError)] = + ): Result[void,AristoError] = ## Convert/merge vertices: ## :: ## current | becomes | condition @@ -205,7 +205,7 @@ proc collapseLeaf( # Grandparent exists let gpr = hike.legs[^4].wp.dup # Writable vertex if gpr.vtx.vType != Branch: - return err((gpr.vid,DelBranchExpexted)) + return err(DelBranchExpexted) db.disposeOfVtx((hike.root, par.vid)) # `par` is obsolete now gpr.vtx.bVid[hike.legs[^4].nibble] = lf.vid db.layersPutVtx((hike.root, gpr.vid), gpr.vtx) @@ -217,7 +217,7 @@ proc collapseLeaf( # Continue below of Leaf: - return err((par.vid,DelLeafUnexpected)) + return err(DelLeafUnexpected) else: # (4) # Replace ^2 by `^2 & vtx` (use `lf` as-is) # `br` is root vertex @@ -261,17 +261,47 @@ proc delSubTreeImpl( ok() +proc delStoTreeImpl( + db: AristoDbRef; # Database, top layer + rvid: RootedVertexID; # Root vertex + accPath: Hash256; + stoPath: NibblesBuf; + ): Result[void,AristoError] = + ## Implementation of *delete* sub-trie. + + let vtx = db.getVtxRc(rvid).valueOr: + if error == GetVtxNotFound: + return ok() + return err(error) + + case vtx.vType + of Branch: + for i in 0..15: + if vtx.bVid[i].isValid: + ? db.delStoTreeImpl( + (rvid.root, vtx.bVid[i]), accPath, + stoPath & NibblesBuf.nibble(byte i)) + of Extension: + ?db.delStoTreeImpl((rvid.root, vtx.eVid), accPath, stoPath & vtx.ePfx) + + of Leaf: + let stoPath = Hash256(data: (stoPath & vtx.lPfx).getBytes()) + db.layersPutStoLeaf(AccountKey.mixUp(accPath, stoPath), nil) + + db.disposeOfVtx(rvid) + + ok() proc deleteImpl( db: AristoDbRef; # Database, top layer hike: Hike; # Fully expanded path - ): Result[void,(VertexID,AristoError)] = + ): Result[void,AristoError] = ## Implementation of *delete* functionality. # Remove leaf entry let lf = hike.legs[^1].wp if lf.vtx.vType != Leaf: - return err((lf.vid,DelLeafExpexted)) + return err(DelLeafExpexted) db.disposeOfVtx((hike.root, lf.vid)) @@ -282,7 +312,7 @@ proc deleteImpl( wp.vtx = wp.vtx.dup # make sure that layers are not impliciteley modified wp if br.vtx.vType != Branch: - return err((br.vid,DelBranchExpexted)) + return err(DelBranchExpexted) # Unlink child vertex from structural table br.vtx.bVid[hike.legs[^2].nibble] = VertexID(0) @@ -296,7 +326,7 @@ proc deleteImpl( let nibble = block: let rc = br.vtx.branchStillNeeded() if rc.isErr: - return err((br.vid,DelBranchWithoutRefs)) + return err(DelBranchWithoutRefs) rc.value # Convert to `Extension` or `Leaf` vertex @@ -306,7 +336,7 @@ proc deleteImpl( let vid = br.vtx.bVid[nibble] VidVtxPair(vid: vid, vtx: db.getVtx (hike.root, vid)) if not nxt.vtx.isValid: - return err((nxt.vid, DelVidStaleVtx)) + return err(DelVidStaleVtx) # Collapse `Branch` vertex `br` depending on `nxt` vertex type case nxt.vtx.vType: @@ -339,10 +369,9 @@ proc deleteAccountRecord*( # Delete storage tree if present if stoID.isValid: - ? db.delSubTreeImpl stoID + ? db.delStoTreeImpl((stoID, stoID), accPath, NibblesBuf()) - db.deleteImpl(hike).isOkOr: - return err(error[1]) + ?db.deleteImpl(hike) db.layersPutAccLeaf(accPath, nil) @@ -375,8 +404,7 @@ proc deleteGenericData*( return err(DelPathNotFound) return err(error[1]) - db.deleteImpl(hike).isOkOr: - return err(error[1]) + ?db.deleteImpl(hike) ok(not db.getVtx((root, root)).isValid) @@ -428,8 +456,9 @@ proc deleteStorageData*( # Mark account path Merkle keys for update db.updateAccountForHasher accHike - db.deleteImpl(stoHike).isOkOr: - return err(error[1]) + ?db.deleteImpl(stoHike) + + db.layersPutStoLeaf(AccountKey.mixUp(accPath, stoPath), nil) # Make sure that an account leaf has no dangling sub-trie if db.getVtx((stoID, stoID)).isValid: @@ -440,7 +469,6 @@ proc deleteStorageData*( leaf.lData.stoID = VertexID(0) db.layersPutAccLeaf(accPath, leaf) db.layersPutVtx((accHike.root, wpAcc.vid), leaf) - db.layersResKey((accHike.root, wpAcc.vid)) ok(true) proc deleteStorageTree*( @@ -464,14 +492,13 @@ proc deleteStorageTree*( # Mark account path Merkle keys for update db.updateAccountForHasher accHike - ? db.delSubTreeImpl stoID + ? db.delStoTreeImpl((stoID, stoID), accPath, NibblesBuf()) # De-register the deleted storage tree from the accounts record let leaf = wpAcc.vtx.dup # Dup on modify leaf.lData.stoID = VertexID(0) db.layersPutAccLeaf(accPath, leaf) db.layersPutVtx((accHike.root, wpAcc.vid), leaf) - db.layersResKey((accHike.root, wpAcc.vid)) ok() # ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_delta.nim b/nimbus/db/aristo/aristo_delta.nim index 73b0adfe0..b10cc6a5e 100644 --- a/nimbus/db/aristo/aristo_delta.nim +++ b/nimbus/db/aristo/aristo_delta.nim @@ -83,10 +83,15 @@ proc deltaPersistent*( ? be.putEndFn writeBatch # Finalise write batch # Copy back updated payloads - for accPath, pyl in db.balancer.accLeaves: + for accPath, vtx in db.balancer.accLeaves: let accKey = accPath.to(AccountKey) - if not db.accLeaves.lruUpdate(accKey, pyl): - discard db.accLeaves.lruAppend(accKey, pyl, accLruSize) + if not db.accLeaves.lruUpdate(accKey, vtx): + discard db.accLeaves.lruAppend(accKey, vtx, accLruSize) + + for mixPath, vtx in db.balancer.stoLeaves: + let mixKey = mixPath.to(AccountKey) + if not db.stoLeaves.lruUpdate(mixKey, vtx): + discard db.stoLeaves.lruAppend(mixKey, vtx, accLruSize) # Update dudes and this descriptor ? updateSiblings.update().commit() diff --git a/nimbus/db/aristo/aristo_desc.nim b/nimbus/db/aristo/aristo_desc.nim index b286eea7e..741bc3b06 100644 --- a/nimbus/db/aristo/aristo_desc.nim +++ b/nimbus/db/aristo/aristo_desc.nim @@ -23,7 +23,7 @@ import std/[hashes, sets, tables], - stew/keyed_queue, + stew/[assign2, keyed_queue], eth/common, results, ./aristo_constants, @@ -92,6 +92,10 @@ type ## TODO a better solution would probably be to cache this in a type ## exposed to the high-level API + stoLeaves*: KeyedQueue[AccountKey, VertexRef] + ## Mixed account/storage path to payload cache - same as above but caches + ## the full lookup of storage slots + AristoDbAction* = proc(db: AristoDbRef) {.gcsafe, raises: [].} ## Generic call back function/closure. @@ -110,6 +114,18 @@ template `==`*(a, b: AccountKey): bool = template to*(a: Hash256, T: type AccountKey): T = AccountKey((ref Hash256)(data: a.data)) +template mixUp*(T: type AccountKey, accPath, stoPath: Hash256): Hash256 = + # Insecure but fast way of mixing the values of two hashes, for the purpose + # of quick lookups - this is certainly not a good idea for general Hash256 + # values but account paths are generated from accounts which would be hard + # to create pre-images for, for the purpose of collisions with a particular + # storage slot + var v {.noinit.}: Hash256 + for i in 0.. VertexRef + accLeaves*: Table[Hash256, VertexRef] ## Account path -> VertexRef + stoLeaves*: Table[Hash256, VertexRef] ## Storage path -> VertexRef LayerRef* = ref LayerObj LayerObj* = object diff --git a/nimbus/db/aristo/aristo_fetch.nim b/nimbus/db/aristo/aristo_fetch.nim index cb09eaff1..ae3e7b33b 100644 --- a/nimbus/db/aristo/aristo_fetch.nim +++ b/nimbus/db/aristo/aristo_fetch.nim @@ -59,27 +59,26 @@ proc retrieveAccountPayload( db: AristoDbRef; accPath: Hash256; ): Result[LeafPayload,AristoError] = - if (let pyl = db.layersGetAccLeaf(accPath); pyl.isSome()): - if not pyl[].isValid(): + if (let leafVtx = db.layersGetAccLeaf(accPath); leafVtx.isSome()): + if not leafVtx[].isValid(): return err(FetchPathNotFound) - return ok pyl[].lData + return ok leafVtx[].lData let accKey = accPath.to(AccountKey) - if (let pyl = db.accLeaves.lruFetch(accKey); pyl.isSome()): - if not pyl[].isValid(): + if (let leafVtx = db.accLeaves.lruFetch(accKey); leafVtx.isSome()): + if not leafVtx[].isValid(): return err(FetchPathNotFound) - return ok pyl[].lData + return ok leafVtx[].lData # Updated payloads are stored in the layers so if we didn't find them there, # it must have been in the database let - payload = db.retrieveLeaf(VertexID(1), accPath.data).valueOr: + leafVtx = db.retrieveLeaf(VertexID(1), accPath.data).valueOr: if error == FetchAccInaccessible: - discard db.accLeaves.lruAppend(accKey, nil, accLruSize) return err(FetchPathNotFound) return err(error) - ok db.accLeaves.lruAppend(accKey, payload, accLruSize).lData + ok db.accLeaves.lruAppend(accKey, leafVtx, accLruSize).lData proc retrieveMerkleHash( db: AristoDbRef; @@ -157,11 +156,7 @@ proc fetchStorageID*( ## Public helper function for retrieving a storage (vertex) ID for a ## given account. let - payload = db.retrieveAccountPayload(accPath).valueOr: - if error == FetchAccInaccessible: - return err(FetchPathNotFound) - return err(error) - + payload = ?db.retrieveAccountPayload(accPath) stoID = payload.stoID if not stoID.isValid: @@ -169,6 +164,42 @@ proc fetchStorageID*( ok stoID +proc retrieveStoragePayload( + db: AristoDbRef; + accPath: Hash256; + stoPath: Hash256; + ): Result[UInt256,AristoError] = + let mixPath = AccountKey.mixUp(accPath, stoPath) + if (let leafVtx = db.layersGetStoLeaf(mixPath); leafVtx.isSome()): + if not leafVtx[].isValid(): + return err(FetchPathNotFound) + return ok leafVtx[].lData.stoData + + let mixKey = mixPath.to(AccountKey) + if (let leafVtx = db.stoLeaves.lruFetch(mixKey); leafVtx.isSome()): + if not leafVtx[].isValid(): + return err(FetchPathNotFound) + return ok leafVtx[].lData.stoData + + # Updated payloads are stored in the layers so if we didn't find them there, + # it must have been in the database + let + leafVtx = db.retrieveLeaf(? db.fetchStorageID(accPath), stoPath.data).valueOr: + return err(error) + + ok db.stoLeaves.lruAppend(mixKey, leafVtx, accLruSize).lData.stoData + +proc hasStoragePayload( + db: AristoDbRef; + accPath: Hash256; + stoPath: Hash256; + ): Result[bool,AristoError] = + let error = db.retrieveStoragePayload(accPath, stoPath).errorOr: + return ok(true) + + if error == FetchPathNotFound: + return ok(false) + err(error) # ------------------------------------------------------------------------------ # Public functions @@ -249,9 +280,9 @@ proc fetchStorageData*( ## For a storage tree related to account `accPath`, fetch the data record ## from the database indexed by `path`. ## - let pyl = ? db.retrieveLeaf(? db.fetchStorageID accPath, stoPath.data) - assert pyl.lData.pType == StoData # debugging only - ok pyl.lData.stoData + let leafVtx = ? db.retrieveLeaf(? db.fetchStorageID accPath, stoPath.data) + assert leafVtx.lData.pType == StoData # debugging only + ok leafVtx.lData.stoData proc fetchStorageState*( db: AristoDbRef; @@ -273,7 +304,7 @@ proc hasPathStorage*( ## For a storage tree related to account `accPath`, query whether the data ## record indexed by `path` exists on the database. ## - db.hasPayload(? db.fetchStorageID accPath, stoPath.data) + db.hasStoragePayload(accPath, stoPath) proc hasStorageData*( db: AristoDbRef; diff --git a/nimbus/db/aristo/aristo_layers.nim b/nimbus/db/aristo/aristo_layers.nim index 494638967..2d978b88d 100644 --- a/nimbus/db/aristo/aristo_layers.nim +++ b/nimbus/db/aristo/aristo_layers.nim @@ -101,6 +101,15 @@ func layersGetAccLeaf*(db: AristoDbRef; accPath: Hash256): Opt[VertexRef] = Opt.none(VertexRef) +func layersGetStoLeaf*(db: AristoDbRef; mixPath: Hash256): Opt[VertexRef] = + db.top.delta.stoLeaves.withValue(mixPath, item): + return Opt.some(item[]) + + for w in db.rstack: + w.delta.stoLeaves.withValue(mixPath, item): + return Opt.some(item[]) + + Opt.none(VertexRef) # ------------------------------------------------------------------------------ # Public functions: setter variants @@ -147,8 +156,11 @@ proc layersUpdateVtx*( db.layersResKey(rvid) -func layersPutAccLeaf*(db: AristoDbRef; accPath: Hash256; pyl: VertexRef) = - db.top.delta.accLeaves[accPath] = pyl +func layersPutAccLeaf*(db: AristoDbRef; accPath: Hash256; leafVtx: VertexRef) = + db.top.delta.accLeaves[accPath] = leafVtx + +func layersPutStoLeaf*(db: AristoDbRef; mixPath: Hash256; leafVtx: VertexRef) = + db.top.delta.stoLeaves[mixPath] = leafVtx # ------------------------------------------------------------------------------ # Public functions @@ -165,8 +177,10 @@ func layersMergeOnto*(src: LayerRef; trg: var LayerObj) = for (vid,key) in src.delta.kMap.pairs: trg.delta.kMap[vid] = key trg.delta.vTop = src.delta.vTop - for (accPath,pyl) in src.delta.accLeaves.pairs: - trg.delta.accLeaves[accPath] = pyl + for (accPath,leafVtx) in src.delta.accLeaves.pairs: + trg.delta.accLeaves[accPath] = leafVtx + for (mixPath,leafVtx) in src.delta.stoLeaves.pairs: + trg.delta.stoLeaves[mixPath] = leafVtx func layersCc*(db: AristoDbRef; level = high(int)): LayerRef = ## Provide a collapsed copy of layers up to a particular transaction level. @@ -183,6 +197,7 @@ func layersCc*(db: AristoDbRef; level = high(int)): LayerRef = kMap: layers[0].delta.kMap, vTop: layers[^1].delta.vTop, accLeaves: layers[0].delta.accLeaves, + stoLeaves: layers[0].delta.stoLeaves, )) # Consecutively merge other layers on top @@ -191,8 +206,10 @@ func layersCc*(db: AristoDbRef; level = high(int)): LayerRef = result.delta.sTab[vid] = vtx for (vid,key) in layers[n].delta.kMap.pairs: result.delta.kMap[vid] = key - for (accPath,pyl) in layers[n].delta.accLeaves.pairs: - result.delta.accLeaves[accPath] = pyl + for (accPath,vtx) in layers[n].delta.accLeaves.pairs: + result.delta.accLeaves[accPath] = vtx + for (mixPath,vtx) in layers[n].delta.stoLeaves.pairs: + result.delta.stoLeaves[mixPath] = vtx # ------------------------------------------------------------------------------ # Public iterators diff --git a/nimbus/db/aristo/aristo_merge.nim b/nimbus/db/aristo/aristo_merge.nim index 4bc0e8cdc..5da23e9c3 100644 --- a/nimbus/db/aristo/aristo_merge.nim +++ b/nimbus/db/aristo/aristo_merge.nim @@ -137,6 +137,8 @@ proc mergeStorageData*( # Mark account path Merkle keys for update resetKeys() + db.layersPutStoLeaf(AccountKey.mixUp(accPath, stoPath), rc.value) + if not stoID.isValid: # Make sure that there is an account that refers to that storage trie let leaf = vtx.dup # Dup on modify