diff --git a/nimbus/db/aristo/README.md b/nimbus/db/aristo/README.md index d1c11ce04..ad97222c5 100644 --- a/nimbus/db/aristo/README.md +++ b/nimbus/db/aristo/README.md @@ -349,7 +349,7 @@ should be allocated in the structural table associated with the zero key. db | stack[n] | | desc | : | | optional passive delta layers, handled by obj | stack[1] | | transaction management (can be used to - | | stack[0] | | successively replace the top layer) + | | stack[0] | | successively recover the top layer) | +----------+ v | +----------+ | | roFilter | optional read-only backend filter @@ -449,11 +449,15 @@ Nevertheless, *(8)* can alse be transformed by committing and saving *tx2* | ø, ø | tx2+PBE | tx3, ~tx2 | - As *(11)* and *(13)* represent the same API, one has +As *(11)* and *(13)* represent the same API, one has - tx2+PBE == tx1+(tx2+~tx1)+PBE because of the middle rows (14) - ~tx2 == ~tx1+~(tx2+~tx1) because of (14) (15) + tx2+PBE =~ tx1+(tx2+~tx1)+PBE because of the middle rows (14) + ~tx2 =~ ~tx1+~(tx2+~tx1) because of (14) (15) - which shows some distributive property in *(14)* and commutative property in - *(15)* for this example. In particulat it might be handy for testing/verifying - against this example. +which looks like some distributive property in *(14)* and commutative +property in *(15)* for this example (but it is not straight algebraically.) +The *=~* operator above indicates that the representations are equivalent in +the sense that they have the same effect on the backend database (looks a +bit like residue classes.) + +It might be handy for testing/verifying an implementation using this example. diff --git a/nimbus/db/aristo/aristo_check/check_be.nim b/nimbus/db/aristo/aristo_check/check_be.nim index 766d5a007..77985ca14 100644 --- a/nimbus/db/aristo/aristo_check/check_be.nim +++ b/nimbus/db/aristo/aristo_check/check_be.nim @@ -129,6 +129,9 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef]( # Check cache against backend if cache: + if db.top.dirty: + return err((VertexID(0),CheckBeCacheIsDirty)) + # Check structural table for (vid,vtx) in db.top.sTab.pairs: # A `kMap[]` entry must exist. diff --git a/nimbus/db/aristo/aristo_debug.nim b/nimbus/db/aristo/aristo_debug.nim index 3b3fde84d..0603453d6 100644 --- a/nimbus/db/aristo/aristo_debug.nim +++ b/nimbus/db/aristo/aristo_debug.nim @@ -200,7 +200,7 @@ proc ppSTab( "{" & sTab.sortedKeys .mapIt((it, sTab.getOrVoid it)) .mapIt("(" & it[0].ppVid & "," & it[1].ppVtx(db,it[0]) & ")") - .join("," & indent.toPfx(1)) & "}" + .join(indent.toPfx(2)) & "}" proc ppLTab( lTab: Table[LeafTie,VertexID]; @@ -210,7 +210,7 @@ proc ppLTab( "{" & lTab.sortedKeys .mapIt((it, lTab.getOrVoid it)) .mapIt("(" & it[0].ppLeafTie(db) & "," & it[1].ppVid & ")") - .join("," & indent.toPfx(1)) & "}" + .join(indent.toPfx(2)) & "}" proc ppPPrf(pPrf: HashSet[VertexID]): string = "{" & pPrf.sortedKeys.mapIt(it.ppVid).join(",") & "}" @@ -324,9 +324,11 @@ proc ppFilter(fl: AristoFilterRef; db: AristoDbRef; indent: int): string = pfx1 = indent.toPfx(1) pfx2 = indent.toPfx(2) result = "" - if db.roFilter.isNil: + if fl.isNil: result &= " n/a" return + result &= pfx & "trg(" & fl.trg.ppKey & ")" + result &= pfx & "src(" & fl.src.ppKey & ")" result &= pfx & "vGen" & pfx1 & "[" if fl.vGen.isSome: result &= fl.vGen.unsafeGet.mapIt(it.ppVid).join(",") @@ -361,7 +363,7 @@ proc ppBeOnly[T](be: T; db: AristoDbRef; indent: int): string = proc ppBe[T](be: T; db: AristoDbRef; indent: int): string = ## backend + filter - db.roFilter.ppFilter(db, indent) & indent.toPfx & be.ppBeOnly(db,indent) + db.roFilter.ppFilter(db, indent+1) & indent.toPfx & be.ppBeOnly(db,indent+1) proc ppLayer( layer: AristoLayerRef; @@ -374,8 +376,8 @@ proc ppLayer( indent = 4; ): string = let - pfx1 = indent.toPfx - pfx2 = indent.toPfx(1) + pfx1 = indent.toPfx(1) + pfx2 = indent.toPfx(2) nOKs = sTabOk.ord + lTabOk.ord + kMapOk.ord + pPrfOk.ord + vGenOk.ord tagOk = 1 < nOKs var @@ -392,6 +394,8 @@ proc ppLayer( rc if not layer.isNil: + if 2 < nOKs: + result &= "".doPrefix(false) if vGenOk: let tLen = layer.vGen.len @@ -613,6 +617,12 @@ proc pp*( ): string = db.top.pp(db, xTabOk=xTabOk, kMapOk=kMapOk, other=other, indent=indent) +proc pp*( + filter: AristoFilterRef; + db = AristoDbRef(); + indent = 4; + ): string = + filter.ppFilter(db, indent) proc pp*( be: TypedBackendRef; diff --git a/nimbus/db/aristo/aristo_desc.nim b/nimbus/db/aristo/aristo_desc.nim index e53d10484..c0386adc8 100644 --- a/nimbus/db/aristo/aristo_desc.nim +++ b/nimbus/db/aristo/aristo_desc.nim @@ -11,7 +11,7 @@ ## Aristo DB -- a Patricia Trie with labeled edges ## =============================================== ## -## These data structures allows to overlay the *Patricia Trie* with *Merkel +## These data structures allow to overlay the *Patricia Trie* with *Merkel ## Trie* hashes. See the `README.md` in the `aristo` folder for documentation. ## ## Some semantic explanations; @@ -22,7 +22,7 @@ {.push raises: [].} import - std/tables, + std/[hashes, sets, tables], eth/common, ./aristo_constants, ./aristo_desc/[ @@ -31,8 +31,8 @@ import from ./aristo_desc/aristo_types_backend import AristoBackendRef +# Not auto-exporting backend export - # Not auto-exporting backend aristo_constants, aristo_error, aristo_types_identifiers, aristo_types_structural @@ -44,9 +44,16 @@ type txUid*: uint ## Unique ID among transactions level*: int ## Stack index for this transaction + AristoDudesRef* = ref object + case rwOk*: bool + of true: + roDudes*: HashSet[AristoDbRef] ## Read-only peers + else: + rwDb*: AristoDbRef ## Link to writable descriptor + AristoDbRef* = ref AristoDbObj AristoDbObj* = object - ## Set of database layers, supporting transaction frames + ## Three tier database object supporting distributed instances. top*: AristoLayerRef ## Database working layer, mutable stack*: seq[AristoLayerRef] ## Stashed immutable parent layers roFilter*: AristoFilterRef ## Apply read filter (locks writing) @@ -54,10 +61,14 @@ type txRef*: AristoTxRef ## Latest active transaction txUidGen*: uint ## Tx-relative unique number generator + dudes*: AristoDudesRef ## Related DB descriptors # Debugging data below, might go away in future xMap*: Table[HashLabel,VertexID] ## For pretty printing, extends `pAmk` + AristoDbAction* = proc(db: AristoDbRef) {.gcsafe, raises: [CatchableError].} + ## Generic call back function/closure. + # ------------------------------------------------------------------------------ # Public helpers # ------------------------------------------------------------------------------ @@ -98,9 +109,13 @@ func isValid*(vid: VertexID): bool = # Public functions, miscellaneous # ------------------------------------------------------------------------------ +# Hash set helper +func hash*(db: AristoDbRef): Hash = + ## Table/KeyedQueue/HashSet mixin + cast[pointer](db).hash + # Note that the below `init()` function cannot go into # `aristo_types_identifiers` as this would result in a circular import. - func init*(key: var HashKey; data: openArray[byte]): bool = ## Import argument `data` into `key` which must have length either `32`, or ## `0`. The latter case is equivalent to an all zero byte array of size `32`. diff --git a/nimbus/db/aristo/aristo_desc/aristo_error.nim b/nimbus/db/aristo/aristo_desc/aristo_error.nim index fd6c50650..80272e4e2 100644 --- a/nimbus/db/aristo/aristo_desc/aristo_error.nim +++ b/nimbus/db/aristo/aristo_desc/aristo_error.nim @@ -92,7 +92,8 @@ type HashifyCannotComplete HashifyCannotHashRoot HashifyExistingHashMismatch - HashifyLeafToRootAllFailed + HashifyDownVtxlevelExceeded + HashifyDownVtxLeafUnexpected HashifyRootHashMismatch HashifyRootVidMismatch HashifyVidCircularDependence @@ -131,6 +132,7 @@ type CheckBeKeyMismatch CheckBeGarbledVGen + CheckBeCacheIsDirty CheckBeCacheKeyMissing CheckBeCacheKeyNonEmpty CheckBeCacheVidUnsynced @@ -167,9 +169,12 @@ type DelVidStaleVtx # Functions from `aristo_filter.nim` + FilRoBackendOrMissing FilStateRootMissing FilStateRootMismatch FilPrettyPointlessLayer + FilDudeFilterUpdateError + FilNotReadOnlyDude # Get functions form `aristo_get.nim` GetLeafNotFound @@ -192,8 +197,9 @@ type # Transaction wrappers TxArgStaleTx - TxBackendMissing + TxRoBackendOrMissing TxNoPendingTx + TxPendingTx TxNotTopTx TxStackGarbled TxStackUnderflow diff --git a/nimbus/db/aristo/aristo_desc/aristo_types_structural.nim b/nimbus/db/aristo/aristo_desc/aristo_types_structural.nim index 51f61b838..f0b0bff37 100644 --- a/nimbus/db/aristo/aristo_desc/aristo_types_structural.nim +++ b/nimbus/db/aristo/aristo_desc/aristo_types_structural.nim @@ -250,6 +250,16 @@ proc dup*(layer: AristoLayerRef): AristoLayerRef = for (k,v) in layer.sTab.pairs: result.sTab[k] = v.dup +proc dup*(filter: AristoFilterRef): AristoFilterRef = + ## Duplicate layer. + result = AristoFilterRef( + src: filter.src, + kMap: filter.kMap, + vGen: filter.vGen, + trg: filter.trg) + for (k,v) in filter.sTab.pairs: + result.sTab[k] = v.dup + proc to*(node: NodeRef; T: type VertexRef): T = ## Extract a copy of the `VertexRef` part from a `NodeRef`. node.VertexRef.dup diff --git a/nimbus/db/aristo/aristo_filter.nim b/nimbus/db/aristo/aristo_filter.nim index 83c1edc0c..5535bb776 100644 --- a/nimbus/db/aristo/aristo_filter.nim +++ b/nimbus/db/aristo/aristo_filter.nim @@ -13,8 +13,9 @@ ## import - std/[options, sequtils, tables], + std/[options, sequtils, sets, tables], results, + ./aristo_desc/aristo_types_backend, "."/[aristo_desc, aristo_get, aristo_vid] type @@ -26,16 +27,6 @@ type # Private helpers # ------------------------------------------------------------------------------ -proc getBeStateRoot( - db: AristoDbRef; - ): Result[HashKey,AristoError] = - let rc = db.getKeyBE VertexID(1) - if rc.isOk: - return ok(rc.value) - if rc.error == GetKeyNotFound: - return ok(VOID_HASH_KEY) - err(rc.error) - proc getLayerStateRoots( db: AristoDbRef; layer: AristoLayerRef; @@ -44,26 +35,115 @@ proc getLayerStateRoots( ## Get the Merkle hash key for target state root to arrive at after this ## reverse filter was applied. var spr: StateRootPair - block: - let rc = db.getBeStateRoot() - if rc.isErr: + + spr.be = block: + let rc = db.getKeyBE VertexID(1) + if rc.isOk: + rc.value + elif rc.error == GetKeyNotFound: + VOID_HASH_KEY + else: return err(rc.error) - spr.be = rc.value + block: spr.fg = layer.kMap.getOrVoid(VertexID 1).key if spr.fg.isValid: return ok(spr) + if chunkedMpt: let vid = layer.pAmk.getOrVoid HashLabel(root: VertexID(1), key: spr.be) if vid == VertexID(1): spr.fg = spr.be return ok(spr) + if layer.sTab.len == 0 and layer.kMap.len == 0 and layer.pAmk.len == 0: return err(FilPrettyPointlessLayer) + err(FilStateRootMismatch) +# ------------------------------------------------------------------------------ +# Private functions +# ------------------------------------------------------------------------------ + +proc merge( + db: AristoDbRef; + upper: AristoFilterRef; # Src filter, `nil` is ok + lower: AristoFilterRef; # Trg filter, `nil` is ok + beStateRoot: HashKey; # Merkle hash key + ): Result[AristoFilterRef,(VertexID,AristoError)] = + ## Merge argument `upper` into the `lower` filter instance. + ## + ## Comparing before and after merge + ## :: + ## current | merged + ## ----------------------------+-------------------------------- + ## trg2 --upper-- (src2==trg1) | + ## | trg2 --newFilter-- (src1==trg0) + ## trg1 --lower-- (src1==trg0) | + ## | + ## trg0 --beStateRoot | trg0 --beStateRoot + ## | + ## + # Degenerate case: `upper` is void + if lower.isNil or lower.vGen.isNone: + if upper.isNil or upper.vGen.isNone: + # Even more degenerate case when both filters are void + return ok AristoFilterRef( + src: beStateRoot, + trg: beStateRoot, + vGen: none(seq[VertexID])) + if upper.src != beStateRoot: + return err((VertexID(1),FilStateRootMismatch)) + return ok(upper) + + # Degenerate case: `upper` is non-trivial and `lower` is void + if upper.isNil or upper.vGen.isNone: + if lower.src != beStateRoot: + return err((VertexID(0), FilStateRootMismatch)) + return ok(lower) + + # Verify stackability + if upper.src != lower.trg or + lower.src != beStateRoot: + return err((VertexID(0), FilStateRootMismatch)) + + # There is no need to deep copy table vertices as they will not be modified. + let newFilter = AristoFilterRef( + src: lower.src, + sTab: lower.sTab, + kMap: lower.kMap, + vGen: upper.vGen, + trg: upper.trg) + + for (vid,vtx) in upper.sTab.pairs: + if vtx.isValid or not newFilter.sTab.hasKey vid: + newFilter.sTab[vid] = vtx + elif newFilter.sTab.getOrVoid(vid).isValid: + let rc = db.getVtxUBE vid + if rc.isOk: + newFilter.sTab[vid] = vtx # VertexRef(nil) + elif rc.error == GetVtxNotFound: + newFilter.sTab.del vid + else: + return err((vid,rc.error)) + + for (vid,key) in upper.kMap.pairs: + if key.isValid or not newFilter.kMap.hasKey vid: + newFilter.kMap[vid] = key + elif newFilter.kMap.getOrVoid(vid).isValid: + let rc = db.getKeyUBE vid + if rc.isOk: + newFilter.kMap[vid] = key # VOID_HASH_KEY + elif rc.error == GetKeyNotFound: + newFilter.kMap.del vid + else: + return err((vid,rc.error)) + + ok newFilter + + # ------------------------------------------------------------------------------ # Public helpers # ------------------------------------------------------------------------------ @@ -122,6 +202,51 @@ proc fwdFilter*( vGen: some(layer.vGen.vidReorg), # Compact recycled IDs trg: trgRoot) + +proc revFilter*( + db: AristoDbRef; + filter: AristoFilterRef; + ): Result[AristoFilterRef,(VertexID,AristoError)] = + ## Assemble reverse filter for the `filter` argument, i.e. changes to the + ## backend that reverse the effect of applying the this read-only filter. + ## + ## This read-only filter is calculated against the current unfiltered + ## backend (excluding optionally installed read-only filter.) + ## + # Register MPT state roots for reverting back + let rev = AristoFilterRef( + src: filter.trg, + trg: filter.src) + + # Get vid generator state on backend + block: + let rc = db.getIdgUBE() + if rc.isErr: + return err((VertexID(0), rc.error)) + rev.vGen = some rc.value + + # Calculate reverse changes for the `sTab[]` structural table + for vid in filter.sTab.keys: + let rc = db.getVtxUBE vid + if rc.isOk: + rev.sTab[vid] = rc.value + elif rc.error == GetVtxNotFound: + rev.sTab[vid] = VertexRef(nil) + else: + return err((vid,rc.error)) + + # Calculate reverse changes for the `kMap` sequence. + for vid in filter.kMap.keys: + let rc = db.getKeyUBE vid + if rc.isOk: + rev.kMap[vid] = rc.value + elif rc.error == GetKeyNotFound: + rev.kMap[vid] = VOID_HASH_KEY + else: + return err((vid,rc.error)) + + ok(rev) + # ------------------------------------------------------------------------------ # Public functions, apply/install filters # ------------------------------------------------------------------------------ @@ -130,81 +255,148 @@ proc merge*( db: AristoDbRef; filter: AristoFilterRef; ): Result[void,(VertexID,AristoError)] = - ## Merge argument `filter` to the filter layer. - ## - ## Comparing before and after merge - ## :: - ## current | merged - ## ----------------------------------+-------------------------------- - ## trg2 --filter-- (src2==trg1) | - ## | trg2 --newFilter-- (src1==trg0) - ## trg1 --db.roFilter-- (src1==trg0) | - ## | - ## trg0 --db.backend | trg0 --db.backend - ## | - let beRoot = block: - let rc = db.getBeStateRoot() + ## Merge the argument `filter` into the read-only filter layer. Note that + ## this function has no control of the filter source. Having merged the + ## argument `filter`, all the `top` and `stack` layers should be cleared. + let ubeRootKey = block: + let rc = db.getKeyUBE VertexID(1) + if rc.isOk: + rc.value + elif rc.error == GetKeyNotFound: + VOID_HASH_KEY + else: + return err((VertexID(1),rc.error)) + + db.roFilter = block: + let rc = db.merge(filter, db.roFilter, ubeRootKey) if rc.isErr: - return err((VertexID(1),FilStateRootMissing)) + return err(rc.error) rc.value - if filter.vGen.isNone: - # Blind argument filter - if db.roFilter.isNil: - # Force read-only system - db.roFilter = AristoFilterRef( - src: beRoot, - trg: beRoot, - vGen: none(seq[VertexID])) - return ok() - - # Simple case: no read-only filter yet - if db.roFilter.isNil or db.roFilter.vGen.isNone: - if filter.src != beRoot: - return err((VertexID(1),FilStateRootMismatch)) - db.roFilter = filter - return ok() - - # Verify merge stackability into existing read-only filter - if filter.src != db.roFilter.trg: - return err((VertexID(1),FilStateRootMismatch)) - - # Merge `filter` into `roFilter` as `newFilter`. There is no need to deep - # copy table vertices as they will not be modified. - let newFilter = AristoFilterRef( - src: db.roFilter.src, - sTab: db.roFilter.sTab, - kMap: db.roFilter.kMap, - vGen: filter.vGen, - trg: filter.trg) - - for (vid,vtx) in filter.sTab.pairs: - if vtx.isValid or not newFilter.sTab.hasKey vid: - newFilter.sTab[vid] = vtx - elif newFilter.sTab.getOrVoid(vid).isValid: - let rc = db.getVtxUBE vid - if rc.isOk: - newFilter.sTab[vid] = vtx # VertexRef(nil) - elif rc.error == GetVtxNotFound: - newFilter.sTab.del vid - else: - return err((vid,rc.error)) - - for (vid,key) in filter.kMap.pairs: - if key.isValid or not newFilter.kMap.hasKey vid: - newFilter.kMap[vid] = key - elif newFilter.kMap.getOrVoid(vid).isValid: - let rc = db.getKeyUBE vid - if rc.isOk: - newFilter.kMap[vid] = key # VOID_HASH_KEY - elif rc.error == GetKeyNotFound: - newFilter.kMap.del vid - else: - return err((vid,rc.error)) - - db.roFilter = newFilter ok() + +proc canResolveBE*(db: AristoDbRef): bool = + ## Check whether the read-only filter can be merged into the backend + if not db.backend.isNil: + if db.dudes.isNil or db.dudes.rwOk: + return true + + +proc resolveBE*(db: AristoDbRef): Result[void,(VertexID,AristoError)] = + ## Resolve the backend filter into the physical backend. This requires that + ## the argument `db` descriptor has read-write permission for the backend + ## (see also the below function `ackqRwMode()`.) + ## + ## For any associated descriptors working on the same backend, their backend + ## filters will be updated so that the change of the backend DB remains + ## unnoticed. + if not db.canResolveBE(): + return err((VertexID(1),FilRoBackendOrMissing)) + + # Blind or missing filter + if db.roFilter.isNil: + return ok() + if db.roFilter.vGen.isNone: + db.roFilter = AristoFilterRef(nil) + return ok() + + let ubeRootKey = block: + let rc = db.getKeyUBE VertexID(1) + if rc.isOk: + rc.value + elif rc.error == GetKeyNotFound: + VOID_HASH_KEY + else: + return err((VertexID(1),rc.error)) + + # Filters rollback helper + var roFilters: seq[(AristoDbRef,AristoFilterRef)] + proc rollback() = + for (d,f) in roFilters: + d.roFilter = f + + # Update dudes + if not db.dudes.isNil: + # Calculate reverse filter from current filter + let rev = block: + let rc = db.revFilter db.roFilter + if rc.isErr: + return err(rc.error) + rc.value + + # Update distributed filters. Note that the physical backend database + # has not been updated, yet. So the new root key for the backend will + # be `db.roFilter.trg`. + for dude in db.dudes.roDudes.items: + let rc = db.merge(dude.roFilter, rev, db.roFilter.trg) + if rc.isErr: + rollback() + return err(rc.error) + roFilters.add (dude, dude.roFilter) + dude.roFilter = rc.value + + # Save structural and other table entries + let + be = db.backend + txFrame = be.putBegFn() + be.putVtxFn(txFrame, db.roFilter.sTab.pairs.toSeq) + be.putKeyFn(txFrame, db.roFilter.kMap.pairs.toSeq) + be.putIdgFn(txFrame, db.roFilter.vGen.unsafeGet) + let w = be.putEndFn txFrame + if w != AristoError(0): + rollback() + return err((VertexID(0),w)) + + ok() + + +proc ackqRwMode*(db: AristoDbRef): Result[void,AristoError] = + ## Re-focus the `db` argument descriptor to backend read-write permission. + if not db.dudes.isNil and not db.dudes.rwOk: + # Steal dudes list, make the rw-parent a read-only dude + let parent = db.dudes.rwDb + db.dudes = parent.dudes + parent.dudes = AristoDudesRef(rwOk: false, rwDb: db) + + # Exclude self + db.dudes.roDudes.excl db + + # Update dudes + for w in db.dudes.roDudes: + # Let all other dudes refer to this one + w.dudes.rwDb = db + + # Update dudes list (parent was alredy updated) + db.dudes.roDudes.incl parent + return ok() + + err(FilNotReadOnlyDude) + + +proc dispose*(db: AristoDbRef): Result[void,AristoError] = + ## Terminate usage of the `db` argument descriptor with backend read-only + ## permission. + ## + ## This type of descriptoy should always be terminated after use. Otherwise + ## it would always be udated when running `resolveBE()` which costs + ## unnecessary computing ressources. Also, the read-only backend filter + ## copies might grow big when it could be avoided. + if not db.isNil and + not db.dudes.isNil and + not db.dudes.rwOk: + # Unlink argument `db` + db.dudes.rwDb.dudes.roDudes.excl db + + # Unlink more so it would not do harm if used wrongly + db.stack.setlen(0) + db.backend = AristoBackendRef(nil) + db.txRef = AristoTxRef(nil) + db.dudes = AristoDudesRef(nil) + return ok() + + err(FilNotReadOnlyDude) + # ------------------------------------------------------------------------------ # End # ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_hashify.nim b/nimbus/db/aristo/aristo_hashify.nim index 3cf2cbf1f..5354ceff6 100644 --- a/nimbus/db/aristo/aristo_hashify.nim +++ b/nimbus/db/aristo/aristo_hashify.nim @@ -58,6 +58,16 @@ type BackVidTab = Table[VertexID,BackVidValRef] + BackWVtxRef = ref object + w: BackVidValRef + vtx: VertexRef + + BackWVtxTab = + Table[VertexID,BackWVtxRef] + +const + SubTreeSearchDepthMax = 64 + logScope: topics = "aristo-hashify" @@ -71,9 +81,15 @@ template logTxt(info: static[string]): static[string] = func getOrVoid(tab: BackVidTab; vid: VertexID): BackVidValRef = tab.getOrDefault(vid, BackVidValRef(nil)) +func getOrVoid(tab: BackWVtxTab; vid: VertexID): BackWVtxRef = + tab.getOrDefault(vid, BackWVtxRef(nil)) + func isValid(brv: BackVidValRef): bool = brv != BackVidValRef(nil) +func isValid(brv: BackWVtxRef): bool = + brv != BackWVtxRef(nil) + # ------------------------------------------------------------------------------ # Private functions # ------------------------------------------------------------------------------ @@ -209,6 +225,119 @@ proc deletedLeafHasher( ok() +# ------------------ + +proc resolveStateRoots( + db: AristoDbRef; # Database, top layer + uVids: BackVidTab; # Unresolved vertex IDs + ): Result[void,(VertexID,AristoError)] = + ## Resolve unresolved nodes. There might be a sub-tree on the backend which + ## blocks resolving the current structure. So search the `uVids` argument + ## list for missing vertices and resolve it. + # + # Update out-of-path hashes, i.e. fill gaps caused by branching out from + # `downMost` table vertices. + # + # Example + # :: + # $1 ^ + # \ | + # $7 -- $6 -- leaf $8 | on top layer, + # \ `--- leaf $9 | $5..$9 were inserted, + # $5 | $1 was redefined + # \ v + # \ + # \ ^ + # $4 -- leaf $2 | from + # `--- leaf $3 | backend (BE) + # v + # backLink[] = {$7} + # downMost[] = {$7} + # top.kMap[] = {£1, £6, £8, £9} + # BE.kMap[] = {£1, £2, £3, £4} + # + # So `$5` (needed for `$7`) cannot be resolved because it is neither on + # the path `($1..$8)`, nor is it on `($1..$9)`. + # + var follow: BackWVtxTab + + proc wVtxRef(db: AristoDbRef; root, vid, toVid: VertexID): BackWVtxRef = + let vtx = db.getVtx vid + if vtx.isValid: + return BackWVtxRef( + vtx: vtx, + w: BackVidValRef( + root: root, + onBe: not db.top.sTab.getOrVoid(vid).isValid, + toVid: toVid)) + + # Init `follow` table by unresolved `Branch` leafs from `vidTab` + for (uVid,uVal) in uVids.pairs: + let uVtx = db.getVtx uVid + if uVtx.isValid and uVtx.vType == Branch: + var didSomething = false + for vid in uVtx.bVid: + if vid.isValid and not db.getKey(vid).isValid: + let w = db.wVtxRef(root=uVal.root, vid=vid, toVid=uVid) + if not w.isNil: + follow[vid] = w + didSomething = true + # Add state root to be resolved, as well + if didSomething and not follow.hasKey uVal.root: + let w = db.wVtxRef(root=uVal.root, vid=uVal.root, toVid=uVal.root) + if not w.isNil: + follow[uVal.root] = w + + # Update and re-collect into `follow` table + var level = 0 + while 0 < follow.len: + var + changes = false + redo: BackWVtxTab + for (fVid,fVal) in follow.pairs: + # Resolve or keep for later + let rc = fVal.vtx.toNode db + if rc.isOk: + # Update Merkle hash + let + key = rc.value.to(HashKey) + rx = db.updateHashKey(fVal.w.root, fVid, key, fVal.w.onBe) + if rx.isErr: + return err((fVid, rx.error)) + changes = true + else: + # Cannot complete with this vertex, so dig deeper and do it later + redo[fVid] = fVal + + case fVal.vtx.vType: + of Branch: + for vid in fVal.vtx.bVid: + if vid.isValid and not db.getKey(vid).isValid: + let w = db.wVtxRef(root=fVal.w.root, vid=vid, toVid=fVid) + if not w.isNil: + changes = true + redo[vid] = w + of Extension: + let vid = fVal.vtx.eVid + if vid.isValid and not db.getKey(vid).isValid: + let w = db.wVtxRef(root=fVal.w.root,vid=vid, toVid=fVid) + if not w.isNil: + changes = true + redo[vid] = w + of Leaf: + # Should habe been hashed earlier + return err((fVid,HashifyDownVtxLeafUnexpected)) + + # Beware of loops + if not changes or SubTreeSearchDepthMax < level: + return err((VertexID(0),HashifyDownVtxlevelExceeded)) + + # Restart with a new instance of `follow` + redo.swap follow + level.inc + + ok() + # ------------------------------------------------------------------------------ # Public functions # ------------------------------------------------------------------------------ @@ -244,7 +373,7 @@ proc hashify*( for (lky,vid) in db.top.lTab.pairs: let hike = lky.hikeUp(db) - # There might be deleted entries on the leaf table. If this is tha case, + # There might be deleted entries on the leaf table. If this is the case, # the Merkle hashes for the vertices in the `hike` can all be compiled. if not vid.isValid: let rc = db.deletedLeafHasher hike @@ -268,12 +397,17 @@ proc hashify*( # Backtrack and register remaining nodes. Note that in case *n == 0*, # the root vertex has not been fully resolved yet. # - # hike.legs: (leg[0], leg[1], .., leg[n-1], leg[n], ..) - # | | | | - # | <---- | <---- | <---- | - # | | | - # | backLink[] | downMost | + # .. unresolved hash keys | all set here .. + # | + # | + # hike.legs: (leg[0], leg[1], ..leg[n-1], leg[n], ..) + # | | | | + # | <---- | <---- | <-------- | + # | | | + # | backLink[] | downMost[] | # + if n+1 < hike.legs.len: + downMost.del hike.legs[n+1].wp.vid downMost[hike.legs[n].wp.vid] = BackVidValRef( root: hike.root, onBe: hike.legs[n].backend, @@ -289,7 +423,9 @@ proc hashify*( # At least one full path leaf..root should have succeeded with labelling # for each root. if completed.len < roots.len: - return err((VertexID(0),HashifyLeafToRootAllFailed)) + let rc = db.resolveStateRoots backLink + if rc.isErr: + return err(rc.error) # Update remaining hashes while 0 < downMost.len: diff --git a/nimbus/db/aristo/aristo_init/memory_only.nim b/nimbus/db/aristo/aristo_init/memory_only.nim index 57acccec5..690975802 100644 --- a/nimbus/db/aristo/aristo_init/memory_only.nim +++ b/nimbus/db/aristo/aristo_init/memory_only.nim @@ -14,6 +14,7 @@ {.push raises: [].} import + std/sets, results, ../aristo_desc, ../aristo_desc/aristo_types_backend, @@ -57,12 +58,22 @@ proc finish*(db: AristoDbRef; flush = false) = ## depending on the type of backend (e.g. the `BackendMemory` backend will ## always flush on close.) ## + ## In case of distributed descriptors accessing the same backend, all + ## distributed descriptors will be destroyed. + ## ## This distructor may be used on already *destructed* descriptors. - if not db.backend.isNil: - db.backend.closeFn flush - db.backend = AristoBackendRef(nil) - db.top = AristoLayerRef(nil) - db.stack.setLen(0) + ## + if not db.isNil: + if not db.backend.isNil: + db.backend.closeFn flush + + if db.dudes.isNil: + db[] = AristoDbObj() + else: + let lebo = if db.dudes.rwOk: db else: db.dudes.rwDb + for w in lebo.dudes.roDudes: + w[] = AristoDbObj() + lebo[] = AristoDbObj() # ----------------- diff --git a/nimbus/db/aristo/aristo_tx.nim b/nimbus/db/aristo/aristo_tx.nim index a0a2395f0..36251167d 100644 --- a/nimbus/db/aristo/aristo_tx.nim +++ b/nimbus/db/aristo/aristo_tx.nim @@ -14,9 +14,9 @@ {.push raises: [].} import - std/[options, sequtils, tables], + std/[options, sets], results, - "."/[aristo_desc, aristo_filter, aristo_hashify] + "."/[aristo_desc, aristo_filter, aristo_get, aristo_hashify] func isTop*(tx: AristoTxRef): bool @@ -32,16 +32,24 @@ func getDbDescFromTopTx(tx: AristoTxRef): Result[AristoDbRef,AristoError] = return err(TxStackUnderflow) ok db -# ------------------------------------------------------------------------------ -# Private functions -# ------------------------------------------------------------------------------ - proc getTxUid(db: AristoDbRef): uint = if db.txUidGen == high(uint): db.txUidGen = 0 db.txUidGen.inc db.txUidGen +proc linkClone(db: AristoDbRef; clone: AristoDbRef) = + ## Link clone to parent + clone.dudes = AristoDudesRef( + rwOk: false, + rwDb: db) + if db.dudes.isNil: + db.dudes = AristoDudesRef( + rwOk: true, + roDudes: @[clone].toHashSet) + else: + db.dudes.roDudes.incl clone + # ------------------------------------------------------------------------------ # Public functions, getters # ------------------------------------------------------------------------------ @@ -75,19 +83,107 @@ func to*(tx: AristoTxRef; T: type[AristoDbRef]): T = ## Getter, retrieves the parent database descriptor from argument `tx` tx.db -proc rebase*( - tx: AristoTxRef; # Some transaction on database - ): Result[void,AristoError] = - ## Revert transaction stack to an earlier point in time. - if not tx.isTop(): - let - db = tx.db - inx = tx.level - if db.stack.len <= inx or db.stack[inx].txUid != tx.txUid: - return err(TxArgStaleTx) - # Roll back to some earlier layer. - db.top = db.stack[inx] - db.stack.setLen(inx) + +proc copyCat*(tx: AristoTxRef): Result[AristoDbRef,AristoError] = + ## Clone a transaction into a new DB descriptor. The new descriptor is linked + ## to the transaction parent and will be updated with functions like + ## `aristo_filter.resolveBE()` or `aristo_filter.ackqRwMode()`. The new + ## descriptor is fully functional apart from the fact that the physical + ## backend cannot be updated (but see `aristo_filter.ackqRwMode()`.) + ## + ## The new DB descriptor contains a copy of the argument transaction `tx` as + ## top layer of level 1 (i.e. this is he only transaction.) Rolling back will + ## end up at the backend layer (incl. backend filter.) + ## + ## Use `aristo_filter.dispose()` to clean up the new DB descriptor. + ## + let db = tx.db + + # Provide new top layer + var topLayer: AristoLayerRef + if db.txRef == tx: + topLayer = db.top.dup + elif tx.level < db.stack.len: + topLayer = db.stack[tx.level].dup + else: + return err(TxArgStaleTx) + if topLayer.txUid != tx.txUid: + return err(TxArgStaleTx) + topLayer.txUid = 1 + + # Empty stack + let stackLayer = block: + let rc = db.getIdgBE() + if rc.isOk: + AristoLayerRef(vGen: rc.value) + elif rc.error == GetIdgNotFound: + AristoLayerRef() + else: + return err(rc.error) + + # Set up clone associated to `db` + let txClone = AristoDbRef( + top: topLayer, # is a deep copy + stack: @[stackLayer], + roFilter: db.roFilter, # no need to copy contents (done when updated) + backend: db.backend, + txUidGen: 1, + dudes: AristoDudesRef( + rwOk: false, + rwDb: db)) + + # Link clone to parent + db.linkClone txClone + + # Install transaction similar to `tx` on clone + txClone.txRef = AristoTxRef( + db: txClone, + txUid: 1, + level: 1) + + ok(txClone) + +proc copyCat*(db: AristoDbRef): Result[AristoDbRef,AristoError] = + ## Variant of `copyCat()`. If there is a transaction pending, then the + ## function returns `db.txTop.value.copyCat()`. Otherwise it returns a + ## clone of the top layer. + ## + ## Use `aristo_filter.dispose()` to clean up the copy cat descriptor. + ## + if db.txRef.isNil: + let dbClone = AristoDbRef( + top: db.top.dup, # is a deep copy + roFilter: db.roFilter, # no need to copy contents (done when updated) + backend: db.backend) + + # Link clone to parent + db.linkClone dbClone + return ok(dbClone) + + db.txRef.copyCat() + + +proc exec*( + tx: AristoTxRef; + action: AristoDbAction; + ): Result[void,AristoError] + {.gcsafe, raises: [CatchableError].} = + ## Execute function argument `action()` on a temporary `tx.copyCat()` + ## transaction database. After return, the temporary database gets + ## destroyed. + ## + let db = block: + let rc = tx.copyCat() + if rc.isErr: + return err(rc.error) + rc.value + + db.action() + + block: + let rc = db.dispose() + if rc.isErr: + return err(rc.error) ok() # ------------------------------------------------------------------------------ @@ -158,7 +254,7 @@ proc commit*( return err((VertexID(0),rc.error)) rc.value - if not dontHashify: + if db.top.dirty and not dontHashify: let rc = db.hashify() if rc.isErr: return err(rc.error) @@ -196,8 +292,8 @@ proc collapse*( if not commit: db.stack[0].swap db.top - if not dontHashify: - var rc = db.hashify() + if db.top.dirty and not dontHashify: + let rc = db.hashify() if rc.isErr: if not commit: db.stack[0].swap db.top # restore @@ -218,8 +314,8 @@ proc stow*( chunkedMpt = false; # Partial data (e.g. from `snap`) ): Result[void,(VertexID,AristoError)] = ## If there is no backend while the `persistent` argument is set `true`, - ## the function returns immediately with an error.The same happens if the - ## backend is locked while `persistent` is set (e.g. by an `exec()` call.) + ## the function returns immediately with an error.The same happens if there + ## is a pending transaction. ## ## The `dontHashify` is treated as described for `commit()`. ## @@ -234,9 +330,17 @@ proc stow*( ## If the argument `persistent` is set `true`, all the staged data are merged ## into the physical backend database and the staged data area is cleared. ## - let be = db.backend - if be.isNil and persistent: - return err((VertexID(0),TxBackendMissing)) + if not db.txRef.isNil: + return err((VertexID(0),TxPendingTx)) + if 0 < db.stack.len: + return err((VertexID(0),TxStackGarbled)) + if persistent and not db.canResolveBE(): + return err((VertexID(0),TxRoBackendOrMissing)) + + if db.top.dirty and not dontHashify: + let rc = db.hashify() + if rc.isErr: + return err(rc.error) let fwd = block: let rc = db.fwdFilter(db.top, chunkedMpt) @@ -246,22 +350,17 @@ proc stow*( if fwd.vGen.isSome: # Otherwise this layer is pointless block: + # Merge `top` layer into `roFilter` let rc = db.merge fwd if rc.isErr: return err(rc.error) - rc.value + db.top = AristoLayerRef(vGen: db.roFilter.vGen.unsafeGet) - if persistent: - # Save structural and other table entries - let txFrame = be.putBegFn() - be.putVtxFn(txFrame, db.roFilter.sTab.pairs.toSeq) - be.putKeyFn(txFrame, db.roFilter.kMap.pairs.toSeq) - be.putIdgFn(txFrame, db.roFilter.vGen.unsafeGet) - let w = be.putEndFn txFrame - if w != AristoError(0): - return err((VertexID(0),w)) - - db.roFilter = AristoFilterRef(nil) + if persistent: + let rc = db.resolveBE() + if rc.isErr: + return err(rc.error) + db.roFilter = AristoFilterRef(nil) # Delete or clear stack and clear top db.stack.setLen(0) diff --git a/tests/test_aristo.nim b/tests/test_aristo.nim index 38f11f68d..9e73a10ef 100644 --- a/tests/test_aristo.nim +++ b/tests/test_aristo.nim @@ -24,7 +24,8 @@ import ../nimbus/sync/snap/worker/db/[rocky_bulk_load, snapdb_accounts, snapdb_desc], ./replay/[pp, undump_accounts, undump_storages], ./test_sync_snap/[snap_test_xx, test_accounts, test_types], - ./test_aristo/[test_backend, test_helpers, test_transcode, test_tx] + ./test_aristo/[ + test_backend, test_filter, test_helpers, test_transcode, test_tx] const baseDir = [".", "..", ".."/"..", $DirSep] @@ -220,6 +221,9 @@ proc accountsRunner( test &"Delete accounts database, successively {accLst.len} entries": check noisy.testTxMergeAndDelete(accLst, dbDir) + test &"Distributed backend access {accLst.len} entries": + check noisy.testDistributedAccess(accLst, dbDir) + proc storagesRunner( noisy = true; @@ -253,6 +257,9 @@ proc storagesRunner( test &"Delete storage database, successively {stoLst.len} entries": check noisy.testTxMergeAndDelete(stoLst, dbDir) + test &"Distributed backend access {stoLst.len} entries": + check noisy.testDistributedAccess(stoLst, dbDir) + # ------------------------------------------------------------------------------ # Main function(s) # ------------------------------------------------------------------------------ diff --git a/tests/test_aristo/test_filter.nim b/tests/test_aristo/test_filter.nim new file mode 100644 index 000000000..f50c9d4b4 --- /dev/null +++ b/tests/test_aristo/test_filter.nim @@ -0,0 +1,423 @@ +# Nimbus - Types, data structures and shared utilities used in network sync +# +# Copyright (c) 2018-2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or +# distributed except according to those terms. + +## Aristo (aka Patricia) DB records distributed backend access test. +## + +import + eth/common, + results, + unittest2, + ../../nimbus/db/aristo/[ + aristo_check, aristo_debug, aristo_desc, aristo_filter, aristo_get, + aristo_merge], + ../../nimbus/db/[aristo, aristo/aristo_init/persistent], + ./test_helpers + +type + LeafTriplet = tuple + a, b, c: seq[LeafTiePayload] + + LeafQuartet = tuple + a, b, c, d: seq[LeafTiePayload] + + DbTriplet = object + db1, db2, db3: AristoDbRef + +# ------------------------------------------------------------------------------ +# Private debugging helpers +# ------------------------------------------------------------------------------ + +proc dump(pfx: string; dx: varargs[AristoDbRef]): string = + proc dump(db: AristoDbRef): string = + db.pp & "\n " & db.to(TypedBackendRef).pp(db) & "\n" + if 0 < dx.len: + result = "\n " + var + pfx = pfx + qfx = "" + if pfx.len == 0: + (pfx,qfx) = ("[","]") + elif 1 < dx.len: + pfx = pfx & "#" + for n in 0 ..< dx.len: + let n1 = n + 1 + result &= pfx + if 1 < dx.len: + result &= $n1 + result &= qfx & "\n " & dx[n].dump + if n1 < dx.len: + result &= " ==========\n " + +proc dump(dx: varargs[AristoDbRef]): string = + "".dump dx + +proc dump(w: DbTriplet): string = + "db".dump(w.db1, w.db2, w.db3) + +# ------------------------------------------------------------------------------ +# Private helpers +# ------------------------------------------------------------------------------ + +iterator quadripartite(td: openArray[ProofTrieData]): LeafQuartet = + ## ... + var collect: seq[seq[LeafTiePayload]] + + for w in td: + let lst = w.kvpLst.mapRootVid VertexID(1) + + if lst.len < 8: + if 2 < collect.len: + yield(collect[0], collect[1], collect[2], lst) + collect.setLen(0) + else: + collect.add lst + else: + if collect.len == 0: + let a = lst.len div 4 + yield(lst[0 ..< a], lst[a ..< 2*a], lst[2*a ..< 3*a], lst[3*a .. ^1]) + else: + if collect.len == 1: + let a = lst.len div 3 + yield(collect[0], lst[0 ..< a], lst[a ..< 2*a], lst[a .. ^1]) + elif collect.len == 2: + let a = lst.len div 2 + yield(collect[0], collect[1], lst[0 ..< a], lst[a .. ^1]) + else: + yield(collect[0], collect[1], collect[2], lst) + collect.setLen(0) + +proc dbTriplet(w: LeafQuartet; rdbPath: string): Result[DbTriplet,AristoError] = + let + db1 = block: + let rc = newAristoDbRef(BackendRocksDB,rdbPath) + if rc.isErr: + check rc.error == 0 + return + rc.value + + # Fill backend + m0 = db1.merge w.a + rc = db1.stow(persistent=true) + + if rc.isErr: + check rc.error == (0,0) + return + + let + db2 = db1.copyCat.value + db3 = db1.copyCat.value + + # Clause (9) from `aristo/README.md` example + m1 = db1.merge w.b + m2 = db2.merge w.c + m3 = db3.merge w.d + + if m1.error == 0 and + m2.error == 0 and + m3.error == 0: + return ok DbTriplet(db1: db1, db2: db2, db3: db3) + + # Failed + db1.finish(flush=true) + + check m1.error == 0 + check m2.error == 0 + check m3.error == 0 + + var error = m1.error + if error != 0: error = m2.error + if error != 0: error = m3.error + err(error) + + +proc checkBeOk( + dx: DbTriplet; + relax = false; + forceCache = false; + noisy = true; + ): bool = + check not dx.db1.top.isNil + block: + let + cache = if forceCache: true else: not dx.db1.top.dirty + rc1 = dx.db1.checkBE(relax=relax, cache=cache) + if rc1.isErr: + noisy.say "***", "db1 check failed (do-cache=", cache, ")" + check rc1.error == (0,0) + return + block: + let + cache = if forceCache: true else: not dx.db2.top.dirty + rc2 = dx.db2.checkBE(relax=relax, cache=cache) + if rc2.isErr: + noisy.say "***", "db2 check failed (do-cache=", cache, ")" + check rc2.error == (0,0) + return + block: + let + cache = if forceCache: true else: not dx.db3.top.dirty + rc3 = dx.db3.checkBE(relax=relax, cache=cache) + if rc3.isErr: + noisy.say "***", "db3 check failed (do-cache=", cache, ")" + check rc3.error == (0,0) + return + true + +# --------- + +proc cleanUp(dx: DbTriplet) = + discard dx.db3.dispose + discard dx.db2.dispose + dx.db1.finish(flush=true) + +proc eq(a, b: AristoFilterRef; db: AristoDbRef; noisy = true): bool = + ## Verify that argument filter `a` has the same effect on the + ## physical/unfiltered backend of `db` as argument filter `b`. + if a.isNil: + return b.isNil + if b.isNil: + return false + if unsafeAddr(a[]) != unsafeAddr(b[]): + if a.src != b.src or + a.trg != b.trg or + a.vGen != b.vGen: + return false + + # Void entries may differ unless on physical backend + var (aTab, bTab) = (a.sTab, b.sTab) + if aTab.len < bTab.len: + aTab.swap bTab + for (vid,aVtx) in aTab.pairs: + let bVtx = bTab.getOrVoid vid + bTab.del vid + + if aVtx != bVtx: + if aVtx.isValid and bVtx.isValid: + return false + # The valid one must match the backend data + let rc = db.getVtxUBE vid + if rc.isErr: + return false + let vtx = if aVtx.isValid: aVtx else: bVtx + if vtx != rc.value: + return false + + elif not vid.isValid and not bTab.hasKey vid: + let rc = db.getVtxUBE vid + if rc.isOk: + return false # Exists on backend but missing on `bTab[]` + elif rc.error != GetKeyNotFound: + return false # general error + + if 0 < bTab.len: + noisy.say "*** eq:", "bTabLen=", bTab.len + return false + + # Similar for `kMap[]` + var (aMap, bMap) = (a.kMap, b.kMap) + if aMap.len < bMap.len: + aMap.swap bMap + for (vid,aKey) in aMap.pairs: + let bKey = bMap.getOrVoid vid + bMap.del vid + + if aKey != bKey: + if aKey.isValid and bKey.isValid: + return false + # The valid one must match the backend data + let rc = db.getKeyUBE vid + if rc.isErr: + return false + let key = if aKey.isValid: aKey else: bKey + if key != rc.value: + return false + + elif not vid.isValid and not bMap.hasKey vid: + let rc = db.getKeyUBE vid + if rc.isOk: + return false # Exists on backend but missing on `bMap[]` + elif rc.error != GetKeyNotFound: + return false # general error + + if 0 < bMap.len: + noisy.say "*** eq:", " bMapLen=", bMap.len + return false + + true + +# ------------------------------------------------------------------------------ +# Public test function +# ------------------------------------------------------------------------------ + +proc testDistributedAccess*( + noisy: bool; + list: openArray[ProofTrieData]; + rdbPath: string; # Rocks DB storage directory + ): bool = + var n = 0 + for w in list.quadripartite: + n.inc + + # Resulting clause (11) filters from `aristo/README.md` example + # which will be used in the second part of the tests + var + c11Filter1 = AristoFilterRef(nil) + c11Filter3 = AristoFilterRef(nil) + + # Work through clauses (8)..(11) from `aristo/README.md` example + block: + + # Clause (8) from `aristo/README.md` example + let + dx = block: + let rc = dbTriplet(w, rdbPath) + if rc.isErr: + return + rc.value + (db1, db2, db3) = (dx.db1, dx.db2, dx.db3) + defer: + dx.cleanUp() + + when false: # or true: + noisy.say "*** testDistributedAccess (1)", "n=", n, dx.dump + + # Clause (9) from `aristo/README.md` example + block: + let rc = db1.stow(persistent=true) + if rc.isErr: + # noisy.say "*** testDistributedAccess (2) n=", n, dx.dump + check rc.error == (0,0) + return + if db1.roFilter != AristoFilterRef(nil): + check db1.roFilter == AristoFilterRef(nil) + return + if db2.roFilter != db3.roFilter: + check db2.roFilter == db3.roFilter + return + + block: + let rc = db2.stow(persistent=false) + if rc.isErr: + noisy.say "*** testDistributedAccess (3)", "n=", n, "db2".dump db2 + check rc.error == (0,0) + return + if db1.roFilter != AristoFilterRef(nil): + check db1.roFilter == AristoFilterRef(nil) + return + if db2.roFilter == db3.roFilter: + check db2.roFilter != db3.roFilter + return + + # Clause (11) from `aristo/README.md` example + block: + let rc = db2.ackqRwMode() + if rc.isErr: + check rc.error == 0 + return + block: + let rc = db2.stow(persistent=true) + if rc.isErr: + check rc.error == (0,0) + return + if db2.roFilter != AristoFilterRef(nil): + check db2.roFilter == AristoFilterRef(nil) + return + + # Check/verify backends + block: + let ok = dx.checkBeOk(noisy=noisy) + if not ok: + noisy.say "*** testDistributedAccess (4)", "n=", n, "db3".dump db3 + check ok + return + + # Capture filters from clause (11) + c11Filter1 = db1.roFilter + c11Filter3 = db3.roFilter + + # Clean up + dx.cleanUp() + + # ---------- + + # Work through clauses (12)..(15) from `aristo/README.md` example + block: + let + dy = block: + let rc = dbTriplet(w, rdbPath) + if rc.isErr: + return + rc.value + (db1, db2, db3) = (dy.db1, dy.db2, dy.db3) + defer: + dy.cleanUp() + + # Build clause (12) from `aristo/README.md` example + block: + let rc = db2.ackqRwMode() + if rc.isErr: + check rc.error == 0 + return + block: + let rc = db2.stow(persistent=true) + if rc.isErr: + check rc.error == (0,0) + return + if db2.roFilter != AristoFilterRef(nil): + check db1.roFilter == AristoFilterRef(nil) + return + if db1.roFilter != db3.roFilter: + check db1.roFilter == db3.roFilter + return + + # Clause (13) from `aristo/README.md` example + block: + let rc = db1.stow(persistent=false) + if rc.isErr: + check rc.error == (0,0) + return + + # Clause (14) from `aristo/README.md` check + block: + let c11Filter1_eq_db1RoFilter = c11Filter1.eq(db1.roFilter, db1, noisy) + if not c11Filter1_eq_db1RoFilter: + noisy.say "*** testDistributedAccess (7)", "n=", n, + "\n c11Filter1=", c11Filter3.pp(db1), + "db1".dump(db1) + check c11Filter1_eq_db1RoFilter + return + + # Clause (15) from `aristo/README.md` check + block: + let c11Filter3_eq_db3RoFilter = c11Filter3.eq(db3. roFilter, db3, noisy) + if not c11Filter3_eq_db3RoFilter: + noisy.say "*** testDistributedAccess (8)", "n=", n, + "\n c11Filter3=", c11Filter3.pp(db3), + "db3".dump(db3) + check c11Filter3_eq_db3RoFilter + return + + # Check/verify backends + block: + let ok = dy.checkBeOk(noisy=noisy) + if not ok: + check ok + return + + when false: # or true: + noisy.say "*** testDistributedAccess (9)", "n=", n, dy.dump + + true + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/tests/test_aristo/test_tx.nim b/tests/test_aristo/test_tx.nim index 18bac22a9..b03a02028 100644 --- a/tests/test_aristo/test_tx.nim +++ b/tests/test_aristo/test_tx.nim @@ -9,12 +9,12 @@ # at your option. This file may not be copied, modified, or # distributed except according to those terms. -## Aristo (aka Patricia) DB records merge test +## Aristo (aka Patricia) DB records transaction based merge test import std/[algorithm, bitops, sequtils, sets, tables], eth/common, - stew/results, + results, unittest2, ../../nimbus/db/aristo/[ aristo_check, aristo_delete, aristo_desc, aristo_get, aristo_merge],