From 03a739ff1b64c928263adef0e32f998fff33a908 Mon Sep 17 00:00:00 2001 From: Kim De Mey Date: Tue, 7 Nov 2023 19:46:26 +0100 Subject: [PATCH] Add contentdb contentCount and contentdb renames/cleanup (#1886) --- fluffy/content_db.nim | 38 ++++++++----- fluffy/network/history/history_network.nim | 1 + fluffy/tests/test_content_db.nim | 65 +++++++++------------- 3 files changed, 52 insertions(+), 52 deletions(-) diff --git a/fluffy/content_db.nim b/fluffy/content_db.nim index 38f6b33c7..b9ad787e2 100644 --- a/fluffy/content_db.nim +++ b/fluffy/content_db.nim @@ -56,7 +56,8 @@ type maxSize: uint32 sizeStmt: SqliteStmt[NoParams, int64] unusedSizeStmt: SqliteStmt[NoParams, int64] - vacStmt: SqliteStmt[NoParams, void] + vacuumStmt: SqliteStmt[NoParams, void] + contentCountStmt: SqliteStmt[NoParams, int64] contentSizeStmt: SqliteStmt[NoParams, int64] getAllOrderedByDistanceStmt: SqliteStmt[array[32, byte], RowInfo] @@ -106,15 +107,15 @@ proc new*( db.registerCustomScalarFunction("xorDistance", xorDistance) .expect("Couldn't register custom xor function") - let getSizeStmt = db.prepareStmt( + let sizeStmt = db.prepareStmt( "SELECT page_count * page_size as size FROM pragma_page_count(), pragma_page_size();", NoParams, int64).get() - let unusedSize = db.prepareStmt( + let unusedSizeStmt = db.prepareStmt( "SELECT freelist_count * page_size as size FROM pragma_freelist_count(), pragma_page_size();", NoParams, int64).get() - let vacStmt = db.prepareStmt( + let vacuumStmt = db.prepareStmt( "VACUUM;", NoParams, void).get() @@ -122,21 +123,24 @@ proc new*( let contentSizeStmt = db.prepareStmt( "SELECT SUM(length(value)) FROM kvstore", - NoParams, int64 - ).get() + NoParams, int64).get() + + let contentCountStmt = db.prepareStmt( + "SELECT COUNT(key) FROM kvstore;", + NoParams, int64).get() let getAllOrderedByDistanceStmt = db.prepareStmt( "SELECT key, length(value), xorDistance(?, key) as distance FROM kvstore ORDER BY distance DESC", - array[32, byte], RowInfo - ).get() + array[32, byte], RowInfo).get() ContentDB( kv: kvStore, maxSize: maxSize, - sizeStmt: getSizeStmt, - vacStmt: vacStmt, - unusedSizeStmt: unusedSize, + sizeStmt: sizeStmt, + unusedSizeStmt: unusedSizeStmt, + vacuumStmt: vacuumStmt, contentSizeStmt: contentSizeStmt, + contentCountStmt: contentCountStmt, getAllOrderedByDistanceStmt: getAllOrderedByDistanceStmt ) @@ -185,7 +189,7 @@ proc reclaimSpace*(db: ContentDB): void = ## Ideal mode of operation, is to run it after several deletes. ## Another option would be to run 'PRAGMA auto_vacuum = FULL;' statement at ## the start of db to leave it up to sqlite to clean up - db.vacStmt.exec().expectDb() + db.vacuumStmt.exec().expectDb() proc size*(db: ContentDB): int64 = ## Retrun current size of DB as product of sqlite page_count and page_size @@ -210,7 +214,7 @@ proc unusedSize(db: ContentDB): int64 = size = res).expectDb() return size -proc realSize*(db: ContentDB): int64 = +proc usedSize*(db: ContentDB): int64 = db.size() - db.unusedSize() proc contentSize*(db: ContentDB): int64 = @@ -220,6 +224,12 @@ proc contentSize*(db: ContentDB): int64 = size = res).expectDb() return size +proc contentCount*(db: ContentDB): int64 = + var count: int64 = 0 + discard (db.contentCountStmt.exec do(res: int64): + count = res).expectDb() + return count + ## Public ContentId based ContentDB calls # TODO: Could also decide to use the ContentKey SSZ bytestring, as this is what @@ -296,7 +306,7 @@ proc put*( # 2. Deal with the edge case where a user configures max db size lower than # current db.size(). With such config the database would try to prune itself # with each addition. - let dbSize = db.realSize() + let dbSize = db.usedSize() if dbSize < int64(db.maxSize): return PutResult(kind: ContentStored) diff --git a/fluffy/network/history/history_network.nim b/fluffy/network/history/history_network.nim index bd379f368..f909a19da 100644 --- a/fluffy/network/history/history_network.nim +++ b/fluffy/network/history/history_network.nim @@ -780,6 +780,7 @@ proc statusLogLoop(n: HistoryNetwork) {.async.} = radius = radiusPercentage.toString(10) & "%", dbSize = $(n.contentDB.size() div 1000) & "kb", contentSize = $(n.contentDB.contentSize() div 1000) & "kb", + contentCount = n.contentDB.contentCount(), routingTableNodes = n.portalProtocol.routingTable.len() await sleepAsync(60.seconds) diff --git a/fluffy/tests/test_content_db.nim b/fluffy/tests/test_content_db.nim index 611b1e1df..f26afe8e2 100644 --- a/fluffy/tests/test_content_db.nim +++ b/fluffy/tests/test_content_db.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2021 Status Research & Development GmbH +# Copyright (c) 2021-2023 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). @@ -14,16 +14,6 @@ import ../content_db, ./test_helpers -proc generateNRandomU256(rng: var HmacDrbgContext, n: int): seq[UInt256] = - var i = 0 - var res = newSeq[UInt256]() - while i < n: - let bytes = rng.generateBytes(32) - let num = UInt256.fromBytesBE(bytes) - res.add(num) - inc i - return res - suite "Content Database": let rng = newRng() let testId = u256(0) @@ -70,57 +60,56 @@ suite "Content Database": let size3 = db.size() discard db.put(u256(2), genByteSeq(numBytes), testId) let size4 = db.size() - let realSize = db.realSize() + let usedSize = db.usedSize() check: size2 > size1 size3 > size2 size3 == size4 - realSize == size4 + usedSize == size4 db.del(u256(2)) db.del(u256(1)) - let realSize1 = db.realSize() + let usedSize1 = db.usedSize() let size5 = db.size() check: size4 == size5 - # real size will be smaller as after del, there are free pages in sqlite - # which can be re-used for further additions - realSize1 < size5 + # The real size will be smaller as after a deletion there are free pages + # in the db which can be re-used for further additions. + usedSize1 < size5 db.reclaimSpace() let size6 = db.size() - let realSize2 = db.realSize() + let usedSize2 = db.usedSize() check: - # After space reclamation size of db should be equal to initial size + # After space reclamation the size of the db back to the initial size. size6 == size1 - realSize2 == size6 + usedSize2 == size6 test "ContentDB pruning": let maxDbSize = uint32(100000) db = ContentDB.new("", maxDbSize, inMemory = true) - let furthestElement = u256(40) - let secondFurthest = u256(30) - let thirdFurthest = u256(20) + furthestElement = u256(40) + secondFurthest = u256(30) + thirdFurthest = u256(20) - - let numBytes = 10000 - let pr1 = db.put(u256(1), genByteSeq(numBytes), u256(0)) - let pr2 = db.put(thirdFurthest, genByteSeq(numBytes), u256(0)) - let pr3 = db.put(u256(3), genByteSeq(numBytes), u256(0)) - let pr4 = db.put(u256(10), genByteSeq(numBytes), u256(0)) - let pr5 = db.put(u256(5), genByteSeq(numBytes), u256(0)) - let pr6 = db.put(u256(10), genByteSeq(numBytes), u256(0)) - let pr7 = db.put(furthestElement, genByteSeq(numBytes), u256(0)) - let pr8 = db.put(secondFurthest, genByteSeq(numBytes), u256(0)) - let pr9 = db.put(u256(2), genByteSeq(numBytes), u256(0)) - let pr10 = db.put(u256(4), genByteSeq(numBytes), u256(0)) + numBytes = 10000 + pr1 = db.put(u256(1), genByteSeq(numBytes), u256(0)) + pr2 = db.put(thirdFurthest, genByteSeq(numBytes), u256(0)) + pr3 = db.put(u256(3), genByteSeq(numBytes), u256(0)) + pr4 = db.put(u256(10), genByteSeq(numBytes), u256(0)) + pr5 = db.put(u256(5), genByteSeq(numBytes), u256(0)) + pr6 = db.put(u256(10), genByteSeq(numBytes), u256(0)) + pr7 = db.put(furthestElement, genByteSeq(numBytes), u256(0)) + pr8 = db.put(secondFurthest, genByteSeq(numBytes), u256(0)) + pr9 = db.put(u256(2), genByteSeq(numBytes), u256(0)) + pr10 = db.put(u256(4), genByteSeq(numBytes), u256(0)) check: pr1.kind == ContentStored @@ -136,9 +125,9 @@ suite "Content Database": check: pr10.numOfDeletedElements == 2 - uint32(db.realSize()) < maxDbSize - # With current settings 2 furthers elements will be delted i.e 30 and 40 - # so the furthest non deleted one will be 20 + uint32(db.usedSize()) < maxDbSize + # With the current settings the 2 furthest elements will be deleted, + # i.e key 30 and 40. The furthest non deleted one will have key 20. pr10.furthestStoredElementDistance == thirdFurthest db.get(furthestElement).isNone() db.get(secondFurthest).isNone()