status-im
/
nimbus-eth1
mirror of https://github.com/status-im/nimbus-eth1.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add contentdb contentCount and contentdb renames/cleanup (#1886)

This commit is contained in:
Kim De Mey 2023-11-07 19:46:26 +01:00 committed by GitHub
parent 9cf3c71a57
commit 03a739ff1b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 52 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
fluffy/content_db.nim
View File

@ -56,7 +56,8 @@ type
maxSize: uint32 maxSize: uint32
sizeStmt: SqliteStmt[NoParams, int64] sizeStmt: SqliteStmt[NoParams, int64]
unusedSizeStmt: SqliteStmt[NoParams, int64] unusedSizeStmt: SqliteStmt[NoParams, int64]
vacStmt: SqliteStmt[NoParams, void] vacuumStmt: SqliteStmt[NoParams, void]
contentCountStmt: SqliteStmt[NoParams, int64]
contentSizeStmt: SqliteStmt[NoParams, int64] contentSizeStmt: SqliteStmt[NoParams, int64]
getAllOrderedByDistanceStmt: SqliteStmt[array[32, byte], RowInfo] getAllOrderedByDistanceStmt: SqliteStmt[array[32, byte], RowInfo]
@ -106,15 +107,15 @@ proc new*(
db.registerCustomScalarFunction("xorDistance", xorDistance) db.registerCustomScalarFunction("xorDistance", xorDistance)
.expect("Couldn't register custom xor function") .expect("Couldn't register custom xor function")
let getSizeStmt = db.prepareStmt( let sizeStmt = db.prepareStmt(
"SELECT page_count * page_size as size FROM pragma_page_count(), pragma_page_size();", "SELECT page_count * page_size as size FROM pragma_page_count(), pragma_page_size();",
NoParams, int64).get() NoParams, int64).get()
let unusedSize = db.prepareStmt( let unusedSizeStmt = db.prepareStmt(
"SELECT freelist_count * page_size as size FROM pragma_freelist_count(), pragma_page_size();", "SELECT freelist_count * page_size as size FROM pragma_freelist_count(), pragma_page_size();",
NoParams, int64).get() NoParams, int64).get()
let vacStmt = db.prepareStmt( let vacuumStmt = db.prepareStmt(
"VACUUM;", "VACUUM;",
NoParams, void).get() NoParams, void).get()
@ -122,21 +123,24 @@ proc new*(
let contentSizeStmt = db.prepareStmt( let contentSizeStmt = db.prepareStmt(
"SELECT SUM(length(value)) FROM kvstore", "SELECT SUM(length(value)) FROM kvstore",
NoParams, int64 NoParams, int64).get()
).get()
let contentCountStmt = db.prepareStmt(
"SELECT COUNT(key) FROM kvstore;",
NoParams, int64).get()
let getAllOrderedByDistanceStmt = db.prepareStmt( let getAllOrderedByDistanceStmt = db.prepareStmt(
"SELECT key, length(value), xorDistance(?, key) as distance FROM kvstore ORDER BY distance DESC", "SELECT key, length(value), xorDistance(?, key) as distance FROM kvstore ORDER BY distance DESC",
array[32, byte], RowInfo array[32, byte], RowInfo).get()
).get()
ContentDB( ContentDB(
kv: kvStore, kv: kvStore,
maxSize: maxSize, maxSize: maxSize,
sizeStmt: getSizeStmt, sizeStmt: sizeStmt,
vacStmt: vacStmt, unusedSizeStmt: unusedSizeStmt,
unusedSizeStmt: unusedSize, vacuumStmt: vacuumStmt,
contentSizeStmt: contentSizeStmt, contentSizeStmt: contentSizeStmt,
contentCountStmt: contentCountStmt,
getAllOrderedByDistanceStmt: getAllOrderedByDistanceStmt getAllOrderedByDistanceStmt: getAllOrderedByDistanceStmt
) )
@ -185,7 +189,7 @@ proc reclaimSpace*(db: ContentDB): void =
## Ideal mode of operation, is to run it after several deletes. ## Ideal mode of operation, is to run it after several deletes.
## Another option would be to run 'PRAGMA auto_vacuum = FULL;' statement at ## Another option would be to run 'PRAGMA auto_vacuum = FULL;' statement at
## the start of db to leave it up to sqlite to clean up ## the start of db to leave it up to sqlite to clean up
db.vacStmt.exec().expectDb() db.vacuumStmt.exec().expectDb()
proc size*(db: ContentDB): int64 = proc size*(db: ContentDB): int64 =
## Retrun current size of DB as product of sqlite page_count and page_size ## Retrun current size of DB as product of sqlite page_count and page_size
@ -210,7 +214,7 @@ proc unusedSize(db: ContentDB): int64 =
size = res).expectDb() size = res).expectDb()
return size return size
proc realSize*(db: ContentDB): int64 = proc usedSize*(db: ContentDB): int64 =
db.size() - db.unusedSize() db.size() - db.unusedSize()
proc contentSize*(db: ContentDB): int64 = proc contentSize*(db: ContentDB): int64 =
@ -220,6 +224,12 @@ proc contentSize*(db: ContentDB): int64 =
size = res).expectDb() size = res).expectDb()
return size return size
proc contentCount*(db: ContentDB): int64 =
var count: int64 = 0
discard (db.contentCountStmt.exec do(res: int64):
count = res).expectDb()
return count
## Public ContentId based ContentDB calls ## Public ContentId based ContentDB calls
# TODO: Could also decide to use the ContentKey SSZ bytestring, as this is what # TODO: Could also decide to use the ContentKey SSZ bytestring, as this is what
@ -296,7 +306,7 @@ proc put*(
# 2. Deal with the edge case where a user configures max db size lower than # 2. Deal with the edge case where a user configures max db size lower than
# current db.size(). With such config the database would try to prune itself # current db.size(). With such config the database would try to prune itself
# with each addition. # with each addition.
let dbSize = db.realSize() let dbSize = db.usedSize()
if dbSize < int64(db.maxSize): if dbSize < int64(db.maxSize):
return PutResult(kind: ContentStored) return PutResult(kind: ContentStored)

1
fluffy/network/history/history_network.nim
View File

@ -780,6 +780,7 @@ proc statusLogLoop(n: HistoryNetwork) {.async.} =
radius = radiusPercentage.toString(10) & "%", radius = radiusPercentage.toString(10) & "%",
dbSize = $(n.contentDB.size() div 1000) & "kb", dbSize = $(n.contentDB.size() div 1000) & "kb",
contentSize = $(n.contentDB.contentSize() div 1000) & "kb", contentSize = $(n.contentDB.contentSize() div 1000) & "kb",
contentCount = n.contentDB.contentCount(),
routingTableNodes = n.portalProtocol.routingTable.len() routingTableNodes = n.portalProtocol.routingTable.len()
await sleepAsync(60.seconds) await sleepAsync(60.seconds)

65
fluffy/tests/test_content_db.nim
View File

@ -1,5 +1,5 @@
# Nimbus # Nimbus
# Copyright (c) 2021 Status Research & Development GmbH # Copyright (c) 2021-2023 Status Research & Development GmbH
# Licensed and distributed under either of # Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@ -14,16 +14,6 @@ import
../content_db, ../content_db,
./test_helpers ./test_helpers
proc generateNRandomU256(rng: var HmacDrbgContext, n: int): seq[UInt256] =
var i = 0
var res = newSeq[UInt256]()
while i < n:
let bytes = rng.generateBytes(32)
let num = UInt256.fromBytesBE(bytes)
res.add(num)
inc i
return res
suite "Content Database": suite "Content Database":
let rng = newRng() let rng = newRng()
let testId = u256(0) let testId = u256(0)
@ -70,57 +60,56 @@ suite "Content Database":
let size3 = db.size() let size3 = db.size()
discard db.put(u256(2), genByteSeq(numBytes), testId) discard db.put(u256(2), genByteSeq(numBytes), testId)
let size4 = db.size() let size4 = db.size()
let realSize = db.realSize() let usedSize = db.usedSize()
check: check:
size2 > size1 size2 > size1
size3 > size2 size3 > size2
size3 == size4 size3 == size4
realSize == size4 usedSize == size4
db.del(u256(2)) db.del(u256(2))
db.del(u256(1)) db.del(u256(1))
let realSize1 = db.realSize() let usedSize1 = db.usedSize()
let size5 = db.size() let size5 = db.size()
check: check:
size4 == size5 size4 == size5
# real size will be smaller as after del, there are free pages in sqlite # The real size will be smaller as after a deletion there are free pages
# which can be re-used for further additions # in the db which can be re-used for further additions.
realSize1 < size5 usedSize1 < size5
db.reclaimSpace() db.reclaimSpace()
let size6 = db.size() let size6 = db.size()
let realSize2 = db.realSize() let usedSize2 = db.usedSize()
check: check:
# After space reclamation size of db should be equal to initial size # After space reclamation the size of the db back to the initial size.
size6 == size1 size6 == size1
realSize2 == size6 usedSize2 == size6
test "ContentDB pruning": test "ContentDB pruning":
let let
maxDbSize = uint32(100000) maxDbSize = uint32(100000)
db = ContentDB.new("", maxDbSize, inMemory = true) db = ContentDB.new("", maxDbSize, inMemory = true)
let furthestElement = u256(40) furthestElement = u256(40)
let secondFurthest = u256(30) secondFurthest = u256(30)
let thirdFurthest = u256(20) thirdFurthest = u256(20)
numBytes = 10000
let numBytes = 10000 pr1 = db.put(u256(1), genByteSeq(numBytes), u256(0))
let pr1 = db.put(u256(1), genByteSeq(numBytes), u256(0)) pr2 = db.put(thirdFurthest, genByteSeq(numBytes), u256(0))
let pr2 = db.put(thirdFurthest, genByteSeq(numBytes), u256(0)) pr3 = db.put(u256(3), genByteSeq(numBytes), u256(0))
let pr3 = db.put(u256(3), genByteSeq(numBytes), u256(0)) pr4 = db.put(u256(10), genByteSeq(numBytes), u256(0))
let pr4 = db.put(u256(10), genByteSeq(numBytes), u256(0)) pr5 = db.put(u256(5), genByteSeq(numBytes), u256(0))
let pr5 = db.put(u256(5), genByteSeq(numBytes), u256(0)) pr6 = db.put(u256(10), genByteSeq(numBytes), u256(0))
let pr6 = db.put(u256(10), genByteSeq(numBytes), u256(0)) pr7 = db.put(furthestElement, genByteSeq(numBytes), u256(0))
let pr7 = db.put(furthestElement, genByteSeq(numBytes), u256(0)) pr8 = db.put(secondFurthest, genByteSeq(numBytes), u256(0))
let pr8 = db.put(secondFurthest, genByteSeq(numBytes), u256(0)) pr9 = db.put(u256(2), genByteSeq(numBytes), u256(0))
let pr9 = db.put(u256(2), genByteSeq(numBytes), u256(0)) pr10 = db.put(u256(4), genByteSeq(numBytes), u256(0))
let pr10 = db.put(u256(4), genByteSeq(numBytes), u256(0))
check: check:
pr1.kind == ContentStored pr1.kind == ContentStored
@ -136,9 +125,9 @@ suite "Content Database":
check: check:
pr10.numOfDeletedElements == 2 pr10.numOfDeletedElements == 2
uint32(db.realSize()) < maxDbSize uint32(db.usedSize()) < maxDbSize
# With current settings 2 furthers elements will be delted i.e 30 and 40 # With the current settings the 2 furthest elements will be deleted,
# so the furthest non deleted one will be 20 # i.e key 30 and 40. The furthest non deleted one will have key 20.
pr10.furthestStoredElementDistance == thirdFurthest pr10.furthestStoredElementDistance == thirdFurthest
db.get(furthestElement).isNone() db.get(furthestElement).isNone()
db.get(secondFurthest).isNone() db.get(secondFurthest).isNone()