status-im
/
nimbus-eth1
mirror of https://github.com/status-im/nimbus-eth1.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add contentdb contentCount and contentdb renames/cleanup (#1886)

This commit is contained in:
Kim De Mey 2023-11-07 19:46:26 +01:00 committed by GitHub
parent 9cf3c71a57
commit 03a739ff1b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 52 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
fluffy/content_db.nim
View File

@ -56,7 +56,8 @@ type
maxSize: uint32
sizeStmt: SqliteStmt[NoParams, int64]
unusedSizeStmt: SqliteStmt[NoParams, int64]
vacStmt: SqliteStmt[NoParams, void]
vacuumStmt: SqliteStmt[NoParams, void]
contentCountStmt: SqliteStmt[NoParams, int64]
contentSizeStmt: SqliteStmt[NoParams, int64]
getAllOrderedByDistanceStmt: SqliteStmt[array[32, byte], RowInfo]
@ -106,15 +107,15 @@ proc new*(
db.registerCustomScalarFunction("xorDistance", xorDistance)
.expect("Couldn't register custom xor function")
let getSizeStmt = db.prepareStmt(
let sizeStmt = db.prepareStmt(
"SELECT page_count * page_size as size FROM pragma_page_count(), pragma_page_size();",
NoParams, int64).get()
let unusedSize = db.prepareStmt(
let unusedSizeStmt = db.prepareStmt(
"SELECT freelist_count * page_size as size FROM pragma_freelist_count(), pragma_page_size();",
NoParams, int64).get()
let vacStmt = db.prepareStmt(
let vacuumStmt = db.prepareStmt(
"VACUUM;",
NoParams, void).get()
@ -122,21 +123,24 @@ proc new*(
let contentSizeStmt = db.prepareStmt(
"SELECT SUM(length(value)) FROM kvstore",
NoParams, int64
).get()
NoParams, int64).get()
let contentCountStmt = db.prepareStmt(
"SELECT COUNT(key) FROM kvstore;",
NoParams, int64).get()
let getAllOrderedByDistanceStmt = db.prepareStmt(
"SELECT key, length(value), xorDistance(?, key) as distance FROM kvstore ORDER BY distance DESC",
array[32, byte], RowInfo
).get()
array[32, byte], RowInfo).get()
ContentDB(
kv: kvStore,
maxSize: maxSize,
sizeStmt: getSizeStmt,
vacStmt: vacStmt,
unusedSizeStmt: unusedSize,
sizeStmt: sizeStmt,
unusedSizeStmt: unusedSizeStmt,
vacuumStmt: vacuumStmt,
contentSizeStmt: contentSizeStmt,
contentCountStmt: contentCountStmt,
getAllOrderedByDistanceStmt: getAllOrderedByDistanceStmt
)
@ -185,7 +189,7 @@ proc reclaimSpace*(db: ContentDB): void =
## Ideal mode of operation, is to run it after several deletes.
## Another option would be to run 'PRAGMA auto_vacuum = FULL;' statement at
## the start of db to leave it up to sqlite to clean up
db.vacStmt.exec().expectDb()
db.vacuumStmt.exec().expectDb()
proc size*(db: ContentDB): int64 =
## Retrun current size of DB as product of sqlite page_count and page_size
@ -210,7 +214,7 @@ proc unusedSize(db: ContentDB): int64 =
size = res).expectDb()
return size
proc realSize*(db: ContentDB): int64 =
proc usedSize*(db: ContentDB): int64 =
db.size() - db.unusedSize()
proc contentSize*(db: ContentDB): int64 =
@ -220,6 +224,12 @@ proc contentSize*(db: ContentDB): int64 =
size = res).expectDb()
return size
proc contentCount*(db: ContentDB): int64 =
var count: int64 = 0
discard (db.contentCountStmt.exec do(res: int64):
count = res).expectDb()
return count
## Public ContentId based ContentDB calls
# TODO: Could also decide to use the ContentKey SSZ bytestring, as this is what
@ -296,7 +306,7 @@ proc put*(
# 2. Deal with the edge case where a user configures max db size lower than
# current db.size(). With such config the database would try to prune itself
# with each addition.
let dbSize = db.realSize()
let dbSize = db.usedSize()
if dbSize < int64(db.maxSize):
return PutResult(kind: ContentStored)

1
fluffy/network/history/history_network.nim
View File

@ -780,6 +780,7 @@ proc statusLogLoop(n: HistoryNetwork) {.async.} =
radius = radiusPercentage.toString(10) & "%",
dbSize = $(n.contentDB.size() div 1000) & "kb",
contentSize = $(n.contentDB.contentSize() div 1000) & "kb",
contentCount = n.contentDB.contentCount(),
routingTableNodes = n.portalProtocol.routingTable.len()
await sleepAsync(60.seconds)

65
fluffy/tests/test_content_db.nim
View File

@ -1,5 +1,5 @@
# Nimbus
# Copyright (c) 2021 Status Research & Development GmbH
# Copyright (c) 2021-2023 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@ -14,16 +14,6 @@ import
../content_db,
./test_helpers
proc generateNRandomU256(rng: var HmacDrbgContext, n: int): seq[UInt256] =
var i = 0
var res = newSeq[UInt256]()
while i < n:
let bytes = rng.generateBytes(32)
let num = UInt256.fromBytesBE(bytes)
res.add(num)
inc i
return res
suite "Content Database":
let rng = newRng()
let testId = u256(0)
@ -70,57 +60,56 @@ suite "Content Database":
let size3 = db.size()
discard db.put(u256(2), genByteSeq(numBytes), testId)
let size4 = db.size()
let realSize = db.realSize()
let usedSize = db.usedSize()
check:
size2 > size1
size3 > size2
size3 == size4
realSize == size4
usedSize == size4
db.del(u256(2))
db.del(u256(1))
let realSize1 = db.realSize()
let usedSize1 = db.usedSize()
let size5 = db.size()
check:
size4 == size5
# real size will be smaller as after del, there are free pages in sqlite
# which can be re-used for further additions
realSize1 < size5
# The real size will be smaller as after a deletion there are free pages
# in the db which can be re-used for further additions.
usedSize1 < size5
db.reclaimSpace()
let size6 = db.size()
let realSize2 = db.realSize()
let usedSize2 = db.usedSize()
check:
# After space reclamation size of db should be equal to initial size
# After space reclamation the size of the db back to the initial size.
size6 == size1
realSize2 == size6
usedSize2 == size6
test "ContentDB pruning":
let
maxDbSize = uint32(100000)
db = ContentDB.new("", maxDbSize, inMemory = true)
let furthestElement = u256(40)
let secondFurthest = u256(30)
let thirdFurthest = u256(20)
furthestElement = u256(40)
secondFurthest = u256(30)
thirdFurthest = u256(20)
let numBytes = 10000
let pr1 = db.put(u256(1), genByteSeq(numBytes), u256(0))
let pr2 = db.put(thirdFurthest, genByteSeq(numBytes), u256(0))
let pr3 = db.put(u256(3), genByteSeq(numBytes), u256(0))
let pr4 = db.put(u256(10), genByteSeq(numBytes), u256(0))
let pr5 = db.put(u256(5), genByteSeq(numBytes), u256(0))
let pr6 = db.put(u256(10), genByteSeq(numBytes), u256(0))
let pr7 = db.put(furthestElement, genByteSeq(numBytes), u256(0))
let pr8 = db.put(secondFurthest, genByteSeq(numBytes), u256(0))
let pr9 = db.put(u256(2), genByteSeq(numBytes), u256(0))
let pr10 = db.put(u256(4), genByteSeq(numBytes), u256(0))
numBytes = 10000
pr1 = db.put(u256(1), genByteSeq(numBytes), u256(0))
pr2 = db.put(thirdFurthest, genByteSeq(numBytes), u256(0))
pr3 = db.put(u256(3), genByteSeq(numBytes), u256(0))
pr4 = db.put(u256(10), genByteSeq(numBytes), u256(0))
pr5 = db.put(u256(5), genByteSeq(numBytes), u256(0))
pr6 = db.put(u256(10), genByteSeq(numBytes), u256(0))
pr7 = db.put(furthestElement, genByteSeq(numBytes), u256(0))
pr8 = db.put(secondFurthest, genByteSeq(numBytes), u256(0))
pr9 = db.put(u256(2), genByteSeq(numBytes), u256(0))
pr10 = db.put(u256(4), genByteSeq(numBytes), u256(0))
check:
pr1.kind == ContentStored
@ -136,9 +125,9 @@ suite "Content Database":
check:
pr10.numOfDeletedElements == 2
uint32(db.realSize()) < maxDbSize
# With current settings 2 furthers elements will be delted i.e 30 and 40
# so the furthest non deleted one will be 20
uint32(db.usedSize()) < maxDbSize
# With the current settings the 2 furthest elements will be deleted,
# i.e key 30 and 40. The furthest non deleted one will have key 20.
pr10.furthestStoredElementDistance == thirdFurthest
db.get(furthestElement).isNone()
db.get(secondFurthest).isNone()