Add contentdb contentCount and contentdb renames/cleanup (#1886)
This commit is contained in:
parent
9cf3c71a57
commit
03a739ff1b
|
@ -56,7 +56,8 @@ type
|
||||||
maxSize: uint32
|
maxSize: uint32
|
||||||
sizeStmt: SqliteStmt[NoParams, int64]
|
sizeStmt: SqliteStmt[NoParams, int64]
|
||||||
unusedSizeStmt: SqliteStmt[NoParams, int64]
|
unusedSizeStmt: SqliteStmt[NoParams, int64]
|
||||||
vacStmt: SqliteStmt[NoParams, void]
|
vacuumStmt: SqliteStmt[NoParams, void]
|
||||||
|
contentCountStmt: SqliteStmt[NoParams, int64]
|
||||||
contentSizeStmt: SqliteStmt[NoParams, int64]
|
contentSizeStmt: SqliteStmt[NoParams, int64]
|
||||||
getAllOrderedByDistanceStmt: SqliteStmt[array[32, byte], RowInfo]
|
getAllOrderedByDistanceStmt: SqliteStmt[array[32, byte], RowInfo]
|
||||||
|
|
||||||
|
@ -106,15 +107,15 @@ proc new*(
|
||||||
db.registerCustomScalarFunction("xorDistance", xorDistance)
|
db.registerCustomScalarFunction("xorDistance", xorDistance)
|
||||||
.expect("Couldn't register custom xor function")
|
.expect("Couldn't register custom xor function")
|
||||||
|
|
||||||
let getSizeStmt = db.prepareStmt(
|
let sizeStmt = db.prepareStmt(
|
||||||
"SELECT page_count * page_size as size FROM pragma_page_count(), pragma_page_size();",
|
"SELECT page_count * page_size as size FROM pragma_page_count(), pragma_page_size();",
|
||||||
NoParams, int64).get()
|
NoParams, int64).get()
|
||||||
|
|
||||||
let unusedSize = db.prepareStmt(
|
let unusedSizeStmt = db.prepareStmt(
|
||||||
"SELECT freelist_count * page_size as size FROM pragma_freelist_count(), pragma_page_size();",
|
"SELECT freelist_count * page_size as size FROM pragma_freelist_count(), pragma_page_size();",
|
||||||
NoParams, int64).get()
|
NoParams, int64).get()
|
||||||
|
|
||||||
let vacStmt = db.prepareStmt(
|
let vacuumStmt = db.prepareStmt(
|
||||||
"VACUUM;",
|
"VACUUM;",
|
||||||
NoParams, void).get()
|
NoParams, void).get()
|
||||||
|
|
||||||
|
@ -122,21 +123,24 @@ proc new*(
|
||||||
|
|
||||||
let contentSizeStmt = db.prepareStmt(
|
let contentSizeStmt = db.prepareStmt(
|
||||||
"SELECT SUM(length(value)) FROM kvstore",
|
"SELECT SUM(length(value)) FROM kvstore",
|
||||||
NoParams, int64
|
NoParams, int64).get()
|
||||||
).get()
|
|
||||||
|
let contentCountStmt = db.prepareStmt(
|
||||||
|
"SELECT COUNT(key) FROM kvstore;",
|
||||||
|
NoParams, int64).get()
|
||||||
|
|
||||||
let getAllOrderedByDistanceStmt = db.prepareStmt(
|
let getAllOrderedByDistanceStmt = db.prepareStmt(
|
||||||
"SELECT key, length(value), xorDistance(?, key) as distance FROM kvstore ORDER BY distance DESC",
|
"SELECT key, length(value), xorDistance(?, key) as distance FROM kvstore ORDER BY distance DESC",
|
||||||
array[32, byte], RowInfo
|
array[32, byte], RowInfo).get()
|
||||||
).get()
|
|
||||||
|
|
||||||
ContentDB(
|
ContentDB(
|
||||||
kv: kvStore,
|
kv: kvStore,
|
||||||
maxSize: maxSize,
|
maxSize: maxSize,
|
||||||
sizeStmt: getSizeStmt,
|
sizeStmt: sizeStmt,
|
||||||
vacStmt: vacStmt,
|
unusedSizeStmt: unusedSizeStmt,
|
||||||
unusedSizeStmt: unusedSize,
|
vacuumStmt: vacuumStmt,
|
||||||
contentSizeStmt: contentSizeStmt,
|
contentSizeStmt: contentSizeStmt,
|
||||||
|
contentCountStmt: contentCountStmt,
|
||||||
getAllOrderedByDistanceStmt: getAllOrderedByDistanceStmt
|
getAllOrderedByDistanceStmt: getAllOrderedByDistanceStmt
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -185,7 +189,7 @@ proc reclaimSpace*(db: ContentDB): void =
|
||||||
## Ideal mode of operation, is to run it after several deletes.
|
## Ideal mode of operation, is to run it after several deletes.
|
||||||
## Another option would be to run 'PRAGMA auto_vacuum = FULL;' statement at
|
## Another option would be to run 'PRAGMA auto_vacuum = FULL;' statement at
|
||||||
## the start of db to leave it up to sqlite to clean up
|
## the start of db to leave it up to sqlite to clean up
|
||||||
db.vacStmt.exec().expectDb()
|
db.vacuumStmt.exec().expectDb()
|
||||||
|
|
||||||
proc size*(db: ContentDB): int64 =
|
proc size*(db: ContentDB): int64 =
|
||||||
## Retrun current size of DB as product of sqlite page_count and page_size
|
## Retrun current size of DB as product of sqlite page_count and page_size
|
||||||
|
@ -210,7 +214,7 @@ proc unusedSize(db: ContentDB): int64 =
|
||||||
size = res).expectDb()
|
size = res).expectDb()
|
||||||
return size
|
return size
|
||||||
|
|
||||||
proc realSize*(db: ContentDB): int64 =
|
proc usedSize*(db: ContentDB): int64 =
|
||||||
db.size() - db.unusedSize()
|
db.size() - db.unusedSize()
|
||||||
|
|
||||||
proc contentSize*(db: ContentDB): int64 =
|
proc contentSize*(db: ContentDB): int64 =
|
||||||
|
@ -220,6 +224,12 @@ proc contentSize*(db: ContentDB): int64 =
|
||||||
size = res).expectDb()
|
size = res).expectDb()
|
||||||
return size
|
return size
|
||||||
|
|
||||||
|
proc contentCount*(db: ContentDB): int64 =
|
||||||
|
var count: int64 = 0
|
||||||
|
discard (db.contentCountStmt.exec do(res: int64):
|
||||||
|
count = res).expectDb()
|
||||||
|
return count
|
||||||
|
|
||||||
## Public ContentId based ContentDB calls
|
## Public ContentId based ContentDB calls
|
||||||
|
|
||||||
# TODO: Could also decide to use the ContentKey SSZ bytestring, as this is what
|
# TODO: Could also decide to use the ContentKey SSZ bytestring, as this is what
|
||||||
|
@ -296,7 +306,7 @@ proc put*(
|
||||||
# 2. Deal with the edge case where a user configures max db size lower than
|
# 2. Deal with the edge case where a user configures max db size lower than
|
||||||
# current db.size(). With such config the database would try to prune itself
|
# current db.size(). With such config the database would try to prune itself
|
||||||
# with each addition.
|
# with each addition.
|
||||||
let dbSize = db.realSize()
|
let dbSize = db.usedSize()
|
||||||
|
|
||||||
if dbSize < int64(db.maxSize):
|
if dbSize < int64(db.maxSize):
|
||||||
return PutResult(kind: ContentStored)
|
return PutResult(kind: ContentStored)
|
||||||
|
|
|
@ -780,6 +780,7 @@ proc statusLogLoop(n: HistoryNetwork) {.async.} =
|
||||||
radius = radiusPercentage.toString(10) & "%",
|
radius = radiusPercentage.toString(10) & "%",
|
||||||
dbSize = $(n.contentDB.size() div 1000) & "kb",
|
dbSize = $(n.contentDB.size() div 1000) & "kb",
|
||||||
contentSize = $(n.contentDB.contentSize() div 1000) & "kb",
|
contentSize = $(n.contentDB.contentSize() div 1000) & "kb",
|
||||||
|
contentCount = n.contentDB.contentCount(),
|
||||||
routingTableNodes = n.portalProtocol.routingTable.len()
|
routingTableNodes = n.portalProtocol.routingTable.len()
|
||||||
|
|
||||||
await sleepAsync(60.seconds)
|
await sleepAsync(60.seconds)
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
# Nimbus
|
# Nimbus
|
||||||
# Copyright (c) 2021 Status Research & Development GmbH
|
# Copyright (c) 2021-2023 Status Research & Development GmbH
|
||||||
# Licensed and distributed under either of
|
# Licensed and distributed under either of
|
||||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
||||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
||||||
|
@ -14,16 +14,6 @@ import
|
||||||
../content_db,
|
../content_db,
|
||||||
./test_helpers
|
./test_helpers
|
||||||
|
|
||||||
proc generateNRandomU256(rng: var HmacDrbgContext, n: int): seq[UInt256] =
|
|
||||||
var i = 0
|
|
||||||
var res = newSeq[UInt256]()
|
|
||||||
while i < n:
|
|
||||||
let bytes = rng.generateBytes(32)
|
|
||||||
let num = UInt256.fromBytesBE(bytes)
|
|
||||||
res.add(num)
|
|
||||||
inc i
|
|
||||||
return res
|
|
||||||
|
|
||||||
suite "Content Database":
|
suite "Content Database":
|
||||||
let rng = newRng()
|
let rng = newRng()
|
||||||
let testId = u256(0)
|
let testId = u256(0)
|
||||||
|
@ -70,57 +60,56 @@ suite "Content Database":
|
||||||
let size3 = db.size()
|
let size3 = db.size()
|
||||||
discard db.put(u256(2), genByteSeq(numBytes), testId)
|
discard db.put(u256(2), genByteSeq(numBytes), testId)
|
||||||
let size4 = db.size()
|
let size4 = db.size()
|
||||||
let realSize = db.realSize()
|
let usedSize = db.usedSize()
|
||||||
|
|
||||||
check:
|
check:
|
||||||
size2 > size1
|
size2 > size1
|
||||||
size3 > size2
|
size3 > size2
|
||||||
size3 == size4
|
size3 == size4
|
||||||
realSize == size4
|
usedSize == size4
|
||||||
|
|
||||||
db.del(u256(2))
|
db.del(u256(2))
|
||||||
db.del(u256(1))
|
db.del(u256(1))
|
||||||
|
|
||||||
let realSize1 = db.realSize()
|
let usedSize1 = db.usedSize()
|
||||||
let size5 = db.size()
|
let size5 = db.size()
|
||||||
|
|
||||||
check:
|
check:
|
||||||
size4 == size5
|
size4 == size5
|
||||||
# real size will be smaller as after del, there are free pages in sqlite
|
# The real size will be smaller as after a deletion there are free pages
|
||||||
# which can be re-used for further additions
|
# in the db which can be re-used for further additions.
|
||||||
realSize1 < size5
|
usedSize1 < size5
|
||||||
|
|
||||||
db.reclaimSpace()
|
db.reclaimSpace()
|
||||||
|
|
||||||
let size6 = db.size()
|
let size6 = db.size()
|
||||||
let realSize2 = db.realSize()
|
let usedSize2 = db.usedSize()
|
||||||
|
|
||||||
check:
|
check:
|
||||||
# After space reclamation size of db should be equal to initial size
|
# After space reclamation the size of the db back to the initial size.
|
||||||
size6 == size1
|
size6 == size1
|
||||||
realSize2 == size6
|
usedSize2 == size6
|
||||||
|
|
||||||
test "ContentDB pruning":
|
test "ContentDB pruning":
|
||||||
let
|
let
|
||||||
maxDbSize = uint32(100000)
|
maxDbSize = uint32(100000)
|
||||||
db = ContentDB.new("", maxDbSize, inMemory = true)
|
db = ContentDB.new("", maxDbSize, inMemory = true)
|
||||||
|
|
||||||
let furthestElement = u256(40)
|
furthestElement = u256(40)
|
||||||
let secondFurthest = u256(30)
|
secondFurthest = u256(30)
|
||||||
let thirdFurthest = u256(20)
|
thirdFurthest = u256(20)
|
||||||
|
|
||||||
|
numBytes = 10000
|
||||||
let numBytes = 10000
|
pr1 = db.put(u256(1), genByteSeq(numBytes), u256(0))
|
||||||
let pr1 = db.put(u256(1), genByteSeq(numBytes), u256(0))
|
pr2 = db.put(thirdFurthest, genByteSeq(numBytes), u256(0))
|
||||||
let pr2 = db.put(thirdFurthest, genByteSeq(numBytes), u256(0))
|
pr3 = db.put(u256(3), genByteSeq(numBytes), u256(0))
|
||||||
let pr3 = db.put(u256(3), genByteSeq(numBytes), u256(0))
|
pr4 = db.put(u256(10), genByteSeq(numBytes), u256(0))
|
||||||
let pr4 = db.put(u256(10), genByteSeq(numBytes), u256(0))
|
pr5 = db.put(u256(5), genByteSeq(numBytes), u256(0))
|
||||||
let pr5 = db.put(u256(5), genByteSeq(numBytes), u256(0))
|
pr6 = db.put(u256(10), genByteSeq(numBytes), u256(0))
|
||||||
let pr6 = db.put(u256(10), genByteSeq(numBytes), u256(0))
|
pr7 = db.put(furthestElement, genByteSeq(numBytes), u256(0))
|
||||||
let pr7 = db.put(furthestElement, genByteSeq(numBytes), u256(0))
|
pr8 = db.put(secondFurthest, genByteSeq(numBytes), u256(0))
|
||||||
let pr8 = db.put(secondFurthest, genByteSeq(numBytes), u256(0))
|
pr9 = db.put(u256(2), genByteSeq(numBytes), u256(0))
|
||||||
let pr9 = db.put(u256(2), genByteSeq(numBytes), u256(0))
|
pr10 = db.put(u256(4), genByteSeq(numBytes), u256(0))
|
||||||
let pr10 = db.put(u256(4), genByteSeq(numBytes), u256(0))
|
|
||||||
|
|
||||||
check:
|
check:
|
||||||
pr1.kind == ContentStored
|
pr1.kind == ContentStored
|
||||||
|
@ -136,9 +125,9 @@ suite "Content Database":
|
||||||
|
|
||||||
check:
|
check:
|
||||||
pr10.numOfDeletedElements == 2
|
pr10.numOfDeletedElements == 2
|
||||||
uint32(db.realSize()) < maxDbSize
|
uint32(db.usedSize()) < maxDbSize
|
||||||
# With current settings 2 furthers elements will be delted i.e 30 and 40
|
# With the current settings the 2 furthest elements will be deleted,
|
||||||
# so the furthest non deleted one will be 20
|
# i.e key 30 and 40. The furthest non deleted one will have key 20.
|
||||||
pr10.furthestStoredElementDistance == thirdFurthest
|
pr10.furthestStoredElementDistance == thirdFurthest
|
||||||
db.get(furthestElement).isNone()
|
db.get(furthestElement).isNone()
|
||||||
db.get(secondFurthest).isNone()
|
db.get(secondFurthest).isNone()
|
||||||
|
|
Loading…
Reference in New Issue