status-im
/
nimbus-eth1
mirror of https://github.com/status-im/nimbus-eth1.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Small cleanup of the ContentDB store handler (#2597)

This commit is contained in:
Kim De Mey 2024-09-07 15:03:13 +02:00 committed by GitHub
parent 0a8986bc77
commit cb94dd0c5b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 41 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
fluffy/database/content_db.nim
View File

@ -417,9 +417,7 @@ proc forcePrune*(db: ContentDB, localId: UInt256, radius: UInt256) =
db.reclaimAndTruncate() db.reclaimAndTruncate()
notice "Finished database pruning" notice "Finished database pruning"
proc put*( proc putAndPrune*(db: ContentDB, key: ContentId, value: openArray[byte]): PutResult =
db: ContentDB, key: ContentId, value: openArray[byte], target: UInt256
): PutResult =
db.put(key, value) db.put(key, value)
# The used size is used as pruning threshold. This means that the database # The used size is used as pruning threshold. This means that the database
@ -449,7 +447,7 @@ proc put*(
# "SELECT key FROM kvstore ORDER BY xorDistance(?, key) DESC LIMIT 1" # "SELECT key FROM kvstore ORDER BY xorDistance(?, key) DESC LIMIT 1"
# Potential adjusting the LIMIT for how many items require deletion. # Potential adjusting the LIMIT for how many items require deletion.
let (distanceOfFurthestElement, deletedBytes, totalContentSize, deletedElements) = let (distanceOfFurthestElement, deletedBytes, totalContentSize, deletedElements) =
db.deleteContentFraction(target, contentDeletionFraction) db.deleteContentFraction(db.localId, contentDeletionFraction)
let deletedFraction = float64(deletedBytes) / float64(totalContentSize) let deletedFraction = float64(deletedBytes) / float64(totalContentSize)
info "Deleted content fraction", deletedBytes, deletedElements, deletedFraction info "Deleted content fraction", deletedBytes, deletedElements, deletedFraction
@ -506,14 +504,12 @@ proc createStoreHandler*(
if p.inRange(contentId): if p.inRange(contentId):
case cfg.kind case cfg.kind
of Dynamic: of Dynamic:
# In case of dynamic radius setting we obey storage limits and adjust # In case of dynamic radius, the radius gets adjusted based on the
# radius to store network fraction corresponding to those storage limits. # to storage capacity and content gets pruned accordingly.
let res = db.put(contentId, content, p.localNode.id) let res = db.putAndPrune(contentId, content)
if res.kind == DbPruned: if res.kind == DbPruned:
portal_pruning_counter.inc(labelValues = [$p.protocolId]) portal_pruning_counter.inc()
portal_pruning_deleted_elements.set( portal_pruning_deleted_elements.set(res.deletedElements.int64)
res.deletedElements.int64, labelValues = [$p.protocolId]
)
if res.deletedFraction > 0.0: if res.deletedFraction > 0.0:
db.adjustRadius(res.deletedFraction, res.distanceOfFurthestElement) db.adjustRadius(res.deletedFraction, res.distanceOfFurthestElement)
@ -526,9 +522,8 @@ proc createStoreHandler*(
info "Database pruning attempt resulted in no content deleted" info "Database pruning attempt resulted in no content deleted"
return return
of Static: of Static:
# If the config is set statically, radius is not adjusted, and is kept # If the radius is static, it may never be adjusted, database capacity
# constant thorugh node life time, also database max size is disabled # is disabled and no pruning is ever done.
# so we will effectivly store fraction of the network
db.put(contentId, content) db.put(contentId, content)
) )

67
fluffy/tests/test_content_db.nim
View File

@ -10,20 +10,18 @@
import import
unittest2, unittest2,
stint, stint,
eth/keys,
../network/state/state_content, ../network/state/state_content,
../database/content_db, ../database/content_db,
./test_helpers ./test_helpers
suite "Content Database": suite "Content Database":
let rng = newRng() const testId = u256(0)
let testId = u256(0)
# Note: We are currently not really testing something new here just basic # Note: We are currently not really testing something new here just basic
# underlying kvstore. # underlying kvstore.
test "ContentDB basic API": test "ContentDB basic API":
let let
db = ContentDB.new( db = ContentDB.new(
"", uint32.high, RadiusConfig(kind: Dynamic), u256(0), inMemory = true "", uint32.high, RadiusConfig(kind: Dynamic), testId, inMemory = true
) )
key = ContentId(UInt256.high()) # Some key key = ContentId(UInt256.high()) # Some key
@ -35,7 +33,7 @@ suite "Content Database":
db.contains(key) == false db.contains(key) == false
block: block:
discard db.put(key, [byte 0, 1, 2, 3], testId) discard db.putAndPrune(key, [byte 0, 1, 2, 3])
let val = db.get(key) let val = db.get(key)
check: check:
@ -53,16 +51,16 @@ suite "Content Database":
test "ContentDB size": test "ContentDB size":
let db = ContentDB.new( let db = ContentDB.new(
"", uint32.high, RadiusConfig(kind: Dynamic), u256(0), inMemory = true "", uint32.high, RadiusConfig(kind: Dynamic), testId, inMemory = true
) )
let numBytes = 10000 let numBytes = 10000
let size1 = db.size() let size1 = db.size()
discard db.put(u256(1), genByteSeq(numBytes), testId) discard db.putAndPrune(u256(1), genByteSeq(numBytes))
let size2 = db.size() let size2 = db.size()
discard db.put(u256(2), genByteSeq(numBytes), testId) discard db.putAndPrune(u256(2), genByteSeq(numBytes))
let size3 = db.size() let size3 = db.size()
discard db.put(u256(2), genByteSeq(numBytes), testId) discard db.putAndPrune(u256(2), genByteSeq(numBytes))
let size4 = db.size() let size4 = db.size()
let usedSize = db.usedSize() let usedSize = db.usedSize()
@ -102,7 +100,7 @@ suite "Content Database":
let let
storageCapacity = 100_000'u64 storageCapacity = 100_000'u64
db = ContentDB.new( db = ContentDB.new(
"", storageCapacity, RadiusConfig(kind: Dynamic), u256(0), inMemory = true "", storageCapacity, RadiusConfig(kind: Dynamic), testId, inMemory = true
) )
furthestElement = u256(40) furthestElement = u256(40)
@ -110,16 +108,16 @@ suite "Content Database":
thirdFurthest = u256(20) thirdFurthest = u256(20)
numBytes = 10_000 numBytes = 10_000
pr1 = db.put(u256(1), genByteSeq(numBytes), u256(0)) pr1 = db.putAndPrune(u256(1), genByteSeq(numBytes))
pr2 = db.put(thirdFurthest, genByteSeq(numBytes), u256(0)) pr2 = db.putAndPrune(thirdFurthest, genByteSeq(numBytes))
pr3 = db.put(u256(3), genByteSeq(numBytes), u256(0)) pr3 = db.putAndPrune(u256(3), genByteSeq(numBytes))
pr4 = db.put(u256(10), genByteSeq(numBytes), u256(0)) pr4 = db.putAndPrune(u256(10), genByteSeq(numBytes))
pr5 = db.put(u256(5), genByteSeq(numBytes), u256(0)) pr5 = db.putAndPrune(u256(5), genByteSeq(numBytes))
pr6 = db.put(u256(11), genByteSeq(numBytes), u256(0)) pr6 = db.putAndPrune(u256(11), genByteSeq(numBytes))
pr7 = db.put(furthestElement, genByteSeq(2000), u256(0)) pr7 = db.putAndPrune(furthestElement, genByteSeq(2000))
pr8 = db.put(secondFurthest, genByteSeq(2000), u256(0)) pr8 = db.putAndPrune(secondFurthest, genByteSeq(2000))
pr9 = db.put(u256(2), genByteSeq(numBytes), u256(0)) pr9 = db.putAndPrune(u256(2), genByteSeq(numBytes))
pr10 = db.put(u256(4), genByteSeq(12000), u256(0)) pr10 = db.putAndPrune(u256(4), genByteSeq(12000))
check: check:
pr1.kind == ContentStored pr1.kind == ContentStored
@ -152,9 +150,8 @@ suite "Content Database":
amountOfItems = 10_000 amountOfItems = 10_000
let let
rng = newRng()
db = ContentDB.new( db = ContentDB.new(
"", startCapacity, RadiusConfig(kind: Dynamic), u256(0), inMemory = true "", startCapacity, RadiusConfig(kind: Dynamic), testId, inMemory = true
) )
localId = UInt256.fromHex( localId = UInt256.fromHex(
"30994892f3e4889d99deb5340050510d1842778acc7a7948adffa475fed51d6e" "30994892f3e4889d99deb5340050510d1842778acc7a7948adffa475fed51d6e"
@ -183,19 +180,19 @@ suite "Content Database":
# Quite a big marging (20%) is added as it is all an approximation. # Quite a big marging (20%) is added as it is all an approximation.
check diff < int64(float(db.storageCapacity) * 0.20) check diff < int64(float(db.storageCapacity) * 0.20)
test "ContentDB radius - start with full radius": test "ContentDB radius - start with full radius":
let let
storageCapacity = 100_000'u64 storageCapacity = 100_000'u64
db = ContentDB.new( db = ContentDB.new(
"", storageCapacity, RadiusConfig(kind: Dynamic), u256(0), inMemory = true "", storageCapacity, RadiusConfig(kind: Dynamic), testId, inMemory = true
) )
radiusHandler = createRadiusHandler(db) radiusHandler = createRadiusHandler(db)
check radiusHandler() == UInt256.high() check radiusHandler() == UInt256.high()
test "ContentDB radius - 0 capacity": test "ContentDB radius - 0 capacity":
let let
db = ContentDB.new("", 0, RadiusConfig(kind: Dynamic), u256(0), inMemory = true) db = ContentDB.new("", 0, RadiusConfig(kind: Dynamic), testId, inMemory = true)
radiusHandler = createRadiusHandler(db) radiusHandler = createRadiusHandler(db)
check radiusHandler() == UInt256.low() check radiusHandler() == UInt256.low()