status-im
/
nimbus-eth1
mirror of https://github.com/status-im/nimbus-eth1.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Small cleanup of the ContentDB store handler (#2597)

This commit is contained in:
Kim De Mey 2024-09-07 15:03:13 +02:00 committed by GitHub
parent 0a8986bc77
commit cb94dd0c5b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 41 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
fluffy/database/content_db.nim
View File

@ -417,9 +417,7 @@ proc forcePrune*(db: ContentDB, localId: UInt256, radius: UInt256) =
db.reclaimAndTruncate()
notice "Finished database pruning"
proc put*(
db: ContentDB, key: ContentId, value: openArray[byte], target: UInt256
): PutResult =
proc putAndPrune*(db: ContentDB, key: ContentId, value: openArray[byte]): PutResult =
db.put(key, value)
# The used size is used as pruning threshold. This means that the database
@ -449,7 +447,7 @@ proc put*(
# "SELECT key FROM kvstore ORDER BY xorDistance(?, key) DESC LIMIT 1"
# Potential adjusting the LIMIT for how many items require deletion.
let (distanceOfFurthestElement, deletedBytes, totalContentSize, deletedElements) =
db.deleteContentFraction(target, contentDeletionFraction)
db.deleteContentFraction(db.localId, contentDeletionFraction)
let deletedFraction = float64(deletedBytes) / float64(totalContentSize)
info "Deleted content fraction", deletedBytes, deletedElements, deletedFraction
@ -506,14 +504,12 @@ proc createStoreHandler*(
if p.inRange(contentId):
case cfg.kind
of Dynamic:
# In case of dynamic radius setting we obey storage limits and adjust
# radius to store network fraction corresponding to those storage limits.
let res = db.put(contentId, content, p.localNode.id)
# In case of dynamic radius, the radius gets adjusted based on the
# to storage capacity and content gets pruned accordingly.
let res = db.putAndPrune(contentId, content)
if res.kind == DbPruned:
portal_pruning_counter.inc(labelValues = [$p.protocolId])
portal_pruning_deleted_elements.set(
res.deletedElements.int64, labelValues = [$p.protocolId]
)
portal_pruning_counter.inc()
portal_pruning_deleted_elements.set(res.deletedElements.int64)
if res.deletedFraction > 0.0:
db.adjustRadius(res.deletedFraction, res.distanceOfFurthestElement)
@ -526,9 +522,8 @@ proc createStoreHandler*(
info "Database pruning attempt resulted in no content deleted"
return
of Static:
# If the config is set statically, radius is not adjusted, and is kept
# constant thorugh node life time, also database max size is disabled
# so we will effectivly store fraction of the network
# If the radius is static, it may never be adjusted, database capacity
# is disabled and no pruning is ever done.
db.put(contentId, content)
)

45
fluffy/tests/test_content_db.nim
View File

@ -10,20 +10,18 @@
import
unittest2,
stint,
eth/keys,
../network/state/state_content,
../database/content_db,
./test_helpers
suite "Content Database":
let rng = newRng()
let testId = u256(0)
const testId = u256(0)
# Note: We are currently not really testing something new here just basic
# underlying kvstore.
test "ContentDB basic API":
let
db = ContentDB.new(
"", uint32.high, RadiusConfig(kind: Dynamic), u256(0), inMemory = true
"", uint32.high, RadiusConfig(kind: Dynamic), testId, inMemory = true
)
key = ContentId(UInt256.high()) # Some key
@ -35,7 +33,7 @@ suite "Content Database":
db.contains(key) == false
block:
discard db.put(key, [byte 0, 1, 2, 3], testId)
discard db.putAndPrune(key, [byte 0, 1, 2, 3])
let val = db.get(key)
check:
@ -53,16 +51,16 @@ suite "Content Database":
test "ContentDB size":
let db = ContentDB.new(
"", uint32.high, RadiusConfig(kind: Dynamic), u256(0), inMemory = true
"", uint32.high, RadiusConfig(kind: Dynamic), testId, inMemory = true
)
let numBytes = 10000
let size1 = db.size()
discard db.put(u256(1), genByteSeq(numBytes), testId)
discard db.putAndPrune(u256(1), genByteSeq(numBytes))
let size2 = db.size()
discard db.put(u256(2), genByteSeq(numBytes), testId)
discard db.putAndPrune(u256(2), genByteSeq(numBytes))
let size3 = db.size()
discard db.put(u256(2), genByteSeq(numBytes), testId)
discard db.putAndPrune(u256(2), genByteSeq(numBytes))
let size4 = db.size()
let usedSize = db.usedSize()
@ -102,7 +100,7 @@ suite "Content Database":
let
storageCapacity = 100_000'u64
db = ContentDB.new(
"", storageCapacity, RadiusConfig(kind: Dynamic), u256(0), inMemory = true
"", storageCapacity, RadiusConfig(kind: Dynamic), testId, inMemory = true
)
furthestElement = u256(40)
@ -110,16 +108,16 @@ suite "Content Database":
thirdFurthest = u256(20)
numBytes = 10_000
pr1 = db.put(u256(1), genByteSeq(numBytes), u256(0))
pr2 = db.put(thirdFurthest, genByteSeq(numBytes), u256(0))
pr3 = db.put(u256(3), genByteSeq(numBytes), u256(0))
pr4 = db.put(u256(10), genByteSeq(numBytes), u256(0))
pr5 = db.put(u256(5), genByteSeq(numBytes), u256(0))
pr6 = db.put(u256(11), genByteSeq(numBytes), u256(0))
pr7 = db.put(furthestElement, genByteSeq(2000), u256(0))
pr8 = db.put(secondFurthest, genByteSeq(2000), u256(0))
pr9 = db.put(u256(2), genByteSeq(numBytes), u256(0))
pr10 = db.put(u256(4), genByteSeq(12000), u256(0))
pr1 = db.putAndPrune(u256(1), genByteSeq(numBytes))
pr2 = db.putAndPrune(thirdFurthest, genByteSeq(numBytes))
pr3 = db.putAndPrune(u256(3), genByteSeq(numBytes))
pr4 = db.putAndPrune(u256(10), genByteSeq(numBytes))
pr5 = db.putAndPrune(u256(5), genByteSeq(numBytes))
pr6 = db.putAndPrune(u256(11), genByteSeq(numBytes))
pr7 = db.putAndPrune(furthestElement, genByteSeq(2000))
pr8 = db.putAndPrune(secondFurthest, genByteSeq(2000))
pr9 = db.putAndPrune(u256(2), genByteSeq(numBytes))
pr10 = db.putAndPrune(u256(4), genByteSeq(12000))
check:
pr1.kind == ContentStored
@ -152,9 +150,8 @@ suite "Content Database":
amountOfItems = 10_000
let
rng = newRng()
db = ContentDB.new(
"", startCapacity, RadiusConfig(kind: Dynamic), u256(0), inMemory = true
"", startCapacity, RadiusConfig(kind: Dynamic), testId, inMemory = true
)
localId = UInt256.fromHex(
"30994892f3e4889d99deb5340050510d1842778acc7a7948adffa475fed51d6e"
@ -187,7 +184,7 @@ test "ContentDB radius - start with full radius":
let
storageCapacity = 100_000'u64
db = ContentDB.new(
"", storageCapacity, RadiusConfig(kind: Dynamic), u256(0), inMemory = true
"", storageCapacity, RadiusConfig(kind: Dynamic), testId, inMemory = true
)
radiusHandler = createRadiusHandler(db)
@ -195,7 +192,7 @@ test "ContentDB radius - start with full radius":
test "ContentDB radius - 0 capacity":
let
db = ContentDB.new("", 0, RadiusConfig(kind: Dynamic), u256(0), inMemory = true)
db = ContentDB.new("", 0, RadiusConfig(kind: Dynamic), testId, inMemory = true)
radiusHandler = createRadiusHandler(db)
check radiusHandler() == UInt256.low()