From c41d531c5175a1f3ce5922d756dfc39015689739 Mon Sep 17 00:00:00 2001 From: Kim De Mey Date: Fri, 10 Nov 2023 17:16:15 +0100 Subject: [PATCH] Adjust storage capacity option (#1891) - Rename storage-size to storage capacity - Change type to uint64 to avoid potential wrap around - Change to MB instead of KB and set default to 2GB --- fluffy/conf.nim | 16 +++++++--------- fluffy/content_db.nim | 30 +++++++++++++++++------------- fluffy/fluffy.nim | 2 +- 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/fluffy/conf.nim b/fluffy/conf.nim index 4a49ff777..99a2279a2 100644 --- a/fluffy/conf.nim +++ b/fluffy/conf.nim @@ -38,10 +38,8 @@ const defaultAdminListenAddressDesc = $defaultAdminListenAddress defaultDataDirDesc = defaultDataDir() defaultClientConfigDesc = $(defaultClientConfig.httpUri) - # 100mb seems a bit smallish we may consider increasing defaults after some - # network measurements - defaultStorageSize* = uint32(1000 * 1000 * 100) - defaultStorageSizeDesc* = $defaultStorageSize + defaultStorageCapacity* = 2000'u32 # 2 GB default + defaultStorageCapacityDesc* = $defaultStorageCapacity defaultTableIpLimitDesc* = $defaultPortalProtocolConfig.tableIpLimits.tableIpLimit @@ -220,12 +218,12 @@ type # TODO maybe it is worth defining minimal storage size and throw error if # value provided is smaller than minimum - storageSize* {. - desc: "Maximum amount (in bytes) of content which will be stored " & + storageCapacityMB* {. + desc: "Maximum amount (in megabytes) of content which will be stored " & "in the local database." - defaultValue: defaultStorageSize - defaultValueDesc: $defaultStorageSizeDesc - name: "storage-size" .}: uint32 + defaultValue: defaultStorageCapacity + defaultValueDesc: $defaultStorageCapacityDesc + name: "storage-capacity" .}: uint64 trustedBlockRoot* {. desc: "Recent trusted finalized block root to initialize the consensus light client from. " & diff --git a/fluffy/content_db.nim b/fluffy/content_db.nim index b9ad787e2..1c0441f21 100644 --- a/fluffy/content_db.nim +++ b/fluffy/content_db.nim @@ -53,7 +53,7 @@ type ContentDB* = ref object kv: KvStoreRef - maxSize: uint32 + storageCapacity*: uint64 sizeStmt: SqliteStmt[NoParams, int64] unusedSizeStmt: SqliteStmt[NoParams, int64] vacuumStmt: SqliteStmt[NoParams, void] @@ -95,8 +95,10 @@ template expectDb(x: auto): untyped = x.expect("working database (disk broken/full?)") proc new*( - T: type ContentDB, path: string, maxSize: uint32, inMemory = false): + T: type ContentDB, path: string, storageCapacity: uint64, inMemory = false): ContentDB = + doAssert(storageCapacity <= uint64(int64.high)) + let db = if inMemory: SqStoreRef.init("", "fluffy-test", inMemory = true).expect( @@ -135,7 +137,7 @@ proc new*( ContentDB( kv: kvStore, - maxSize: maxSize, + storageCapacity: storageCapacity, sizeStmt: sizeStmt, unusedSizeStmt: unusedSizeStmt, vacuumStmt: vacuumStmt, @@ -192,14 +194,14 @@ proc reclaimSpace*(db: ContentDB): void = db.vacuumStmt.exec().expectDb() proc size*(db: ContentDB): int64 = - ## Retrun current size of DB as product of sqlite page_count and page_size + ## Return current size of DB as product of sqlite page_count and page_size: ## https://www.sqlite.org/pragma.html#pragma_page_count ## https://www.sqlite.org/pragma.html#pragma_page_size - ## It returns total size of db i.e both data and metadata used to store content - ## also it is worth noting that when deleting content, size may lags behind due + ## It returns the total size of db on the disk, i.e both data and metadata + ## used to store content. + ## It is worth noting that when deleting content, the size may lag behind due ## to the way how deleting works in sqlite. ## Good description can be found in: https://www.sqlite.org/lang_vacuum.html - var size: int64 = 0 discard (db.sizeStmt.exec do(res: int64): size = res).expectDb() @@ -208,17 +210,18 @@ proc size*(db: ContentDB): int64 = proc unusedSize(db: ContentDB): int64 = ## Returns the total size of the pages which are unused by the database, ## i.e they can be re-used for new content. - var size: int64 = 0 discard (db.unusedSizeStmt.exec do(res: int64): size = res).expectDb() return size proc usedSize*(db: ContentDB): int64 = + ## Returns the total size of the database (data + metadata) minus the unused + ## pages. db.size() - db.unusedSize() proc contentSize*(db: ContentDB): int64 = - ## Returns total size of content stored in DB + ## Returns total size of the content stored in DB. var size: int64 = 0 discard (db.contentSizeStmt.exec do(res: int64): size = res).expectDb() @@ -296,9 +299,10 @@ proc put*( db.put(key, value) - # We use real size for our pruning threshold, which means that database file - # will reach size specified in db.maxSize, and will stay that size thorough - # node life time, as after content deletion free pages will be re used. + # The used size is used as pruning threshold. This means that the database + # size will reach the size specified in db.storageCapacity and will stay + # around that size throughout the node's lifetime, as after content deletion + # due to pruning, the free pages will be re-used. # TODO: # 1. Devise vacuum strategy - after few pruning cycles database can become # fragmented which may impact performance, so at some point in time `VACUUM` @@ -308,7 +312,7 @@ proc put*( # with each addition. let dbSize = db.usedSize() - if dbSize < int64(db.maxSize): + if dbSize < int64(db.storageCapacity): return PutResult(kind: ContentStored) else: # TODO Add some configuration for this magic number diff --git a/fluffy/fluffy.nim b/fluffy/fluffy.nim index 2b8ab29de..4f84c3737 100644 --- a/fluffy/fluffy.nim +++ b/fluffy/fluffy.nim @@ -129,7 +129,7 @@ proc run(config: PortalConf) {.raises: [CatchableError].} = let db = ContentDB.new(config.dataDir / "db" / "contentdb_" & d.localNode.id.toBytesBE().toOpenArray(0, 8).toHex(), - maxSize = config.storageSize) + storageCapacity = config.storageCapacityMB * 1_000_000) portalConfig = PortalProtocolConfig.init( config.tableIpLimit,