From 711e5e09d1a1b9cc64a8c1f4bb0d2356a623840e Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 6 Jul 2023 16:23:27 -0700 Subject: [PATCH] Plumbing in conf types (#472) Goal is to provide documentation and to enable conf utils byte params. * Create byte units NByte and plumb through in size locations. * add json serde * plumb parseDuration to conf --- codex.nim | 1 + codex/blocktype.nim | 6 +- codex/chunker.nim | 8 +- codex/codex.nim | 8 +- codex/conf.nim | 74 +++++++++++++---- codex/erasure/erasure.nim | 12 +-- codex/manifest/coders.nim | 4 +- codex/manifest/manifest.nim | 13 +-- codex/manifest/types.nim | 7 +- codex/node.nim | 6 +- codex/rest/api.nim | 2 +- codex/stores/cachestore.nim | 33 +++++--- codex/streams/storestream.nim | 10 ++- codex/units.nim | 83 +++++++++++++++++++ codex/utils.nim | 75 ++++++++++++++++- .../blockexchange/engine/testblockexc.nim | 6 +- .../codex/blockexchange/engine/testengine.nim | 6 +- tests/codex/helpers.nim | 2 +- tests/codex/helpers/randomchunker.nim | 8 +- tests/codex/storageproofs/testnetwork.nim | 6 +- tests/codex/storageproofs/testpor.nim | 12 +-- tests/codex/storageproofs/teststpstore.nim | 6 +- tests/codex/stores/testcachestore.nim | 8 +- tests/codex/testchunking.nim | 8 +- tests/codex/testerasure.nim | 2 +- tests/codex/testnode.nim | 8 +- tests/codex/teststorestream.nim | 2 +- 27 files changed, 318 insertions(+), 98 deletions(-) create mode 100644 codex/units.nim diff --git a/codex.nim b/codex.nim index be012992..073331f1 100644 --- a/codex.nim +++ b/codex.nim @@ -21,6 +21,7 @@ import pkg/libp2p import ./codex/conf import ./codex/codex +import ./codex/units import ./codex/utils/keyutils export codex, conf, libp2p, chronos, chronicles diff --git a/codex/blocktype.nim b/codex/blocktype.nim index fb9cb481..bb904617 100644 --- a/codex/blocktype.nim +++ b/codex/blocktype.nim @@ -20,15 +20,17 @@ import pkg/questionable import pkg/questionable/results import pkg/chronicles +import ./units +import ./utils import ./formats import ./errors -export errors, formats +export errors, formats, units const # Size of blocks for storage / network exchange, # should be divisible by 31 for PoR and by 64 for Leopard ECC - BlockSize* = 31 * 64 * 33 + DefaultBlockSize* = NBytes 31 * 64 * 33 type Block* = ref object of RootObj diff --git a/codex/chunker.nim b/codex/chunker.nim index 436db0fc..58fcb4a3 100644 --- a/codex/chunker.nim +++ b/codex/chunker.nim @@ -24,7 +24,7 @@ import ./blocktype export blocktype const - DefaultChunkSize* = BlockSize + DefaultChunkSize* = DefaultBlockSize type # default reader type @@ -35,7 +35,7 @@ type Chunker* = ref object reader*: Reader # Procedure called to actually read the data offset*: int # Bytes read so far (position in the stream) - chunkSize*: Natural # Size of each chunk + chunkSize*: NBytes # Size of each chunk pad*: bool # Pad last chunk to chunkSize? FileChunker* = Chunker @@ -46,7 +46,7 @@ proc getBytes*(c: Chunker): Future[seq[byte]] {.async.} = ## the instantiated chunker ## - var buff = newSeq[byte](c.chunkSize) + var buff = newSeq[byte](c.chunkSize.int) let read = await c.reader(cast[ChunkBuffer](addr buff[0]), buff.len) if read <= 0: @@ -59,7 +59,7 @@ proc getBytes*(c: Chunker): Future[seq[byte]] {.async.} = return move buff -func new*( +proc new*( T: type Chunker, reader: Reader, chunkSize = DefaultChunkSize, diff --git a/codex/codex.nim b/codex/codex.nim index f470b738..43fcf151 100644 --- a/codex/codex.nim +++ b/codex/codex.nim @@ -176,8 +176,8 @@ proc new*( var cache: CacheStore = nil - if config.cacheSize > 0: - cache = CacheStore.new(cacheSize = config.cacheSize * MiB) + if config.cacheSize > 0'nb: + cache = CacheStore.new(cacheSize = config.cacheSize) ## Is unused? let @@ -215,11 +215,11 @@ proc new*( metaDs = SQLiteDatastore.new(config.dataDir / CodexMetaNamespace) .expect("Should create meta data store!"), quotaMaxBytes = config.storageQuota.uint, - blockTtl = config.blockTtlSeconds.seconds) + blockTtl = config.blockTtl) maintenance = BlockMaintainer.new( repoStore, - interval = config.blockMaintenanceIntervalSeconds.seconds, + interval = config.blockMaintenanceInterval, numberOfBlocksPerInterval = config.blockMaintenanceNumberOfBlocks) peerStore = PeerCtxStore.new() diff --git a/codex/conf.nim b/codex/conf.nim index 43c9f23f..5eedd463 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -27,13 +27,17 @@ import pkg/toml_serialization import pkg/metrics import pkg/metrics/chronos_httpserver import pkg/stew/shims/net as stewnet +import pkg/stew/shims/parseutils import pkg/libp2p import pkg/ethers import ./discovery import ./stores +import ./units +import ./utils -export DefaultCacheSizeMiB, net, DefaultQuotaBytes, DefaultBlockTtl, DefaultBlockMaintenanceInterval, DefaultNumberOfBlocksToMaintainPerInterval +export units +export net, DefaultQuotaBytes, DefaultBlockTtl, DefaultBlockMaintenanceInterval, DefaultNumberOfBlocksToMaintainPerInterval const codex_enable_api_debug_peers* {.booldefine.} = false @@ -161,10 +165,10 @@ type apiPort* {. desc: "The REST Api port", - defaultValue: 8080 + defaultValue: 8080.Port defaultValueDesc: "8080" name: "api-port" - abbr: "p" }: int + abbr: "p" }: Port repoKind* {. desc: "backend for main repo store (fs, sqlite)" @@ -177,20 +181,20 @@ type defaultValue: DefaultQuotaBytes defaultValueDesc: $DefaultQuotaBytes name: "storage-quota" - abbr: "q" }: Natural + abbr: "q" }: NBytes - blockTtlSeconds* {. + blockTtl* {. desc: "Default block timeout in seconds - 0 disables the ttl" - defaultValue: DefaultBlockTtl.seconds + defaultValue: DefaultBlockTtl defaultValueDesc: $DefaultBlockTtl name: "block-ttl" - abbr: "t" }: int + abbr: "t" }: Duration - blockMaintenanceIntervalSeconds* {. + blockMaintenanceInterval* {. desc: "Time interval in seconds - determines frequency of block maintenance cycle: how often blocks are checked for expiration and cleanup." - defaultValue: DefaultBlockMaintenanceInterval.seconds + defaultValue: DefaultBlockMaintenanceInterval defaultValueDesc: $DefaultBlockMaintenanceInterval - name: "block-mi" }: int + name: "block-mi" }: Duration blockMaintenanceNumberOfBlocks* {. desc: "Number of blocks to check every maintenance cycle." @@ -199,11 +203,11 @@ type name: "block-mn" }: int cacheSize* {. - desc: "The size in MiB of the block cache, 0 disables the cache - might help on slow hardrives" + desc: "The size of the block cache, 0 disables the cache - might help on slow hardrives" defaultValue: 0 defaultValueDesc: "0" name: "cache-size" - abbr: "c" }: Natural + abbr: "c" }: NBytes persistence* {. desc: "Enables persistence mechanism, requires an Ethereum node" @@ -288,7 +292,7 @@ proc defaultDataDir*(): string = getHomeDir() / dataDir -func parseCmdArg*(T: type MultiAddress, input: string): T +proc parseCmdArg*(T: type MultiAddress, input: string): T {.upraises: [ValueError, LPError].} = MultiAddress.init($input).tryGet() @@ -303,9 +307,25 @@ proc parseCmdArg*(T: type SignedPeerRecord, uri: string): T = quit QuitFailure res -func parseCmdArg*(T: type EthAddress, address: string): T = +proc parseCmdArg*(T: type EthAddress, address: string): T = EthAddress.init($address).get() +proc parseCmdArg*(T: type NBytes, val: string): T = + var num = 0'i64 + let count = parseSize(val, num, alwaysBin = true) + if count == 0: + warn "Invalid number of bytes", nbytes=val + quit QuitFailure + NBytes(num) + +proc parseCmdArg*(T: type Duration, val: string): T = + var dur: Duration + let count = parseDuration(val, dur) + if count == 0: + warn "Invalid duration parse", dur=dur + quit QuitFailure + dur + proc readValue*(r: var TomlReader, val: var EthAddress) {.upraises: [SerializationError, IOError].} = val = EthAddress.init(r.readValue(string)).get() @@ -317,10 +337,36 @@ proc readValue*(r: var TomlReader, val: var SignedPeerRecord) = val = SignedPeerRecord.parseCmdArg(uri) +proc readValue*(r: var TomlReader, val: var NBytes) + {.upraises: [SerializationError, IOError].} = + var value = 0'i64 + var str = r.readValue(string) + let count = parseSize(str, value, alwaysBin = true) + if count == 0: + error "invalid number of bytes for configuration value", value = str + quit QuitFailure + val = NBytes(value) + +proc readValue*(r: var TomlReader, val: var Duration) + {.upraises: [SerializationError, IOError].} = + var str = r.readValue(string) + var dur: Duration + let count = parseDuration(str, dur) + if count == 0: + error "Invalid duration parse", value = str + quit QuitFailure + val = dur + # no idea why confutils needs this: proc completeCmdArg*(T: type EthAddress; val: string): seq[string] = discard +proc completeCmdArg*(T: type NBytes; val: string): seq[string] = + discard + +proc completeCmdArg*(T: type Duration; val: string): seq[string] = + discard + # silly chronicles, colors is a compile-time property proc stripAnsi(v: string): string = var diff --git a/codex/erasure/erasure.nim b/codex/erasure/erasure.nim index 5d9c60fa..9b4df58d 100644 --- a/codex/erasure/erasure.nim +++ b/codex/erasure/erasure.nim @@ -92,14 +92,14 @@ proc encode*( new_manifest = encoded.len var - encoder = self.encoderProvider(manifest.blockSize, blocks, parity) + encoder = self.encoderProvider(manifest.blockSize.int, blocks, parity) try: for i in 0.. self.size: trace "Block size is larger than cache size", blk = blkSize, cache = self.size @@ -142,7 +141,7 @@ func putBlockSync(self: CacheStore, blk: Block): bool = while self.currentSize + blkSize > self.size: try: let removed = self.cache.removeLru() - self.currentSize -= removed.data.len + self.currentSize -= removed.data.len.NBytes except EmptyLruCacheError as exc: # if the cache is empty, can't remove anything, so break and add item # to the cache @@ -179,7 +178,7 @@ method delBlock*(self: CacheStore, cid: Cid): Future[?!void] {.async.} = let removed = self.cache.del(cid) if removed.isSome: - self.currentSize -= removed.get.data.len + self.currentSize -= removed.get.data.len.NBytes return success() @@ -189,11 +188,11 @@ method close*(self: CacheStore): Future[void] {.async.} = discard -func new*( +proc new*( _: type CacheStore, blocks: openArray[Block] = [], - cacheSize: Positive = DefaultCacheSize, # in bytes - chunkSize: Positive = DefaultChunkSize # in bytes + cacheSize: NBytes = DefaultCacheSize, + chunkSize: NBytes = DefaultChunkSize ): CacheStore {.raises: [Defect, ValueError].} = ## Create a new CacheStore instance ## @@ -203,9 +202,9 @@ func new*( if cacheSize < chunkSize: raise newException(ValueError, "cacheSize cannot be less than chunkSize") - var currentSize = 0 let - size = cacheSize div chunkSize + currentSize = 0'nb + size = int(cacheSize div chunkSize) cache = newLruCache[Cid, Block](size) store = CacheStore( cache: cache, @@ -216,3 +215,11 @@ func new*( discard store.putBlockSync(blk) return store + +proc new*( + _: type CacheStore, + blocks: openArray[Block] = [], + cacheSize: int, + chunkSize: int +): CacheStore {.raises: [Defect, ValueError].} = + CacheStore.new(blocks, NBytes cacheSize, NBytes chunkSize) diff --git a/codex/streams/storestream.nim b/codex/streams/storestream.nim index c6621d72..0f174aee 100644 --- a/codex/streams/storestream.nim +++ b/codex/streams/storestream.nim @@ -63,7 +63,7 @@ proc new*( result.initStream() method `size`*(self: StoreStream): int = - bytes(self.manifest, self.pad) + bytes(self.manifest, self.pad).int proc `size=`*(self: StoreStream, size: int) {.error: "Setting the size is forbidden".} = @@ -93,9 +93,11 @@ method readOnce*( # Compute from the current stream position `self.offset` the block num/offset to read # Compute how many bytes to read from this block let - blockNum = self.offset div self.manifest.blockSize - blockOffset = self.offset mod self.manifest.blockSize - readBytes = min([self.size - self.offset, nbytes - read, self.manifest.blockSize - blockOffset]) + blockNum = self.offset div self.manifest.blockSize.int + blockOffset = self.offset mod self.manifest.blockSize.int + readBytes = min([self.size - self.offset, + nbytes - read, + self.manifest.blockSize.int - blockOffset]) # Read contents of block `blockNum` without blk =? await self.store.getBlock(self.manifest[blockNum]), error: diff --git a/codex/units.nim b/codex/units.nim new file mode 100644 index 00000000..dcda8152 --- /dev/null +++ b/codex/units.nim @@ -0,0 +1,83 @@ +## Nim-Codex +## Copyright (c) 2023 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. +## + +import std/hashes +import std/strutils + +import pkg/upraises +import pkg/json_serialization +import pkg/json_serialization/std/options + +type + NBytes* = distinct Natural + +template basicMaths(T: untyped) = + proc `+` *(x: T, y: static[int]): T = T(`+`(x.Natural, y.Natural)) + proc `-` *(x: T, y: static[int]): T = T(`-`(x.Natural, y.Natural)) + proc `*` *(x: T, y: static[int]): T = T(`*`(x.Natural, y.Natural)) + proc `+` *(x, y: T): T = T(`+`(x.Natural, y.Natural)) + proc `-` *(x, y: T): T = T(`-`(x.Natural, y.Natural)) + proc `*` *(x, y: T): T = T(`*`(x.Natural, y.Natural)) + proc `<` *(x, y: T): bool {.borrow.} + proc `<=` *(x, y: T): bool {.borrow.} + proc `==` *(x, y: T): bool {.borrow.} + proc `+=` *(x: var T, y: T) {.borrow.} + proc `-=` *(x: var T, y: T) {.borrow.} + proc `hash` *(x: T): Hash {.borrow.} +template divMaths(T: untyped) = + proc `mod` *(x, y: T): T = T(`mod`(x.Natural, y.Natural)) + proc `div` *(x, y: T): Natural = `div`(x.Natural, y.Natural) + # proc `/` *(x, y: T): Natural = `/`(x.Natural, y.Natural) + +basicMaths(NBytes) +divMaths(NBytes) + +proc `$`*(ts: NBytes): string = $(int(ts)) & "'NByte" +proc `'nb`*(n: string): NBytes = parseInt(n).NBytes + + +const + MiB = 1024.NBytes * 1024.NBytes # ByteSz, 1 mebibyte = 1,048,576 ByteSz + +proc MiBs*(v: Natural): NBytes = v.NBytes * MiB + +func divUp*[T: NBytes](a, b : T): int = + ## Division with result rounded up (rather than truncated as in 'div') + assert(b != T(0)) + if a==T(0): int(0) else: int( ((a - T(1)) div b) + 1 ) + +proc writeValue*( + writer: var JsonWriter, + value: NBytes +) {.upraises:[IOError].} = + writer.writeValue value.int + +proc readValue*( + reader: var JsonReader, + value: var NBytes +) {.upraises: [SerializationError, IOError].} = + value = NBytes reader.readValue(int) + +when isMainModule: + + import unittest2 + + suite "maths": + test "basics": + let x = 5.NBytes + let y = 10.NBytes + check x + y == 15.NBytes + expect RangeDefect: + check x - y == 10.NBytes + check y - x == 5.NBytes + check x * y == 50.NBytes + check y div x == 2 + check y > x == true + check y <= x == false diff --git a/codex/utils.nim b/codex/utils.nim index e5f21ef2..70547c5a 100644 --- a/codex/utils.nim +++ b/codex/utils.nim @@ -1,15 +1,84 @@ +## Nim-Codex +## Copyright (c) 2023 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. +## + +import std/parseutils + +import pkg/chronos + import ./utils/asyncheapqueue import ./utils/fileutils export asyncheapqueue, fileutils -func divUp*[T](a, b : T): T = +func divUp*[T: SomeInteger](a, b : T): T = ## Division with result rounded up (rather than truncated as in 'div') - assert(b != 0) - if a==0: 0 else: ((a - 1) div b) + 1 + assert(b != T(0)) + if a==T(0): T(0) else: ((a - T(1)) div b) + T(1) func roundUp*[T](a, b : T): T = ## Round up 'a' to the next value divisible by 'b' divUp(a,b) * b +when not declared(parseDuration): # Odd code formatting to minimize diff v. mainLine + const Whitespace = {' ', '\t', '\v', '\r', '\l', '\f'} + + func toLowerAscii(c: char): char = + if c in {'A'..'Z'}: char(uint8(c) xor 0b0010_0000'u8) else: c + + func parseDuration*(s: string, size: var Duration): int = + ## Parse a size qualified by simple time into `Duration`. + ## + runnableExamples: + var res: Duration # caller must still know if 'b' refers to bytes|bits + doAssert parseDuration("10H", res) == 3 + doAssert res == initDuration(hours=10) + doAssert parseDuration("64m", res) == 6 + doAssert res == initDuration(minutes=64) + doAssert parseDuration("7m/block", res) == 2 # '/' stops parse + doAssert res == initDuration(minutes=7) # 1 shl 30, forced binary metric + doAssert parseDuration("3d", res) == 2 # '/' stops parse + doAssert res == initDuration(days=3) # 1 shl 30, forced binary metric + + const prefix = "s" & "mhdw" # byte|bit & lowCase metric-ish prefixes + const timeScale = [1.0, 60.0, 3600.0, 86_400.0, 604_800.0] + + var number: float + var scale = 1.0 + result = parseFloat(s, number) + if number < 0: # While parseFloat accepts negatives .. + result = 0 #.. we do not since sizes cannot be < 0 + else: + let start = result # Save spot to maybe unwind white to EOS + while result < s.len and s[result] in Whitespace: + inc result + if result < s.len: # Illegal starting char => unity + if (let si = prefix.find(s[result].toLowerAscii); si >= 0): + inc result # Now parse the scale + scale = timeScale[si] + else: # Unwind result advancement when there.. + result = start #..is no unit to the end of `s`. + var sizeF = number * scale + 0.5 # Saturate to int64.high when too big + size = seconds(int(sizeF)) + +when isMainModule: + import unittest2 + + suite "time parse": + test "parseDuration": + var res: Duration # caller must still know if 'b' refers to bytes|bits + check parseDuration("10Hr", res) == 3 + check res == hours(10) + check parseDuration("64min", res) == 3 + check res == minutes(64) + check parseDuration("7m/block", res) == 2 # '/' stops parse + check res == minutes(7) # 1 shl 30, forced binary metric + check parseDuration("3d", res) == 2 # '/' stops parse + check res == days(3) # 1 shl 30, forced binary metric diff --git a/tests/codex/blockexchange/engine/testblockexc.nim b/tests/codex/blockexchange/engine/testblockexc.nim index 1ed776d2..4b9e7de4 100644 --- a/tests/codex/blockexchange/engine/testblockexc.nim +++ b/tests/codex/blockexchange/engine/testblockexc.nim @@ -20,8 +20,8 @@ import ../../helpers asyncchecksuite "NetworkStore engine - 2 nodes": let - chunker1 = RandomChunker.new(Rng.instance(), size = 2048, chunkSize = 256) - chunker2 = RandomChunker.new(Rng.instance(), size = 2048, chunkSize = 256) + chunker1 = RandomChunker.new(Rng.instance(), size = 2048, chunkSize = 256'nb) + chunker2 = RandomChunker.new(Rng.instance(), size = 2048, chunkSize = 256'nb) var nodeCmps1, nodeCmps2: NodesComponents @@ -182,7 +182,7 @@ asyncchecksuite "NetworkStore engine - 2 nodes": asyncchecksuite "NetworkStore - multiple nodes": let - chunker = RandomChunker.new(Rng.instance(), size = 4096, chunkSize = 256) + chunker = RandomChunker.new(Rng.instance(), size = 4096, chunkSize = 256'nb) var switch: seq[Switch] diff --git a/tests/codex/blockexchange/engine/testengine.nim b/tests/codex/blockexchange/engine/testengine.nim index 20e3808d..a7c8786a 100644 --- a/tests/codex/blockexchange/engine/testengine.nim +++ b/tests/codex/blockexchange/engine/testengine.nim @@ -37,7 +37,7 @@ asyncchecksuite "NetworkStore engine basic": rng = Rng.instance() seckey = PrivateKey.random(rng[]).tryGet() peerId = PeerId.init(seckey.getPublicKey().tryGet()).tryGet() - chunker = RandomChunker.new(Rng.instance(), size = 1024, chunkSize = 256) + chunker = RandomChunker.new(Rng.instance(), size = 1024'nb, chunkSize = 256'nb) wallet = WalletRef.example blockDiscovery = Discovery.new() peerStore = PeerCtxStore.new() @@ -144,7 +144,7 @@ asyncchecksuite "NetworkStore engine handlers": setup: rng = Rng.instance() - chunker = RandomChunker.new(rng, size = 1024, chunkSize = 256) + chunker = RandomChunker.new(rng, size = 1024'nb, chunkSize = 256'nb) while true: let chunk = await chunker.getBytes() @@ -373,7 +373,7 @@ asyncchecksuite "Task Handler": setup: rng = Rng.instance() - chunker = RandomChunker.new(rng, size = 1024, chunkSize = 256) + chunker = RandomChunker.new(rng, size = 1024, chunkSize = 256'nb) while true: let chunk = await chunker.getBytes() if chunk.len <= 0: diff --git a/tests/codex/helpers.nim b/tests/codex/helpers.nim index 76498ecd..60b1cd14 100644 --- a/tests/codex/helpers.nim +++ b/tests/codex/helpers.nim @@ -12,7 +12,7 @@ import ./helpers/mockdiscovery import ./helpers/eventually import ../checktest -export randomchunker, nodeutils, mockdiscovery, eventually, checktest +export randomchunker, nodeutils, mockdiscovery, eventually, checktest, manifest # NOTE: The meaning of equality for blocks # is changed here, because blocks are now `ref` diff --git a/tests/codex/helpers/randomchunker.nim b/tests/codex/helpers/randomchunker.nim index 008491d1..90fe14c4 100644 --- a/tests/codex/helpers/randomchunker.nim +++ b/tests/codex/helpers/randomchunker.nim @@ -13,13 +13,17 @@ type proc new*( T: type RandomChunker, rng: Rng, - chunkSize = DefaultChunkSize, - size: int, + chunkSize: int | NBytes, + size: int | NBytes, pad = false ): RandomChunker = ## Create a chunker that produces random data ## + let + size = size.int + chunkSize = chunkSize.NBytes + var consumed = 0 proc reader(data: ChunkBuffer, len: int): Future[int] {.async, gcsafe, raises: [Defect].} = diff --git a/tests/codex/storageproofs/testnetwork.nim b/tests/codex/storageproofs/testnetwork.nim index 9f1710a9..a56d7d63 100644 --- a/tests/codex/storageproofs/testnetwork.nim +++ b/tests/codex/storageproofs/testnetwork.nim @@ -20,7 +20,7 @@ import ../examples import ../helpers const - BlockSize = 31 * 64 + BlockSize = 31'nb * 64 DataSetSize = BlockSize * 100 asyncchecksuite "Storage Proofs Network": @@ -48,7 +48,7 @@ asyncchecksuite "Storage Proofs Network": tags: seq[Tag] setup: - chunker = RandomChunker.new(Rng.instance(), size = DataSetSize, chunkSize = BlockSize) + chunker = RandomChunker.new(Rng.instance(), size = DataSetSize.int, chunkSize = BlockSize) store = CacheStore.new(cacheSize = DataSetSize, chunkSize = BlockSize) manifest = Manifest.new(blockSize = BlockSize).tryGet() (spk, ssk) = st.keyGen() @@ -66,7 +66,7 @@ asyncchecksuite "Storage Proofs Network": por = await PoR.init( porStream, ssk, spk, - BlockSize) + BlockSize.int) porMsg = por.toMessage() tags = blocks.mapIt( diff --git a/tests/codex/storageproofs/testpor.nim b/tests/codex/storageproofs/testpor.nim index 86b15480..e0c8ff58 100644 --- a/tests/codex/storageproofs/testpor.nim +++ b/tests/codex/storageproofs/testpor.nim @@ -14,8 +14,8 @@ import pkg/codex/blocktype as bt import ../helpers const - BlockSize = 31 * 4 - SectorSize = 31 + BlockSize = 31'nb * 4 + SectorSize = 31'nb SectorsPerBlock = BlockSize div SectorSize DataSetSize = BlockSize * 100 @@ -30,7 +30,7 @@ asyncchecksuite "BLS PoR": proofStream: StoreStream setup: - chunker = RandomChunker.new(Rng.instance(), size = DataSetSize, chunkSize = BlockSize) + chunker = RandomChunker.new(Rng.instance(), size = DataSetSize.int, chunkSize = BlockSize) store = CacheStore.new(cacheSize = DataSetSize, chunkSize = BlockSize) manifest = Manifest.new(blockSize = BlockSize).tryGet() (spk, ssk) = st.keyGen() @@ -55,7 +55,7 @@ asyncchecksuite "BLS PoR": porStream, ssk, spk, - BlockSize) + BlockSize.int) proc createProof(por: PoR, q: seq[QElement]): Future[Proof] = return generateProof( @@ -96,7 +96,7 @@ asyncchecksuite "Test Serialization": proofStream: StoreStream setup: - chunker = RandomChunker.new(Rng.instance(), size = DataSetSize, chunkSize = BlockSize) + chunker = RandomChunker.new(Rng.instance(), size = DataSetSize.int, chunkSize = BlockSize) store = CacheStore.new(cacheSize = DataSetSize, chunkSize = BlockSize) manifest = Manifest.new(blockSize = BlockSize).tryGet() @@ -114,7 +114,7 @@ asyncchecksuite "Test Serialization": porStream, ssk, spk, - BlockSize) + BlockSize.int) q = generateQuery(por.tau, 22) proofStream = StoreStream.new(store, manifest) proof = await generateProof( diff --git a/tests/codex/storageproofs/teststpstore.nim b/tests/codex/storageproofs/teststpstore.nim index cc02bf29..a7542665 100644 --- a/tests/codex/storageproofs/teststpstore.nim +++ b/tests/codex/storageproofs/teststpstore.nim @@ -12,7 +12,7 @@ import pkg/codex/blocktype as bt import ../helpers const - BlockSize = 31 * 64 + BlockSize = 31'nb * 64'nb DataSetSize = BlockSize * 100 asyncchecksuite "Test PoR store": @@ -34,7 +34,7 @@ asyncchecksuite "Test PoR store": tags: seq[Tag] setup: - chunker = RandomChunker.new(Rng.instance(), size = DataSetSize, chunkSize = BlockSize) + chunker = RandomChunker.new(Rng.instance(), size = DataSetSize.int, chunkSize = BlockSize) store = CacheStore.new(cacheSize = DataSetSize, chunkSize = BlockSize) manifest = Manifest.new(blockSize = BlockSize).tryGet() (spk, ssk) = st.keyGen() @@ -52,7 +52,7 @@ asyncchecksuite "Test PoR store": por = await PoR.init( porStream, ssk, spk, - BlockSize) + BlockSize.int) porMsg = por.toMessage() tags = blocks.mapIt( diff --git a/tests/codex/stores/testcachestore.nim b/tests/codex/stores/testcachestore.nim index b2d374c4..7f36741c 100644 --- a/tests/codex/stores/testcachestore.nim +++ b/tests/codex/stores/testcachestore.nim @@ -30,10 +30,10 @@ checksuite "Cache Store": discard CacheStore.new(cacheSize = 1, chunkSize = 2) store = CacheStore.new(cacheSize = 100, chunkSize = 1) - check store.currentSize == 0 + check store.currentSize == 0'nb store = CacheStore.new(@[newBlock1, newBlock2, newBlock3]) - check store.currentSize == 300 + check store.currentSize == 300'nb # initial cache blocks total more than cache size, currentSize should # never exceed max cache size @@ -41,7 +41,7 @@ checksuite "Cache Store": blocks = @[newBlock1, newBlock2, newBlock3], cacheSize = 200, chunkSize = 1) - check store.currentSize == 200 + check store.currentSize == 200'nb # cache size cannot be less than chunks size expect ValueError: @@ -67,7 +67,7 @@ checksuite "Cache Store": not (await store.hasBlock(newBlock1.cid)).tryGet() (await store.hasBlock(newBlock2.cid)).tryGet() (await store.hasBlock(newBlock2.cid)).tryGet() - store.currentSize == newBlock2.data.len + newBlock3.data.len # 200 + store.currentSize.int == newBlock2.data.len + newBlock3.data.len # 200 commonBlockStoreTests( "Cache", proc: BlockStore = diff --git a/tests/codex/testchunking.nim b/tests/codex/testchunking.nim index 10a0b453..efeada27 100644 --- a/tests/codex/testchunking.nim +++ b/tests/codex/testchunking.nim @@ -24,7 +24,7 @@ asyncchecksuite "Chunking": let chunker = Chunker.new( reader = reader, - chunkSize = 2) + chunkSize = 2'nb) check: (await chunker.getBytes()) == [1.byte, 2] @@ -39,7 +39,7 @@ asyncchecksuite "Chunking": let stream = BufferStream.new() let chunker = LPStreamChunker.new( stream = stream, - chunkSize = 2) + chunkSize = 2'nb) proc writer() {.async.} = for d in [@[1.byte, 2, 3, 4], @[5.byte, 6, 7, 8], @[9.byte, 0]]: @@ -63,7 +63,7 @@ asyncchecksuite "Chunking": let (path, _, _) = instantiationInfo(-2, fullPaths = true) # get this file's name file = open(path) - fileChunker = FileChunker.new(file = file, chunkSize = 256, pad = false) + fileChunker = FileChunker.new(file = file, chunkSize = 256'nb, pad = false) var data: seq[byte] while true: @@ -71,7 +71,7 @@ asyncchecksuite "Chunking": if buff.len <= 0: break - check buff.len <= fileChunker.chunkSize + check buff.len <= fileChunker.chunkSize.int data.add(buff) check: diff --git a/tests/codex/testerasure.nim b/tests/codex/testerasure.nim index 84f011c8..96dd4c47 100644 --- a/tests/codex/testerasure.nim +++ b/tests/codex/testerasure.nim @@ -14,7 +14,7 @@ import pkg/codex/rng import ./helpers asyncchecksuite "Erasure encode/decode": - const BlockSize = 1024 + const BlockSize = 1024'nb const dataSetSize = BlockSize * 123 # weird geometry var rng: Rng diff --git a/tests/codex/testnode.nim b/tests/codex/testnode.nim index 01f2b1bb..abe33763 100644 --- a/tests/codex/testnode.nim +++ b/tests/codex/testnode.nim @@ -57,7 +57,7 @@ asyncchecksuite "Test Node": proc retrieve(cid: Cid): Future[seq[byte]] {.async.} = # Retrieve an entire file contents by file Cid let - oddChunkSize = math.trunc(BlockSize/1.359).int # Let's check that node.retrieve can correctly rechunk data + oddChunkSize = math.trunc(DefaultBlockSize.float/1.359).int # Let's check that node.retrieve can correctly rechunk data stream = (await node.retrieve(cid)).tryGet() var data: seq[byte] @@ -76,7 +76,7 @@ asyncchecksuite "Test Node": setup: file = open(path.splitFile().dir /../ "fixtures" / "test.jpg") - chunker = FileChunker.new(file = file, chunkSize = BlockSize) + chunker = FileChunker.new(file = file, chunkSize = DefaultBlockSize) switch = newStandardSwitch() wallet = WalletRef.new(EthPrivateKey.random()) network = BlockExcNetwork.new(switch) @@ -132,7 +132,7 @@ asyncchecksuite "Test Node": let stream = BufferStream.new() storeFut = node.store(stream) - oddChunkSize = math.trunc(BlockSize/3.14).int # Let's check that node.store can correctly rechunk these odd chunks + oddChunkSize = math.trunc(DefaultBlockSize.float/3.14).NBytes # Let's check that node.store can correctly rechunk these odd chunks oddChunker = FileChunker.new(file = file, chunkSize = oddChunkSize, pad = false) # TODO: doesn't work with pad=tue var original: seq[byte] @@ -159,7 +159,7 @@ asyncchecksuite "Test Node": let data = await retrieve(manifestCid) check: - data.len == localManifest.originalBytes + data.len == localManifest.originalBytes.int data.len == original.len sha256.digest(data) == sha256.digest(original) diff --git a/tests/codex/teststorestream.nim b/tests/codex/teststorestream.nim index c4df3667..4edcc1ff 100644 --- a/tests/codex/teststorestream.nim +++ b/tests/codex/teststorestream.nim @@ -42,7 +42,7 @@ asyncchecksuite "StoreStream": setup: store = CacheStore.new() - manifest = Manifest.new(blockSize = 10).tryGet() + manifest = Manifest.new(blockSize = 10'nb).tryGet() stream = StoreStream.new(store, manifest) for d in data: