diff --git a/storage/blockexchange/engine/engine.nim b/storage/blockexchange/engine/engine.nim index b1aa6b8a..baa993b7 100644 --- a/storage/blockexchange/engine/engine.nim +++ b/storage/blockexchange/engine/engine.nim @@ -573,18 +573,18 @@ proc broadcastWantHave( try: await self.network.request - .sendWantList( - peerCtx.id, - @[rangeAddress], - priority = 0, - cancel = false, - wantType = WantType.WantHave, - full = false, - sendDontHave = false, - rangeCount = count, - downloadId = download.id, - ) - .wait(DefaultWantHaveSendTimeout) + .sendWantList( + peerCtx.id, + @[rangeAddress], + priority = 0, + cancel = false, + wantType = WantType.WantHave, + full = false, + sendDontHave = false, + rangeCount = count, + downloadId = download.id, + ) + .wait(DefaultWantHaveSendTimeout) except AsyncTimeoutError: warn "Want-have send timed out", peer = peerCtx.id except CatchableError as err: @@ -786,18 +786,18 @@ proc downloadWorker( for peerId in staleUnknown: try: await self.network.request - .sendWantList( - peerId, - @[rangeAddress], - priority = 0, - cancel = false, - wantType = WantType.WantHave, - full = false, - sendDontHave = false, - rangeCount = count, - downloadId = download.id, - ) - .wait(DefaultWantHaveSendTimeout) + .sendWantList( + peerId, + @[rangeAddress], + priority = 0, + cancel = false, + wantType = WantType.WantHave, + full = false, + sendDontHave = false, + rangeCount = count, + downloadId = download.id, + ) + .wait(DefaultWantHaveSendTimeout) except AsyncTimeoutError: trace "Re-query stale unknown peer send timed out", peer = peerId except CatchableError as err: diff --git a/storage/blockexchange/protocol/wantblocks.nim b/storage/blockexchange/protocol/wantblocks.nim index ce2b7101..fd05a565 100644 --- a/storage/blockexchange/protocol/wantblocks.nim +++ b/storage/blockexchange/protocol/wantblocks.nim @@ -201,11 +201,10 @@ proc decodeRequest*(data: openArray[byte]): WantBlocksResult[WantBlocksRequest] if offset + cidLen + SizeRangeCount > data.len: return err(wantBlocksError(RequestTruncated, "Request truncated (CID)")) - let cid = - ?Cid.init(data.toOpenArray(offset, offset + cidLen - 1)).mapErr( - proc(e: auto): ref WantBlocksError = - wantBlocksError(InvalidCid, "Invalid CID: " & $e) - ) + let cid = ?Cid.init(data.toOpenArray(offset, offset + cidLen - 1)).mapErr( + proc(e: auto): ref WantBlocksError = + wantBlocksError(InvalidCid, "Invalid CID: " & $e) + ) offset += cidLen let rangeCount = @@ -353,11 +352,10 @@ proc decodeResponseMetadata( if offset + cidLen + SizeBlockCount > data.len: return err(wantBlocksError(MetadataTruncated, "Metadata truncated at CID")) - let treeCid = - ?Cid.init(data.toOpenArray(offset, offset + cidLen - 1)).mapErr( - proc(e: auto): ref WantBlocksError = - wantBlocksError(InvalidCid, "Invalid CID: " & $e) - ) + let treeCid = ?Cid.init(data.toOpenArray(offset, offset + cidLen - 1)).mapErr( + proc(e: auto): ref WantBlocksError = + wantBlocksError(InvalidCid, "Invalid CID: " & $e) + ) offset += cidLen let blockCount = uint32.fromBytes(data.toOpenArray(offset, offset + 3), littleEndian) @@ -392,11 +390,10 @@ proc decodeResponseMetadata( if offset + blockCidLen > data.len: return err(wantBlocksError(MetadataTruncated, "Metadata truncated at block CID " & $i)) - let blockCid = - ?Cid.init(data.toOpenArray(offset, offset + blockCidLen - 1)).mapErr( - proc(e: auto): ref WantBlocksError = - wantBlocksError(InvalidCid, "Invalid block CID at " & $i & ": " & $e) - ) + let blockCid = ?Cid.init(data.toOpenArray(offset, offset + blockCidLen - 1)).mapErr( + proc(e: auto): ref WantBlocksError = + wantBlocksError(InvalidCid, "Invalid block CID at " & $i & ": " & $e) + ) offset += blockCidLen if offset + SizeDataLen > data.len: diff --git a/storage/conf.nim b/storage/conf.nim index e591858d..56c4d131 100644 --- a/storage/conf.nim +++ b/storage/conf.nim @@ -136,8 +136,7 @@ type .}: OutDir listenIp* {. - desc: - "IP address to listen on for remote peer connections, can be ipv4 or ipv6", + desc: "IP address to listen on for remote peer connections, can be ipv4 or ipv6", defaultValue: "0.0.0.0".parseIpAddress, defaultValueDesc: "Listens on all addresses.", abbr: "i", diff --git a/storage/node.nim b/storage/node.nim index 65ea39fb..180dfb66 100644 --- a/storage/node.nim +++ b/storage/node.nim @@ -136,10 +136,9 @@ proc fetchDatasetAsync*( ): Future[?!void] {.async: (raises: [CancelledError]).} = let treeCid = manifest.treeCid - download = - ?self.engine.startTreeDownloadOpaque( - treeCid, manifest.blockSize.uint32, manifest.blocksCount.uint64 - ) + download = ?self.engine.startTreeDownloadOpaque( + treeCid, manifest.blockSize.uint32, manifest.blocksCount.uint64 + ) try: trace "Starting tree download", treeCid = treeCid, totalBlocks = manifest.blocksCount diff --git a/storage/rest/api.nim b/storage/rest/api.nim index 07623174..b6e34b1d 100644 --- a/storage/rest/api.nim +++ b/storage/rest/api.nim @@ -408,13 +408,12 @@ proc initDataApi(node: StorageNodeRef, repoStore: RepoStore, router: var RestRou return RestApiResponse.response($json, contentType = "application/json") router.api(MethodGet, "/api/storage/v1/space") do() -> RestApiResponse: - let json = - %RestRepoStore( - totalBlocks: repoStore.totalBlocks, - quotaMaxBytes: repoStore.quotaMaxBytes, - quotaUsedBytes: repoStore.quotaUsedBytes, - quotaReservedBytes: repoStore.quotaReservedBytes, - ) + let json = %RestRepoStore( + totalBlocks: repoStore.totalBlocks, + quotaMaxBytes: repoStore.quotaMaxBytes, + quotaUsedBytes: repoStore.quotaUsedBytes, + quotaReservedBytes: repoStore.quotaReservedBytes, + ) return RestApiResponse.response($json, contentType = "application/json") proc initNodeApi(node: StorageNodeRef, conf: StorageConf, router: var RestRouter) = @@ -512,20 +511,16 @@ proc initDebugApi(node: StorageNodeRef, conf: StorageConf, router: var RestRoute try: let table = RestRoutingTable.init(node.discovery.protocol.routingTable) - let json = - %*{ - "id": $node.switch.peerInfo.peerId, - "addrs": node.switch.peerInfo.addrs.mapIt($it), - "repo": $conf.dataDir, - "spr": - if node.discovery.dhtRecord.isSome: - node.discovery.dhtRecord.get.toURI - else: - "", - "announceAddresses": node.discovery.announceAddrs, - "table": table, - "storage": {"version": $storageVersion, "revision": $storageRevision}, - } + let json = %*{ + "id": $node.switch.peerInfo.peerId, + "addrs": node.switch.peerInfo.addrs.mapIt($it), + "repo": $conf.dataDir, + "spr": + if node.discovery.dhtRecord.isSome: node.discovery.dhtRecord.get.toURI else: "", + "announceAddresses": node.discovery.announceAddrs, + "table": table, + "storage": {"version": $storageVersion, "revision": $storageRevision}, + } # return pretty json for human readability return RestApiResponse.response( diff --git a/storage/storage.nim b/storage/storage.nim index 4cde9125..41980e20 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -95,13 +95,12 @@ proc stop*(s: StorageServer) {.async.} = notice "Stopping Storage node" - var futures = - @[ - s.storageNode.switch.stop(), - s.storageNode.stop(), - s.repoStore.stop(), - s.maintenance.stop(), - ] + var futures = @[ + s.storageNode.switch.stop(), + s.storageNode.stop(), + s.repoStore.stop(), + s.maintenance.stop(), + ] if s.restServer != nil: futures.add(s.restServer.stop()) diff --git a/tests/integration/5_minutes/testrestapi.nim b/tests/integration/5_minutes/testrestapi.nim index d7a33f3f..17ce1e3e 100644 --- a/tests/integration/5_minutes/testrestapi.nim +++ b/tests/integration/5_minutes/testrestapi.nim @@ -56,11 +56,10 @@ twonodessuite "REST API": check (await response.body) != "" test "node retrieve the metadata", twoNodesConfig: - let headers = - @[ - ("Content-Type", "text/plain"), - ("Content-Disposition", "attachment; filename=\"example.txt\""), - ] + let headers = @[ + ("Content-Type", "text/plain"), + ("Content-Disposition", "attachment; filename=\"example.txt\""), + ] let uploadResponse = await client1.uploadRaw("some file contents", headers) let cid = await uploadResponse.body let listResponse = await client1.listRaw() @@ -81,11 +80,10 @@ twonodessuite "REST API": check manifest["mimetype"].getStr() == "text/plain" test "node set the headers when for download", twoNodesConfig: - let headers = - @[ - ("Content-Disposition", "attachment; filename=\"example.txt\""), - ("Content-Type", "text/plain"), - ] + let headers = @[ + ("Content-Disposition", "attachment; filename=\"example.txt\""), + ("Content-Type", "text/plain"), + ] let uploadResponse = await client1.uploadRaw("some file contents", headers) let cid = await uploadResponse.body diff --git a/tests/storage/blockexchange/engine/testengine.nim b/tests/storage/blockexchange/engine/testengine.nim index fc860ff6..0cff45a9 100644 --- a/tests/storage/blockexchange/engine/testengine.nim +++ b/tests/storage/blockexchange/engine/testengine.nim @@ -197,17 +197,16 @@ asyncchecksuite "NetworkStore engine handlers": (await localStore.putCidAndProof(rootCid, i, blk.cid, proof)).tryGet() let wantList = WantList( - entries: - @[ - WantListEntry( - address: BlockAddress(treeCid: rootCid, index: 0), - priority: 0, - cancel: false, - wantType: WantType.WantHave, - sendDontHave: false, - rangeCount: blocks.len.uint64, - ) - ], + entries: @[ + WantListEntry( + address: BlockAddress(treeCid: rootCid, index: 0), + priority: 0, + cancel: false, + wantType: WantType.WantHave, + sendDontHave: false, + rangeCount: blocks.len.uint64, + ) + ], full: false, ) @@ -238,17 +237,16 @@ asyncchecksuite "NetworkStore engine handlers": (await localStore.putCidAndProof(rootCid, i, blocks[i].cid, proof)).tryGet() let wantList = WantList( - entries: - @[ - WantListEntry( - address: BlockAddress(treeCid: rootCid, index: 0), - priority: 0, - cancel: false, - wantType: WantType.WantHave, - sendDontHave: false, - rangeCount: blocks.len.uint64, - ) - ], + entries: @[ + WantListEntry( + address: BlockAddress(treeCid: rootCid, index: 0), + priority: 0, + cancel: false, + wantType: WantType.WantHave, + sendDontHave: false, + rangeCount: blocks.len.uint64, + ) + ], full: false, ) diff --git a/tests/storage/blockexchange/engine/testscheduler.nim b/tests/storage/blockexchange/engine/testscheduler.nim index 7fb68294..31726208 100644 --- a/tests/storage/blockexchange/engine/testscheduler.nim +++ b/tests/storage/blockexchange/engine/testscheduler.nim @@ -367,13 +367,12 @@ suite "Scheduler": check batch.get.start == 0 check batch.get.count == 8 - let missingRanges = - @[ - (start: 1'u64, count: 1'u64), - (start: 3'u64, count: 1'u64), - (start: 5'u64, count: 1'u64), - (start: 7'u64, count: 1'u64), - ] + let missingRanges = @[ + (start: 1'u64, count: 1'u64), + (start: 3'u64, count: 1'u64), + (start: 5'u64, count: 1'u64), + (start: 7'u64, count: 1'u64), + ] scheduler.partialComplete(0, missingRanges) check scheduler.completedWatermark() == 0 diff --git a/tests/storage/blockexchange/protocol/testmessage.nim b/tests/storage/blockexchange/protocol/testmessage.nim index 1e914859..98e0a27d 100644 --- a/tests/storage/blockexchange/protocol/testmessage.nim +++ b/tests/storage/blockexchange/protocol/testmessage.nim @@ -113,25 +113,24 @@ suite "WantList protobuf encoding": let treeCid = Cid.example wantList = WantList( - entries: - @[ - WantListEntry( - address: BlockAddress(treeCid: treeCid, index: 0), - priority: 1, - cancel: false, - wantType: WantType.WantHave, - sendDontHave: false, - rangeCount: 10, - ), - WantListEntry( - address: BlockAddress(treeCid: treeCid, index: 1), - priority: 2, - cancel: false, - wantType: WantType.WantHave, - sendDontHave: true, - rangeCount: 0, - ), - ], + entries: @[ + WantListEntry( + address: BlockAddress(treeCid: treeCid, index: 0), + priority: 1, + cancel: false, + wantType: WantType.WantHave, + sendDontHave: false, + rangeCount: 10, + ), + WantListEntry( + address: BlockAddress(treeCid: treeCid, index: 1), + priority: 2, + cancel: false, + wantType: WantType.WantHave, + sendDontHave: true, + rangeCount: 0, + ), + ], full: true, ) @@ -232,17 +231,16 @@ suite "Full Message protobuf encoding": treeCid = Cid.example msg = Message( wantList: WantList( - entries: - @[ - WantListEntry( - address: BlockAddress(treeCid: treeCid, index: 0), - priority: 1, - cancel: false, - wantType: WantType.WantHave, - sendDontHave: false, - rangeCount: 100, - ) - ], + entries: @[ + WantListEntry( + address: BlockAddress(treeCid: treeCid, index: 0), + priority: 1, + cancel: false, + wantType: WantType.WantHave, + sendDontHave: false, + rangeCount: 100, + ) + ], full: false, ), blockPresences: @[], @@ -259,14 +257,13 @@ suite "Full Message protobuf encoding": treeCid = Cid.example msg = Message( wantList: WantList(entries: @[], full: false), - blockPresences: - @[ - BlockPresence( - address: BlockAddress(treeCid: treeCid, index: 0), - kind: BlockPresenceType.HaveRange, - ranges: @[(start: 0'u64, count: 500'u64)], - ) - ], + blockPresences: @[ + BlockPresence( + address: BlockAddress(treeCid: treeCid, index: 0), + kind: BlockPresenceType.HaveRange, + ranges: @[(start: 0'u64, count: 500'u64)], + ) + ], ) encoded = msg.protobufEncode() decoded = Message.protobufDecode(encoded) diff --git a/tests/storage/node/helpers.nim b/tests/storage/node/helpers.nim index d6118500..ad87f6cc 100644 --- a/tests/storage/node/helpers.nim +++ b/tests/storage/node/helpers.nim @@ -96,10 +96,9 @@ template setupAndTearDown*() {.dirty.} = blockDiscovery = Discovery.new( switch.peerInfo.privateKey, - announceAddrs = - @[ - MultiAddress.init("/ip4/127.0.0.1/tcp/0").expect("Should return multiaddress") - ], + announceAddrs = @[ + MultiAddress.init("/ip4/127.0.0.1/tcp/0").expect("Should return multiaddress") + ], ) peerStore = PeerContextStore.new() downloadManager = DownloadManager.new() diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index 08005bca..21faa156 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -21,18 +21,16 @@ suite "NAT Address Tests": # Expected results let - expectedDiscoveryAddrs = - @[ - MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), - MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), - MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), - ] - expectedlibp2pAddrs = - @[ - MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), - MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), - MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), - ] + expectedDiscoveryAddrs = @[ + MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), + MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), + MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), + ] + expectedlibp2pAddrs = @[ + MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), + MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), + MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), + ] #ipv6Addr = MultiAddress.init("/ip6/::1/tcp/5000").expect("valid multiaddr") addrs = @[localAddr, anyAddr, publicAddr] diff --git a/tests/storage/utils/testutils.nim b/tests/storage/utils/testutils.nim index 3da911d4..4e2b4427 100644 --- a/tests/storage/utils/testutils.nim +++ b/tests/storage/utils/testutils.nim @@ -7,13 +7,12 @@ suite "FindIt": type AnObject = object attribute1*: int - var objList = - @[ - AnObject(attribute1: 1), - AnObject(attribute1: 3), - AnObject(attribute1: 5), - AnObject(attribute1: 3), - ] + var objList = @[ + AnObject(attribute1: 1), + AnObject(attribute1: 3), + AnObject(attribute1: 5), + AnObject(attribute1: 3), + ] test "should retur index of first object matching predicate": assert objList.findIt(it.attribute1 == 3) == 1