From e50ea884114d08c82f5e0136b59a16d28e7031ad Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 27 Oct 2022 07:41:34 -0600 Subject: [PATCH] Remove protobuf serialization (#289) * add format for cid * cid formatIt change * track nim-libp2p-unstable * rework probuf serialization for por * add missing include * removing nim protobuf serialization * rollback to dht to main * remove protobuf serialization import --- .gitmodules | 5 - codex/blockexchange/engine/discovery.nim | 16 +- codex/blockexchange/engine/engine.nim | 6 +- codex/blockexchange/protobuf/payments.nim | 1 - codex/blocktype.nim | 4 + codex/discovery.nim | 7 +- codex/formats.nim | 28 +++ codex/node.nim | 3 +- codex/rest/api.nim | 2 +- codex/storageproofs/por/por.nim | 2 +- .../por/serialization/messages.nim | 185 ++++++++++++++++++ .../storageproofs/por/serialization/por.proto | 33 ---- .../por/serialization/serialization.nim | 12 +- codex/storageproofs/stp.proto | 15 -- codex/storageproofs/stpnetwork.nim | 12 +- codex/storageproofs/stpproto.nim | 8 +- codex/storageproofs/stpproto/messages.nim | 68 +++++++ codex/storageproofs/stpstore.nim | 16 +- codex/stores/blockstore.nim | 1 - codex/stores/localstore.nim | 118 +++++++++++ codex/utils/fileutils.nim | 3 +- tests/codex/storageproofs/testnetwork.nim | 1 - vendor/nim-protobuf-serialization | 1 - 23 files changed, 444 insertions(+), 103 deletions(-) create mode 100644 codex/formats.nim create mode 100644 codex/storageproofs/por/serialization/messages.nim delete mode 100644 codex/storageproofs/por/serialization/por.proto delete mode 100644 codex/storageproofs/stp.proto create mode 100644 codex/storageproofs/stpproto/messages.nim create mode 100644 codex/stores/localstore.nim delete mode 160000 vendor/nim-protobuf-serialization diff --git a/.gitmodules b/.gitmodules index 28007257..9abd397f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -33,11 +33,6 @@ url = https://github.com/status-im/nim-stew.git ignore = untracked branch = master -[submodule "vendor/nim-protobuf-serialization"] - path = vendor/nim-protobuf-serialization - url = https://github.com/status-im/nim-protobuf-serialization.git - ignore = untracked - branch = master [submodule "vendor/nim-nitro"] path = vendor/nim-nitro url = https://github.com/status-im/nim-nitro.git diff --git a/codex/blockexchange/engine/discovery.nim b/codex/blockexchange/engine/discovery.nim index 90009241..882a29e0 100644 --- a/codex/blockexchange/engine/discovery.nim +++ b/codex/blockexchange/engine/discovery.nim @@ -103,19 +103,19 @@ proc advertiseTaskLoop(b: DiscoveryEngine) {.async.} = cid = await b.advertiseQueue.get() if cid in b.inFlightAdvReqs: - trace "Advertise request already in progress", cid = $cid + trace "Advertise request already in progress", cid continue try: let request = b.discovery.provide(cid) b.inFlightAdvReqs[cid] = request codex_inflight_discovery.set(b.inFlightAdvReqs.len.int64) - trace "Advertising block", cid = $cid, inflight = b.inFlightAdvReqs.len + trace "Advertising block", cid, inflight = b.inFlightAdvReqs.len await request finally: b.inFlightAdvReqs.del(cid) codex_inflight_discovery.set(b.inFlightAdvReqs.len.int64) - trace "Advertised block", cid = $cid, inflight = b.inFlightAdvReqs.len + trace "Advertised block", cid, inflight = b.inFlightAdvReqs.len except CatchableError as exc: trace "Exception in advertise task runner", exc = exc.msg @@ -131,15 +131,15 @@ proc discoveryTaskLoop(b: DiscoveryEngine) {.async.} = cid = await b.discoveryQueue.get() if cid in b.inFlightDiscReqs: - trace "Discovery request already in progress", cid = $cid + trace "Discovery request already in progress", cid continue let haves = b.peers.peersHave(cid) - trace "Current number of peers for block", cid = $cid, count = haves.len + trace "Current number of peers for block", cid, count = haves.len if haves.len < b.minPeersPerBlock: - trace "Discovering block", cid = $cid + trace "Discovering block", cid try: let request = b.discovery @@ -173,7 +173,7 @@ proc queueFindBlocksReq*(b: DiscoveryEngine, cids: seq[Cid]) {.inline.} = try: for cid in cids: if cid notin b.discoveryQueue: - trace "Queueing find block request", cid = $cid + trace "Queueing find block request", cid await b.discoveryQueue.put(cid) except CatchableError as exc: trace "Exception queueing discovery request", exc = exc.msg @@ -185,7 +185,7 @@ proc queueProvideBlocksReq*(b: DiscoveryEngine, cids: seq[Cid]) {.inline.} = try: for cid in cids: if cid notin b.advertiseQueue: - trace "Queueing provide block request", cid = $cid + trace "Queueing provide block request", cid await b.advertiseQueue.put(cid) except CatchableError as exc: trace "Exception queueing discovery request", exc = exc.msg diff --git a/codex/blockexchange/engine/engine.nim b/codex/blockexchange/engine/engine.nim index e2826ced..fca42f64 100644 --- a/codex/blockexchange/engine/engine.nim +++ b/codex/blockexchange/engine/engine.nim @@ -122,7 +122,7 @@ proc requestBlock*( ## Request a block from remotes ## - trace "Requesting block", cid = $cid + trace "Requesting block", cid if cid in b.pendingBlocks: return await b.pendingBlocks.getWantHandle(cid, timeout) @@ -136,7 +136,7 @@ proc requestBlock*( if peers.len <= 0: peers = toSeq(b.peers) # Get any peer if peers.len <= 0: - trace "No peers to request blocks from", cid = $cid + trace "No peers to request blocks from", cid b.discovery.queueFindBlocksReq(@[cid]) return await blk @@ -150,7 +150,7 @@ proc requestBlock*( wantType = WantType.wantBlock) # we want this remote to send us a block if (peers.len - 1) == 0: - trace "Not enough peers to send want list to", cid = $cid + trace "Not enough peers to send want list to", cid b.discovery.queueFindBlocksReq(@[cid]) return await blk # no peers to send wants to diff --git a/codex/blockexchange/protobuf/payments.nim b/codex/blockexchange/protobuf/payments.nim index bf1ee0f2..61321d2e 100644 --- a/codex/blockexchange/protobuf/payments.nim +++ b/codex/blockexchange/protobuf/payments.nim @@ -1,4 +1,3 @@ -import pkg/protobuf_serialization import pkg/stew/byteutils import pkg/stint import pkg/nitro diff --git a/codex/blocktype.nim b/codex/blocktype.nim index ce04644f..0911c22a 100644 --- a/codex/blocktype.nim +++ b/codex/blocktype.nim @@ -18,9 +18,13 @@ import pkg/libp2p import pkg/stew/byteutils import pkg/questionable import pkg/questionable/results +import pkg/chronicles +import ./formats import ./errors +export errors, formats + const # Size of blocks for storage / network exchange, # should be divisible by 31 for PoR and by 64 for Leopard ECC diff --git a/codex/discovery.nim b/codex/discovery.nim index 2f9917f3..f2135bae 100644 --- a/codex/discovery.nim +++ b/codex/discovery.nim @@ -22,6 +22,7 @@ import pkg/libp2pdht/discv5/protocol as discv5 import ./rng import ./errors +import ./formats export discv5 @@ -82,10 +83,10 @@ method find*( ## Find block providers ## - trace "Finding providers for block", cid = $cid + trace "Finding providers for block", cid without providers =? (await d.protocol.getProviders(cid.toNodeId())).mapFailure, error: - trace "Error finding providers for block", cid = $cid, error = error.msg + trace "Error finding providers for block", cid, error = error.msg return providers @@ -93,7 +94,7 @@ method provide*(d: Discovery, cid: Cid) {.async, base.} = ## Provide a bock Cid ## - trace "Providing block", cid = $cid + trace "Providing block", cid let nodes = await d.protocol.addProvider( cid.toNodeId(), diff --git a/codex/formats.nim b/codex/formats.nim new file mode 100644 index 00000000..ec79dabe --- /dev/null +++ b/codex/formats.nim @@ -0,0 +1,28 @@ +## Nim-Codex +## Copyright (c) 2022 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + +import std/strutils + +import pkg/chronicles +import pkg/libp2p + +func shortLog*(cid: Cid): string = + ## Returns compact string representation of ``pid``. + var scid = $cid + if len(scid) > 10: + scid[3] = '*' + + when (NimMajor, NimMinor) > (1, 4): + scid.delete(4 .. scid.high - 6) + else: + scid.delete(4, scid.high - 6) + + scid + +chronicles.formatIt(Cid): shortLog(it) diff --git a/codex/node.nim b/codex/node.nim index ee15154f..f53de6f5 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -10,6 +10,7 @@ import std/options import std/tables import std/sequtils +import std/strformat import pkg/questionable import pkg/questionable/results @@ -188,7 +189,7 @@ proc store*( blockManifest.add(blk.cid) if isErr (await node.blockStore.putBlock(blk)): # trace "Unable to store block", cid = blk.cid - return failure("Unable to store block " & $blk.cid) + return failure(&"Unable to store block {blk.cid}") except CancelledError as exc: raise exc diff --git a/codex/rest/api.nim b/codex/rest/api.nim index 9918460b..0f98d71d 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -247,7 +247,7 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter = trace "Error uploading file", exc = error.msg return RestApiResponse.error(Http500, error.msg) - trace "Uploaded file", cid = $cid + trace "Uploaded file", cid return RestApiResponse.response($cid) except CancelledError as exc: return RestApiResponse.error(Http500) diff --git a/codex/storageproofs/por/por.nim b/codex/storageproofs/por/por.nim index 03207e32..f2f465a3 100644 --- a/codex/storageproofs/por/por.nim +++ b/codex/storageproofs/por/por.nim @@ -1,4 +1,4 @@ -## Nim-POS +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/codex/storageproofs/por/serialization/messages.nim b/codex/storageproofs/por/serialization/messages.nim new file mode 100644 index 00000000..0602ccc7 --- /dev/null +++ b/codex/storageproofs/por/serialization/messages.nim @@ -0,0 +1,185 @@ +## Nim-Codex +## Copyright (c) 2022 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + +import pkg/questionable/results +import pkg/libp2p/protobuf/minprotobuf + +type + TauZeroMessage* = object + name*: seq[byte] + n*: int64 + u*: seq[seq[byte]] + + TauMessage* = object + t*: TauZeroMessage + signature*: seq[byte] + + PubKeyMessage* = object + signkey*: seq[byte] + key*: seq[byte] + + PorMessage* = object + tau*: TauMessage + spk*: PubKeyMessage + authenticators*: seq[seq[byte]] + + ProofMessage* = object + mu*: seq[seq[byte]] + sigma*: seq[byte] + + PoREnvelope* = object + por*: PorMessage + proof*: ProofMessage + +func write*(pb: var ProtoBuffer, field: int, value: TauZeroMessage) = + var ipb = initProtoBuffer() + ipb.write(1, value.name) + ipb.write(2, value.n.uint64) + + for u in value.u: + ipb.write(3, u) + + ipb.finish() + pb.write(field, ipb) + +func write*(pb: var ProtoBuffer, field: int, value: TauMessage) = + var ipb = initProtoBuffer() + ipb.write(1, value.t) + ipb.write(2, value.signature) + ipb.finish() + + pb.write(field, ipb) + +func write*(pb: var ProtoBuffer, field: int, value: PubKeyMessage) = + var ipb = initProtoBuffer() + ipb.write(1, value.signkey) + ipb.write(2, value.key) + ipb.finish() + pb.write(field, ipb) + +func write*(pb: var ProtoBuffer, field: int, value: PorMessage) = + var ipb = initProtoBuffer() + ipb.write(1, value.tau) + ipb.write(2, value.spk) + + for a in value.authenticators: + ipb.write(3, a) + + ipb.finish() + pb.write(field, ipb) + +func encode*(msg: PorMessage): seq[byte] = + var ipb = initProtoBuffer() + ipb.write(1, msg.tau) + ipb.write(2, msg.spk) + + for a in msg.authenticators: + ipb.write(3, a) + + ipb.finish + ipb.buffer + +func write*(pb: var ProtoBuffer, field: int, value: ProofMessage) = + var ipb = initProtoBuffer() + for mu in value.mu: + ipb.write(1, mu) + + ipb.write(2, value.sigma) + ipb.finish() + pb.write(field, ipb) + +func encode*(message: PoREnvelope): seq[byte] = + var ipb = initProtoBuffer() + ipb.write(1, message.por) + ipb.write(2, message.proof) + ipb.finish + ipb.buffer + +proc decode*(_: type TauZeroMessage, pb: ProtoBuffer): ProtoResult[TauZeroMessage] = + var + value = TauZeroMessage() + + discard ? pb.getField(1, value.name) + + var val: uint64 + discard ? pb.getField(2, val) + value.n = val.int64 + + var bytes: seq[seq[byte]] + discard ? pb.getRepeatedField(3, bytes) + + for b in bytes: + value.u.add(b) + + ok(value) + +proc decode*(_: type TauMessage, pb: ProtoBuffer): ProtoResult[TauMessage] = + var + value = TauMessage() + ipb: ProtoBuffer + + discard ? pb.getField(1, ipb) + + value.t = ? TauZeroMessage.decode(ipb) + + discard ? pb.getField(2, value.signature) + + ok(value) + +proc decode*(_: type PubKeyMessage, pb: ProtoBuffer): ProtoResult[PubKeyMessage] = + var + value = PubKeyMessage() + + discard ? pb.getField(1, value.signkey) + discard ? pb.getField(2, value.key) + + ok(value) + +proc decode*(_: type PorMessage, pb: ProtoBuffer): ProtoResult[PorMessage] = + var + value = PorMessage() + ipb: ProtoBuffer + + discard ? pb.getField(1, ipb) + value.tau = ? TauMessage.decode(ipb) + + discard ? pb.getField(2, ipb) + value.spk = ? PubKeyMessage.decode(ipb) + + var + bytes: seq[seq[byte]] + + discard ? pb.getRepeatedField(3, bytes) + + for b in bytes: + value.authenticators.add(b) + + ok(value) + +proc decode*(_: type PorMessage, msg: seq[byte]): ProtoResult[PorMessage] = + PorMessage.decode(initProtoBuffer(msg)) + +proc decode*(_: type ProofMessage, pb: ProtoBuffer): ProtoResult[ProofMessage] = + var + value = ProofMessage() + + discard ? pb.getField(1, value.mu) + discard ? pb.getField(2, value.sigma) + + ok(value) + +func decode*(_: type PoREnvelope, msg: openArray[byte]): ?!PoREnvelope = + var + value = PoREnvelope() + pb = initProtoBuffer(msg) + + discard ? pb.getField(1, ? value.por.decode) + discard ? pb.getField(2, ? value.proof.decode) + + ok(value) diff --git a/codex/storageproofs/por/serialization/por.proto b/codex/storageproofs/por/serialization/por.proto deleted file mode 100644 index 82670e18..00000000 --- a/codex/storageproofs/por/serialization/por.proto +++ /dev/null @@ -1,33 +0,0 @@ -syntax = "proto3"; - -message PoREnvelope { - message TauZeroMessage { - bytes name = 1; - int64 n = 2; - repeated bytes u = 3; - } - - message TauMessage { - TauZeroMessage t = 1; - bytes signature = 2; - } - - message PubKeyMessage { - bytes signkey = 1; - bytes key = 2; - } - - message PorMessage { - TauMessage tau = 1; - PubKeyMessage spk = 2; - repeated bytes authenticators = 3; - } - - message ProofMessage { - repeated bytes mu = 1; - bytes sigma = 2; - } - - PorMessage por = 1; - ProofMessage proof = 2; -} diff --git a/codex/storageproofs/por/serialization/serialization.nim b/codex/storageproofs/por/serialization/serialization.nim index c22e7e1e..10214bed 100644 --- a/codex/storageproofs/por/serialization/serialization.nim +++ b/codex/storageproofs/por/serialization/serialization.nim @@ -1,4 +1,4 @@ -## Nim-POS +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -9,19 +9,14 @@ import std/sequtils -import pkg/protobuf_serialization import pkg/stew/results import pkg/stew/objects import pkg/blscurve import pkg/blscurve/blst/blst_abi -import_proto3 "por.proto" +import ./messages -export TauZeroMessage -export TauMessage -export ProofMessage -export PorMessage -export PoREnvelope +export messages import ../por @@ -33,6 +28,7 @@ func toMessage*(self: Proof): ProofMessage = for mu in self.mu: var serialized: array[32, byte] + blst_bendian_from_scalar(serialized, mu) message.mu.add(toSeq(serialized)) diff --git a/codex/storageproofs/stp.proto b/codex/storageproofs/stp.proto deleted file mode 100644 index 800e9035..00000000 --- a/codex/storageproofs/stp.proto +++ /dev/null @@ -1,15 +0,0 @@ -syntax = "proto3"; - -message StorageProofsMessage { - message Tag { - int64 idx = 1; - bytes tag = 2; - } - - message TagsMessage { - bytes cid = 1; - repeated Tag tags = 2; - } - - TagsMessage tagsMsg = 1; -} diff --git a/codex/storageproofs/stpnetwork.nim b/codex/storageproofs/stpnetwork.nim index afafb7b5..99abd990 100644 --- a/codex/storageproofs/stpnetwork.nim +++ b/codex/storageproofs/stpnetwork.nim @@ -15,10 +15,10 @@ import pkg/chronicles import pkg/questionable import pkg/questionable/results import pkg/contractabi/address as ca -import pkg/protobuf_serialization import ./stpproto import ../discovery +import ../formats const Codec* = "/dagger/storageproofs/1.0.0" @@ -59,8 +59,7 @@ proc uploadTags*( conn = await connFut try: - await conn.writeLp( - Protobuf.encode(StorageProofsMessage(tagsMsg: msg))) + await conn.writeLp(msg.encode) except CancelledError as exc: raise exc except CatchableError as exc: @@ -79,10 +78,11 @@ method init*(self: StpNetwork) = try: let msg = await conn.readLp(MaxMessageSize) - message = Protobuf.decode(msg, StorageProofsMessage) + res = TagsMessage.decode(msg) - if message.tagsMsg.tags.len > 0 and not self.tagsHandle.isNil: - await self.tagsHandle(message.tagsMsg) + if not self.tagsHandle.isNil: + if res.isOk and res.get.tags.len > 0: + await self.tagsHandle(res.get) except CatchableError as exc: trace "Exception handling Storage Proofs message", exc = exc.msg finally: diff --git a/codex/storageproofs/stpproto.nim b/codex/storageproofs/stpproto.nim index 39303099..364582be 100644 --- a/codex/storageproofs/stpproto.nim +++ b/codex/storageproofs/stpproto.nim @@ -1,7 +1,3 @@ -import pkg/protobuf_serialization +import ./stpproto/messages -import_proto3 "stp.proto" - -export StorageProofsMessage -export TagsMessage -export Tag +export messages diff --git a/codex/storageproofs/stpproto/messages.nim b/codex/storageproofs/stpproto/messages.nim new file mode 100644 index 00000000..d5294bbb --- /dev/null +++ b/codex/storageproofs/stpproto/messages.nim @@ -0,0 +1,68 @@ +## Nim-Codex +## Copyright (c) 2022 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + +import pkg/questionable/results +import pkg/libp2p/protobuf/minprotobuf + +import ../../errors + +type + Tag* = object + idx*: int64 + tag*: seq[byte] + + TagsMessage* = object + cid*: seq[byte] + tags*: seq[Tag] + +func write*(pb: var ProtoBuffer, field: int, value: Tag) = + var ipb = initProtoBuffer() + ipb.write(1, value.idx.uint64) + ipb.write(2, value.tag) + ipb.finish() + pb.write(field, ipb) + +func encode*(msg: TagsMessage): seq[byte] = + var ipb = initProtoBuffer() + ipb.write(1, msg.cid) + + for tag in msg.tags: + ipb.write(2, tag) + + ipb.finish() + ipb.buffer + +func decode*(_: type Tag, pb: ProtoBuffer): ProtoResult[Tag] = + var + value = Tag() + idx: uint64 + + discard ? pb.getField(1, idx) + value.idx = idx.int64 + + discard ? pb.getField(2, value.tag) + + ok(value) + +func decode*(_: type TagsMessage, msg: openArray[byte]): ProtoResult[TagsMessage] = + var + value = TagsMessage() + pb = initProtoBuffer(msg) + + discard ? pb.getField(1, value.cid) + + var + bytes: seq[seq[byte]] + + discard ? pb.getRepeatedField(2, bytes) + + for b in bytes: + value.tags.add(? Tag.decode(initProtoBuffer(b))) + + ok(value) diff --git a/codex/storageproofs/stpstore.nim b/codex/storageproofs/stpstore.nim index e64cd107..b4ab14bd 100644 --- a/codex/storageproofs/stpstore.nim +++ b/codex/storageproofs/stpstore.nim @@ -16,7 +16,9 @@ import pkg/chronicles import pkg/stew/io2 import pkg/questionable import pkg/questionable/results -import pkg/protobuf_serialization + +import ../errors +import ../formats import ./stpproto import ./por @@ -44,7 +46,7 @@ proc retrieve*( trace "Cannot retrieve storage proof data from fs", path , error return failure("Cannot retrieve storage proof data from fs") - return Protobuf.decode(data, PorMessage).success + return PorMessage.decode(data).mapFailure proc store*( self: StpStore, @@ -62,12 +64,12 @@ proc store*( let path = dir / "por" if ( - let res = io2.writeFile(path, Protobuf.encode(por)); + let res = io2.writeFile(path, por.encode()); res.isErr): let error = io2.ioErrorMsg(res.error) - trace "Unable to store storage proofs", path, cid = cid, error + trace "Unable to store storage proofs", path, cid, error return failure( - &"Unable to store storage proofs - path = ${path} cid = ${$cid} error = ${error}") + &"Unable to store storage proofs - path = ${path} cid = ${cid} error = ${error}") return success() @@ -106,9 +108,9 @@ proc store*( let res = io2.writeFile(path, t.tag); res.isErr): let error = io2.ioErrorMsg(res.error) - trace "Unable to store tags", path, cid = cid, error + trace "Unable to store tags", path, cid, error return failure( - &"Unable to store tags - path = ${path} cid = ${$cid} error = ${error}") + &"Unable to store tags - path = ${path} cid = ${cid} error = ${error}") return success() diff --git a/codex/stores/blockstore.nim b/codex/stores/blockstore.nim index bda25c23..54f796f1 100644 --- a/codex/stores/blockstore.nim +++ b/codex/stores/blockstore.nim @@ -13,7 +13,6 @@ push: {.upraises: [].} import pkg/chronos import pkg/libp2p -import pkg/questionable import pkg/questionable/results import ../blocktype diff --git a/codex/stores/localstore.nim b/codex/stores/localstore.nim new file mode 100644 index 00000000..900825cd --- /dev/null +++ b/codex/stores/localstore.nim @@ -0,0 +1,118 @@ +## Nim-Codex +## Copyright (c) 2022 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + +import std/os + +import pkg/upraises + +push: {.upraises: [].} + +import pkg/chronos +import pkg/libp2p +import pkg/questionable +import pkg/questionable/results +import pkg/datastore + +import ./blockstore +import ../blocktype +import ../namespaces +import ../manifest + +export blocktype, libp2p + +const + CacheBytesKey* = CodexMetaNamespace / "bytes" / "cache" + CachePersistentKey* = CodexMetaNamespace / "bytes" / "persistent" + +type + LocalStore* = ref object of BlockStore + ds*: Datastore + blocksRepo*: BlockStore # TODO: Should be a Datastore + manifestRepo*: BlockStore # TODO: Should be a Datastore + cacheBytes*: uint + persistBytes*: uint + +method getBlock*(self: LocalStore, cid: Cid): Future[?!Block] = + ## Get a block from the blockstore + ## + + if cid.isManifest: + self.manifestRepo.getBlock(cid) + else: + self.blocksRepo.getBlock(cid) + +method putBlock*(self: LocalStore, blk: Block): Future[?!void] = + ## Put a block to the blockstore + ## + + if blk.cid.isManifest: + self.manifestRepo.putBlock(blk) + else: + self.blocksRepo.putBlock(blk) + +method delBlock*(self: LocalStore, cid: Cid): Future[?!void] = + ## Delete a block from the blockstore + ## + + if cid.isManifest: + self.manifestRepo.delBlock(cid) + else: + self.blocksRepo.delBlock(cid) + +method hasBlock*(self: LocalStore, cid: Cid): Future[?!bool] = + ## Check if the block exists in the blockstore + ## + + if cid.isManifest: + self.manifestRepo.hasBlock(cid) + else: + self.blocksRepo.hasBlock(cid) + +method listBlocks*( + self: LocalStore, + blkType: MultiCodec, + batch = 100, + onBlock: OnBlock): Future[?!void] = + ## Get the list of blocks in the LocalStore. + ## This is an intensive operation + ## + + if $blkType in ManifestContainers: + self.manifestRepo.listBlocks(blkType, batch, onBlock) + else: + self.blocksRepo.listBlocks(onBlock) + +method close*(self: LocalStore) {.async.} = + ## Close the blockstore, cleaning up resources managed by it. + ## For some implementations this may be a no-op + ## + + await self.manifestRepo.close() + await self.blocksRepo.close() + +proc contains*(self: LocalStore, blk: Cid): Future[bool] {.async.} = + ## Check if the block exists in the blockstore. + ## Return false if error encountered + ## + + return (await self.hasBlock(blk)) |? false + +func new*( + T: type LocalStore, + datastore: Datastore, + blocksRepo: BlockStore, + manifestRepo: BlockStore, + cacheBytes: uint, + persistBytes: uint): T = + T( + datastore: datastore, + blocksRepo: blocksRepo, + manifestRepo: manifestRepo, + cacheBytes: cacheBytes, + persistBytes: persistBytes) diff --git a/codex/utils/fileutils.nim b/codex/utils/fileutils.nim index a900460a..d96e646c 100644 --- a/codex/utils/fileutils.nim +++ b/codex/utils/fileutils.nim @@ -9,12 +9,11 @@ ## Partially taken from nim beacon chain +import std/strutils import pkg/upraises push: {.upraises: [].} -import std/strutils - import pkg/chronicles import stew/io2 diff --git a/tests/codex/storageproofs/testnetwork.nim b/tests/codex/storageproofs/testnetwork.nim index ad59a95f..4f103b87 100644 --- a/tests/codex/storageproofs/testnetwork.nim +++ b/tests/codex/storageproofs/testnetwork.nim @@ -5,7 +5,6 @@ import pkg/asynctest import pkg/chronos import pkg/libp2p import pkg/libp2p/errors -import pkg/protobuf_serialization import pkg/contractabi as ca import pkg/codex/rng diff --git a/vendor/nim-protobuf-serialization b/vendor/nim-protobuf-serialization deleted file mode 160000 index f7d671f8..00000000 --- a/vendor/nim-protobuf-serialization +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f7d671f877e01213494aac7903421ccdbe70616f