status-im/nim-codex
2
0
Fork 0
mirror of https://github.com/status-im/nim-codex.git synced 2025-02-22 23:48:51 +00:00
Code Issues Projects Releases Wiki Activity

style: nph formatting (#1067)

Browse Source 
* style: nph setup

* chore: formates codex/ and tests/ folder with nph 0.6.1
This commit is contained in:
Adam Uhlíř 2025-01-21 21:54:46 +01:00 committed by GitHub
parent d114e6e942
commit e5df8c50d3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
272 changed files with 7290 additions and 7562 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
.github/workflows/ci.yml vendored
View File

@ -47,6 +47,19 @@ jobs:
matrix: ${{ needs.matrix.outputs.matrix }}
cache_nonce: ${{ needs.matrix.outputs.cache_nonce }}
linting:
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'
steps:
- uses: actions/checkout@v4
- name: Check `nph` formatting
uses: arnetheduck/nph-action@v1
with:
version: 0.6.1
options: "codex/ tests/"
fail: true
suggest: true
coverage:
# Force to stick to ubuntu 20.04 for coverage because
# lcov was updated to 2.x version in ubuntu-latest

3
.gitmodules vendored
View File

@ -218,3 +218,6 @@
[submodule "vendor/nim-zippy"]
path = vendor/nim-zippy
url = https://github.com/status-im/nim-zippy.git
[submodule "vendor/nph"]
path = vendor/nph
url = https://github.com/arnetheduck/nph.git

39
Makefile
View File

@ -17,6 +17,7 @@
# version pinned by nimbus-build-system.
#PINNED_NIM_VERSION := 38640664088251bbc88917b4bacfd86ec53014b8 # 1.6.21
PINNED_NIM_VERSION := v2.0.14
ifeq ($(NIM_COMMIT),)
NIM_COMMIT := $(PINNED_NIM_VERSION)
else ifeq ($(NIM_COMMIT),pinned)
@ -199,4 +200,42 @@ ifneq ($(USE_LIBBACKTRACE), 0)
+ $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT)
endif
############
## Format ##
############
.PHONY: build-nph install-nph-hook clean-nph print-nph-path
# Default location for nph binary shall be next to nim binary to make it available on the path.
NPH:=$(shell dirname $(NIM_BINARY))/nph
build-nph:
ifeq ("$(wildcard $(NPH))","")
$(ENV_SCRIPT) nim c vendor/nph/src/nph.nim && \
mv vendor/nph/src/nph $(shell dirname $(NPH))
echo "nph utility is available at " $(NPH)
endif
GIT_PRE_COMMIT_HOOK := .git/hooks/pre-commit
install-nph-hook: build-nph
ifeq ("$(wildcard $(GIT_PRE_COMMIT_HOOK))","")
cp ./tools/scripts/git_pre_commit_format.sh $(GIT_PRE_COMMIT_HOOK)
else
echo "$(GIT_PRE_COMMIT_HOOK) already present, will NOT override"
exit 1
endif
nph/%: build-nph
echo -e $(FORMAT_MSG) "nph/$*" && \
$(NPH) $*
clean-nph:
rm -f $(NPH)
# To avoid hardcoding nph binary location in several places
print-nph-path:
echo "$(NPH)"
clean: | clean-nph
endif # "variables.mk" was not included

13
README.md
View File

@ -31,6 +31,7 @@ Run the client with:
```bash
build/codex
```
## Configuration
It is possible to configure a Codex node in several ways:
@ -51,3 +52,15 @@ To get acquainted with Codex, consider:
## API
The client exposes a REST API that can be used to interact with the clients. Overview of the API can be found on [api.codex.storage](https://api.codex.storage).
## Contributing and development
Feel free to dive in, contributions are welcomed! Open an issue or submit PRs.
### Linting and formatting
`nim-codex` uses [nph](https://github.com/arnetheduck/nph) for formatting our code and it is requrired to adhere to its styling.
If you are setting up fresh setup, in order to get `nph` run `make build-nph`.
In order to format files run `make nph/<file/folder you want to format>`.
If you want you can install Git pre-commit hook using `make install-nph-commit`, which will format modified files prior commiting them.
If you are using VSCode and the [NimLang](https://marketplace.visualstudio.com/items?itemName=NimLang.nimlang) extension you can enable "Format On Save" (eq. the `nim.formatOnSave` property) that will format the files using `nph`.

9
codex/blockexchange.nim
View File

@ -1,10 +1,5 @@
import ./blockexchange/[
network,
engine,
peers]
import ./blockexchange/[network, engine, peers]
import ./blockexchange/protobuf/[
blockexc,
presence]
import ./blockexchange/protobuf/[blockexc, presence]
export network, engine, blockexc, presence, peers

16
codex/blockexchange/engine/advertiser.nim
View File

@ -34,8 +34,7 @@ const
DefaultConcurrentAdvertRequests = 10
DefaultAdvertiseLoopSleep = 30.minutes
type
Advertiser* = ref object of RootObj
type Advertiser* = ref object of RootObj
localStore*: BlockStore # Local block store for this instance
discovery*: Discovery # Discovery interface
@ -83,7 +82,6 @@ proc advertiseLocalStoreLoop(b: Advertiser) {.async: (raises: []).} =
trace "Advertiser iterating blocks finished."
await sleepAsync(b.advertiseLocalStoreLoopSleep)
except CancelledError:
break # do not propagate as advertiseLocalStoreLoop was asyncSpawned
except CatchableError as e:
@ -94,20 +92,17 @@ proc advertiseLocalStoreLoop(b: Advertiser) {.async: (raises: []).} =
proc processQueueLoop(b: Advertiser) {.async: (raises: []).} =
while b.advertiserRunning:
try:
let
cid = await b.advertiseQueue.get()
let cid = await b.advertiseQueue.get()
if cid in b.inFlightAdvReqs:
continue
try:
let
request = b.discovery.provide(cid)
let request = b.discovery.provide(cid)
b.inFlightAdvReqs[cid] = request
codex_inflight_advertise.set(b.inFlightAdvReqs.len.int64)
await request
finally:
b.inFlightAdvReqs.del(cid)
codex_inflight_advertise.set(b.inFlightAdvReqs.len.int64)
@ -166,7 +161,7 @@ proc new*(
localStore: BlockStore,
discovery: Discovery,
concurrentAdvReqs = DefaultConcurrentAdvertRequests,
advertiseLocalStoreLoopSleep = DefaultAdvertiseLoopSleep
advertiseLocalStoreLoopSleep = DefaultAdvertiseLoopSleep,
): Advertiser =
## Create a advertiser instance
##
@ -177,4 +172,5 @@ proc new*(
advertiseQueue: newAsyncQueue[Cid](concurrentAdvReqs),
trackedFutures: TrackedFutures.new(),
inFlightAdvReqs: initTable[Cid, Future[void]](),
advertiseLocalStoreLoopSleep: advertiseLocalStoreLoopSleep)
advertiseLocalStoreLoopSleep: advertiseLocalStoreLoopSleep,
)

30
codex/blockexchange/engine/discovery.nim
View File

@ -40,8 +40,7 @@ const
DefaultMinPeersPerBlock = 3
DefaultDiscoveryLoopSleep = 3.seconds
type
DiscoveryEngine* = ref object of RootObj
type DiscoveryEngine* = ref object of RootObj
localStore*: BlockStore # Local block store for this instance
peers*: PeerCtxStore # Peer context store
network*: BlockExcNetwork # Network interface
@ -54,7 +53,8 @@ type
trackedFutures*: TrackedFutures # Tracked Discovery tasks futures
minPeersPerBlock*: int # Max number of peers with block
discoveryLoopSleep: Duration # Discovery loop sleep
inFlightDiscReqs*: Table[Cid, Future[seq[SignedPeerRecord]]] # Inflight discovery requests
inFlightDiscReqs*: Table[Cid, Future[seq[SignedPeerRecord]]]
# Inflight discovery requests
proc discoveryQueueLoop(b: DiscoveryEngine) {.async: (raises: []).} =
while b.discEngineRunning:
@ -81,36 +81,27 @@ proc discoveryTaskLoop(b: DiscoveryEngine) {.async: (raises: []).} =
while b.discEngineRunning:
try:
let
cid = await b.discoveryQueue.get()
let cid = await b.discoveryQueue.get()
if cid in b.inFlightDiscReqs:
trace "Discovery request already in progress", cid
continue
let
haves = b.peers.peersHave(cid)
let haves = b.peers.peersHave(cid)
if haves.len < b.minPeersPerBlock:
try:
let
request = b.discovery
.find(cid)
.wait(DefaultDiscoveryTimeout)
let request = b.discovery.find(cid).wait(DefaultDiscoveryTimeout)
b.inFlightDiscReqs[cid] = request
codex_inflight_discovery.set(b.inFlightDiscReqs.len.int64)
let
peers = await request
let peers = await request
let
dialed = await allFinished(
peers.mapIt( b.network.dialPeer(it.data) ))
let dialed = await allFinished(peers.mapIt(b.network.dialPeer(it.data)))
for i, f in dialed:
if f.failed:
await b.discovery.removeProvider(peers[i].data.peerId)
finally:
b.inFlightDiscReqs.del(cid)
codex_inflight_discovery.set(b.inFlightDiscReqs.len.int64)
@ -180,7 +171,7 @@ proc new*(
pendingBlocks: PendingBlocksManager,
concurrentDiscReqs = DefaultConcurrentDiscRequests,
discoveryLoopSleep = DefaultDiscoveryLoopSleep,
minPeersPerBlock = DefaultMinPeersPerBlock
minPeersPerBlock = DefaultMinPeersPerBlock,
): DiscoveryEngine =
## Create a discovery engine instance for advertising services
##
@ -195,4 +186,5 @@ proc new*(
trackedFutures: TrackedFutures.new(),
inFlightDiscReqs: initTable[Cid, Future[seq[SignedPeerRecord]]](),
discoveryLoopSleep: discoveryLoopSleep,
minPeersPerBlock: minPeersPerBlock)
minPeersPerBlock: minPeersPerBlock,
)

252
codex/blockexchange/engine/engine.nim
View File

@ -44,12 +44,24 @@ export peers, pendingblocks, payments, discovery
logScope:
topics = "codex blockexcengine"
declareCounter(codex_block_exchange_want_have_lists_sent, "codex blockexchange wantHave lists sent")
declareCounter(codex_block_exchange_want_have_lists_received, "codex blockexchange wantHave lists received")
declareCounter(codex_block_exchange_want_block_lists_sent, "codex blockexchange wantBlock lists sent")
declareCounter(codex_block_exchange_want_block_lists_received, "codex blockexchange wantBlock lists received")
declareCounter(
codex_block_exchange_want_have_lists_sent, "codex blockexchange wantHave lists sent"
)
declareCounter(
codex_block_exchange_want_have_lists_received,
"codex blockexchange wantHave lists received",
)
declareCounter(
codex_block_exchange_want_block_lists_sent, "codex blockexchange wantBlock lists sent"
)
declareCounter(
codex_block_exchange_want_block_lists_received,
"codex blockexchange wantBlock lists received",
)
declareCounter(codex_block_exchange_blocks_sent, "codex blockexchange blocks sent")
declareCounter(codex_block_exchange_blocks_received, "codex blockexchange blocks received")
declareCounter(
codex_block_exchange_blocks_received, "codex blockexchange blocks received"
)
const
DefaultMaxPeersPerRequest* = 10
@ -70,7 +82,8 @@ type
localStore*: BlockStore # Local block store for this instance
network*: BlockExcNetwork # Petwork interface
peers*: PeerCtxStore # Peers we're currently actively exchanging with
taskQueue*: AsyncHeapQueue[BlockExcPeerCtx] # Peers we're currently processing tasks for
taskQueue*: AsyncHeapQueue[BlockExcPeerCtx]
# Peers we're currently processing tasks for
concurrentTasks: int # Number of concurrent peers we're serving at any given time
trackedFutures: TrackedFutures # Tracks futures of blockexc tasks
blockexcRunning: bool # Indicates if the blockexc task is running
@ -87,7 +100,7 @@ type
price*: UInt256
# attach task scheduler to engine
proc scheduleTask(b: BlockExcEngine, task: BlockExcPeerCtx): bool {.gcsafe} =
proc scheduleTask(b: BlockExcEngine, task: BlockExcPeerCtx): bool {.gcsafe.} =
b.taskQueue.pushOrUpdateNoWait(task).isOk()
proc blockexcTaskRunner(b: BlockExcEngine) {.async: (raises: []).}
@ -128,35 +141,26 @@ proc stop*(b: BlockExcEngine) {.async.} =
trace "NetworkStore stopped"
proc sendWantHave(
b: BlockExcEngine,
addresses: seq[BlockAddress],
peers: seq[BlockExcPeerCtx]): Future[void] {.async.} =
b: BlockExcEngine, addresses: seq[BlockAddress], peers: seq[BlockExcPeerCtx]
): Future[void] {.async.} =
for p in peers:
let toAsk = addresses.filterIt(it notin p.peerHave)
trace "Sending wantHave request", toAsk, peer = p.id
await b.network.request.sendWantList(
p.id,
toAsk,
wantType = WantType.WantHave)
await b.network.request.sendWantList(p.id, toAsk, wantType = WantType.WantHave)
codex_block_exchange_want_have_lists_sent.inc()
proc sendWantBlock(
b: BlockExcEngine,
addresses: seq[BlockAddress],
blockPeer: BlockExcPeerCtx): Future[void] {.async.} =
b: BlockExcEngine, addresses: seq[BlockAddress], blockPeer: BlockExcPeerCtx
): Future[void] {.async.} =
trace "Sending wantBlock request to", addresses, peer = blockPeer.id
await b.network.request.sendWantList(
blockPeer.id,
addresses,
wantType = WantType.WantBlock) # we want this remote to send us a block
blockPeer.id, addresses, wantType = WantType.WantBlock
) # we want this remote to send us a block
codex_block_exchange_want_block_lists_sent.inc()
proc monitorBlockHandle(
b: BlockExcEngine,
handle: Future[Block],
address: BlockAddress,
peerId: PeerId) {.async.} =
b: BlockExcEngine, handle: Future[Block], address: BlockAddress, peerId: PeerId
) {.async.} =
try:
discard await handle
except CancelledError as exc:
@ -175,12 +179,13 @@ proc monitorBlockHandle(
await b.network.switch.disconnect(peerId)
b.discovery.queueFindBlocksReq(@[address.cidOrTreeCid])
proc pickPseudoRandom(address: BlockAddress, peers: seq[BlockExcPeerCtx]): BlockExcPeerCtx =
proc pickPseudoRandom(
address: BlockAddress, peers: seq[BlockExcPeerCtx]
): BlockExcPeerCtx =
return peers[hash(address) mod peers.len]
proc requestBlock*(
b: BlockExcEngine,
address: BlockAddress,
b: BlockExcEngine, address: BlockAddress
): Future[?!Block] {.async.} =
let blockFuture = b.pendingBlocks.getWantHandle(address, b.blockFetchTimeout)
@ -204,16 +209,12 @@ proc requestBlock*(
except AsyncTimeoutError as err:
failure err
proc requestBlock*(
b: BlockExcEngine,
cid: Cid
): Future[?!Block] =
proc requestBlock*(b: BlockExcEngine, cid: Cid): Future[?!Block] =
b.requestBlock(BlockAddress.init(cid))
proc blockPresenceHandler*(
b: BlockExcEngine,
peer: PeerId,
blocks: seq[BlockPresence]) {.async.} =
b: BlockExcEngine, peer: PeerId, blocks: seq[BlockPresence]
) {.async.} =
let
peerCtx = b.peers.get(peer)
wantList = toSeq(b.pendingBlocks.wantList)
@ -227,17 +228,12 @@ proc blockPresenceHandler*(
let
peerHave = peerCtx.peerHave
dontWantCids = peerHave.filterIt(
it notin wantList
)
dontWantCids = peerHave.filterIt(it notin wantList)
if dontWantCids.len > 0:
peerCtx.cleanPresence(dontWantCids)
let
wantCids = wantList.filterIt(
it in peerHave
)
let wantCids = wantList.filterIt(it in peerHave)
if wantCids.len > 0:
trace "Peer has blocks in our wantList", peer, wants = wantCids
@ -246,13 +242,12 @@ proc blockPresenceHandler*(
# if none of the connected peers report our wants in their have list,
# fire up discovery
b.discovery.queueFindBlocksReq(
toSeq(b.pendingBlocks.wantListCids)
.filter do(cid: Cid) -> bool:
not b.peers.anyIt( cid in it.peerHaveCids ))
toSeq(b.pendingBlocks.wantListCids).filter do(cid: Cid) -> bool:
not b.peers.anyIt(cid in it.peerHaveCids)
)
proc scheduleTasks(b: BlockExcEngine, blocksDelivery: seq[BlockDelivery]) {.async.} =
let
cids = blocksDelivery.mapIt( it.blk.cid )
let cids = blocksDelivery.mapIt(it.blk.cid)
# schedule any new peers to provide blocks to
for p in b.peers:
@ -270,14 +265,16 @@ proc scheduleTasks(b: BlockExcEngine, blocksDelivery: seq[BlockDelivery]) {.asyn
proc cancelBlocks(b: BlockExcEngine, addrs: seq[BlockAddress]) {.async.} =
## Tells neighboring peers that we're no longer interested in a block.
trace "Sending block request cancellations to peers", addrs, peers = b.peers.mapIt($it.id)
trace "Sending block request cancellations to peers",
addrs, peers = b.peers.mapIt($it.id)
let failed = (await allFinished(
let failed = (
await allFinished(
b.peers.mapIt(
b.network.request.sendWantCancellations(
peer = it.id,
addresses = addrs))))
.filterIt(it.failed)
b.network.request.sendWantCancellations(peer = it.id, addresses = addrs)
)
)
).filterIt(it.failed)
if failed.len > 0:
warn "Failed to send block request cancellations to peers", peers = failed.len
@ -290,12 +287,13 @@ proc resolveBlocks*(b: BlockExcEngine, blocksDelivery: seq[BlockDelivery]) {.asy
proc resolveBlocks*(b: BlockExcEngine, blocks: seq[Block]) {.async.} =
await b.resolveBlocks(
blocks.mapIt(
BlockDelivery(blk: it, address: BlockAddress(leaf: false, cid: it.cid)
)))
BlockDelivery(blk: it, address: BlockAddress(leaf: false, cid: it.cid))
)
)
proc payForBlocks(engine: BlockExcEngine,
peer: BlockExcPeerCtx,
blocksDelivery: seq[BlockDelivery]) {.async.} =
proc payForBlocks(
engine: BlockExcEngine, peer: BlockExcPeerCtx, blocksDelivery: seq[BlockDelivery]
) {.async.} =
let
sendPayment = engine.network.request.sendPayment
price = peer.price(blocksDelivery.mapIt(it.address))
@ -304,9 +302,7 @@ proc payForBlocks(engine: BlockExcEngine,
trace "Sending payment for blocks", price, len = blocksDelivery.len
await sendPayment(peer.id, payment)
proc validateBlockDelivery(
b: BlockExcEngine,
bd: BlockDelivery): ?!void =
proc validateBlockDelivery(b: BlockExcEngine, bd: BlockDelivery): ?!void =
if bd.address notin b.pendingBlocks:
return failure("Received block is not currently a pending block")
@ -315,27 +311,30 @@ proc validateBlockDelivery(
return failure("Missing proof")
if proof.index != bd.address.index:
return failure("Proof index " & $proof.index & " doesn't match leaf index " & $bd.address.index)
return failure(
"Proof index " & $proof.index & " doesn't match leaf index " & $bd.address.index
)
without leaf =? bd.blk.cid.mhash.mapFailure, err:
return failure("Unable to get mhash from cid for block, nested err: " & err.msg)
without treeRoot =? bd.address.treeCid.mhash.mapFailure, err:
return failure("Unable to get mhash from treeCid for block, nested err: " & err.msg)
return
failure("Unable to get mhash from treeCid for block, nested err: " & err.msg)
if err =? proof.verify(leaf, treeRoot).errorOption:
return failure("Unable to verify proof for block, nested err: " & err.msg)
else: # not leaf
if bd.address.cid != bd.blk.cid:
return failure("Delivery cid " & $bd.address.cid & " doesn't match block cid " & $bd.blk.cid)
return failure(
"Delivery cid " & $bd.address.cid & " doesn't match block cid " & $bd.blk.cid
)
return success()
proc blocksDeliveryHandler*(
b: BlockExcEngine,
peer: PeerId,
blocksDelivery: seq[BlockDelivery]) {.async.} =
b: BlockExcEngine, peer: PeerId, blocksDelivery: seq[BlockDelivery]
) {.async.} =
trace "Received blocks from peer", peer, blocks = (blocksDelivery.mapIt(it.address))
var validatedBlocksDelivery: seq[BlockDelivery]
@ -356,12 +355,11 @@ proc blocksDeliveryHandler*(
without proof =? bd.proof:
error "Proof expected for a leaf block delivery"
continue
if err =? (await b.localStore.putCidAndProof(
bd.address.treeCid,
bd.address.index,
bd.blk.cid,
proof)).errorOption:
if err =? (
await b.localStore.putCidAndProof(
bd.address.treeCid, bd.address.index, bd.blk.cid, proof
)
).errorOption:
error "Unable to store proof and cid for a block"
continue
@ -370,20 +368,15 @@ proc blocksDeliveryHandler*(
await b.resolveBlocks(validatedBlocksDelivery)
codex_block_exchange_blocks_received.inc(validatedBlocksDelivery.len.int64)
let
peerCtx = b.peers.get(peer)
let peerCtx = b.peers.get(peer)
if peerCtx != nil:
await b.payForBlocks(peerCtx, blocksDelivery)
## shouldn't we remove them from the want-list instead of this:
peerCtx.cleanPresence(blocksDelivery.mapIt(it.address))
proc wantListHandler*(
b: BlockExcEngine,
peer: PeerId,
wantList: WantList) {.async.} =
let
peerCtx = b.peers.get(peer)
proc wantListHandler*(b: BlockExcEngine, peer: PeerId, wantList: WantList) {.async.} =
let peerCtx = b.peers.get(peer)
if peerCtx.isNil:
return
@ -393,8 +386,7 @@ proc wantListHandler*(
schedulePeer = false
for e in wantList.entries:
let
idx = peerCtx.peerWants.findIt(it.address == e.address)
let idx = peerCtx.peerWants.findIt(it.address == e.address)
logScope:
peer = peerCtx.id
@ -404,24 +396,22 @@ proc wantListHandler*(
if idx < 0: # Adding new entry to peer wants
let
have = await e.address in b.localStore
price = @(
b.pricing.get(Pricing(price: 0.u256))
.price.toBytesBE)
price = @(b.pricing.get(Pricing(price: 0.u256)).price.toBytesBE)
if e.wantType == WantType.WantHave:
if have:
presence.add(
BlockPresence(
address: e.address,
`type`: BlockPresenceType.Have,
price: price))
address: e.address, `type`: BlockPresenceType.Have, price: price
)
)
else:
if e.sendDontHave:
presence.add(
BlockPresence(
address: e.address,
`type`: BlockPresenceType.DontHave,
price: price))
address: e.address, `type`: BlockPresenceType.DontHave, price: price
)
)
peerCtx.peerWants.add(e)
codex_block_exchange_want_have_lists_received.inc()
@ -446,31 +436,24 @@ proc wantListHandler*(
if not b.scheduleTask(peerCtx):
warn "Unable to schedule task for peer", peer
proc accountHandler*(
engine: BlockExcEngine,
peer: PeerId,
account: Account) {.async.} =
let
context = engine.peers.get(peer)
proc accountHandler*(engine: BlockExcEngine, peer: PeerId, account: Account) {.async.} =
let context = engine.peers.get(peer)
if context.isNil:
return
context.account = account.some
proc paymentHandler*(
engine: BlockExcEngine,
peer: PeerId,
payment: SignedState) {.async.} =
engine: BlockExcEngine, peer: PeerId, payment: SignedState
) {.async.} =
trace "Handling payments", peer
without context =? engine.peers.get(peer).option and
account =? context.account:
without context =? engine.peers.get(peer).option and account =? context.account:
trace "No context or account for peer", peer
return
if channel =? context.paymentChannel:
let
sender = account.address
let sender = account.address
discard engine.wallet.acceptPayment(channel, Asset, sender, payment)
else:
context.paymentChannel = engine.wallet.acceptChannel(payment).option
@ -484,17 +467,14 @@ proc setupPeer*(b: BlockExcEngine, peer: PeerId) {.async.} =
if peer notin b.peers:
trace "Setting up new peer", peer
b.peers.add(BlockExcPeerCtx(
id: peer
))
b.peers.add(BlockExcPeerCtx(id: peer))
trace "Added peer", peers = b.peers.len
# broadcast our want list, the other peer will do the same
if b.pendingBlocks.wantListLen > 0:
trace "Sending our want list to a peer", peer
let cids = toSeq(b.pendingBlocks.wantList)
await b.network.request.sendWantList(
peer, cids, full = true)
await b.network.request.sendWantList(peer, cids, full = true)
if address =? b.pricing .? address:
await b.network.request.sendAccount(peer, Account(address: address))
@ -515,10 +495,8 @@ proc taskHandler*(b: BlockExcEngine, task: BlockExcPeerCtx) {.gcsafe, async.} =
# TODO: There should be all sorts of accounting of
# bytes sent/received here
var
wantsBlocks = task.peerWants.filterIt(
it.wantType == WantType.WantBlock and not it.inFlight
)
var wantsBlocks =
task.peerWants.filterIt(it.wantType == WantType.WantBlock and not it.inFlight)
proc updateInFlight(addresses: seq[BlockAddress], inFlight: bool) =
for peerWant in task.peerWants.mitems:
@ -535,18 +513,20 @@ proc taskHandler*(b: BlockExcEngine, task: BlockExcPeerCtx) {.gcsafe, async.} =
if e.address.leaf:
(await b.localStore.getBlockAndProof(e.address.treeCid, e.address.index)).map(
(blkAndProof: (Block, CodexProof)) =>
BlockDelivery(address: e.address, blk: blkAndProof[0], proof: blkAndProof[1].some)
BlockDelivery(
address: e.address, blk: blkAndProof[0], proof: blkAndProof[1].some
)
)
else:
(await b.localStore.getBlock(e.address)).map(
(blk: Block) => BlockDelivery(address: e.address, blk: blk, proof: CodexProof.none)
(blk: Block) =>
BlockDelivery(address: e.address, blk: blk, proof: CodexProof.none)
)
let
blocksDeliveryFut = await allFinished(wantsBlocks.map(localLookup))
blocksDelivery = blocksDeliveryFut
.filterIt(it.completed and it.read.isOk)
.mapIt(it.read.get)
blocksDelivery =
blocksDeliveryFut.filterIt(it.completed and it.read.isOk).mapIt(it.read.get)
# All the wants that failed local lookup must be set to not-in-flight again.
let
@ -555,11 +535,9 @@ proc taskHandler*(b: BlockExcEngine, task: BlockExcPeerCtx) {.gcsafe, async.} =
updateInFlight(failedAddresses, false)
if blocksDelivery.len > 0:
trace "Sending blocks to peer", peer = task.id, blocks = (blocksDelivery.mapIt(it.address))
await b.network.request.sendBlocksDelivery(
task.id,
blocksDelivery
)
trace "Sending blocks to peer",
peer = task.id, blocks = (blocksDelivery.mapIt(it.address))
await b.network.request.sendBlocksDelivery(task.id, blocksDelivery)
codex_block_exchange_blocks_sent.inc(blocksDelivery.len.int64)
@ -572,8 +550,7 @@ proc blockexcTaskRunner(b: BlockExcEngine) {.async: (raises: []).} =
trace "Starting blockexc task runner"
while b.blockexcRunning:
try:
let
peerCtx = await b.taskQueue.pop()
let peerCtx = await b.taskQueue.pop()
await b.taskHandler(peerCtx)
except CancelledError:
@ -599,8 +576,7 @@ proc new*(
## Create new block exchange engine instance
##
let
engine = BlockExcEngine(
let engine = BlockExcEngine(
localStore: localStore,
peers: peerStore,
pendingBlocks: pendingBlocks,
@ -612,7 +588,8 @@ proc new*(
taskQueue: newAsyncHeapQueue[BlockExcPeerCtx](DefaultTaskQueueSize),
discovery: discovery,
advertiser: advertiser,
blockFetchTimeout: blockFetchTimeout)
blockFetchTimeout: blockFetchTimeout,
)
proc peerEventHandler(peerId: PeerId, event: PeerEvent) {.async.} =
if event.kind == PeerEventKind.Joined:
@ -624,19 +601,17 @@ proc new*(
network.switch.addPeerEventHandler(peerEventHandler, PeerEventKind.Joined)
network.switch.addPeerEventHandler(peerEventHandler, PeerEventKind.Left)
proc blockWantListHandler(
peer: PeerId,
wantList: WantList): Future[void] {.gcsafe.} =
proc blockWantListHandler(peer: PeerId, wantList: WantList): Future[void] {.gcsafe.} =
engine.wantListHandler(peer, wantList)
proc blockPresenceHandler(
peer: PeerId,
presence: seq[BlockPresence]): Future[void] {.gcsafe.} =
peer: PeerId, presence: seq[BlockPresence]
): Future[void] {.gcsafe.} =
engine.blockPresenceHandler(peer, presence)
proc blocksDeliveryHandler(
peer: PeerId,
blocksDelivery: seq[BlockDelivery]): Future[void] {.gcsafe.} =
peer: PeerId, blocksDelivery: seq[BlockDelivery]
): Future[void] {.gcsafe.} =
engine.blocksDeliveryHandler(peer, blocksDelivery)
proc accountHandler(peer: PeerId, account: Account): Future[void] {.gcsafe.} =
@ -650,6 +625,7 @@ proc new*(
onBlocksDelivery: blocksDeliveryHandler,
onPresence: blockPresenceHandler,
onAccount: accountHandler,
onPayment: paymentHandler)
onPayment: paymentHandler,
)
return engine

13
codex/blockexchange/engine/payments.nim
View File

@ -15,15 +15,16 @@ import ../peers
export nitro
export results
push: {.upraises: [].}
push:
{.upraises: [].}
const ChainId* = 0.u256 # invalid chain id for now
const Asset* = EthAddress.zero # invalid ERC20 asset address for now
const AmountPerChannel = (10'u64 ^ 18).u256 # 1 asset, ERC20 default is 18 decimals
func openLedgerChannel*(wallet: WalletRef,
hub: EthAddress,
asset: EthAddress): ?!ChannelId =
func openLedgerChannel*(
wallet: WalletRef, hub: EthAddress, asset: EthAddress
): ?!ChannelId =
wallet.openLedgerChannel(hub, ChainId, asset, AmountPerChannel)
func getOrOpenChannel(wallet: WalletRef, peer: BlockExcPeerCtx): ?!ChannelId =
@ -36,9 +37,7 @@ func getOrOpenChannel(wallet: WalletRef, peer: BlockExcPeerCtx): ?!ChannelId =
else:
failure "no account set for peer"
func pay*(wallet: WalletRef,
peer: BlockExcPeerCtx,
amount: UInt256): ?!SignedState =
func pay*(wallet: WalletRef, peer: BlockExcPeerCtx, amount: UInt256): ?!SignedState =
if account =? peer.account:
let asset = Asset
let receiver = account.address

40
codex/blockexchange/engine/pendingblocks.nim
View File

@ -12,7 +12,8 @@ import std/monotimes
import pkg/upraises
push: {.upraises: [].}
push:
{.upraises: [].}
import pkg/chronos
import pkg/libp2p
@ -25,11 +26,15 @@ import ../../logutils
logScope:
topics = "codex pendingblocks"
declareGauge(codex_block_exchange_pending_block_requests, "codex blockexchange pending block requests")
declareGauge(codex_block_exchange_retrieval_time_us, "codex blockexchange block retrieval time us")
declareGauge(
codex_block_exchange_pending_block_requests,
"codex blockexchange pending block requests",
)
declareGauge(
codex_block_exchange_retrieval_time_us, "codex blockexchange block retrieval time us"
)
const
DefaultBlockTimeout* = 10.minutes
const DefaultBlockTimeout* = 10.minutes
type
BlockReq* = object
@ -47,7 +52,8 @@ proc getWantHandle*(
p: PendingBlocksManager,
address: BlockAddress,
timeout = DefaultBlockTimeout,
inFlight = false): Future[Block] {.async.} =
inFlight = false,
): Future[Block] {.async.} =
## Add an event for a block
##
@ -56,7 +62,8 @@ proc getWantHandle*(
p.blocks[address] = BlockReq(
handle: newFuture[Block]("pendingBlocks.getWantHandle"),
inFlight: inFlight,
startTime: getMonoTime().ticks)
startTime: getMonoTime().ticks,
)
p.updatePendingBlockGauge()
return await p.blocks[address].handle.wait(timeout)
@ -72,15 +79,13 @@ proc getWantHandle*(
p.updatePendingBlockGauge()
proc getWantHandle*(
p: PendingBlocksManager,
cid: Cid,
timeout = DefaultBlockTimeout,
inFlight = false): Future[Block] =
p: PendingBlocksManager, cid: Cid, timeout = DefaultBlockTimeout, inFlight = false
): Future[Block] =
p.getWantHandle(BlockAddress.init(cid), timeout, inFlight)
proc resolve*(
p: PendingBlocksManager,
blocksDelivery: seq[BlockDelivery]) {.gcsafe, raises: [].} =
p: PendingBlocksManager, blocksDelivery: seq[BlockDelivery]
) {.gcsafe, raises: [].} =
## Resolve pending blocks
##
@ -101,19 +106,14 @@ proc resolve*(
else:
trace "Block handle already finished", address = bd.address
proc setInFlight*(
p: PendingBlocksManager,
address: BlockAddress,
inFlight = true) =
proc setInFlight*(p: PendingBlocksManager, address: BlockAddress, inFlight = true) =
## Set inflight status for a block
##
p.blocks.withValue(address, pending):
pending[].inFlight = inFlight
proc isInFlight*(
p: PendingBlocksManager,
address: BlockAddress): bool =
proc isInFlight*(p: PendingBlocksManager, address: BlockAddress): bool =
## Check if a block is in flight
##

113
codex/blockexchange/network/network.nim
View File

@ -35,8 +35,10 @@ const
type
WantListHandler* = proc(peer: PeerId, wantList: WantList): Future[void] {.gcsafe.}
BlocksDeliveryHandler* = proc(peer: PeerId, blocks: seq[BlockDelivery]): Future[void] {.gcsafe.}
BlockPresenceHandler* = proc(peer: PeerId, precense: seq[BlockPresence]): Future[void] {.gcsafe.}
BlocksDeliveryHandler* =
proc(peer: PeerId, blocks: seq[BlockDelivery]): Future[void] {.gcsafe.}
BlockPresenceHandler* =
proc(peer: PeerId, precense: seq[BlockPresence]): Future[void] {.gcsafe.}
AccountHandler* = proc(peer: PeerId, account: Account): Future[void] {.gcsafe.}
PaymentHandler* = proc(peer: PeerId, payment: SignedState): Future[void] {.gcsafe.}
@ -54,10 +56,14 @@ type
cancel: bool = false,
wantType: WantType = WantType.WantHave,
full: bool = false,
sendDontHave: bool = false): Future[void] {.gcsafe.}
WantCancellationSender* = proc(peer: PeerId, addresses: seq[BlockAddress]): Future[void] {.gcsafe.}
BlocksDeliverySender* = proc(peer: PeerId, blocksDelivery: seq[BlockDelivery]): Future[void] {.gcsafe.}
PresenceSender* = proc(peer: PeerId, presence: seq[BlockPresence]): Future[void] {.gcsafe.}
sendDontHave: bool = false,
): Future[void] {.gcsafe.}
WantCancellationSender* =
proc(peer: PeerId, addresses: seq[BlockAddress]): Future[void] {.gcsafe.}
BlocksDeliverySender* =
proc(peer: PeerId, blocksDelivery: seq[BlockDelivery]): Future[void] {.gcsafe.}
PresenceSender* =
proc(peer: PeerId, presence: seq[BlockPresence]): Future[void] {.gcsafe.}
AccountSender* = proc(peer: PeerId, account: Account): Future[void] {.gcsafe.}
PaymentSender* = proc(peer: PeerId, payment: SignedState): Future[void] {.gcsafe.}
@ -108,10 +114,7 @@ proc send*(b: BlockExcNetwork, id: PeerId, msg: pb.Message) {.async.} =
finally:
b.inflightSema.release()
proc handleWantList(
b: BlockExcNetwork,
peer: NetworkPeer,
list: WantList) {.async.} =
proc handleWantList(b: BlockExcNetwork, peer: NetworkPeer, list: WantList) {.async.} =
## Handle incoming want list
##
@ -126,7 +129,8 @@ proc sendWantList*(
cancel: bool = false,
wantType: WantType = WantType.WantHave,
full: bool = false,
sendDontHave: bool = false): Future[void] =
sendDontHave: bool = false,
): Future[void] =
## Send a want message to peer
##
@ -137,43 +141,41 @@ proc sendWantList*(
priority: priority,
cancel: cancel,
wantType: wantType,
sendDontHave: sendDontHave) ),
full: full)
sendDontHave: sendDontHave,
)
),
full: full,
)
b.send(id, Message(wantlist: msg))
proc sendWantCancellations*(
b: BlockExcNetwork,
id: PeerId,
addresses: seq[BlockAddress]): Future[void] {.async.} =
b: BlockExcNetwork, id: PeerId, addresses: seq[BlockAddress]
): Future[void] {.async.} =
## Informs a remote peer that we're no longer interested in a set of blocks
##
await b.sendWantList(id = id, addresses = addresses, cancel = true)
proc handleBlocksDelivery(
b: BlockExcNetwork,
peer: NetworkPeer,
blocksDelivery: seq[BlockDelivery]) {.async.} =
b: BlockExcNetwork, peer: NetworkPeer, blocksDelivery: seq[BlockDelivery]
) {.async.} =
## Handle incoming blocks
##
if not b.handlers.onBlocksDelivery.isNil:
await b.handlers.onBlocksDelivery(peer.id, blocksDelivery)
proc sendBlocksDelivery*(
b: BlockExcNetwork,
id: PeerId,
blocksDelivery: seq[BlockDelivery]): Future[void] =
b: BlockExcNetwork, id: PeerId, blocksDelivery: seq[BlockDelivery]
): Future[void] =
## Send blocks to remote
##
b.send(id, pb.Message(payload: blocksDelivery))
proc handleBlockPresence(
b: BlockExcNetwork,
peer: NetworkPeer,
presence: seq[BlockPresence]) {.async.} =
b: BlockExcNetwork, peer: NetworkPeer, presence: seq[BlockPresence]
) {.async.} =
## Handle block presence
##
@ -181,56 +183,44 @@ proc handleBlockPresence(
await b.handlers.onPresence(peer.id, presence)
proc sendBlockPresence*(
b: BlockExcNetwork,
id: PeerId,
presence: seq[BlockPresence]): Future[void] =
b: BlockExcNetwork, id: PeerId, presence: seq[BlockPresence]
): Future[void] =
## Send presence to remote
##
b.send(id, Message(blockPresences: @presence))
proc handleAccount(
network: BlockExcNetwork,
peer: NetworkPeer,
account: Account) {.async.} =
network: BlockExcNetwork, peer: NetworkPeer, account: Account
) {.async.} =
## Handle account info
##
if not network.handlers.onAccount.isNil:
await network.handlers.onAccount(peer.id, account)
proc sendAccount*(
b: BlockExcNetwork,
id: PeerId,
account: Account): Future[void] =
proc sendAccount*(b: BlockExcNetwork, id: PeerId, account: Account): Future[void] =
## Send account info to remote
##
b.send(id, Message(account: AccountMessage.init(account)))
proc sendPayment*(
b: BlockExcNetwork,
id: PeerId,
payment: SignedState): Future[void] =
proc sendPayment*(b: BlockExcNetwork, id: PeerId, payment: SignedState): Future[void] =
## Send payment to remote
##
b.send(id, Message(payment: StateChannelUpdate.init(payment)))
proc handlePayment(
network: BlockExcNetwork,
peer: NetworkPeer,
payment: SignedState) {.async.} =
network: BlockExcNetwork, peer: NetworkPeer, payment: SignedState
) {.async.} =
## Handle payment
##
if not network.handlers.onPayment.isNil:
await network.handlers.onPayment(peer.id, payment)
proc rpcHandler(
b: BlockExcNetwork,
peer: NetworkPeer,
msg: Message) {.raises: [].} =
proc rpcHandler(b: BlockExcNetwork, peer: NetworkPeer, msg: Message) {.raises: [].} =
## handle rpc messages
##
if msg.wantList.entries.len > 0:
@ -325,15 +315,14 @@ proc new*(
T: type BlockExcNetwork,
switch: Switch,
connProvider: ConnProvider = nil,
maxInflight = MaxInflight): BlockExcNetwork =
maxInflight = MaxInflight,
): BlockExcNetwork =
## Create a new BlockExcNetwork instance
##
let
self = BlockExcNetwork(
switch: switch,
getConn: connProvider,
inflightSema: newAsyncSemaphore(maxInflight))
let self = BlockExcNetwork(
switch: switch, getConn: connProvider, inflightSema: newAsyncSemaphore(maxInflight)
)
proc sendWantList(
id: PeerId,
@ -342,15 +331,18 @@ proc new*(
cancel: bool = false,
wantType: WantType = WantType.WantHave,
full: bool = false,
sendDontHave: bool = false): Future[void] {.gcsafe.} =
self.sendWantList(
id, cids, priority, cancel,
wantType, full, sendDontHave)
sendDontHave: bool = false,
): Future[void] {.gcsafe.} =
self.sendWantList(id, cids, priority, cancel, wantType, full, sendDontHave)
proc sendWantCancellations(id: PeerId, addresses: seq[BlockAddress]): Future[void] {.gcsafe.} =
proc sendWantCancellations(
id: PeerId, addresses: seq[BlockAddress]
): Future[void] {.gcsafe.} =
self.sendWantCancellations(id, addresses)
proc sendBlocksDelivery(id: PeerId, blocksDelivery: seq[BlockDelivery]): Future[void] {.gcsafe.} =
proc sendBlocksDelivery(
id: PeerId, blocksDelivery: seq[BlockDelivery]
): Future[void] {.gcsafe.} =
self.sendBlocksDelivery(id, blocksDelivery)
proc sendPresence(id: PeerId, presence: seq[BlockPresence]): Future[void] {.gcsafe.} =
@ -368,7 +360,8 @@ proc new*(
sendBlocksDelivery: sendBlocksDelivery,
sendPresence: sendPresence,
sendAccount: sendAccount,
sendPayment: sendPayment)
sendPayment: sendPayment,
)
self.init()
return self

18
codex/blockexchange/network/networkpeer.nim
View File

@ -8,7 +8,8 @@
## those terms.
import pkg/upraises
push: {.upraises: [].}
push:
{.upraises: [].}
import pkg/chronos
import pkg/libp2p
@ -33,8 +34,7 @@ type
getConn: ConnProvider
proc connected*(b: NetworkPeer): bool =
not(isNil(b.sendConn)) and
not(b.sendConn.closed or b.sendConn.atEof)
not (isNil(b.sendConn)) and not (b.sendConn.closed or b.sendConn.atEof)
proc readLoop*(b: NetworkPeer, conn: Connection) {.async.} =
if isNil(conn):
@ -83,12 +83,8 @@ func new*(
T: type NetworkPeer,
peer: PeerId,
connProvider: ConnProvider,
rpcHandler: RPCHandler): NetworkPeer =
rpcHandler: RPCHandler,
): NetworkPeer =
doAssert(not isNil(connProvider), "should supply connection provider")
doAssert(not isNil(connProvider),
"should supply connection provider")
NetworkPeer(
id: peer,
getConn: connProvider,
handler: rpcHandler)
NetworkPeer(id: peer, getConn: connProvider, handler: rpcHandler)

3
codex/blockexchange/peers/peercontext.nim
View File

@ -25,8 +25,7 @@ import ../../logutils
export payments, nitro
type
BlockExcPeerCtx* = ref object of RootObj
type BlockExcPeerCtx* = ref object of RootObj
id*: PeerId
blocks*: Table[BlockAddress, Presence] # remote peer have list including price
peerWants*: seq[WantListEntry] # remote peers want lists

5
codex/blockexchange/peers/peerctxstore.nim
View File

@ -13,7 +13,8 @@ import std/algorithm
import pkg/upraises
push: {.upraises: [].}
push:
{.upraises: [].}
import pkg/chronos
import pkg/libp2p
@ -22,7 +23,6 @@ import ../protobuf/blockexc
import ../../blocktype
import ../../logutils
import ./peercontext
export peercontext
@ -32,6 +32,7 @@ logScope:
type
PeerCtxStore* = ref object of RootObj
peers*: OrderedTable[PeerId, BlockExcPeerCtx]
PeersForBlock* = object of RootObj
with*: seq[BlockExcPeerCtx]
without*: seq[BlockExcPeerCtx]

1
codex/blockexchange/protobuf/blockexc.nim
View File

@ -42,7 +42,6 @@ proc `==`*(a: WantListEntry, b: BlockAddress): bool =
proc `<`*(a, b: WantListEntry): bool =
a.priority < b.priority
proc `==`*(a: BlockPresence, b: BlockAddress): bool =
return a.address == b

22
codex/blockexchange/protobuf/message.nim
View File

@ -20,7 +20,7 @@ const
type
WantType* = enum
WantBlock = 0,
WantBlock = 0
WantHave = 1
WantListEntry* = object
@ -41,7 +41,7 @@ type
proof*: ?CodexProof # Present only if `address.leaf` is true
BlockPresenceType* = enum
Have = 0,
Have = 0
DontHave = 1
BlockPresence* = object
@ -140,7 +140,6 @@ proc protobufEncode*(value: Message): seq[byte] =
ipb.finish()
ipb.buffer
#
# Decoding Message from seq[byte] in Protobuf format
#
@ -211,7 +210,8 @@ proc decode*(_: type BlockDelivery, pb: ProtoBuffer): ProtoResult[BlockDelivery]
if ?pb.getField(1, cidBuf):
cid = ?Cid.init(cidBuf).mapErr(x => ProtoError.IncorrectBlob)
if ?pb.getField(2, dataBuf):
value.blk = ? Block.new(cid, dataBuf, verify = true).mapErr(x => ProtoError.IncorrectBlob)
value.blk =
?Block.new(cid, dataBuf, verify = true).mapErr(x => ProtoError.IncorrectBlob)
if ?pb.getField(3, ipb):
value.address = ?BlockAddress.decode(ipb)
@ -240,14 +240,14 @@ proc decode*(_: type BlockPresence, pb: ProtoBuffer): ProtoResult[BlockPresence]
ok(value)
proc decode*(_: type AccountMessage, pb: ProtoBuffer): ProtoResult[AccountMessage] =
var
value = AccountMessage()
var value = AccountMessage()
discard ?pb.getField(1, value.address)
ok(value)
proc decode*(_: type StateChannelUpdate, pb: ProtoBuffer): ProtoResult[StateChannelUpdate] =
var
value = StateChannelUpdate()
proc decode*(
_: type StateChannelUpdate, pb: ProtoBuffer
): ProtoResult[StateChannelUpdate] =
var value = StateChannelUpdate()
discard ?pb.getField(1, value.update)
ok(value)
@ -261,7 +261,9 @@ proc protobufDecode*(_: type Message, msg: seq[byte]): ProtoResult[Message] =
value.wantList = ?WantList.decode(ipb)
if ?pb.getRepeatedField(3, sublist):
for item in sublist:
value.payload.add(? BlockDelivery.decode(initProtoBuffer(item, maxSize = MaxBlockSize)))
value.payload.add(
?BlockDelivery.decode(initProtoBuffer(item, maxSize = MaxBlockSize))
)
if ?pb.getRepeatedField(4, sublist):
for item in sublist:
value.blockPresences.add(?BlockPresence.decode(initProtoBuffer(item)))

6
codex/blockexchange/protobuf/payments.nim
View File

@ -11,10 +11,10 @@ export StateChannelUpdate
export stint
export nitro
push: {.upraises: [].}
push:
{.upraises: [].}
type
Account* = object
type Account* = object
address*: EthAddress
func init*(_: type AccountMessage, account: Account): AccountMessage =

12
codex/blockexchange/protobuf/presence.nim
View File

@ -11,7 +11,8 @@ export questionable
export stint
export BlockPresenceType
upraises.push: {.upraises: [].}
upraises.push:
{.upraises: [].}
type
PresenceMessage* = blockexc.BlockPresence
@ -32,15 +33,12 @@ func init*(_: type Presence, message: PresenceMessage): ?Presence =
some Presence(
address: message.address,
have: message.`type` == BlockPresenceType.Have,
price: price
price: price,
)
func init*(_: type PresenceMessage, presence: Presence): PresenceMessage =
PresenceMessage(
address: presence.address,
`type`: if presence.have:
BlockPresenceType.Have
else:
BlockPresenceType.DontHave,
price: @(presence.price.toBytesBE)
`type`: if presence.have: BlockPresenceType.Have else: BlockPresenceType.DontHave,
price: @(presence.price.toBytesBE),
)

48
codex/blocktype.nim
View File

@ -14,7 +14,8 @@ export tables
import pkg/upraises
push: {.upraises: [].}
push:
{.upraises: [].}
import pkg/libp2p/[cid, multicodec, multihash]
import pkg/stew/byteutils
@ -49,11 +50,11 @@ logutils.formatIt(LogFormat.textLines, BlockAddress):
else:
"cid: " & shortLog($it.cid)
logutils.formatIt(LogFormat.json, BlockAddress): %it
logutils.formatIt(LogFormat.json, BlockAddress):
%it
proc `==`*(a, b: BlockAddress): bool =
a.leaf == b.leaf and
(
a.leaf == b.leaf and (
if a.leaf:
a.treeCid == b.treeCid and a.index == b.index
else:
@ -67,10 +68,7 @@ proc `$`*(a: BlockAddress): string =
"cid: " & $a.cid
proc cidOrTreeCid*(a: BlockAddress): Cid =
if a.leaf:
a.treeCid
else:
a.cid
if a.leaf: a.treeCid else: a.cid
proc address*(b: Block): BlockAddress =
BlockAddress(leaf: false, cid: b.cid)
@ -90,7 +88,8 @@ func new*(
data: openArray[byte] = [],
version = CIDv1,
mcodec = Sha256HashCodec,
codec = BlockCodec): ?!Block =
codec = BlockCodec,
): ?!Block =
## creates a new block for both storage and network IO
##
@ -100,15 +99,11 @@ func new*(
# TODO: If the hash is `>=` to the data,
# use the Cid as a container!
Block(
cid: cid,
data: @data).success
Block(cid: cid, data: @data).success
proc new*(
T: type Block,
cid: Cid,
data: openArray[byte],
verify: bool = true
T: type Block, cid: Cid, data: openArray[byte], verify: bool = true
): ?!Block =
## creates a new block for both storage and network IO
##
@ -121,22 +116,23 @@ proc new*(
if computedCid != cid:
return "Cid doesn't match the data".failure
return Block(
cid: cid,
data: @data
).success
return Block(cid: cid, data: @data).success
proc emptyBlock*(version: CidVersion, hcodec: MultiCodec): ?!Block =
emptyCid(version, hcodec, BlockCodec)
.flatMap((cid: Cid) => Block.new(cid = cid, data = @[]))
emptyCid(version, hcodec, BlockCodec).flatMap(
(cid: Cid) => Block.new(cid = cid, data = @[])
)
proc emptyBlock*(cid: Cid): ?!Block =
cid.mhash.mapFailure.flatMap((mhash: MultiHash) =>
emptyBlock(cid.cidver, mhash.mcodec))
cid.mhash.mapFailure.flatMap(
(mhash: MultiHash) => emptyBlock(cid.cidver, mhash.mcodec)
)
proc isEmpty*(cid: Cid): bool =
success(cid) == cid.mhash.mapFailure.flatMap((mhash: MultiHash) =>
emptyCid(cid.cidver, mhash.mcodec, cid.mcodec))
success(cid) ==
cid.mhash.mapFailure.flatMap(
(mhash: MultiHash) => emptyCid(cid.cidver, mhash.mcodec, cid.mcodec)
)
proc isEmpty*(blk: Block): bool =
blk.cid.isEmpty

47
codex/chunker.nim
View File

@ -11,7 +11,8 @@
import pkg/upraises
push: {.upraises: [].}
push:
{.upraises: [].}
import pkg/questionable
import pkg/questionable/results
@ -23,8 +24,7 @@ import ./logutils
export blocktype
const
DefaultChunkSize* = DefaultBlockSize
const DefaultChunkSize* = DefaultBlockSize
type
# default reader type
@ -60,30 +60,21 @@ proc getBytes*(c: Chunker): Future[seq[byte]] {.async.} =
return move buff
proc new*(
T: type Chunker,
reader: Reader,
chunkSize = DefaultChunkSize,
pad = true
T: type Chunker, reader: Reader, chunkSize = DefaultChunkSize, pad = true
): Chunker =
## create a new Chunker instance
##
Chunker(
reader: reader,
offset: 0,
chunkSize: chunkSize,
pad: pad)
Chunker(reader: reader, offset: 0, chunkSize: chunkSize, pad: pad)
proc new*(
T: type LPStreamChunker,
stream: LPStream,
chunkSize = DefaultChunkSize,
pad = true
T: type LPStreamChunker, stream: LPStream, chunkSize = DefaultChunkSize, pad = true
): LPStreamChunker =
## create the default File chunker
##
proc reader(data: ChunkBuffer, len: int): Future[int]
{.gcsafe, async, raises: [Defect].} =
proc reader(
data: ChunkBuffer, len: int
): Future[int] {.gcsafe, async, raises: [Defect].} =
var res = 0
try:
while res < len:
@ -101,22 +92,17 @@ proc new*(
return res
LPStreamChunker.new(
reader = reader,
chunkSize = chunkSize,
pad = pad)
LPStreamChunker.new(reader = reader, chunkSize = chunkSize, pad = pad)
proc new*(
T: type FileChunker,
file: File,
chunkSize = DefaultChunkSize,
pad = true
T: type FileChunker, file: File, chunkSize = DefaultChunkSize, pad = true
): FileChunker =
## create the default File chunker
##
proc reader(data: ChunkBuffer, len: int): Future[int]
{.gcsafe, async, raises: [Defect].} =
proc reader(
data: ChunkBuffer, len: int
): Future[int] {.gcsafe, async, raises: [Defect].} =
var total = 0
try:
while total < len:
@ -135,7 +121,4 @@ proc new*(
return total
FileChunker.new(
reader = reader,
chunkSize = chunkSize,
pad = pad)
FileChunker.new(reader = reader, chunkSize = chunkSize, pad = pad)

6
codex/clock.nim
View File

@ -20,9 +20,9 @@ method start*(clock: Clock) {.base, async.} =
method stop*(clock: Clock) {.base, async.} =
discard
proc withTimeout*(future: Future[void],
clock: Clock,
expiry: SecondsSince1970) {.async.} =
proc withTimeout*(
future: Future[void], clock: Clock, expiry: SecondsSince1970
) {.async.} =
let timeout = clock.waitUntil(expiry)
try:
await future or timeout

111
codex/codex.nim
View File

@ -68,8 +68,7 @@ proc waitForSync(provider: Provider): Future[void] {.async.} =
inc sleepTime
trace "Ethereum provider is synced."
proc bootstrapInteractions(
s: CodexServer): Future[void] {.async.} =
proc bootstrapInteractions(s: CodexServer): Future[void] {.async.} =
## bootstrap interactions and return contracts
## using clients, hosts, validators pairings
##
@ -137,10 +136,10 @@ proc bootstrapInteractions(
host = some HostInteractions.new(clock, sales)
if config.validator:
without validationConfig =? ValidationConfig.init(
config.validatorMaxSlots,
config.validatorGroups,
config.validatorGroupIndex), err:
without validationConfig =?
ValidationConfig.init(
config.validatorMaxSlots, config.validatorGroups, config.validatorGroupIndex
), err:
error "Invalid validation parameters", err = err.msg
quit QuitFailure
let validation = Validation.new(clock, market, validationConfig)
@ -157,9 +156,8 @@ proc start*(s: CodexServer) {.async.} =
await s.codexNode.switch.start()
let (announceAddrs, discoveryAddrs) = nattedAddress(
s.config.nat,
s.codexNode.switch.peerInfo.addrs,
s.config.discoveryPort)
s.config.nat, s.codexNode.switch.peerInfo.addrs, s.config.discoveryPort
)
s.codexNode.discovery.updateAnnounceRecord(announceAddrs)
s.codexNode.discovery.updateDhtRecord(discoveryAddrs)
@ -176,15 +174,14 @@ proc stop*(s: CodexServer) {.async.} =
s.codexNode.switch.stop(),
s.codexNode.stop(),
s.repoStore.stop(),
s.maintenance.stop())
s.maintenance.stop(),
)
proc new*(
T: type CodexServer,
config: CodexConf,
privateKey: CodexPrivateKey): CodexServer =
T: type CodexServer, config: CodexConf, privateKey: CodexPrivateKey
): CodexServer =
## create CodexServer including setting up datastore, repostore, etc
let
switch = SwitchBuilder
let switch = SwitchBuilder
.new()
.withPrivateKey(privateKey)
.withAddresses(config.listenAddrs)
@ -197,64 +194,88 @@ proc new*(
.withTcpTransport({ServerFlags.ReuseAddr})
.build()
var
cache: CacheStore = nil
var cache: CacheStore = nil
if config.cacheSize > 0'nb:
cache = CacheStore.new(cacheSize = config.cacheSize)
## Is unused?
let
discoveryDir = config.dataDir / CodexDhtNamespace
let discoveryDir = config.dataDir / CodexDhtNamespace
if io2.createPath(discoveryDir).isErr:
trace "Unable to create discovery directory for block store", discoveryDir = discoveryDir
trace "Unable to create discovery directory for block store",
discoveryDir = discoveryDir
raise (ref Defect)(
msg: "Unable to create discovery directory for block store: " & discoveryDir)
msg: "Unable to create discovery directory for block store: " & discoveryDir
)
let
discoveryStore = Datastore(
LevelDbDatastore.new(config.dataDir / CodexDhtProvidersNamespace)
.expect("Should create discovery datastore!"))
LevelDbDatastore.new(config.dataDir / CodexDhtProvidersNamespace).expect(
"Should create discovery datastore!"
)
)
discovery = Discovery.new(
switch.peerInfo.privateKey,
announceAddrs = config.listenAddrs,
bindPort = config.discoveryPort,
bootstrapNodes = config.bootstrapNodes,
store = discoveryStore)
store = discoveryStore,
)
wallet = WalletRef.new(EthPrivateKey.random())
network = BlockExcNetwork.new(switch)
repoData = case config.repoKind
of repoFS: Datastore(FSDatastore.new($config.dataDir, depth = 5)
.expect("Should create repo file data store!"))
of repoSQLite: Datastore(SQLiteDatastore.new($config.dataDir)
.expect("Should create repo SQLite data store!"))
of repoLevelDb: Datastore(LevelDbDatastore.new($config.dataDir)
.expect("Should create repo LevelDB data store!"))
repoData =
case config.repoKind
of repoFS:
Datastore(
FSDatastore.new($config.dataDir, depth = 5).expect(
"Should create repo file data store!"
)
)
of repoSQLite:
Datastore(
SQLiteDatastore.new($config.dataDir).expect(
"Should create repo SQLite data store!"
)
)
of repoLevelDb:
Datastore(
LevelDbDatastore.new($config.dataDir).expect(
"Should create repo LevelDB data store!"
)
)
repoStore = RepoStore.new(
repoDs = repoData,
metaDs = LevelDbDatastore.new(config.dataDir / CodexMetaNamespace)
.expect("Should create metadata store!"),
metaDs = LevelDbDatastore.new(config.dataDir / CodexMetaNamespace).expect(
"Should create metadata store!"
),
quotaMaxBytes = config.storageQuota,
blockTtl = config.blockTtl)
blockTtl = config.blockTtl,
)
maintenance = BlockMaintainer.new(
repoStore,
interval = config.blockMaintenanceInterval,
numberOfBlocksPerInterval = config.blockMaintenanceNumberOfBlocks)
numberOfBlocksPerInterval = config.blockMaintenanceNumberOfBlocks,
)
peerStore = PeerCtxStore.new()
pendingBlocks = PendingBlocksManager.new()
advertiser = Advertiser.new(repoStore, discovery)
blockDiscovery = DiscoveryEngine.new(repoStore, peerStore, network, discovery, pendingBlocks)
engine = BlockExcEngine.new(repoStore, wallet, network, blockDiscovery, advertiser, peerStore, pendingBlocks)
blockDiscovery =
DiscoveryEngine.new(repoStore, peerStore, network, discovery, pendingBlocks)
engine = BlockExcEngine.new(
repoStore, wallet, network, blockDiscovery, advertiser, peerStore, pendingBlocks
)
store = NetworkStore.new(engine, repoStore)
prover = if config.prover:
let backend = config.initializeBackend().expect("Unable to create prover backend.")
prover =
if config.prover:
let backend =
config.initializeBackend().expect("Unable to create prover backend.")
some Prover.new(store, backend, config.numProofSamples)
else:
none Prover
@ -264,13 +285,16 @@ proc new*(
networkStore = store,
engine = engine,
discovery = discovery,
prover = prover)
prover = prover,
)
restServer = RestServerRef.new(
restServer = RestServerRef
.new(
codexNode.initRestApi(config, repoStore, config.apiCorsAllowedOrigin),
initTAddress(config.apiBindAddress, config.apiPort),
bufferSize = (1024 * 64),
maxRequestBodySize = int.high)
maxRequestBodySize = int.high,
)
.expect("Should start rest server!")
switch.mount(network)
@ -280,4 +304,5 @@ proc new*(
codexNode: codexNode,
restServer: restServer,
repoStore: repoStore,
maintenance: maintenance)
maintenance: maintenance,
)

32
codex/codextypes.nim
View File

@ -48,18 +48,10 @@ const
SlotProvingRootCodec* = multiCodec("codex-proving-root")
CodexSlotCellCodec* = multiCodec("codex-slot-cell")
CodexHashesCodecs* = [
Sha256HashCodec,
Pos2Bn128SpngCodec,
Pos2Bn128MrklCodec
]
CodexHashesCodecs* = [Sha256HashCodec, Pos2Bn128SpngCodec, Pos2Bn128MrklCodec]
CodexPrimitivesCodecs* = [
ManifestCodec,
DatasetRootCodec,
BlockCodec,
SlotRootCodec,
SlotProvingRootCodec,
ManifestCodec, DatasetRootCodec, BlockCodec, SlotRootCodec, SlotProvingRootCodec,
CodexSlotCellCodec,
]
@ -78,24 +70,19 @@ proc initEmptyCidTable(): ?!Table[(CidVersion, MultiCodec, MultiCodec), Cid] =
Sha512HashCodec: ?MultiHash.digest($Sha512HashCodec, emptyData).mapFailure,
}.toTable
var
table = initTable[(CidVersion, MultiCodec, MultiCodec), Cid]()
var table = initTable[(CidVersion, MultiCodec, MultiCodec), Cid]()
for hcodec, mhash in PadHashes.pairs:
table[(CIDv1, hcodec, BlockCodec)] = ?Cid.init(CIDv1, BlockCodec, mhash).mapFailure
success table
proc emptyCid*(
version: CidVersion,
hcodec: MultiCodec,
dcodec: MultiCodec): ?!Cid =
proc emptyCid*(version: CidVersion, hcodec: MultiCodec, dcodec: MultiCodec): ?!Cid =
## Returns cid representing empty content,
## given cid version, hash codec and data codec
##
var
table {.global, threadvar.}: Table[(CidVersion, MultiCodec, MultiCodec), Cid]
var table {.global, threadvar.}: Table[(CidVersion, MultiCodec, MultiCodec), Cid]
once:
table = ?initEmptyCidTable()
@ -103,11 +90,10 @@ proc emptyCid*(
table[(version, hcodec, dcodec)].catch
proc emptyDigest*(
version: CidVersion,
hcodec: MultiCodec,
dcodec: MultiCodec): ?!MultiHash =
version: CidVersion, hcodec: MultiCodec, dcodec: MultiCodec
): ?!MultiHash =
## Returns hash representing empty content,
## given cid version, hash codec and data codec
##
emptyCid(version, hcodec, dcodec)
.flatMap((cid: Cid) => cid.mhash.mapFailure)
emptyCid(version, hcodec, dcodec).flatMap((cid: Cid) => cid.mhash.mapFailure)

484
codex/conf.nim
View File

@ -50,13 +50,12 @@ export units, net, codextypes, logutils, completeCmdArg, parseCmdArg, NatConfig
export ValidationGroups, MaxSlots
export
DefaultQuotaBytes,
DefaultBlockTtl,
DefaultBlockMaintenanceInterval,
DefaultQuotaBytes, DefaultBlockTtl, DefaultBlockMaintenanceInterval,
DefaultNumberOfBlocksToMaintainPerInterval
proc defaultDataDir*(): string =
let dataDir = when defined(windows):
let dataDir =
when defined(windows):
"AppData" / "Roaming" / "Codex"
elif defined(macosx):
"Library" / "Application Support" / "Codex"
@ -96,320 +95,341 @@ type
CodexConf* = object
configFile* {.
desc: "Loads the configuration from a TOML file"
defaultValueDesc: "none"
defaultValue: InputFile.none
name: "config-file" }: Option[InputFile]
desc: "Loads the configuration from a TOML file",
defaultValueDesc: "none",
defaultValue: InputFile.none,
name: "config-file"
.}: Option[InputFile]
logLevel* {.
defaultValue: "info"
desc: "Sets the log level",
name: "log-level" }: string
logLevel* {.defaultValue: "info", desc: "Sets the log level", name: "log-level".}:
string
logFormat* {.
desc: "Specifies what kind of logs should be written to stdout (auto, " &
"colors, nocolors, json)"
defaultValueDesc: "auto"
defaultValue: LogKind.Auto
name: "log-format" }: LogKind
desc:
"Specifies what kind of logs should be written to stdout (auto, " &
"colors, nocolors, json)",
defaultValueDesc: "auto",
defaultValue: LogKind.Auto,
name: "log-format"
.}: LogKind
metricsEnabled* {.
desc: "Enable the metrics server"
defaultValue: false
name: "metrics" }: bool
desc: "Enable the metrics server", defaultValue: false, name: "metrics"
.}: bool
metricsAddress* {.
desc: "Listening address of the metrics server"
defaultValue: defaultAddress(config)
defaultValueDesc: "127.0.0.1"
name: "metrics-address" }: IpAddress
desc: "Listening address of the metrics server",
defaultValue: defaultAddress(config),
defaultValueDesc: "127.0.0.1",
name: "metrics-address"
.}: IpAddress
metricsPort* {.
desc: "Listening HTTP port of the metrics server"
defaultValue: 8008
name: "metrics-port" }: Port
desc: "Listening HTTP port of the metrics server",
defaultValue: 8008,
name: "metrics-port"
.}: Port
dataDir* {.
desc: "The directory where codex will store configuration and data"
defaultValue: DefaultDataDir
defaultValueDesc: $DefaultDataDir
abbr: "d"
name: "data-dir" }: OutDir
desc: "The directory where codex will store configuration and data",
defaultValue: DefaultDataDir,
defaultValueDesc: $DefaultDataDir,
abbr: "d",
name: "data-dir"
.}: OutDir
listenAddrs* {.
desc: "Multi Addresses to listen on"
defaultValue: @[
MultiAddress.init("/ip4/0.0.0.0/tcp/0")
.expect("Should init multiaddress")]
defaultValueDesc: "/ip4/0.0.0.0/tcp/0"
abbr: "i"
name: "listen-addrs" }: seq[MultiAddress]
desc: "Multi Addresses to listen on",
defaultValue:
@[MultiAddress.init("/ip4/0.0.0.0/tcp/0").expect("Should init multiaddress")],
defaultValueDesc: "/ip4/0.0.0.0/tcp/0",
abbr: "i",
name: "listen-addrs"
.}: seq[MultiAddress]
nat* {.
desc: "Specify method to use for determining public address. " &
"Must be one of: any, none, upnp, pmp, extip:<IP>"
defaultValue: defaultNatConfig()
defaultValueDesc: "any"
name: "nat" }: NatConfig
desc:
"Specify method to use for determining public address. " &
"Must be one of: any, none, upnp, pmp, extip:<IP>",
defaultValue: defaultNatConfig(),
defaultValueDesc: "any",
name: "nat"
.}: NatConfig
discoveryPort* {.
desc: "Discovery (UDP) port"
defaultValue: 8090.Port
defaultValueDesc: "8090"
abbr: "u"
name: "disc-port" }: Port
desc: "Discovery (UDP) port",
defaultValue: 8090.Port,
defaultValueDesc: "8090",
abbr: "u",
name: "disc-port"
.}: Port
netPrivKeyFile* {.
desc: "Source of network (secp256k1) private key file path or name"
defaultValue: "key"
name: "net-privkey" }: string
desc: "Source of network (secp256k1) private key file path or name",
defaultValue: "key",
name: "net-privkey"
.}: string
bootstrapNodes* {.
desc: "Specifies one or more bootstrap nodes to use when " &
"connecting to the network"
abbr: "b"
name: "bootstrap-node" }: seq[SignedPeerRecord]
desc:
"Specifies one or more bootstrap nodes to use when " &
"connecting to the network",
abbr: "b",
name: "bootstrap-node"
.}: seq[SignedPeerRecord]
maxPeers* {.
desc: "The maximum number of peers to connect to"
defaultValue: 160
name: "max-peers" }: int
desc: "The maximum number of peers to connect to",
defaultValue: 160,
name: "max-peers"
.}: int
agentString* {.
defaultValue: "Codex"
desc: "Node agent string which is used as identifier in network"
name: "agent-string" }: string
defaultValue: "Codex",
desc: "Node agent string which is used as identifier in network",
name: "agent-string"
.}: string
apiBindAddress* {.
desc: "The REST API bind address"
defaultValue: "127.0.0.1"
name: "api-bindaddr"
}: string
desc: "The REST API bind address", defaultValue: "127.0.0.1", name: "api-bindaddr"
.}: string
apiPort* {.
desc: "The REST Api port",
defaultValue: 8080.Port
defaultValueDesc: "8080"
name: "api-port"
abbr: "p" }: Port
defaultValue: 8080.Port,
defaultValueDesc: "8080",
name: "api-port",
abbr: "p"
.}: Port
apiCorsAllowedOrigin* {.
desc: "The REST Api CORS allowed origin for downloading data. " &
desc:
"The REST Api CORS allowed origin for downloading data. " &
"'*' will allow all origins, '' will allow none.",
defaultValue: string.none
defaultValueDesc: "Disallow all cross origin requests to download data"
name: "api-cors-origin" }: Option[string]
repoKind* {.
desc: "Backend for main repo store (fs, sqlite, leveldb)"
defaultValueDesc: "fs"
defaultValue: repoFS
name: "repo-kind" }: RepoKind
storageQuota* {.
desc: "The size of the total storage quota dedicated to the node"
defaultValue: DefaultQuotaBytes
defaultValueDesc: $DefaultQuotaBytes
name: "storage-quota"
abbr: "q" }: NBytes
blockTtl* {.
desc: "Default block timeout in seconds - 0 disables the ttl"
defaultValue: DefaultBlockTtl
defaultValueDesc: $DefaultBlockTtl
name: "block-ttl"
abbr: "t" }: Duration
blockMaintenanceInterval* {.
desc: "Time interval in seconds - determines frequency of block " &
"maintenance cycle: how often blocks are checked " &
"for expiration and cleanup"
defaultValue: DefaultBlockMaintenanceInterval
defaultValueDesc: $DefaultBlockMaintenanceInterval
name: "block-mi" }: Duration
blockMaintenanceNumberOfBlocks* {.
desc: "Number of blocks to check every maintenance cycle"
defaultValue: DefaultNumberOfBlocksToMaintainPerInterval
defaultValueDesc: $DefaultNumberOfBlocksToMaintainPerInterval
name: "block-mn" }: int
cacheSize* {.
desc: "The size of the block cache, 0 disables the cache - " &
"might help on slow hardrives"
defaultValue: 0
defaultValueDesc: "0"
name: "cache-size"
abbr: "c" }: NBytes
logFile* {.
desc: "Logs to file"
defaultValue: string.none
name: "log-file"
hidden
defaultValue: string.none,
defaultValueDesc: "Disallow all cross origin requests to download data",
name: "api-cors-origin"
.}: Option[string]
case cmd* {.
defaultValue: noCmd
command }: StartUpCmd
repoKind* {.
desc: "Backend for main repo store (fs, sqlite, leveldb)",
defaultValueDesc: "fs",
defaultValue: repoFS,
name: "repo-kind"
.}: RepoKind
storageQuota* {.
desc: "The size of the total storage quota dedicated to the node",
defaultValue: DefaultQuotaBytes,
defaultValueDesc: $DefaultQuotaBytes,
name: "storage-quota",
abbr: "q"
.}: NBytes
blockTtl* {.
desc: "Default block timeout in seconds - 0 disables the ttl",
defaultValue: DefaultBlockTtl,
defaultValueDesc: $DefaultBlockTtl,
name: "block-ttl",
abbr: "t"
.}: Duration
blockMaintenanceInterval* {.
desc:
"Time interval in seconds - determines frequency of block " &
"maintenance cycle: how often blocks are checked " & "for expiration and cleanup",
defaultValue: DefaultBlockMaintenanceInterval,
defaultValueDesc: $DefaultBlockMaintenanceInterval,
name: "block-mi"
.}: Duration
blockMaintenanceNumberOfBlocks* {.
desc: "Number of blocks to check every maintenance cycle",
defaultValue: DefaultNumberOfBlocksToMaintainPerInterval,
defaultValueDesc: $DefaultNumberOfBlocksToMaintainPerInterval,
name: "block-mn"
.}: int
cacheSize* {.
desc:
"The size of the block cache, 0 disables the cache - " &
"might help on slow hardrives",
defaultValue: 0,
defaultValueDesc: "0",
name: "cache-size",
abbr: "c"
.}: NBytes
logFile* {.
desc: "Logs to file", defaultValue: string.none, name: "log-file", hidden
.}: Option[string]
case cmd* {.defaultValue: noCmd, command.}: StartUpCmd
of persistence:
ethProvider* {.
desc: "The URL of the JSON-RPC API of the Ethereum node"
defaultValue: "ws://localhost:8545"
desc: "The URL of the JSON-RPC API of the Ethereum node",
defaultValue: "ws://localhost:8545",
name: "eth-provider"
.}: string
ethAccount* {.
desc: "The Ethereum account that is used for storage contracts"
defaultValue: EthAddress.none
defaultValueDesc: ""
desc: "The Ethereum account that is used for storage contracts",
defaultValue: EthAddress.none,
defaultValueDesc: "",
name: "eth-account"
.}: Option[EthAddress]
ethPrivateKey* {.
desc: "File containing Ethereum private key for storage contracts"
defaultValue: string.none
defaultValueDesc: ""
desc: "File containing Ethereum private key for storage contracts",
defaultValue: string.none,
defaultValueDesc: "",
name: "eth-private-key"
.}: Option[string]
marketplaceAddress* {.
desc: "Address of deployed Marketplace contract"
defaultValue: EthAddress.none
defaultValueDesc: ""
desc: "Address of deployed Marketplace contract",
defaultValue: EthAddress.none,
defaultValueDesc: "",
name: "marketplace-address"
.}: Option[EthAddress]
# TODO: should go behind a feature flag
simulateProofFailures* {.
desc: "Simulates proof failures once every N proofs. 0 = disabled."
defaultValue: 0
name: "simulate-proof-failures"
desc: "Simulates proof failures once every N proofs. 0 = disabled.",
defaultValue: 0,
name: "simulate-proof-failures",
hidden
.}: int
validator* {.
desc: "Enables validator, requires an Ethereum node"
defaultValue: false
desc: "Enables validator, requires an Ethereum node",
defaultValue: false,
name: "validator"
.}: bool
validatorMaxSlots* {.
desc: "Maximum number of slots that the validator monitors"
longDesc: "If set to 0, the validator will not limit " &
"the maximum number of slots it monitors"
defaultValue: 1000
desc: "Maximum number of slots that the validator monitors",
longDesc:
"If set to 0, the validator will not limit " &
"the maximum number of slots it monitors",
defaultValue: 1000,
name: "validator-max-slots"
.}: MaxSlots
validatorGroups* {.
desc: "Slot validation groups"
longDesc: "A number indicating total number of groups into " &
desc: "Slot validation groups",
longDesc:
"A number indicating total number of groups into " &
"which the whole slot id space will be divided. " &
"The value must be in the range [2, 65535]. " &
"If not provided, the validator will observe " &
"the whole slot id space and the value of " &
"the --validator-group-index parameter will be ignored. " &
"Powers of twos are advised for even distribution"
defaultValue: ValidationGroups.none
"Powers of twos are advised for even distribution",
defaultValue: ValidationGroups.none,
name: "validator-groups"
.}: Option[ValidationGroups]
validatorGroupIndex* {.
desc: "Slot validation group index"
longDesc: "The value provided must be in the range " &
desc: "Slot validation group index",
longDesc:
"The value provided must be in the range " &
"[0, validatorGroups). Ignored when --validator-groups " &
"is not provided. Only slot ids satisfying condition " &
"[(slotId mod validationGroups) == groupIndex] will be " &
"observed by the validator"
defaultValue: 0
"observed by the validator",
defaultValue: 0,
name: "validator-group-index"
.}: uint16
rewardRecipient* {.
desc: "Address to send payouts to (eg rewards and refunds)"
desc: "Address to send payouts to (eg rewards and refunds)",
name: "reward-recipient"
.}: Option[EthAddress]
case persistenceCmd* {.
defaultValue: noCmd
command }: PersistenceCmd
case persistenceCmd* {.defaultValue: noCmd, command.}: PersistenceCmd
of PersistenceCmd.prover:
circuitDir* {.
desc: "Directory where Codex will store proof circuit data"
defaultValue: DefaultCircuitDir
defaultValueDesc: $DefaultCircuitDir
abbr: "cd"
name: "circuit-dir" }: OutDir
desc: "Directory where Codex will store proof circuit data",
defaultValue: DefaultCircuitDir,
defaultValueDesc: $DefaultCircuitDir,
abbr: "cd",
name: "circuit-dir"
.}: OutDir
circomR1cs* {.
desc: "The r1cs file for the storage circuit"
defaultValue: $DefaultCircuitDir / "proof_main.r1cs"
defaultValueDesc: $DefaultCircuitDir & "/proof_main.r1cs"
desc: "The r1cs file for the storage circuit",
defaultValue: $DefaultCircuitDir / "proof_main.r1cs",
defaultValueDesc: $DefaultCircuitDir & "/proof_main.r1cs",
name: "circom-r1cs"
.}: InputFile
circomWasm* {.
desc: "The wasm file for the storage circuit"
defaultValue: $DefaultCircuitDir / "proof_main.wasm"
defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.wasm"
desc: "The wasm file for the storage circuit",
defaultValue: $DefaultCircuitDir / "proof_main.wasm",
defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.wasm",
name: "circom-wasm"
.}: InputFile
circomZkey* {.
desc: "The zkey file for the storage circuit"
defaultValue: $DefaultCircuitDir / "proof_main.zkey"
defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.zkey"
desc: "The zkey file for the storage circuit",
defaultValue: $DefaultCircuitDir / "proof_main.zkey",
defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.zkey",
name: "circom-zkey"
.}: InputFile
# TODO: should probably be hidden and behind a feature flag
circomNoZkey* {.
desc: "Ignore the zkey file - use only for testing!"
defaultValue: false
desc: "Ignore the zkey file - use only for testing!",
defaultValue: false,
name: "circom-no-zkey"
.}: bool
numProofSamples* {.
desc: "Number of samples to prove"
defaultValue: DefaultSamplesNum
defaultValueDesc: $DefaultSamplesNum
name: "proof-samples" }: int
desc: "Number of samples to prove",
defaultValue: DefaultSamplesNum,
defaultValueDesc: $DefaultSamplesNum,
name: "proof-samples"
.}: int
maxSlotDepth* {.
desc: "The maximum depth of the slot tree"
defaultValue: DefaultMaxSlotDepth
defaultValueDesc: $DefaultMaxSlotDepth
name: "max-slot-depth" }: int
desc: "The maximum depth of the slot tree",
defaultValue: DefaultMaxSlotDepth,
defaultValueDesc: $DefaultMaxSlotDepth,
name: "max-slot-depth"
.}: int
maxDatasetDepth* {.
desc: "The maximum depth of the dataset tree"
defaultValue: DefaultMaxDatasetDepth
defaultValueDesc: $DefaultMaxDatasetDepth
name: "max-dataset-depth" }: int
desc: "The maximum depth of the dataset tree",
defaultValue: DefaultMaxDatasetDepth,
defaultValueDesc: $DefaultMaxDatasetDepth,
name: "max-dataset-depth"
.}: int
maxBlockDepth* {.
desc: "The maximum depth of the network block merkle tree"
defaultValue: DefaultBlockDepth
defaultValueDesc: $DefaultBlockDepth
name: "max-block-depth" }: int
desc: "The maximum depth of the network block merkle tree",
defaultValue: DefaultBlockDepth,
defaultValueDesc: $DefaultBlockDepth,
name: "max-block-depth"
.}: int
maxCellElms* {.
desc: "The maximum number of elements in a cell"
defaultValue: DefaultCellElms
defaultValueDesc: $DefaultCellElms
name: "max-cell-elements" }: int
desc: "The maximum number of elements in a cell",
defaultValue: DefaultCellElms,
defaultValueDesc: $DefaultCellElms,
name: "max-cell-elements"
.}: int
of PersistenceCmd.noCmd:
discard
of StartUpCmd.noCmd:
discard # end of persistence
EthAddress* = ethers.Address
logutils.formatIt(LogFormat.textLines, EthAddress): it.short0xHexLog
logutils.formatIt(LogFormat.json, EthAddress): %it
logutils.formatIt(LogFormat.textLines, EthAddress):
it.short0xHexLog
logutils.formatIt(LogFormat.json, EthAddress):
%it
func defaultAddress*(conf: CodexConf): IpAddress =
result = static parseIpAddress("127.0.0.1")
@ -443,13 +463,12 @@ const
nimBanner* = getNimBanner()
codexFullVersion* =
"Codex version: " & codexVersion & "\p" &
"Codex revision: " & codexRevision & "\p" &
"Codex version: " & codexVersion & "\p" & "Codex revision: " & codexRevision & "\p" &
nimBanner
proc parseCmdArg*(T: typedesc[MultiAddress],
input: string): MultiAddress
{.upraises: [ValueError] .} =
proc parseCmdArg*(
T: typedesc[MultiAddress], input: string
): MultiAddress {.upraises: [ValueError].} =
var ma: MultiAddress
try:
let res = MultiAddress.init(input)
@ -478,7 +497,7 @@ proc parseCmdArg*(T: type SignedPeerRecord, uri: string): T =
res
func parseCmdArg*(T: type NatConfig, p: string): T {.raises: [ValueError].} =
case p.toLowerAscii:
case p.toLowerAscii
of "any":
NatConfig(hasExtIp: false, nat: NatStrategy.NatAny)
of "none":
@ -499,7 +518,7 @@ func parseCmdArg*(T: type NatConfig, p: string): T {.raises: [ValueError].} =
let error = "Not a valid NAT option: " & p
raise newException(ValueError, error)
proc completeCmdArg*(T: type NatConfig; val: string): seq[string] =
proc completeCmdArg*(T: type NatConfig, val: string): seq[string] =
return @[]
proc parseCmdArg*(T: type EthAddress, address: string): T =
@ -521,8 +540,9 @@ proc parseCmdArg*(T: type Duration, val: string): T =
quit QuitFailure
dur
proc readValue*(r: var TomlReader, val: var EthAddress)
{.upraises: [SerializationError, IOError].} =
proc readValue*(
r: var TomlReader, val: var EthAddress
) {.upraises: [SerializationError, IOError].} =
val = EthAddress.init(r.readValue(string)).get()
proc readValue*(r: var TomlReader, val: var SignedPeerRecord) =
@ -548,8 +568,9 @@ proc readValue*(r: var TomlReader, val: var MultiAddress) =
warn "Invalid MultiAddress", input = input, error = res.error()
quit QuitFailure
proc readValue*(r: var TomlReader, val: var NBytes)
{.upraises: [SerializationError, IOError].} =
proc readValue*(
r: var TomlReader, val: var NBytes
) {.upraises: [SerializationError, IOError].} =
var value = 0'i64
var str = r.readValue(string)
let count = parseSize(str, value, alwaysBin = true)
@ -558,8 +579,9 @@ proc readValue*(r: var TomlReader, val: var NBytes)
quit QuitFailure
val = NBytes(value)
proc readValue*(r: var TomlReader, val: var Duration)
{.upraises: [SerializationError, IOError].} =
proc readValue*(
r: var TomlReader, val: var Duration
) {.upraises: [SerializationError, IOError].} =
var str = r.readValue(string)
var dur: Duration
let count = parseDuration(str, dur)
@ -568,20 +590,23 @@ proc readValue*(r: var TomlReader, val: var Duration)
quit QuitFailure
val = dur
proc readValue*(r: var TomlReader, val: var NatConfig)
{.raises: [SerializationError].} =
val = try: parseCmdArg(NatConfig, r.readValue(string))
proc readValue*(
r: var TomlReader, val: var NatConfig
) {.raises: [SerializationError].} =
val =
try:
parseCmdArg(NatConfig, r.readValue(string))
except CatchableError as err:
raise newException(SerializationError, err.msg)
# no idea why confutils needs this:
proc completeCmdArg*(T: type EthAddress; val: string): seq[string] =
proc completeCmdArg*(T: type EthAddress, val: string): seq[string] =
discard
proc completeCmdArg*(T: type NBytes; val: string): seq[string] =
proc completeCmdArg*(T: type NBytes, val: string): seq[string] =
discard
proc completeCmdArg*(T: type Duration; val: string): seq[string] =
proc completeCmdArg*(T: type Duration, val: string): seq[string] =
discard
# silly chronicles, colors is a compile-time property
@ -627,8 +652,8 @@ proc updateLogLevel*(logLevel: string) {.upraises: [ValueError].} =
setLogLevel(parseEnum[LogLevel](directives[0].toUpperAscii))
except ValueError:
raise (ref ValueError)(
msg: "Please specify one of: trace, debug, " &
"info, notice, warn, error or fatal"
msg:
"Please specify one of: trace, debug, " & "info, notice, warn, error or fatal"
)
if directives.len > 1:
@ -641,7 +666,9 @@ proc setupLogging*(conf: CodexConf) =
warn "Logging configuration options not enabled in the current build"
else:
var logFile: ?IoHandle
proc noOutput(logLevel: LogLevel, msg: LogOutputStr) = discard
proc noOutput(logLevel: LogLevel, msg: LogOutputStr) =
discard
proc writeAndFlush(f: File, msg: LogOutputStr) =
try:
f.write(msg)
@ -662,14 +689,11 @@ proc setupLogging*(conf: CodexConf) =
defaultChroniclesStream.outputs[2].writer = noOutput
if logFilePath =? conf.logFile and logFilePath.len > 0:
let logFileHandle = openFile(
logFilePath,
{OpenFlags.Write, OpenFlags.Create, OpenFlags.Truncate}
)
let logFileHandle =
openFile(logFilePath, {OpenFlags.Write, OpenFlags.Create, OpenFlags.Truncate})
if logFileHandle.isErr:
error "failed to open log file",
path = logFilePath,
errorCode = $logFileHandle.error
path = logFilePath, errorCode = $logFileHandle.error
else:
logFile = logFileHandle.option
defaultChroniclesStream.outputs[2].writer = fileFlush
@ -677,14 +701,13 @@ proc setupLogging*(conf: CodexConf) =
defaultChroniclesStream.outputs[1].writer = noOutput
let writer =
case conf.logFormat:
case conf.logFormat
of LogKind.Auto:
if isatty(stdout):
if isatty(stdout): stdoutFlush else: noColorsFlush
of LogKind.Colors:
stdoutFlush
else:
of LogKind.NoColors:
noColorsFlush
of LogKind.Colors: stdoutFlush
of LogKind.NoColors: noColorsFlush
of LogKind.Json:
defaultChroniclesStream.outputs[1].writer = stdoutFlush
noOutput
@ -697,6 +720,7 @@ proc setupLogging*(conf: CodexConf) =
inc(counter)
let withoutNewLine = msg[0 ..^ 2]
writer(logLevel, withoutNewLine & " count=" & $counter & "\n")
defaultChroniclesStream.outputs[0].writer = numberedWriter
else:
defaultChroniclesStream.outputs[0].writer = writer

6
codex/contracts/clock.nim
View File

@ -11,8 +11,7 @@ export clock
logScope:
topics = "contracts clock"
type
OnChainClock* = ref object of Clock
type OnChainClock* = ref object of Clock
provider: Provider
subscription: Subscription
offset: times.Duration
@ -29,7 +28,8 @@ proc update(clock: OnChainClock, blck: Block) =
let computerTime = getTime()
clock.offset = blockTime - computerTime
clock.blockNumber = number
trace "updated clock", blockTime=blck.timestamp, blockNumber=number, offset=clock.offset
trace "updated clock",
blockTime = blck.timestamp, blockNumber = number, offset = clock.offset
clock.newBlock.fire()
proc update(clock: OnChainClock) {.async.} =

15
codex/contracts/config.nim
View File

@ -8,11 +8,14 @@ type
MarketplaceConfig* = object
collateral*: CollateralConfig
proofs*: ProofConfig
CollateralConfig* = object
repairRewardPercentage*: uint8 # percentage of remaining collateral slot has after it has been freed
repairRewardPercentage*: uint8
# percentage of remaining collateral slot has after it has been freed
maxNumberOfSlashes*: uint8 # frees slot when the number of slashes reaches this value
slashCriterion*: uint16 # amount of proofs missed that lead to slashing
slashPercentage*: uint8 # percentage of the collateral that is slashed
ProofConfig* = object
period*: UInt256 # proofs requirements are calculated per period (in seconds)
timeout*: UInt256 # mark proofs as missing before the timeout (in seconds)
@ -23,14 +26,13 @@ type
# blocks. Should be a prime number to ensure there are no cycles.
downtimeProduct*: uint8
func fromTuple(_: type ProofConfig, tupl: tuple): ProofConfig =
ProofConfig(
period: tupl[0],
timeout: tupl[1],
downtime: tupl[2],
zkeyHash: tupl[3],
downtimeProduct: tupl[4]
downtimeProduct: tupl[4],
)
func fromTuple(_: type CollateralConfig, tupl: tuple): CollateralConfig =
@ -38,14 +40,11 @@ func fromTuple(_: type CollateralConfig, tupl: tuple): CollateralConfig =
repairRewardPercentage: tupl[0],
maxNumberOfSlashes: tupl[1],
slashCriterion: tupl[2],
slashPercentage: tupl[3]
slashPercentage: tupl[3],
)
func fromTuple(_: type MarketplaceConfig, tupl: tuple): MarketplaceConfig =
MarketplaceConfig(
collateral: tupl[0],
proofs: tupl[1]
)
MarketplaceConfig(collateral: tupl[0], proofs: tupl[1])
func solidityType*(_: type ProofConfig): string =
solidityType(ProofConfig.fieldTypes)

15
codex/contracts/deployment.nim
View File

@ -13,17 +13,14 @@ type Deployment* = ref object
const knownAddresses = {
# Hardhat localhost network
"31337": {
"Marketplace": Address.init("0x322813Fd9A801c5507c9de605d63CEA4f2CE6c44"),
}.toTable,
"31337":
{"Marketplace": Address.init("0x322813Fd9A801c5507c9de605d63CEA4f2CE6c44")}.toTable,
# Taiko Alpha-3 Testnet
"167005": {
"Marketplace": Address.init("0x948CF9291b77Bd7ad84781b9047129Addf1b894F")
}.toTable,
"167005":
{"Marketplace": Address.init("0x948CF9291b77Bd7ad84781b9047129Addf1b894F")}.toTable,
# Codex Testnet - Nov 25 2024 18:41:29 PM (+00:00 UTC)
"789987": {
"Marketplace": Address.init("0xAB03b6a58C5262f530D54146DA2a552B1C0F7648")
}.toTable
"789987":
{"Marketplace": Address.init("0xAB03b6a58C5262f530D54146DA2a552B1C0F7648")}.toTable,
}.toTable
proc getKnownAddress(T: type, chainId: UInt256): ?Address =

9
codex/contracts/interactions/clientinteractions.nim
View File

@ -9,13 +9,12 @@ import ./interactions
export purchasing
export logutils
type
ClientInteractions* = ref object of ContractInteractions
type ClientInteractions* = ref object of ContractInteractions
purchasing*: Purchasing
proc new*(_: type ClientInteractions,
clock: OnChainClock,
purchasing: Purchasing): ClientInteractions =
proc new*(
_: type ClientInteractions, clock: OnChainClock, purchasing: Purchasing
): ClientInteractions =
ClientInteractions(clock: clock, purchasing: purchasing)
proc start*(self: ClientInteractions) {.async.} =

9
codex/contracts/interactions/hostinteractions.nim
View File

@ -7,15 +7,10 @@ import ./interactions
export sales
export logutils
type
HostInteractions* = ref object of ContractInteractions
type HostInteractions* = ref object of ContractInteractions
sales*: Sales
proc new*(
_: type HostInteractions,
clock: Clock,
sales: Sales
): HostInteractions =
proc new*(_: type HostInteractions, clock: Clock, sales: Sales): HostInteractions =
## Create a new HostInteractions instance
##
HostInteractions(clock: clock, sales: sales)

3
codex/contracts/interactions/interactions.nim
View File

@ -5,8 +5,7 @@ import ../market
export clock
type
ContractInteractions* = ref object of RootObj
type ContractInteractions* = ref object of RootObj
clock*: Clock
method start*(self: ContractInteractions) {.async, base.} =

9
codex/contracts/interactions/validatorinteractions.nim
View File

@ -3,13 +3,12 @@ import ../../validation
export validation
type
ValidatorInteractions* = ref object of ContractInteractions
type ValidatorInteractions* = ref object of ContractInteractions
validation: Validation
proc new*(_: type ValidatorInteractions,
clock: OnChainClock,
validation: Validation): ValidatorInteractions =
proc new*(
_: type ValidatorInteractions, clock: OnChainClock, validation: Validation
): ValidatorInteractions =
ValidatorInteractions(clock: clock, validation: validation)
proc start*(self: ValidatorInteractions) {.async.} =

231
codex/contracts/market.nim
View File

@ -27,18 +27,12 @@ type
eventSubscription: EventSubscription
func new*(
_: type OnChainMarket,
contract: Marketplace,
rewardRecipient = Address.none): OnChainMarket =
_: type OnChainMarket, contract: Marketplace, rewardRecipient = Address.none
): OnChainMarket =
without signer =? contract.signer:
raiseAssert("Marketplace contract should have a signer")
OnChainMarket(
contract: contract,
signer: signer,
rewardRecipient: rewardRecipient
)
OnChainMarket(contract: contract, signer: signer, rewardRecipient: rewardRecipient)
proc raiseMarketError(message: string) {.raises: [MarketError].} =
raise newException(MarketError, message)
@ -115,16 +109,18 @@ method requestStorage(market: OnChainMarket, request: StorageRequest){.async.} =
await market.approveFunds(request.price())
discard await market.contract.requestStorage(request).confirm(1)
method getRequest*(market: OnChainMarket,
id: RequestId): Future[?StorageRequest] {.async.} =
method getRequest*(
market: OnChainMarket, id: RequestId
): Future[?StorageRequest] {.async.} =
convertEthersError:
try:
return some await market.contract.getRequest(id)
except Marketplace_UnknownRequest:
return none StorageRequest
method requestState*(market: OnChainMarket,
requestId: RequestId): Future[?RequestState] {.async.} =
method requestState*(
market: OnChainMarket, requestId: RequestId
): Future[?RequestState] {.async.} =
convertEthersError:
try:
let overrides = CallOverrides(blockTag: some BlockTag.pending)
@ -132,25 +128,26 @@ method requestState*(market: OnChainMarket,
except Marketplace_UnknownRequest:
return none RequestState
method slotState*(market: OnChainMarket,
slotId: SlotId): Future[SlotState] {.async.} =
method slotState*(market: OnChainMarket, slotId: SlotId): Future[SlotState] {.async.} =
convertEthersError:
let overrides = CallOverrides(blockTag: some BlockTag.pending)
return await market.contract.slotState(slotId, overrides)
method getRequestEnd*(market: OnChainMarket,
id: RequestId): Future[SecondsSince1970] {.async.} =
method getRequestEnd*(
market: OnChainMarket, id: RequestId
): Future[SecondsSince1970] {.async.} =
convertEthersError:
return await market.contract.requestEnd(id)
method requestExpiresAt*(market: OnChainMarket,
id: RequestId): Future[SecondsSince1970] {.async.} =
method requestExpiresAt*(
market: OnChainMarket, id: RequestId
): Future[SecondsSince1970] {.async.} =
convertEthersError:
return await market.contract.requestExpiry(id)
method getHost(market: OnChainMarket,
requestId: RequestId,
slotIndex: UInt256): Future[?Address] {.async.} =
method getHost(
market: OnChainMarket, requestId: RequestId, slotIndex: UInt256
): Future[?Address] {.async.} =
convertEthersError:
let slotId = slotId(requestId, slotIndex)
let address = await market.contract.getHost(slotId)
@ -159,19 +156,20 @@ method getHost(market: OnChainMarket,
else:
return none Address
method getActiveSlot*(market: OnChainMarket,
slotId: SlotId): Future[?Slot] {.async.} =
method getActiveSlot*(market: OnChainMarket, slotId: SlotId): Future[?Slot] {.async.} =
convertEthersError:
try:
return some await market.contract.getActiveSlot(slotId)
except Marketplace_SlotIsFree:
return none Slot
method fillSlot(market: OnChainMarket,
method fillSlot(
market: OnChainMarket,
requestId: RequestId,
slotIndex: UInt256,
proof: Groth16Proof,
collateral: UInt256) {.async.} =
collateral: UInt256,
) {.async.} =
convertEthersError:
logScope:
requestId
@ -192,8 +190,8 @@ method freeSlot*(market: OnChainMarket, slotId: SlotId) {.async.} =
freeSlot = market.contract.freeSlot(
slotId,
rewardRecipient, # --reward-recipient
collateralRecipient) # SP's address
collateralRecipient,
) # SP's address
else:
# Otherwise, use the SP's address as both the reward and collateral
# recipient (the contract will use msg.sender for both)
@ -201,14 +199,11 @@ method freeSlot*(market: OnChainMarket, slotId: SlotId) {.async.} =
discard await freeSlot.confirm(1)
method withdrawFunds(market: OnChainMarket,
requestId: RequestId) {.async.} =
method withdrawFunds(market: OnChainMarket, requestId: RequestId) {.async.} =
convertEthersError:
discard await market.contract.withdrawFunds(requestId).confirm(1)
method isProofRequired*(market: OnChainMarket,
id: SlotId): Future[bool] {.async.} =
method isProofRequired*(market: OnChainMarket, id: SlotId): Future[bool] {.async.} =
convertEthersError:
try:
let overrides = CallOverrides(blockTag: some BlockTag.pending)
@ -216,8 +211,7 @@ method isProofRequired*(market: OnChainMarket,
except Marketplace_SlotIsFree:
return false
method willProofBeRequired*(market: OnChainMarket,
id: SlotId): Future[bool] {.async.} =
method willProofBeRequired*(market: OnChainMarket, id: SlotId): Future[bool] {.async.} =
convertEthersError:
try:
let overrides = CallOverrides(blockTag: some BlockTag.pending)
@ -225,27 +219,25 @@ method willProofBeRequired*(market: OnChainMarket,
except Marketplace_SlotIsFree:
return false
method getChallenge*(market: OnChainMarket, id: SlotId): Future[ProofChallenge] {.async.} =
method getChallenge*(
market: OnChainMarket, id: SlotId
): Future[ProofChallenge] {.async.} =
convertEthersError:
let overrides = CallOverrides(blockTag: some BlockTag.pending)
return await market.contract.getChallenge(id, overrides)
method submitProof*(market: OnChainMarket,
id: SlotId,
proof: Groth16Proof) {.async.} =
method submitProof*(market: OnChainMarket, id: SlotId, proof: Groth16Proof) {.async.} =
convertEthersError:
discard await market.contract.submitProof(id, proof).confirm(1)
method markProofAsMissing*(market: OnChainMarket,
id: SlotId,
period: Period) {.async.} =
method markProofAsMissing*(
market: OnChainMarket, id: SlotId, period: Period
) {.async.} =
convertEthersError:
discard await market.contract.markProofAsMissing(id, period).confirm(1)
method canProofBeMarkedAsMissing*(
market: OnChainMarket,
id: SlotId,
period: Period
market: OnChainMarket, id: SlotId, period: Period
): Future[bool] {.async.} =
let provider = market.contract.provider
let contractWithoutSigner = market.contract.connect(provider)
@ -258,45 +250,41 @@ method canProofBeMarkedAsMissing*(
return false
method reserveSlot*(
market: OnChainMarket,
requestId: RequestId,
slotIndex: UInt256) {.async.} =
market: OnChainMarket, requestId: RequestId, slotIndex: UInt256
) {.async.} =
convertEthersError:
discard await market.contract.reserveSlot(
discard await market.contract
.reserveSlot(
requestId,
slotIndex,
# reserveSlot runs out of gas for unknown reason, but 100k gas covers it
TransactionOverrides(gasLimit: some 100000.u256)
).confirm(1)
TransactionOverrides(gasLimit: some 100000.u256),
)
.confirm(1)
method canReserveSlot*(
market: OnChainMarket,
requestId: RequestId,
slotIndex: UInt256): Future[bool] {.async.} =
market: OnChainMarket, requestId: RequestId, slotIndex: UInt256
): Future[bool] {.async.} =
convertEthersError:
return await market.contract.canReserveSlot(requestId, slotIndex)
method subscribeRequests*(market: OnChainMarket,
callback: OnRequest):
Future[MarketSubscription] {.async.} =
method subscribeRequests*(
market: OnChainMarket, callback: OnRequest
): Future[MarketSubscription] {.async.} =
proc onEvent(eventResult: ?!StorageRequested) {.upraises: [].} =
without event =? eventResult, eventErr:
error "There was an error in Request subscription", msg = eventErr.msg
return
callback(event.requestId,
event.ask,
event.expiry)
callback(event.requestId, event.ask, event.expiry)
convertEthersError:
let subscription = await market.contract.subscribe(StorageRequested, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
method subscribeSlotFilled*(market: OnChainMarket,
callback: OnSlotFilled):
Future[MarketSubscription] {.async.} =
method subscribeSlotFilled*(
market: OnChainMarket, callback: OnSlotFilled
): Future[MarketSubscription] {.async.} =
proc onEvent(eventResult: ?!SlotFilled) {.upraises: [].} =
without event =? eventResult, eventErr:
error "There was an error in SlotFilled subscription", msg = eventErr.msg
@ -308,11 +296,12 @@ method subscribeSlotFilled*(market: OnChainMarket,
let subscription = await market.contract.subscribe(SlotFilled, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
method subscribeSlotFilled*(market: OnChainMarket,
method subscribeSlotFilled*(
market: OnChainMarket,
requestId: RequestId,
slotIndex: UInt256,
callback: OnSlotFilled):
Future[MarketSubscription] {.async.} =
callback: OnSlotFilled,
): Future[MarketSubscription] {.async.} =
proc onSlotFilled(eventRequestId: RequestId, eventSlotIndex: UInt256) =
if eventRequestId == requestId and eventSlotIndex == slotIndex:
callback(requestId, slotIndex)
@ -320,9 +309,9 @@ method subscribeSlotFilled*(market: OnChainMarket,
convertEthersError:
return await market.subscribeSlotFilled(onSlotFilled)
method subscribeSlotFreed*(market: OnChainMarket,
callback: OnSlotFreed):
Future[MarketSubscription] {.async.} =
method subscribeSlotFreed*(
market: OnChainMarket, callback: OnSlotFreed
): Future[MarketSubscription] {.async.} =
proc onEvent(eventResult: ?!SlotFreed) {.upraises: [].} =
without event =? eventResult, eventErr:
error "There was an error in SlotFreed subscription", msg = eventErr.msg
@ -335,12 +324,12 @@ method subscribeSlotFreed*(market: OnChainMarket,
return OnChainMarketSubscription(eventSubscription: subscription)
method subscribeSlotReservationsFull*(
market: OnChainMarket,
callback: OnSlotReservationsFull): Future[MarketSubscription] {.async.} =
market: OnChainMarket, callback: OnSlotReservationsFull
): Future[MarketSubscription] {.async.} =
proc onEvent(eventResult: ?!SlotReservationsFull) {.upraises: [].} =
without event =? eventResult, eventErr:
error "There was an error in SlotReservationsFull subscription", msg = eventErr.msg
error "There was an error in SlotReservationsFull subscription",
msg = eventErr.msg
return
callback(event.requestId, event.slotIndex)
@ -349,9 +338,9 @@ method subscribeSlotReservationsFull*(
let subscription = await market.contract.subscribe(SlotReservationsFull, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
method subscribeFulfillment(market: OnChainMarket,
callback: OnFulfillment):
Future[MarketSubscription] {.async.} =
method subscribeFulfillment(
market: OnChainMarket, callback: OnFulfillment
): Future[MarketSubscription] {.async.} =
proc onEvent(eventResult: ?!RequestFulfilled) {.upraises: [].} =
without event =? eventResult, eventErr:
error "There was an error in RequestFulfillment subscription", msg = eventErr.msg
@ -363,10 +352,9 @@ method subscribeFulfillment(market: OnChainMarket,
let subscription = await market.contract.subscribe(RequestFulfilled, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
method subscribeFulfillment(market: OnChainMarket,
requestId: RequestId,
callback: OnFulfillment):
Future[MarketSubscription] {.async.} =
method subscribeFulfillment(
market: OnChainMarket, requestId: RequestId, callback: OnFulfillment
): Future[MarketSubscription] {.async.} =
proc onEvent(eventResult: ?!RequestFulfilled) {.upraises: [].} =
without event =? eventResult, eventErr:
error "There was an error in RequestFulfillment subscription", msg = eventErr.msg
@ -379,9 +367,9 @@ method subscribeFulfillment(market: OnChainMarket,
let subscription = await market.contract.subscribe(RequestFulfilled, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
method subscribeRequestCancelled*(market: OnChainMarket,
callback: OnRequestCancelled):
Future[MarketSubscription] {.async.} =
method subscribeRequestCancelled*(
market: OnChainMarket, callback: OnRequestCancelled
): Future[MarketSubscription] {.async.} =
proc onEvent(eventResult: ?!RequestCancelled) {.upraises: [].} =
without event =? eventResult, eventErr:
error "There was an error in RequestCancelled subscription", msg = eventErr.msg
@ -393,10 +381,9 @@ method subscribeRequestCancelled*(market: OnChainMarket,
let subscription = await market.contract.subscribe(RequestCancelled, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
method subscribeRequestCancelled*(market: OnChainMarket,
requestId: RequestId,
callback: OnRequestCancelled):
Future[MarketSubscription] {.async.} =
method subscribeRequestCancelled*(
market: OnChainMarket, requestId: RequestId, callback: OnRequestCancelled
): Future[MarketSubscription] {.async.} =
proc onEvent(eventResult: ?!RequestCancelled) {.upraises: [].} =
without event =? eventResult, eventErr:
error "There was an error in RequestCancelled subscription", msg = eventErr.msg
@ -409,10 +396,10 @@ method subscribeRequestCancelled*(market: OnChainMarket,
let subscription = await market.contract.subscribe(RequestCancelled, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
method subscribeRequestFailed*(market: OnChainMarket,
callback: OnRequestFailed):
Future[MarketSubscription] {.async.} =
proc onEvent(eventResult: ?!RequestFailed) {.upraises:[]} =
method subscribeRequestFailed*(
market: OnChainMarket, callback: OnRequestFailed
): Future[MarketSubscription] {.async.} =
proc onEvent(eventResult: ?!RequestFailed) {.upraises: [].} =
without event =? eventResult, eventErr:
error "There was an error in RequestFailed subscription", msg = eventErr.msg
return
@ -423,11 +410,10 @@ method subscribeRequestFailed*(market: OnChainMarket,
let subscription = await market.contract.subscribe(RequestFailed, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
method subscribeRequestFailed*(market: OnChainMarket,
requestId: RequestId,
callback: OnRequestFailed):
Future[MarketSubscription] {.async.} =
proc onEvent(eventResult: ?!RequestFailed) {.upraises:[]} =
method subscribeRequestFailed*(
market: OnChainMarket, requestId: RequestId, callback: OnRequestFailed
): Future[MarketSubscription] {.async.} =
proc onEvent(eventResult: ?!RequestFailed) {.upraises: [].} =
without event =? eventResult, eventErr:
error "There was an error in RequestFailed subscription", msg = eventErr.msg
return
@ -439,9 +425,9 @@ method subscribeRequestFailed*(market: OnChainMarket,
let subscription = await market.contract.subscribe(RequestFailed, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
method subscribeProofSubmission*(market: OnChainMarket,
callback: OnProofSubmitted):
Future[MarketSubscription] {.async.} =
method subscribeProofSubmission*(
market: OnChainMarket, callback: OnProofSubmitted
): Future[MarketSubscription] {.async.} =
proc onEvent(eventResult: ?!ProofSubmitted) {.upraises: [].} =
without event =? eventResult, eventErr:
error "There was an error in ProofSubmitted subscription", msg = eventErr.msg
@ -457,48 +443,37 @@ method unsubscribe*(subscription: OnChainMarketSubscription) {.async.} =
await subscription.eventSubscription.unsubscribe()
method queryPastSlotFilledEvents*(
market: OnChainMarket,
fromBlock: BlockTag): Future[seq[SlotFilled]] {.async.} =
market: OnChainMarket, fromBlock: BlockTag
): Future[seq[SlotFilled]] {.async.} =
convertEthersError:
return await market.contract.queryFilter(SlotFilled,
fromBlock,
BlockTag.latest)
return await market.contract.queryFilter(SlotFilled, fromBlock, BlockTag.latest)
method queryPastSlotFilledEvents*(
market: OnChainMarket,
blocksAgo: int): Future[seq[SlotFilled]] {.async.} =
market: OnChainMarket, blocksAgo: int
): Future[seq[SlotFilled]] {.async.} =
convertEthersError:
let fromBlock =
await market.contract.provider.pastBlockTag(blocksAgo)
let fromBlock = await market.contract.provider.pastBlockTag(blocksAgo)
return await market.queryPastSlotFilledEvents(fromBlock)
method queryPastSlotFilledEvents*(
market: OnChainMarket,
fromTime: SecondsSince1970): Future[seq[SlotFilled]] {.async.} =
market: OnChainMarket, fromTime: SecondsSince1970
): Future[seq[SlotFilled]] {.async.} =
convertEthersError:
let fromBlock =
await market.contract.provider.blockNumberForEpoch(fromTime)
let fromBlock = await market.contract.provider.blockNumberForEpoch(fromTime)
return await market.queryPastSlotFilledEvents(BlockTag.init(fromBlock))
method queryPastStorageRequestedEvents*(
market: OnChainMarket,
fromBlock: BlockTag): Future[seq[StorageRequested]] {.async.} =
market: OnChainMarket, fromBlock: BlockTag
): Future[seq[StorageRequested]] {.async.} =
convertEthersError:
return await market.contract.queryFilter(StorageRequested,
fromBlock,
BlockTag.latest)
return
await market.contract.queryFilter(StorageRequested, fromBlock, BlockTag.latest)
method queryPastStorageRequestedEvents*(
market: OnChainMarket,
blocksAgo: int): Future[seq[StorageRequested]] {.async.} =
market: OnChainMarket, blocksAgo: int
): Future[seq[StorageRequested]] {.async.} =
convertEthersError:
let fromBlock =
await market.contract.provider.pastBlockTag(blocksAgo)
let fromBlock = await market.contract.provider.pastBlockTag(blocksAgo)
return await market.queryPastStorageRequestedEvents(fromBlock)

130
codex/contracts/marketplace.nim
View File

@ -52,22 +52,96 @@ proc slashMisses*(marketplace: Marketplace): UInt256 {.contract, view.}
proc slashPercentage*(marketplace: Marketplace): UInt256 {.contract, view.}
proc minCollateralThreshold*(marketplace: Marketplace): UInt256 {.contract, view.}
proc requestStorage*(marketplace: Marketplace, request: StorageRequest): Confirmable {.contract, errors:[Marketplace_InvalidClientAddress, Marketplace_RequestAlreadyExists, Marketplace_InvalidExpiry, Marketplace_InsufficientSlots, Marketplace_InvalidMaxSlotLoss].}
proc fillSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256, proof: Groth16Proof): Confirmable {.contract, errors:[Marketplace_InvalidSlot, Marketplace_ReservationRequired, Marketplace_SlotNotFree, Marketplace_StartNotBeforeExpiry, Marketplace_UnknownRequest].}
proc withdrawFunds*(marketplace: Marketplace, requestId: RequestId): Confirmable {.contract, errors:[Marketplace_InvalidClientAddress, Marketplace_InvalidState, Marketplace_NothingToWithdraw, Marketplace_UnknownRequest].}
proc withdrawFunds*(marketplace: Marketplace, requestId: RequestId, withdrawAddress: Address): Confirmable {.contract, errors:[Marketplace_InvalidClientAddress, Marketplace_InvalidState, Marketplace_NothingToWithdraw, Marketplace_UnknownRequest].}
proc freeSlot*(marketplace: Marketplace, id: SlotId): Confirmable {.contract, errors:[Marketplace_InvalidSlotHost, Marketplace_AlreadyPaid, Marketplace_StartNotBeforeExpiry, Marketplace_UnknownRequest, Marketplace_SlotIsFree].}
proc freeSlot*(marketplace: Marketplace, id: SlotId, rewardRecipient: Address, collateralRecipient: Address): Confirmable {.contract, errors:[Marketplace_InvalidSlotHost, Marketplace_AlreadyPaid, Marketplace_StartNotBeforeExpiry, Marketplace_UnknownRequest, Marketplace_SlotIsFree].}
proc getRequest*(marketplace: Marketplace, id: RequestId): StorageRequest {.contract, view, errors:[Marketplace_UnknownRequest].}
proc requestStorage*(
marketplace: Marketplace, request: StorageRequest
): Confirmable {.
contract,
errors: [
Marketplace_InvalidClientAddress, Marketplace_RequestAlreadyExists,
Marketplace_InvalidExpiry, Marketplace_InsufficientSlots,
Marketplace_InvalidMaxSlotLoss,
]
.}
proc fillSlot*(
marketplace: Marketplace,
requestId: RequestId,
slotIndex: UInt256,
proof: Groth16Proof,
): Confirmable {.
contract,
errors: [
Marketplace_InvalidSlot, Marketplace_ReservationRequired, Marketplace_SlotNotFree,
Marketplace_StartNotBeforeExpiry, Marketplace_UnknownRequest,
]
.}
proc withdrawFunds*(
marketplace: Marketplace, requestId: RequestId
): Confirmable {.
contract,
errors: [
Marketplace_InvalidClientAddress, Marketplace_InvalidState,
Marketplace_NothingToWithdraw, Marketplace_UnknownRequest,
]
.}
proc withdrawFunds*(
marketplace: Marketplace, requestId: RequestId, withdrawAddress: Address
): Confirmable {.
contract,
errors: [
Marketplace_InvalidClientAddress, Marketplace_InvalidState,
Marketplace_NothingToWithdraw, Marketplace_UnknownRequest,
]
.}
proc freeSlot*(
marketplace: Marketplace, id: SlotId
): Confirmable {.
contract,
errors: [
Marketplace_InvalidSlotHost, Marketplace_AlreadyPaid,
Marketplace_StartNotBeforeExpiry, Marketplace_UnknownRequest, Marketplace_SlotIsFree,
]
.}
proc freeSlot*(
marketplace: Marketplace,
id: SlotId,
rewardRecipient: Address,
collateralRecipient: Address,
): Confirmable {.
contract,
errors: [
Marketplace_InvalidSlotHost, Marketplace_AlreadyPaid,
Marketplace_StartNotBeforeExpiry, Marketplace_UnknownRequest, Marketplace_SlotIsFree,
]
.}
proc getRequest*(
marketplace: Marketplace, id: RequestId
): StorageRequest {.contract, view, errors: [Marketplace_UnknownRequest].}
proc getHost*(marketplace: Marketplace, id: SlotId): Address {.contract, view.}
proc getActiveSlot*(marketplace: Marketplace, id: SlotId): Slot {.contract, view, errors:[Marketplace_SlotIsFree].}
proc getActiveSlot*(
marketplace: Marketplace, id: SlotId
): Slot {.contract, view, errors: [Marketplace_SlotIsFree].}
proc myRequests*(marketplace: Marketplace): seq[RequestId] {.contract, view.}
proc mySlots*(marketplace: Marketplace): seq[SlotId] {.contract, view.}
proc requestState*(marketplace: Marketplace, requestId: RequestId): RequestState {.contract, view, errors:[Marketplace_UnknownRequest].}
proc requestState*(
marketplace: Marketplace, requestId: RequestId
): RequestState {.contract, view, errors: [Marketplace_UnknownRequest].}
proc slotState*(marketplace: Marketplace, slotId: SlotId): SlotState {.contract, view.}
proc requestEnd*(marketplace: Marketplace, requestId: RequestId): SecondsSince1970 {.contract, view.}
proc requestExpiry*(marketplace: Marketplace, requestId: RequestId): SecondsSince1970 {.contract, view.}
proc requestEnd*(
marketplace: Marketplace, requestId: RequestId
): SecondsSince1970 {.contract, view.}
proc requestExpiry*(
marketplace: Marketplace, requestId: RequestId
): SecondsSince1970 {.contract, view.}
proc proofTimeout*(marketplace: Marketplace): UInt256 {.contract, view.}
@ -75,11 +149,35 @@ proc proofEnd*(marketplace: Marketplace, id: SlotId): UInt256 {.contract, view.}
proc missingProofs*(marketplace: Marketplace, id: SlotId): UInt256 {.contract, view.}
proc isProofRequired*(marketplace: Marketplace, id: SlotId): bool {.contract, view.}
proc willProofBeRequired*(marketplace: Marketplace, id: SlotId): bool {.contract, view.}
proc getChallenge*(marketplace: Marketplace, id: SlotId): array[32, byte] {.contract, view.}
proc getChallenge*(
marketplace: Marketplace, id: SlotId
): array[32, byte] {.contract, view.}
proc getPointer*(marketplace: Marketplace, id: SlotId): uint8 {.contract, view.}
proc submitProof*(marketplace: Marketplace, id: SlotId, proof: Groth16Proof): Confirmable {.contract, errors:[Proofs_ProofAlreadySubmitted, Proofs_InvalidProof, Marketplace_UnknownRequest].}
proc markProofAsMissing*(marketplace: Marketplace, id: SlotId, period: UInt256): Confirmable {.contract, errors:[Marketplace_SlotNotAcceptingProofs, Marketplace_StartNotBeforeExpiry, Proofs_PeriodNotEnded, Proofs_ValidationTimedOut, Proofs_ProofNotMissing, Proofs_ProofNotRequired, Proofs_ProofAlreadyMarkedMissing].}
proc submitProof*(
marketplace: Marketplace, id: SlotId, proof: Groth16Proof
): Confirmable {.
contract,
errors:
[Proofs_ProofAlreadySubmitted, Proofs_InvalidProof, Marketplace_UnknownRequest]
.}
proc reserveSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256): Confirmable {.contract.}
proc canReserveSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256): bool {.contract, view.}
proc markProofAsMissing*(
marketplace: Marketplace, id: SlotId, period: UInt256
): Confirmable {.
contract,
errors: [
Marketplace_SlotNotAcceptingProofs, Marketplace_StartNotBeforeExpiry,
Proofs_PeriodNotEnded, Proofs_ValidationTimedOut, Proofs_ProofNotMissing,
Proofs_ProofNotRequired, Proofs_ProofAlreadyMarkedMissing,
]
.}
proc reserveSlot*(
marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256
): Confirmable {.contract.}
proc canReserveSlot*(
marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256
): bool {.contract, view.}

3
codex/contracts/proofs.nim
View File

@ -7,13 +7,16 @@ type
a*: G1Point
b*: G2Point
c*: G1Point
G1Point* = object
x*: UInt256
y*: UInt256
# A field element F_{p^2} encoded as `real + i * imag`
Fp2Element* = object
real*: UInt256
imag*: UInt256
G2Point* = object
x*: Fp2Element
y*: Fp2Element

45
codex/contracts/provider.nim
View File

@ -12,8 +12,9 @@ logScope:
proc raiseProviderError(message: string) {.raises: [ProviderError].} =
raise newException(ProviderError, message)
proc blockNumberAndTimestamp*(provider: Provider, blockTag: BlockTag):
Future[(UInt256, UInt256)] {.async: (raises: [ProviderError]).} =
proc blockNumberAndTimestamp*(
provider: Provider, blockTag: BlockTag
): Future[(UInt256, UInt256)] {.async: (raises: [ProviderError]).} =
without latestBlock =? await provider.getBlock(blockTag):
raiseProviderError("Could not get latest block")
@ -23,14 +24,10 @@ proc blockNumberAndTimestamp*(provider: Provider, blockTag: BlockTag):
return (latestBlockNumber, latestBlock.timestamp)
proc binarySearchFindClosestBlock(
provider: Provider,
epochTime: int,
low: UInt256,
high: UInt256): Future[UInt256] {.async: (raises: [ProviderError]).} =
let (_, lowTimestamp) =
await provider.blockNumberAndTimestamp(BlockTag.init(low))
let (_, highTimestamp) =
await provider.blockNumberAndTimestamp(BlockTag.init(high))
provider: Provider, epochTime: int, low: UInt256, high: UInt256
): Future[UInt256] {.async: (raises: [ProviderError]).} =
let (_, lowTimestamp) = await provider.blockNumberAndTimestamp(BlockTag.init(low))
let (_, highTimestamp) = await provider.blockNumberAndTimestamp(BlockTag.init(high))
if abs(lowTimestamp.truncate(int) - epochTime) <
abs(highTimestamp.truncate(int) - epochTime):
return low
@ -41,8 +38,8 @@ proc binarySearchBlockNumberForEpoch(
provider: Provider,
epochTime: UInt256,
latestBlockNumber: UInt256,
earliestBlockNumber: UInt256): Future[UInt256]
{.async: (raises: [ProviderError]).} =
earliestBlockNumber: UInt256,
): Future[UInt256] {.async: (raises: [ProviderError]).} =
var low = earliestBlockNumber
var high = latestBlockNumber
@ -63,12 +60,12 @@ proc binarySearchBlockNumberForEpoch(
# low is always greater than high - this is why we use high, where
# intuitively we would use low:
await provider.binarySearchFindClosestBlock(
epochTime.truncate(int), low=high, high=low)
epochTime.truncate(int), low = high, high = low
)
proc blockNumberForEpoch*(
provider: Provider,
epochTime: SecondsSince1970): Future[UInt256]
{.async: (raises: [ProviderError]).} =
provider: Provider, epochTime: SecondsSince1970
): Future[UInt256] {.async: (raises: [ProviderError]).} =
let epochTimeUInt256 = epochTime.u256
let (latestBlockNumber, latestBlockTimestamp) =
await provider.blockNumberAndTimestamp(BlockTag.latest)
@ -110,17 +107,17 @@ proc blockNumberForEpoch*(
if earliestBlockNumber > 0 and earliestBlockTimestamp > epochTimeUInt256:
let availableHistoryInDays =
(latestBlockTimestamp - earliestBlockTimestamp) div
1.days.secs.u256
warn "Short block history detected.", earliestBlockTimestamp =
earliestBlockTimestamp, days = availableHistoryInDays
(latestBlockTimestamp - earliestBlockTimestamp) div 1.days.secs.u256
warn "Short block history detected.",
earliestBlockTimestamp = earliestBlockTimestamp, days = availableHistoryInDays
return earliestBlockNumber
return await provider.binarySearchBlockNumberForEpoch(
epochTimeUInt256, latestBlockNumber, earliestBlockNumber)
epochTimeUInt256, latestBlockNumber, earliestBlockNumber
)
proc pastBlockTag*(provider: Provider,
blocksAgo: int):
Future[BlockTag] {.async: (raises: [ProviderError]).} =
proc pastBlockTag*(
provider: Provider, blocksAgo: int
): Future[BlockTag] {.async: (raises: [ProviderError]).} =
let head = await provider.getBlockNumber()
return BlockTag.init(head - blocksAgo.abs.u256)

41
codex/contracts/requests.nim
View File

@ -18,6 +18,7 @@ type
content* {.serialize.}: StorageContent
expiry* {.serialize.}: UInt256
nonce*: Nonce
StorageAsk* = object
slots* {.serialize.}: uint64
slotSize* {.serialize.}: UInt256
@ -26,12 +27,15 @@ type
reward* {.serialize.}: UInt256
collateral* {.serialize.}: UInt256
maxSlotLoss* {.serialize.}: uint64
StorageContent* = object
cid* {.serialize.}: string
merkleRoot*: array[32, byte]
Slot* = object
request* {.serialize.}: StorageRequest
slotIndex* {.serialize.}: UInt256
SlotId* = distinct array[32, byte]
RequestId* = distinct array[32, byte]
Nonce* = distinct array[32, byte]
@ -41,6 +45,7 @@ type
Cancelled
Finished
Failed
SlotState* {.pure.} = enum
Free
Filled
@ -80,27 +85,26 @@ proc toHex*[T: distinct](id: T): string =
type baseType = T.distinctBase
baseType(id).toHex
logutils.formatIt(LogFormat.textLines, Nonce): it.short0xHexLog
logutils.formatIt(LogFormat.textLines, RequestId): it.short0xHexLog
logutils.formatIt(LogFormat.textLines, SlotId): it.short0xHexLog
logutils.formatIt(LogFormat.json, Nonce): it.to0xHexLog
logutils.formatIt(LogFormat.json, RequestId): it.to0xHexLog
logutils.formatIt(LogFormat.json, SlotId): it.to0xHexLog
logutils.formatIt(LogFormat.textLines, Nonce):
it.short0xHexLog
logutils.formatIt(LogFormat.textLines, RequestId):
it.short0xHexLog
logutils.formatIt(LogFormat.textLines, SlotId):
it.short0xHexLog
logutils.formatIt(LogFormat.json, Nonce):
it.to0xHexLog
logutils.formatIt(LogFormat.json, RequestId):
it.to0xHexLog
logutils.formatIt(LogFormat.json, SlotId):
it.to0xHexLog
func fromTuple(_: type StorageRequest, tupl: tuple): StorageRequest =
StorageRequest(
client: tupl[0],
ask: tupl[1],
content: tupl[2],
expiry: tupl[3],
nonce: tupl[4]
client: tupl[0], ask: tupl[1], content: tupl[2], expiry: tupl[3], nonce: tupl[4]
)
func fromTuple(_: type Slot, tupl: tuple): Slot =
Slot(
request: tupl[0],
slotIndex: tupl[1]
)
Slot(request: tupl[0], slotIndex: tupl[1])
func fromTuple(_: type StorageAsk, tupl: tuple): StorageAsk =
StorageAsk(
@ -110,14 +114,11 @@ func fromTuple(_: type StorageAsk, tupl: tuple): StorageAsk =
proofProbability: tupl[3],
reward: tupl[4],
collateral: tupl[5],
maxSlotLoss: tupl[6]
maxSlotLoss: tupl[6],
)
func fromTuple(_: type StorageContent, tupl: tuple): StorageContent =
StorageContent(
cid: tupl[0],
merkleRoot: tupl[1]
)
StorageContent(cid: tupl[0], merkleRoot: tupl[1])
func solidityType*(_: type StorageContent): string =
solidityType(StorageContent.fieldTypes)

73
codex/discovery.nim
View File

@ -32,13 +32,13 @@ export discv5
logScope:
topics = "codex discovery"
type
Discovery* = ref object of RootObj
type Discovery* = ref object of RootObj
protocol*: discv5.Protocol # dht protocol
key: PrivateKey # private key
peerId: PeerId # the peer id of the local node
announceAddrs*: seq[MultiAddress] # addresses announced as part of the provider records
providerRecord*: ?SignedPeerRecord # record to advertice node connection information, this carry any
providerRecord*: ?SignedPeerRecord
# record to advertice node connection information, this carry any
# address that the node can be connected on
dhtRecord*: ?SignedPeerRecord # record to advertice DHT connection information
@ -54,14 +54,11 @@ proc toNodeId*(host: ca.Address): NodeId =
readUintBE[256](keccak256.digest(host.toArray).data)
proc findPeer*(
d: Discovery,
peerId: PeerId): Future[?PeerRecord] {.async.} =
proc findPeer*(d: Discovery, peerId: PeerId): Future[?PeerRecord] {.async.} =
trace "protocol.resolve..."
## Find peer using the given Discovery object
##
let
node = await d.protocol.resolve(toNodeId(peerId))
let node = await d.protocol.resolve(toNodeId(peerId))
return
if node.isSome():
@ -69,13 +66,10 @@ proc findPeer*(
else:
PeerRecord.none
method find*(
d: Discovery,
cid: Cid): Future[seq[SignedPeerRecord]] {.async, base.} =
method find*(d: Discovery, cid: Cid): Future[seq[SignedPeerRecord]] {.async, base.} =
## Find block providers
##
without providers =?
(await d.protocol.getProviders(cid.toNodeId())).mapFailure, error:
without providers =? (await d.protocol.getProviders(cid.toNodeId())).mapFailure, error:
warn "Error finding providers for block", cid, error = error.msg
return providers.filterIt(not (it.data.peerId == d.peerId))
@ -83,23 +77,20 @@ method find*(
method provide*(d: Discovery, cid: Cid) {.async, base.} =
## Provide a block Cid
##
let
nodes = await d.protocol.addProvider(
cid.toNodeId(), d.providerRecord.get)
let nodes = await d.protocol.addProvider(cid.toNodeId(), d.providerRecord.get)
if nodes.len <= 0:
warn "Couldn't provide to any nodes!"
method find*(
d: Discovery,
host: ca.Address): Future[seq[SignedPeerRecord]] {.async, base.} =
d: Discovery, host: ca.Address
): Future[seq[SignedPeerRecord]] {.async, base.} =
## Find host providers
##
trace "Finding providers for host", host = $host
without var providers =?
(await d.protocol.getProviders(host.toNodeId())).mapFailure, error:
without var providers =? (await d.protocol.getProviders(host.toNodeId())).mapFailure,
error:
trace "Error finding providers for host", host = $host, exc = error.msg
return
@ -117,15 +108,11 @@ method provide*(d: Discovery, host: ca.Address) {.async, base.} =
##
trace "Providing host", host = $host
let
nodes = await d.protocol.addProvider(
host.toNodeId(), d.providerRecord.get)
let nodes = await d.protocol.addProvider(host.toNodeId(), d.providerRecord.get)
if nodes.len > 0:
trace "Provided to nodes", nodes = nodes.len
method removeProvider*(
d: Discovery,
peerId: PeerId): Future[void] {.base, gcsafe.} =
method removeProvider*(d: Discovery, peerId: PeerId): Future[void] {.base, gcsafe.} =
## Remove provider from providers table
##
@ -139,26 +126,24 @@ proc updateAnnounceRecord*(d: Discovery, addrs: openArray[MultiAddress]) =
d.announceAddrs = @addrs
trace "Updating announce record", addrs = d.announceAddrs
d.providerRecord = SignedPeerRecord.init(
d.key, PeerRecord.init(d.peerId, d.announceAddrs))
d.providerRecord = SignedPeerRecord
.init(d.key, PeerRecord.init(d.peerId, d.announceAddrs))
.expect("Should construct signed record").some
if not d.protocol.isNil:
d.protocol.updateRecord(d.providerRecord)
.expect("Should update SPR")
d.protocol.updateRecord(d.providerRecord).expect("Should update SPR")
proc updateDhtRecord*(d: Discovery, addrs: openArray[MultiAddress]) =
## Update providers record
##
trace "Updating Dht record", addrs = addrs
d.dhtRecord = SignedPeerRecord.init(
d.key, PeerRecord.init(d.peerId, @addrs))
d.dhtRecord = SignedPeerRecord
.init(d.key, PeerRecord.init(d.peerId, @addrs))
.expect("Should construct signed record").some
if not d.protocol.isNil:
d.protocol.updateRecord(d.dhtRecord)
.expect("Should update SPR")
d.protocol.updateRecord(d.dhtRecord).expect("Should update SPR")
proc start*(d: Discovery) {.async.} =
d.protocol.open()
@ -174,15 +159,13 @@ proc new*(
bindPort = 0.Port,
announceAddrs: openArray[MultiAddress],
bootstrapNodes: openArray[SignedPeerRecord] = [],
store: Datastore = SQLiteDatastore.new(Memory).expect("Should not fail!")
store: Datastore = SQLiteDatastore.new(Memory).expect("Should not fail!"),
): Discovery =
## Create a new Discovery node instance for the given key and datastore
##
var
self = Discovery(
key: key,
peerId: PeerId.init(key).expect("Should construct PeerId"))
var self =
Discovery(key: key, peerId: PeerId.init(key).expect("Should construct PeerId"))
self.updateAnnounceRecord(announceAddrs)
@ -190,11 +173,8 @@ proc new*(
# FIXME disable IP limits temporarily so we can run our workshop. Re-enable
# and figure out proper solution.
let discoveryConfig = DiscoveryConfig(
tableIpLimits: TableIpLimits(
tableIpLimit: high(uint),
bucketIpLimit:high(uint)
),
bitsPerHop: DefaultBitsPerHop
tableIpLimits: TableIpLimits(tableIpLimit: high(uint), bucketIpLimit: high(uint)),
bitsPerHop: DefaultBitsPerHop,
)
# --------------------------------------------------------------------------
@ -206,6 +186,7 @@ proc new*(
bootstrapRecords = bootstrapNodes,
rng = Rng.instance(),
providers = ProvidersManager.new(store),
config = discoveryConfig)
config = discoveryConfig,
)
self

12
codex/erasure/backend.nim
View File

@ -9,7 +9,8 @@
import pkg/upraises
push: {.upraises: [].}
push:
{.upraises: [].}
import ../stores
@ -28,19 +29,14 @@ method release*(self: ErasureBackend) {.base, gcsafe.} =
raiseAssert("not implemented!")
method encode*(
self: EncoderBackend,
buffers,
parity: var openArray[seq[byte]]
self: EncoderBackend, buffers, parity: var openArray[seq[byte]]
): Result[void, cstring] {.base, gcsafe.} =
## encode buffers using a backend
##
raiseAssert("not implemented!")
method decode*(
self: DecoderBackend,
buffers,
parity,
recovered: var openArray[seq[byte]]
self: DecoderBackend, buffers, parity, recovered: var openArray[seq[byte]]
): Result[void, cstring] {.base, gcsafe.} =
## decode buffers using a backend
##

46
codex/erasure/backends/leopard.nim
View File

@ -22,19 +22,16 @@ type
decoder*: Option[LeoDecoder]
method encode*(
self: LeoEncoderBackend,
data,
parity: var openArray[seq[byte]]): Result[void, cstring] =
self: LeoEncoderBackend, data, parity: var openArray[seq[byte]]
): Result[void, cstring] =
## Encode data using Leopard backend
if parity.len == 0:
return ok()
var encoder = if self.encoder.isNone:
self.encoder = (? LeoEncoder.init(
self.blockSize,
self.buffers,
self.parity)).some
var encoder =
if self.encoder.isNone:
self.encoder = (?LeoEncoder.init(self.blockSize, self.buffers, self.parity)).some
self.encoder.get()
else:
self.encoder.get()
@ -42,18 +39,13 @@ method encode*(
encoder.encode(data, parity)
method decode*(
self: LeoDecoderBackend,
data,
parity,
recovered: var openArray[seq[byte]]): Result[void, cstring] =
self: LeoDecoderBackend, data, parity, recovered: var openArray[seq[byte]]
): Result[void, cstring] =
## Decode data using given Leopard backend
var decoder =
if self.decoder.isNone:
self.decoder = (? LeoDecoder.init(
self.blockSize,
self.buffers,
self.parity)).some
self.decoder = (?LeoDecoder.init(self.blockSize, self.buffers, self.parity)).some
self.decoder.get()
else:
self.decoder.get()
@ -69,25 +61,15 @@ method release*(self: LeoDecoderBackend) =
self.decoder.get().free()
proc new*(
T: type LeoEncoderBackend,
blockSize,
buffers,
parity: int): LeoEncoderBackend =
T: type LeoEncoderBackend, blockSize, buffers, parity: int
): LeoEncoderBackend =
## Create an instance of an Leopard Encoder backend
##
LeoEncoderBackend(
blockSize: blockSize,
buffers: buffers,
parity: parity)
LeoEncoderBackend(blockSize: blockSize, buffers: buffers, parity: parity)
proc new*(
T: type LeoDecoderBackend,
blockSize,
buffers,
parity: int): LeoDecoderBackend =
T: type LeoDecoderBackend, blockSize, buffers, parity: int
): LeoDecoderBackend =
## Create an instance of an Leopard Decoder backend
##
LeoDecoderBackend(
blockSize: blockSize,
buffers: buffers,
parity: parity)
LeoDecoderBackend(blockSize: blockSize, buffers: buffers, parity: parity)

131
codex/erasure/erasure.nim
View File

@ -9,7 +9,8 @@
import pkg/upraises
push: {.upraises: [].}
push:
{.upraises: [].}
import std/sequtils
import std/sugar
@ -60,12 +61,11 @@ type
## columns (with up to M blocks missing per column),
## or any combination there of.
##
EncoderProvider* =
proc(size, blocks, parity: int): EncoderBackend {.raises: [Defect], noSideEffect.}
EncoderProvider* = proc(size, blocks, parity: int): EncoderBackend
{.raises: [Defect], noSideEffect.}
DecoderProvider* = proc(size, blocks, parity: int): DecoderBackend
{.raises: [Defect], noSideEffect.}
DecoderProvider* =
proc(size, blocks, parity: int): DecoderBackend {.raises: [Defect], noSideEffect.}
Erasure* = ref object
encoderProvider*: EncoderProvider
@ -98,21 +98,22 @@ func indexToPos(steps, idx, step: int): int {.inline.} =
(idx - step) div steps
proc getPendingBlocks(
self: Erasure,
manifest: Manifest,
indicies: seq[int]): AsyncIter[(?!bt.Block, int)] =
self: Erasure, manifest: Manifest, indicies: seq[int]
): AsyncIter[(?!bt.Block, int)] =
## Get pending blocks iterator
##
var
# request blocks from the store
pendingBlocks = indicies.map( (i: int) =>
self.store.getBlock(
BlockAddress.init(manifest.treeCid, i)
).map((r: ?!bt.Block) => (r, i)) # Get the data blocks (first K)
pendingBlocks = indicies.map(
(i: int) =>
self.store.getBlock(BlockAddress.init(manifest.treeCid, i)).map(
(r: ?!bt.Block) => (r, i)
) # Get the data blocks (first K)
)
proc isFinished(): bool = pendingBlocks.len == 0
proc isFinished(): bool =
pendingBlocks.len == 0
proc genNext(): Future[(?!bt.Block, int)] {.async.} =
let completedFut = await one(pendingBlocks)
@ -123,7 +124,9 @@ proc getPendingBlocks(
let (_, index) = await completedFut
raise newException(
CatchableError,
"Future for block id not found, tree cid: " & $manifest.treeCid & ", index: " & $index)
"Future for block id not found, tree cid: " & $manifest.treeCid & ", index: " &
$index,
)
AsyncIter[(?!bt.Block, int)].new(genNext, isFinished)
@ -134,18 +137,18 @@ proc prepareEncodingData(
step: Natural,
data: ref seq[seq[byte]],
cids: ref seq[Cid],
emptyBlock: seq[byte]): Future[?!Natural] {.async.} =
emptyBlock: seq[byte],
): Future[?!Natural] {.async.} =
## Prepare data for encoding
##
let
strategy = params.strategy.init(
firstIndex = 0,
lastIndex = params.rounded - 1,
iterations = params.steps
firstIndex = 0, lastIndex = params.rounded - 1, iterations = params.steps
)
indicies = toSeq(strategy.getIndicies(step))
pendingBlocksIter = self.getPendingBlocks(manifest, indicies.filterIt(it < manifest.blocksCount))
pendingBlocksIter =
self.getPendingBlocks(manifest, indicies.filterIt(it < manifest.blocksCount))
var resolved = 0
for fut in pendingBlocksIter:
@ -164,7 +167,8 @@ proc prepareEncodingData(
let pos = indexToPos(params.steps, idx, step)
trace "Padding with empty block", idx
shallowCopy(data[pos], emptyBlock)
without emptyBlockCid =? emptyCid(manifest.version, manifest.hcodec, manifest.codec), err:
without emptyBlockCid =? emptyCid(manifest.version, manifest.hcodec, manifest.codec),
err:
return failure(err)
cids[idx] = emptyBlockCid
@ -177,7 +181,8 @@ proc prepareDecodingData(
data: ref seq[seq[byte]],
parityData: ref seq[seq[byte]],
cids: ref seq[Cid],
emptyBlock: seq[byte]): Future[?!(Natural, Natural)] {.async.} =
emptyBlock: seq[byte],
): Future[?!(Natural, Natural)] {.async.} =
## Prepare data for decoding
## `encoded` - the encoded manifest
## `step` - the current step
@ -189,9 +194,7 @@ proc prepareDecodingData(
let
strategy = encoded.protectedStrategy.init(
firstIndex = 0,
lastIndex = encoded.blocksCount - 1,
iterations = encoded.steps
firstIndex = 0, lastIndex = encoded.blocksCount - 1, iterations = encoded.steps
)
indicies = toSeq(strategy.getIndicies(step))
pendingBlocksIter = self.getPendingBlocks(encoded, indicies)
@ -211,8 +214,7 @@ proc prepareDecodingData(
trace "Failed retreiving a block", idx, treeCid = encoded.treeCid, msg = err.msg
continue
let
pos = indexToPos(encoded.steps, idx, step)
let pos = indexToPos(encoded.steps, idx, step)
logScope:
cid = blk.cid
@ -224,7 +226,9 @@ proc prepareDecodingData(
cids[idx] = blk.cid
if idx >= encoded.rounded:
trace "Retrieved parity block"
shallowCopy(parityData[pos - encoded.ecK], if blk.isEmpty: emptyBlock else: blk.data)
shallowCopy(
parityData[pos - encoded.ecK], if blk.isEmpty: emptyBlock else: blk.data
)
parityPieces.inc
else:
trace "Retrieved data block"
@ -238,15 +242,17 @@ proc prepareDecodingData(
proc init*(
_: type EncodingParams,
manifest: Manifest,
ecK: Natural, ecM: Natural,
strategy: StrategyType): ?!EncodingParams =
ecK: Natural,
ecM: Natural,
strategy: StrategyType,
): ?!EncodingParams =
if ecK > manifest.blocksCount:
let exc = (ref InsufficientBlocksError)(
msg: "Unable to encode manifest, not enough blocks, ecK = " &
$ecK &
", blocksCount = " &
$manifest.blocksCount,
minSize: ecK.NBytes * manifest.blockSize)
msg:
"Unable to encode manifest, not enough blocks, ecK = " & $ecK &
", blocksCount = " & $manifest.blocksCount,
minSize: ecK.NBytes * manifest.blockSize,
)
return failure(exc)
let
@ -260,13 +266,11 @@ proc init*(
rounded: rounded,
steps: steps,
blocksCount: blocksCount,
strategy: strategy
strategy: strategy,
)
proc encodeData(
self: Erasure,
manifest: Manifest,
params: EncodingParams
self: Erasure, manifest: Manifest, params: EncodingParams
): Future[?!Manifest] {.async.} =
## Encode blocks pointed to by the protected manifest
##
@ -292,7 +296,8 @@ proc encodeData(
# TODO: Don't allocate a new seq every time, allocate once and zero out
var
data = seq[seq[byte]].new() # number of blocks to encode
parityData = newSeqWith[seq[byte]](params.ecM, newSeq[byte](manifest.blockSize.int))
parityData =
newSeqWith[seq[byte]](params.ecM, newSeq[byte](manifest.blockSize.int))
data[].setLen(params.ecK)
# TODO: this is a tight blocking loop so we sleep here to allow
@ -301,15 +306,14 @@ proc encodeData(
await sleepAsync(10.millis)
without resolved =?
(await self.prepareEncodingData(manifest, params, step, data, cids, emptyBlock)), err:
(await self.prepareEncodingData(manifest, params, step, data, cids, emptyBlock)),
err:
trace "Unable to prepare data", error = err.msg
return failure(err)
trace "Erasure coding data", data = data[].len, parity = parityData.len
if (
let res = encoder.encode(data[], parityData);
res.isErr):
if (let res = encoder.encode(data[], parityData); res.isErr):
trace "Unable to encode manifest!", error = $res.error
return failure($res.error)
@ -341,7 +345,7 @@ proc encodeData(
datasetSize = (manifest.blockSize.int * params.blocksCount).NBytes,
ecK = params.ecK,
ecM = params.ecM,
strategy = params.strategy
strategy = params.strategy,
)
trace "Encoded data successfully", treeCid, blocksCount = params.blocksCount
@ -360,7 +364,8 @@ proc encode*(
manifest: Manifest,
blocks: Natural,
parity: Natural,
strategy = SteppedStrategy): Future[?!Manifest] {.async.} =
strategy = SteppedStrategy,
): Future[?!Manifest] {.async.} =
## Encode a manifest into one that is erasure protected.
##
## `manifest` - the original manifest to be encoded
@ -376,9 +381,7 @@ proc encode*(
return success encodedManifest
proc decode*(
self: Erasure,
encoded: Manifest): Future[?!Manifest] {.async.} =
proc decode*(self: Erasure, encoded: Manifest): Future[?!Manifest] {.async.} =
## Decode a protected manifest into it's original
## manifest
##
@ -408,13 +411,17 @@ proc decode*(
var
data = seq[seq[byte]].new()
parityData = seq[seq[byte]].new()
recovered = newSeqWith[seq[byte]](encoded.ecK, newSeq[byte](encoded.blockSize.int))
recovered =
newSeqWith[seq[byte]](encoded.ecK, newSeq[byte](encoded.blockSize.int))
data[].setLen(encoded.ecK) # set len to K
parityData[].setLen(encoded.ecM) # set len to M
without (dataPieces, _) =?
(await self.prepareDecodingData(encoded, step, data, parityData, cids, emptyBlock)), err:
without (dataPieces, _) =? (
await self.prepareDecodingData(
encoded, step, data, parityData, cids, emptyBlock
)
), err:
trace "Unable to prepare data", error = err.msg
return failure(err)
@ -424,9 +431,7 @@ proc decode*(
trace "Erasure decoding data"
if (
let err = decoder.decode(data[], parityData[], recovered);
err.isErr):
if (let err = decoder.decode(data[], parityData[], recovered); err.isErr):
trace "Unable to decode data!", err = $err.error
return failure($err.error)
@ -460,10 +465,12 @@ proc decode*(
return failure(err)
if treeCid != encoded.originalTreeCid:
return failure("Original tree root differs from the tree root computed out of recovered data")
return failure(
"Original tree root differs from the tree root computed out of recovered data"
)
let idxIter = Iter[Natural].new(recoveredIndices)
.filter((i: Natural) => i < tree.leavesCount)
let idxIter =
Iter[Natural].new(recoveredIndices).filter((i: Natural) => i < tree.leavesCount)
if err =? (await self.store.putSomeProofs(tree, idxIter)).errorOption:
return failure(err)
@ -482,11 +489,11 @@ proc new*(
T: type Erasure,
store: BlockStore,
encoderProvider: EncoderProvider,
decoderProvider: DecoderProvider): Erasure =
decoderProvider: DecoderProvider,
): Erasure =
## Create a new Erasure instance for encoding and decoding manifests
##
Erasure(
store: store,
encoderProvider: encoderProvider,
decoderProvider: decoderProvider)
store: store, encoderProvider: encoderProvider, decoderProvider: decoderProvider
)

8
codex/errors.nim
View File

@ -20,13 +20,15 @@ type
CodexResult*[T] = Result[T, ref CodexError]
template mapFailure*[T, V, E](
exp: Result[T, V],
exc: typedesc[E],
exp: Result[T, V], exc: typedesc[E]
): Result[T, ref CatchableError] =
## Convert `Result[T, E]` to `Result[E, ref CatchableError]`
##
exp.mapErr(proc (e: V): ref CatchableError = (ref exc)(msg: $e))
exp.mapErr(
proc(e: V): ref CatchableError =
(ref exc)(msg: $e)
)
template mapFailure*[T, V](exp: Result[T, V]): Result[T, ref CatchableError] =
mapFailure(exp, CodexError)

39
codex/indexingstrategy.nim
View File

@ -10,7 +10,7 @@ type
# 0 => 0, 1, 2
# 1 => 3, 4, 5
# 2 => 6, 7, 8
LinearStrategy,
LinearStrategy
# Stepped indexing:
# 0 => 0, 3, 6
@ -21,7 +21,6 @@ type
# Representing a strategy for grouping indices (of blocks usually)
# Given an interation-count as input, will produce a seq of
# selected indices.
IndexingError* = object of CodexError
IndexingWrongIndexError* = object of IndexingError
IndexingWrongIterationsError* = object of IndexingError
@ -33,19 +32,21 @@ type
iterations*: int # getIndices(iteration) will run from 0 ..< iterations
step*: int
func checkIteration(self: IndexingStrategy, iteration: int): void {.raises: [IndexingError].} =
func checkIteration(
self: IndexingStrategy, iteration: int
): void {.raises: [IndexingError].} =
if iteration >= self.iterations:
raise newException(
IndexingError,
"Indexing iteration can't be greater than or equal to iterations.")
IndexingError, "Indexing iteration can't be greater than or equal to iterations."
)
func getIter(first, last, step: int): Iter[int] =
{.cast(noSideEffect).}:
Iter[int].new(first, last, step)
func getLinearIndicies(
self: IndexingStrategy,
iteration: int): Iter[int] {.raises: [IndexingError].} =
self: IndexingStrategy, iteration: int
): Iter[int] {.raises: [IndexingError].} =
self.checkIteration(iteration)
let
@ -55,8 +56,8 @@ func getLinearIndicies(
getIter(first, last, 1)
func getSteppedIndicies(
self: IndexingStrategy,
iteration: int): Iter[int] {.raises: [IndexingError].} =
self: IndexingStrategy, iteration: int
): Iter[int] {.raises: [IndexingError].} =
self.checkIteration(iteration)
let
@ -66,9 +67,8 @@ func getSteppedIndicies(
getIter(first, last, self.iterations)
func getIndicies*(
self: IndexingStrategy,
iteration: int): Iter[int] {.raises: [IndexingError].} =
self: IndexingStrategy, iteration: int
): Iter[int] {.raises: [IndexingError].} =
case self.strategyType
of StrategyType.LinearStrategy:
self.getLinearIndicies(iteration)
@ -76,22 +76,25 @@ func getIndicies*(
self.getSteppedIndicies(iteration)
func init*(
strategy: StrategyType,
firstIndex, lastIndex, iterations: int): IndexingStrategy {.raises: [IndexingError].} =
strategy: StrategyType, firstIndex, lastIndex, iterations: int
): IndexingStrategy {.raises: [IndexingError].} =
if firstIndex > lastIndex:
raise newException(
IndexingWrongIndexError,
"firstIndex (" & $firstIndex & ") can't be greater than lastIndex (" & $lastIndex & ")")
"firstIndex (" & $firstIndex & ") can't be greater than lastIndex (" & $lastIndex &
")",
)
if iterations <= 0:
raise newException(
IndexingWrongIterationsError,
"iterations (" & $iterations & ") must be greater than zero.")
"iterations (" & $iterations & ") must be greater than zero.",
)
IndexingStrategy(
strategyType: strategy,
firstIndex: firstIndex,
lastIndex: lastIndex,
iterations: iterations,
step: divUp((lastIndex - firstIndex + 1), iterations))
step: divUp((lastIndex - firstIndex + 1), iterations),
)

43
codex/logutils.nim
View File

@ -123,7 +123,8 @@ func shortLog*(long: string, ellipses = "*", start = 3, stop = 6): string =
short
func shortHexLog*(long: string): string =
if long[0..1] == "0x": result &= "0x"
if long[0 .. 1] == "0x":
result &= "0x"
result &= long[2 .. long.high].shortLog("..", 4, 4)
func short0xHexLog*[N: static[int], T: array[N, byte]](v: T): string =
@ -182,12 +183,16 @@ template formatIt*(format: LogFormat, T: typedesc, body: untyped) =
let v = opts.map(opt => opt.formatJsonOption)
setProperty(r, key, json.`%`(v))
proc setProperty*(r: var JsonRecord, key: string, val: seq[T]) {.raises:[ValueError, IOError].} =
proc setProperty*(
r: var JsonRecord, key: string, val: seq[T]
) {.raises: [ValueError, IOError].} =
var it {.inject, used.}: T
let v = val.map(it => body)
setProperty(r, key, json.`%`(v))
proc setProperty*(r: var JsonRecord, key: string, val: T) {.raises:[ValueError, IOError].} =
proc setProperty*(
r: var JsonRecord, key: string, val: T
) {.raises: [ValueError, IOError].} =
var it {.inject, used.}: T = val
let v = body
setProperty(r, key, json.`%`(v))
@ -218,23 +223,35 @@ template formatIt*(format: LogFormat, T: typedesc, body: untyped) =
let v = opts.map(opt => opt.formatTextLineOption)
setProperty(r, key, v.formatTextLineSeq)
proc setProperty*(r: var TextLineRecord, key: string, val: seq[T]) {.raises:[ValueError, IOError].} =
proc setProperty*(
r: var TextLineRecord, key: string, val: seq[T]
) {.raises: [ValueError, IOError].} =
var it {.inject, used.}: T
let v = val.map(it => body)
setProperty(r, key, v.formatTextLineSeq)
proc setProperty*(r: var TextLineRecord, key: string, val: T) {.raises:[ValueError, IOError].} =
proc setProperty*(
r: var TextLineRecord, key: string, val: T
) {.raises: [ValueError, IOError].} =
var it {.inject, used.}: T = val
let v = body
setProperty(r, key, v)
template formatIt*(T: type, body: untyped) {.dirty.} =
formatIt(LogFormat.textLines, T): body
formatIt(LogFormat.json, T): body
formatIt(LogFormat.textLines, T):
body
formatIt(LogFormat.json, T):
body
formatIt(LogFormat.textLines, Cid): shortLog($it)
formatIt(LogFormat.json, Cid): $it
formatIt(UInt256): $it
formatIt(MultiAddress): $it
formatIt(LogFormat.textLines, array[32, byte]): it.short0xHexLog
formatIt(LogFormat.json, array[32, byte]): it.to0xHex
formatIt(LogFormat.textLines, Cid):
shortLog($it)
formatIt(LogFormat.json, Cid):
$it
formatIt(UInt256):
$it
formatIt(MultiAddress):
$it
formatIt(LogFormat.textLines, array[32, byte]):
it.short0xHexLog
formatIt(LogFormat.json, array[32, byte]):
it.to0xHex

18
codex/manifest/coders.nim
View File

@ -12,7 +12,8 @@
import pkg/upraises
import times
push: {.upraises: [].}
push:
{.upraises: [].}
import std/tables
import std/sequtils
@ -206,15 +207,14 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
if pbVerificationInfo.getField(4, verifiableStrategy).isErr:
return failure("Unable to decode `verifiableStrategy` from manifest!")
let
treeCid = ? Cid.init(treeCidBuf).mapFailure
let treeCid = ?Cid.init(treeCidBuf).mapFailure
var filenameOption = if filename.len == 0: string.none else: filename.some
var mimetypeOption = if mimetype.len == 0: string.none else: mimetype.some
var uploadedAtOption = if uploadedAt == 0: int64.none else: uploadedAt.int64.some
let
self = if protected:
let self =
if protected:
Manifest.new(
treeCid = treeCid,
datasetSize = datasetSize.NBytes,
@ -229,7 +229,8 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
strategy = StrategyType(protectedStrategy),
filename = filenameOption,
mimetype = mimetypeOption,
uploadedAt = uploadedAtOption)
uploadedAt = uploadedAtOption,
)
else:
Manifest.new(
treeCid = treeCid,
@ -240,7 +241,8 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
codec = codec.MultiCodec,
filename = filenameOption,
mimetype = mimetypeOption,
uploadedAt = uploadedAtOption)
uploadedAt = uploadedAtOption,
)
?self.verify()
@ -254,7 +256,7 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
verifyRoot = verifyRootCid,
slotRoots = slotRootCids,
cellSize = cellSize.NBytes,
strategy = StrategyType(verifiableStrategy)
strategy = StrategyType(verifiableStrategy),
)
self.success

133
codex/manifest/manifest.nim
View File

@ -11,7 +11,8 @@
import pkg/upraises
push: {.upraises: [].}
push:
{.upraises: [].}
import pkg/libp2p/protobuf/minprotobuf
import pkg/libp2p/[cid, multihash, multicodec]
@ -25,14 +26,13 @@ import ../blocktype
import ../indexingstrategy
import ../logutils
# TODO: Manifest should be reworked to more concrete types,
# perhaps using inheritance
type
Manifest* = ref object of RootObj
type Manifest* = ref object of RootObj
treeCid {.serialize.}: Cid # Root of the merkle tree
datasetSize {.serialize.}: NBytes # Total size of all blocks
blockSize {.serialize.}: NBytes # Size of each contained block (might not be needed if blocks are len-prefixed)
blockSize {.serialize.}: NBytes
# Size of each contained block (might not be needed if blocks are len-prefixed)
codec: MultiCodec # Dataset codec
hcodec: MultiCodec # Multihash codec
version: CidVersion # Cid version
@ -46,7 +46,8 @@ type
originalTreeCid: Cid # The original root of the dataset being erasure coded
originalDatasetSize: NBytes
protectedStrategy: StrategyType # Indexing strategy used to build the slot roots
case verifiable {.serialize.}: bool # Verifiable datasets can be used to generate storage proofs
case verifiable {.serialize.}: bool
# Verifiable datasets can be used to generate storage proofs
of true:
verifyRoot: Cid # Root of the top level merkle tree built from slot roots
slotRoots: seq[Cid] # Individual slot root built from the original dataset blocks
@ -159,7 +160,8 @@ func verify*(self: Manifest): ?!void =
##
if self.protected and (self.blocksCount != self.steps * (self.ecK + self.ecM)):
return failure newException(CodexError, "Broken manifest: wrong originalBlocksCount")
return
failure newException(CodexError, "Broken manifest: wrong originalBlocksCount")
return success()
@ -167,41 +169,32 @@ func cid*(self: Manifest): ?!Cid {.deprecated: "use treeCid instead".} =
self.treeCid.success
func `==`*(a, b: Manifest): bool =
(a.treeCid == b.treeCid) and
(a.datasetSize == b.datasetSize) and
(a.blockSize == b.blockSize) and
(a.version == b.version) and
(a.hcodec == b.hcodec) and
(a.codec == b.codec) and
(a.protected == b.protected) and
(a.filename == b.filename) and
(a.mimetype == b.mimetype) and
(a.uploadedAt == b.uploadedAt) and
(if a.protected:
(a.ecK == b.ecK) and
(a.ecM == b.ecM) and
(a.originalTreeCid == b.originalTreeCid) and
(a.treeCid == b.treeCid) and (a.datasetSize == b.datasetSize) and
(a.blockSize == b.blockSize) and (a.version == b.version) and (a.hcodec == b.hcodec) and
(a.codec == b.codec) and (a.protected == b.protected) and (a.filename == b.filename) and
(a.mimetype == b.mimetype) and (a.uploadedAt == b.uploadedAt) and (
if a.protected:
(a.ecK == b.ecK) and (a.ecM == b.ecM) and (a.originalTreeCid == b.originalTreeCid) and
(a.originalDatasetSize == b.originalDatasetSize) and
(a.protectedStrategy == b.protectedStrategy) and
(a.verifiable == b.verifiable) and
(if a.verifiable:
(a.verifyRoot == b.verifyRoot) and
(a.slotRoots == b.slotRoots) and
(a.cellSize == b.cellSize) and
(a.verifiableStrategy == b.verifiableStrategy)
(a.protectedStrategy == b.protectedStrategy) and (a.verifiable == b.verifiable) and
(
if a.verifiable:
(a.verifyRoot == b.verifyRoot) and (a.slotRoots == b.slotRoots) and
(a.cellSize == b.cellSize) and (
a.verifiableStrategy == b.verifiableStrategy
)
else:
true)
true
)
else:
true)
true
)
func `$`*(self: Manifest): string =
result = "treeCid: " & $self.treeCid &
", datasetSize: " & $self.datasetSize &
", blockSize: " & $self.blockSize &
", version: " & $self.version &
", hcodec: " & $self.hcodec &
", codec: " & $self.codec &
", protected: " & $self.protected
result =
"treeCid: " & $self.treeCid & ", datasetSize: " & $self.datasetSize & ", blockSize: " &
$self.blockSize & ", version: " & $self.version & ", hcodec: " & $self.hcodec &
", codec: " & $self.codec & ", protected: " & $self.protected
if self.filename.isSome:
result &= ", filename: " & $self.filename
@ -212,19 +205,19 @@ func `$`*(self: Manifest): string =
if self.uploadedAt.isSome:
result &= ", uploadedAt: " & $self.uploadedAt
result &= (if self.protected:
", ecK: " & $self.ecK &
", ecM: " & $self.ecM &
", originalTreeCid: " & $self.originalTreeCid &
", originalDatasetSize: " & $self.originalDatasetSize &
", verifiable: " & $self.verifiable &
(if self.verifiable:
", verifyRoot: " & $self.verifyRoot &
", slotRoots: " & $self.slotRoots
result &= (
if self.protected:
", ecK: " & $self.ecK & ", ecM: " & $self.ecM & ", originalTreeCid: " &
$self.originalTreeCid & ", originalDatasetSize: " & $self.originalDatasetSize &
", verifiable: " & $self.verifiable & (
if self.verifiable:
", verifyRoot: " & $self.verifyRoot & ", slotRoots: " & $self.slotRoots
else:
"")
""
)
else:
"")
""
)
return result
@ -243,8 +236,8 @@ func new*(
protected = false,
filename: ?string = string.none,
mimetype: ?string = string.none,
uploadedAt: ?int64 = int64.none): Manifest =
uploadedAt: ?int64 = int64.none,
): Manifest =
T(
treeCid: treeCid,
blockSize: blockSize,
@ -255,7 +248,8 @@ func new*(
protected: protected,
filename: filename,
mimetype: mimetype,
uploadedAt: uploadedAt)
uploadedAt: uploadedAt,
)
func new*(
T: type Manifest,
@ -263,7 +257,8 @@ func new*(
treeCid: Cid,
datasetSize: NBytes,
ecK, ecM: int,
strategy = SteppedStrategy): Manifest =
strategy = SteppedStrategy,
): Manifest =
## Create an erasure protected dataset from an
## unprotected one
##
@ -276,18 +271,17 @@ func new*(
hcodec: manifest.hcodec,
blockSize: manifest.blockSize,
protected: true,
ecK: ecK, ecM: ecM,
ecK: ecK,
ecM: ecM,
originalTreeCid: manifest.treeCid,
originalDatasetSize: manifest.datasetSize,
protectedStrategy: strategy,
filename: manifest.filename,
mimetype: manifest.mimetype,
uploadedAt: manifest.uploadedAt
uploadedAt: manifest.uploadedAt,
)
func new*(
T: type Manifest,
manifest: Manifest): Manifest =
func new*(T: type Manifest, manifest: Manifest): Manifest =
## Create an unprotected dataset from an
## erasure protected one
##
@ -302,7 +296,8 @@ func new*(
protected: false,
filename: manifest.filename,
mimetype: manifest.mimetype,
uploadedAt: manifest.uploadedAt)
uploadedAt: manifest.uploadedAt,
)
func new*(
T: type Manifest,
@ -319,8 +314,8 @@ func new*(
strategy = SteppedStrategy,
filename: ?string = string.none,
mimetype: ?string = string.none,
uploadedAt: ?int64 = int64.none): Manifest =
uploadedAt: ?int64 = int64.none,
): Manifest =
Manifest(
treeCid: treeCid,
datasetSize: datasetSize,
@ -336,7 +331,8 @@ func new*(
protectedStrategy: strategy,
filename: filename,
mimetype: mimetype,
uploadedAt: uploadedAt)
uploadedAt: uploadedAt,
)
func new*(
T: type Manifest,
@ -344,18 +340,19 @@ func new*(
verifyRoot: Cid,
slotRoots: openArray[Cid],
cellSize = DefaultCellSize,
strategy = LinearStrategy): ?!Manifest =
strategy = LinearStrategy,
): ?!Manifest =
## Create a verifiable dataset from an
## protected one
##
if not manifest.protected:
return failure newException(
CodexError, "Can create verifiable manifest only from protected manifest.")
CodexError, "Can create verifiable manifest only from protected manifest."
)
if slotRoots.len != manifest.numSlots:
return failure newException(
CodexError, "Wrong number of slot roots.")
return failure newException(CodexError, "Wrong number of slot roots.")
success Manifest(
treeCid: manifest.treeCid,
@ -377,12 +374,10 @@ func new*(
verifiableStrategy: strategy,
filename: manifest.filename,
mimetype: manifest.mimetype,
uploadedAt: manifest.uploadedAt
uploadedAt: manifest.uploadedAt,
)
func new*(
T: type Manifest,
data: openArray[byte]): ?!Manifest =
func new*(T: type Manifest, data: openArray[byte]): ?!Manifest =
## Create a manifest instance from given data
##

199
codex/market.nim
View File

@ -19,13 +19,14 @@ type
Market* = ref object of RootObj
MarketError* = object of CodexError
Subscription* = ref object of RootObj
OnRequest* = proc(id: RequestId,
ask: StorageAsk,
expiry: UInt256) {.gcsafe, upraises:[].}
OnRequest* =
proc(id: RequestId, ask: StorageAsk, expiry: UInt256) {.gcsafe, upraises: [].}
OnFulfillment* = proc(requestId: RequestId) {.gcsafe, upraises: [].}
OnSlotFilled* = proc(requestId: RequestId, slotIndex: UInt256) {.gcsafe, upraises:[].}
OnSlotFilled* =
proc(requestId: RequestId, slotIndex: UInt256) {.gcsafe, upraises: [].}
OnSlotFreed* = proc(requestId: RequestId, slotIndex: UInt256) {.gcsafe, upraises: [].}
OnSlotReservationsFull* = proc(requestId: RequestId, slotIndex: UInt256) {.gcsafe, upraises: [].}
OnSlotReservationsFull* =
proc(requestId: RequestId, slotIndex: UInt256) {.gcsafe, upraises: [].}
OnRequestCancelled* = proc(requestId: RequestId) {.gcsafe, upraises: [].}
OnRequestFailed* = proc(requestId: RequestId) {.gcsafe, upraises: [].}
OnProofSubmitted* = proc(id: SlotId) {.gcsafe, upraises: [].}
@ -37,21 +38,28 @@ type
requestId*: RequestId
ask*: StorageAsk
expiry*: UInt256
SlotFilled* = object of MarketplaceEvent
requestId* {.indexed.}: RequestId
slotIndex*: UInt256
SlotFreed* = object of MarketplaceEvent
requestId* {.indexed.}: RequestId
slotIndex*: UInt256
SlotReservationsFull* = object of MarketplaceEvent
requestId* {.indexed.}: RequestId
slotIndex*: UInt256
RequestFulfilled* = object of MarketplaceEvent
requestId* {.indexed.}: RequestId
RequestCancelled* = object of MarketplaceEvent
requestId* {.indexed.}: RequestId
RequestFailed* = object of MarketplaceEvent
requestId* {.indexed.}: RequestId
ProofSubmitted* = object of MarketplaceEvent
id*: SlotId
@ -81,8 +89,7 @@ proc inDowntime*(market: Market, slotId: SlotId): Future[bool] {.async.} =
let pntr = await market.getPointer(slotId)
return pntr < downtime
method requestStorage*(market: Market,
request: StorageRequest) {.base, async.} =
method requestStorage*(market: Market, request: StorageRequest) {.base, async.} =
raiseAssert("not implemented")
method myRequests*(market: Market): Future[seq[RequestId]] {.base, async.} =
@ -91,182 +98,168 @@ method myRequests*(market: Market): Future[seq[RequestId]] {.base, async.} =
method mySlots*(market: Market): Future[seq[SlotId]] {.base, async.} =
raiseAssert("not implemented")
method getRequest*(market: Market,
id: RequestId):
Future[?StorageRequest] {.base, async.} =
method getRequest*(
market: Market, id: RequestId
): Future[?StorageRequest] {.base, async.} =
raiseAssert("not implemented")
method requestState*(market: Market,
requestId: RequestId): Future[?RequestState] {.base, async.} =
method requestState*(
market: Market, requestId: RequestId
): Future[?RequestState] {.base, async.} =
raiseAssert("not implemented")
method slotState*(market: Market,
slotId: SlotId): Future[SlotState] {.base, async.} =
method slotState*(market: Market, slotId: SlotId): Future[SlotState] {.base, async.} =
raiseAssert("not implemented")
method getRequestEnd*(market: Market,
id: RequestId): Future[SecondsSince1970] {.base, async.} =
method getRequestEnd*(
market: Market, id: RequestId
): Future[SecondsSince1970] {.base, async.} =
raiseAssert("not implemented")
method requestExpiresAt*(market: Market,
id: RequestId): Future[SecondsSince1970] {.base, async.} =
method requestExpiresAt*(
market: Market, id: RequestId
): Future[SecondsSince1970] {.base, async.} =
raiseAssert("not implemented")
method getHost*(market: Market,
requestId: RequestId,
slotIndex: UInt256): Future[?Address] {.base, async.} =
method getHost*(
market: Market, requestId: RequestId, slotIndex: UInt256
): Future[?Address] {.base, async.} =
raiseAssert("not implemented")
method getActiveSlot*(
method getActiveSlot*(market: Market, slotId: SlotId): Future[?Slot] {.base, async.} =
raiseAssert("not implemented")
method fillSlot*(
market: Market,
slotId: SlotId): Future[?Slot] {.base, async.} =
raiseAssert("not implemented")
method fillSlot*(market: Market,
requestId: RequestId,
slotIndex: UInt256,
proof: Groth16Proof,
collateral: UInt256) {.base, async.} =
collateral: UInt256,
) {.base, async.} =
raiseAssert("not implemented")
method freeSlot*(market: Market, slotId: SlotId) {.base, async.} =
raiseAssert("not implemented")
method withdrawFunds*(market: Market,
requestId: RequestId) {.base, async.} =
method withdrawFunds*(market: Market, requestId: RequestId) {.base, async.} =
raiseAssert("not implemented")
method subscribeRequests*(market: Market,
callback: OnRequest):
Future[Subscription] {.base, async.} =
method subscribeRequests*(
market: Market, callback: OnRequest
): Future[Subscription] {.base, async.} =
raiseAssert("not implemented")
method isProofRequired*(market: Market,
id: SlotId): Future[bool] {.base, async.} =
method isProofRequired*(market: Market, id: SlotId): Future[bool] {.base, async.} =
raiseAssert("not implemented")
method willProofBeRequired*(market: Market,
id: SlotId): Future[bool] {.base, async.} =
method willProofBeRequired*(market: Market, id: SlotId): Future[bool] {.base, async.} =
raiseAssert("not implemented")
method getChallenge*(market: Market, id: SlotId): Future[ProofChallenge] {.base, async.} =
method getChallenge*(
market: Market, id: SlotId
): Future[ProofChallenge] {.base, async.} =
raiseAssert("not implemented")
method submitProof*(market: Market,
id: SlotId,
proof: Groth16Proof) {.base, async.} =
method submitProof*(market: Market, id: SlotId, proof: Groth16Proof) {.base, async.} =
raiseAssert("not implemented")
method markProofAsMissing*(market: Market,
id: SlotId,
period: Period) {.base, async.} =
method markProofAsMissing*(market: Market, id: SlotId, period: Period) {.base, async.} =
raiseAssert("not implemented")
method canProofBeMarkedAsMissing*(market: Market,
id: SlotId,
period: Period): Future[bool] {.base, async.} =
method canProofBeMarkedAsMissing*(
market: Market, id: SlotId, period: Period
): Future[bool] {.base, async.} =
raiseAssert("not implemented")
method reserveSlot*(
market: Market,
requestId: RequestId,
slotIndex: UInt256) {.base, async.} =
market: Market, requestId: RequestId, slotIndex: UInt256
) {.base, async.} =
raiseAssert("not implemented")
method canReserveSlot*(
market: Market,
requestId: RequestId,
slotIndex: UInt256): Future[bool] {.base, async.} =
market: Market, requestId: RequestId, slotIndex: UInt256
): Future[bool] {.base, async.} =
raiseAssert("not implemented")
method subscribeFulfillment*(market: Market,
callback: OnFulfillment):
Future[Subscription] {.base, async.} =
method subscribeFulfillment*(
market: Market, callback: OnFulfillment
): Future[Subscription] {.base, async.} =
raiseAssert("not implemented")
method subscribeFulfillment*(market: Market,
requestId: RequestId,
callback: OnFulfillment):
Future[Subscription] {.base, async.} =
method subscribeFulfillment*(
market: Market, requestId: RequestId, callback: OnFulfillment
): Future[Subscription] {.base, async.} =
raiseAssert("not implemented")
method subscribeSlotFilled*(market: Market,
callback: OnSlotFilled):
Future[Subscription] {.base, async.} =
method subscribeSlotFilled*(
market: Market, callback: OnSlotFilled
): Future[Subscription] {.base, async.} =
raiseAssert("not implemented")
method subscribeSlotFilled*(market: Market,
requestId: RequestId,
slotIndex: UInt256,
callback: OnSlotFilled):
Future[Subscription] {.base, async.} =
method subscribeSlotFilled*(
market: Market, requestId: RequestId, slotIndex: UInt256, callback: OnSlotFilled
): Future[Subscription] {.base, async.} =
raiseAssert("not implemented")
method subscribeSlotFreed*(market: Market,
callback: OnSlotFreed):
Future[Subscription] {.base, async.} =
method subscribeSlotFreed*(
market: Market, callback: OnSlotFreed
): Future[Subscription] {.base, async.} =
raiseAssert("not implemented")
method subscribeSlotReservationsFull*(
market: Market,
callback: OnSlotReservationsFull): Future[Subscription] {.base, async.} =
market: Market, callback: OnSlotReservationsFull
): Future[Subscription] {.base, async.} =
raiseAssert("not implemented")
method subscribeRequestCancelled*(market: Market,
callback: OnRequestCancelled):
Future[Subscription] {.base, async.} =
method subscribeRequestCancelled*(
market: Market, callback: OnRequestCancelled
): Future[Subscription] {.base, async.} =
raiseAssert("not implemented")
method subscribeRequestCancelled*(market: Market,
requestId: RequestId,
callback: OnRequestCancelled):
Future[Subscription] {.base, async.} =
method subscribeRequestCancelled*(
market: Market, requestId: RequestId, callback: OnRequestCancelled
): Future[Subscription] {.base, async.} =
raiseAssert("not implemented")
method subscribeRequestFailed*(market: Market,
callback: OnRequestFailed):
Future[Subscription] {.base, async.} =
method subscribeRequestFailed*(
market: Market, callback: OnRequestFailed
): Future[Subscription] {.base, async.} =
raiseAssert("not implemented")
method subscribeRequestFailed*(market: Market,
requestId: RequestId,
callback: OnRequestFailed):
Future[Subscription] {.base, async.} =
method subscribeRequestFailed*(
market: Market, requestId: RequestId, callback: OnRequestFailed
): Future[Subscription] {.base, async.} =
raiseAssert("not implemented")
method subscribeProofSubmission*(market: Market,
callback: OnProofSubmitted):
Future[Subscription] {.base, async.} =
method subscribeProofSubmission*(
market: Market, callback: OnProofSubmitted
): Future[Subscription] {.base, async.} =
raiseAssert("not implemented")
method unsubscribe*(subscription: Subscription) {.base, async, upraises: [].} =
raiseAssert("not implemented")
method queryPastSlotFilledEvents*(
market: Market,
fromBlock: BlockTag): Future[seq[SlotFilled]] {.base, async.} =
market: Market, fromBlock: BlockTag
): Future[seq[SlotFilled]] {.base, async.} =
raiseAssert("not implemented")
method queryPastSlotFilledEvents*(
market: Market,
blocksAgo: int): Future[seq[SlotFilled]] {.base, async.} =
market: Market, blocksAgo: int
): Future[seq[SlotFilled]] {.base, async.} =
raiseAssert("not implemented")
method queryPastSlotFilledEvents*(
market: Market,
fromTime: SecondsSince1970): Future[seq[SlotFilled]] {.base, async.} =
market: Market, fromTime: SecondsSince1970
): Future[seq[SlotFilled]] {.base, async.} =
raiseAssert("not implemented")
method queryPastStorageRequestedEvents*(
market: Market,
fromBlock: BlockTag): Future[seq[StorageRequested]] {.base, async.} =
market: Market, fromBlock: BlockTag
): Future[seq[StorageRequested]] {.base, async.} =
raiseAssert("not implemented")
method queryPastStorageRequestedEvents*(
market: Market,
blocksAgo: int): Future[seq[StorageRequested]] {.base, async.} =
market: Market, blocksAgo: int
): Future[seq[StorageRequested]] {.base, async.} =
raiseAssert("not implemented")

11
codex/merkletree/codex/coders.nim
View File

@ -9,7 +9,8 @@
import pkg/upraises
push: {.upraises: [].}
push:
{.upraises: [].}
import pkg/libp2p
import pkg/questionable
@ -103,10 +104,7 @@ proc decode*(_: type CodexProof, data: seq[byte]): ?!CodexProof =
CodexProof.init(mcodec, index.int, nleaves.int, nodes)
proc fromJson*(
_: type CodexProof,
json: JsonNode
): ?!CodexProof =
proc fromJson*(_: type CodexProof, json: JsonNode): ?!CodexProof =
expectJsonKind(Cid, JString, json)
var bytes: seq[byte]
try:
@ -116,4 +114,5 @@ proc fromJson*(
CodexProof.decode(bytes)
func `%`*(proof: CodexProof): JsonNode = % byteutils.toHex(proof.encode())
func `%`*(proof: CodexProof): JsonNode =
%byteutils.toHex(proof.encode())

82
codex/merkletree/codex/codex.nim
View File

@ -56,8 +56,7 @@ proc initMultiHashCodeTable(): Table[MultiCodec, MHash] {.compileTime.} =
const CodeHashes = initMultiHashCodeTable()
func mhash*(mcodec: MultiCodec): ?!MHash =
let
mhash = CodeHashes.getOrDefault(mcodec)
let mhash = CodeHashes.getOrDefault(mcodec)
if isNil(mhash.coder):
return failure "Invalid multihash codec"
@ -71,8 +70,7 @@ func digestSize*(self: (CodexTree or CodexProof)): int =
self.mhash.size
func getProof*(self: CodexTree, index: int): ?!CodexProof =
var
proof = CodexProof(mcodec: self.mcodec)
var proof = CodexProof(mcodec: self.mcodec)
?self.getProof(index, proof)
@ -86,12 +84,10 @@ func verify*(self: CodexProof, leaf: MultiHash, root: MultiHash): ?!bool =
rootBytes = root.digestBytes
leafBytes = leaf.digestBytes
if self.mcodec != root.mcodec or
self.mcodec != leaf.mcodec:
if self.mcodec != root.mcodec or self.mcodec != leaf.mcodec:
return failure "Hash codec mismatch"
if rootBytes.len != root.size and
leafBytes.len != leaf.size:
if rootBytes.len != root.size and leafBytes.len != leaf.size:
return failure "Invalid hash length"
self.verify(leafBytes, rootBytes)
@ -99,25 +95,17 @@ func verify*(self: CodexProof, leaf: MultiHash, root: MultiHash): ?!bool =
func verify*(self: CodexProof, leaf: Cid, root: Cid): ?!bool =
self.verify(?leaf.mhash.mapFailure, ?leaf.mhash.mapFailure)
proc rootCid*(
self: CodexTree,
version = CIDv1,
dataCodec = DatasetRootCodec): ?!Cid =
proc rootCid*(self: CodexTree, version = CIDv1, dataCodec = DatasetRootCodec): ?!Cid =
if (?self.root).len == 0:
return failure "Empty root"
let
mhash = ? MultiHash.init(self.mcodec, ? self.root).mapFailure
let mhash = ?MultiHash.init(self.mcodec, ?self.root).mapFailure
Cid.init(version, DatasetRootCodec, mhash).mapFailure
func getLeafCid*(
self: CodexTree,
i: Natural,
version = CIDv1,
dataCodec = BlockCodec): ?!Cid =
self: CodexTree, i: Natural, version = CIDv1, dataCodec = BlockCodec
): ?!Cid =
if i >= self.leavesCount:
return failure "Invalid leaf index " & $i
@ -128,24 +116,19 @@ func getLeafCid*(
Cid.init(version, dataCodec, mhash).mapFailure
proc `$`*(self: CodexTree): string =
let root = if self.root.isOk: byteutils.toHex(self.root.get) else: "none"
"CodexTree(" &
" root: " & root &
", leavesCount: " & $self.leavesCount &
", levels: " & $self.levels &
", mcodec: " & $self.mcodec & " )"
let root =
if self.root.isOk:
byteutils.toHex(self.root.get)
else:
"none"
"CodexTree(" & " root: " & root & ", leavesCount: " & $self.leavesCount & ", levels: " &
$self.levels & ", mcodec: " & $self.mcodec & " )"
proc `$`*(self: CodexProof): string =
"CodexProof(" &
" nleaves: " & $self.nleaves &
", index: " & $self.index &
", path: " & $self.path.mapIt( byteutils.toHex(it) ) &
", mcodec: " & $self.mcodec & " )"
"CodexProof(" & " nleaves: " & $self.nleaves & ", index: " & $self.index & ", path: " &
$self.path.mapIt(byteutils.toHex(it)) & ", mcodec: " & $self.mcodec & " )"
func compress*(
x, y: openArray[byte],
key: ByteTreeKey,
mhash: MHash): ?!ByteHash =
func compress*(x, y: openArray[byte], key: ByteTreeKey, mhash: MHash): ?!ByteHash =
## Compress two hashes
##
@ -154,10 +137,8 @@ func compress*(
success digest
func init*(
_: type CodexTree,
mcodec: MultiCodec = Sha256HashCodec,
leaves: openArray[ByteHash]): ?!CodexTree =
_: type CodexTree, mcodec: MultiCodec = Sha256HashCodec, leaves: openArray[ByteHash]
): ?!CodexTree =
if leaves.len == 0:
return failure "Empty leaves"
@ -170,16 +151,12 @@ func init*(
if mhash.size != leaves[0].len:
return failure "Invalid hash length"
var
self = CodexTree(mcodec: mcodec, compress: compressor, zero: Zero)
var self = CodexTree(mcodec: mcodec, compress: compressor, zero: Zero)
self.layers = ?merkleTreeWorker(self, leaves, isBottomLayer = true)
success self
func init*(
_: type CodexTree,
leaves: openArray[MultiHash]): ?!CodexTree =
func init*(_: type CodexTree, leaves: openArray[MultiHash]): ?!CodexTree =
if leaves.len == 0:
return failure "Empty leaves"
@ -189,9 +166,7 @@ func init*(
CodexTree.init(mcodec, leaves)
func init*(
_: type CodexTree,
leaves: openArray[Cid]): ?!CodexTree =
func init*(_: type CodexTree, leaves: openArray[Cid]): ?!CodexTree =
if leaves.len == 0:
return failure "Empty leaves"
@ -205,8 +180,8 @@ proc fromNodes*(
_: type CodexTree,
mcodec: MultiCodec = Sha256HashCodec,
nodes: openArray[ByteHash],
nleaves: int): ?!CodexTree =
nleaves: int,
): ?!CodexTree =
if nodes.len == 0:
return failure "Empty nodes"
@ -243,8 +218,8 @@ func init*(
mcodec: MultiCodec = Sha256HashCodec,
index: int,
nleaves: int,
nodes: openArray[ByteHash]): ?!CodexProof =
nodes: openArray[ByteHash],
): ?!CodexProof =
if nodes.len == 0:
return failure "Empty nodes"
@ -260,4 +235,5 @@ func init*(
mcodec: mcodec,
index: index,
nleaves: nleaves,
path: @nodes)
path: @nodes,
)

20
codex/merkletree/merkletree.nim
View File

@ -59,9 +59,8 @@ func root*[H, K](self: MerkleTree[H, K]): ?!H =
return success last[0]
func getProof*[H, K](
self: MerkleTree[H, K],
index: int,
proof: MerkleProof[H, K]): ?!void =
self: MerkleTree[H, K], index: int, proof: MerkleProof[H, K]
): ?!void =
let depth = self.depth
let nleaves = self.leavesCount
@ -73,7 +72,11 @@ func getProof*[H, K](
var m = nleaves
for i in 0 ..< depth:
let j = k xor 1
path[i] = if (j < m): self.layers[i][j] else: self.zero
path[i] =
if (j < m):
self.layers[i][j]
else:
self.zero
k = k shr 1
m = (m + 1) shr 1
@ -85,8 +88,7 @@ func getProof*[H, K](
success()
func getProof*[H, K](self: MerkleTree[H, K], index: int): ?!MerkleProof[H, K] =
var
proof = MerkleProof[H, K]()
var proof = MerkleProof[H, K]()
?self.getProof(index, proof)
@ -121,10 +123,8 @@ func verify*[H, K](proof: MerkleProof[H, K], leaf: H, root: H): ?!bool =
success bool(root == ?proof.reconstructRoot(leaf))
func merkleTreeWorker*[H, K](
self: MerkleTree[H, K],
xs: openArray[H],
isBottomLayer: static bool): ?!seq[seq[H]] =
self: MerkleTree[H, K], xs: openArray[H], isBottomLayer: static bool
): ?!seq[seq[H]] =
let a = low(xs)
let b = high(xs)
let m = b - a + 1

62
codex/merkletree/poseidon2.nim
View File

@ -46,64 +46,49 @@ type
proc `$`*(self: Poseidon2Tree): string =
let root = if self.root.isOk: self.root.get.toHex else: "none"
"Poseidon2Tree(" &
" root: " & root &
", leavesCount: " & $self.leavesCount &
"Poseidon2Tree(" & " root: " & root & ", leavesCount: " & $self.leavesCount &
", levels: " & $self.levels & " )"
proc `$`*(self: Poseidon2Proof): string =
"Poseidon2Proof(" &
" nleaves: " & $self.nleaves &
", index: " & $self.index &
"Poseidon2Proof(" & " nleaves: " & $self.nleaves & ", index: " & $self.index &
", path: " & $self.path.mapIt(it.toHex) & " )"
func toArray32*(bytes: openArray[byte]): array[32, byte] =
result[0 ..< bytes.len] = bytes[0 ..< bytes.len]
converter toKey*(key: PoseidonKeysEnum): Poseidon2Hash =
case key:
case key
of KeyNone: KeyNoneF
of KeyBottomLayer: KeyBottomLayerF
of KeyOdd: KeyOddF
of KeyOddAndBottomLayer: KeyOddAndBottomLayerF
func init*(
_: type Poseidon2Tree,
leaves: openArray[Poseidon2Hash]): ?!Poseidon2Tree =
func init*(_: type Poseidon2Tree, leaves: openArray[Poseidon2Hash]): ?!Poseidon2Tree =
if leaves.len == 0:
return failure "Empty leaves"
let
compressor = proc(
x, y: Poseidon2Hash,
key: PoseidonKeysEnum): ?!Poseidon2Hash {.noSideEffect.} =
let compressor = proc(
x, y: Poseidon2Hash, key: PoseidonKeysEnum
): ?!Poseidon2Hash {.noSideEffect.} =
success compress(x, y, key.toKey)
var
self = Poseidon2Tree(compress: compressor, zero: Poseidon2Zero)
var self = Poseidon2Tree(compress: compressor, zero: Poseidon2Zero)
self.layers = ?merkleTreeWorker(self, leaves, isBottomLayer = true)
success self
func init*(
_: type Poseidon2Tree,
leaves: openArray[array[31, byte]]): ?!Poseidon2Tree =
Poseidon2Tree.init(
leaves.mapIt( Poseidon2Hash.fromBytes(it) ))
func init*(_: type Poseidon2Tree, leaves: openArray[array[31, byte]]): ?!Poseidon2Tree =
Poseidon2Tree.init(leaves.mapIt(Poseidon2Hash.fromBytes(it)))
proc fromNodes*(
_: type Poseidon2Tree,
nodes: openArray[Poseidon2Hash],
nleaves: int): ?!Poseidon2Tree =
_: type Poseidon2Tree, nodes: openArray[Poseidon2Hash], nleaves: int
): ?!Poseidon2Tree =
if nodes.len == 0:
return failure "Empty nodes"
let
compressor = proc(
x, y: Poseidon2Hash,
key: PoseidonKeysEnum): ?!Poseidon2Hash {.noSideEffect.} =
let compressor = proc(
x, y: Poseidon2Hash, key: PoseidonKeysEnum
): ?!Poseidon2Hash {.noSideEffect.} =
success compress(x, y, key.toKey)
var
@ -126,18 +111,14 @@ proc fromNodes*(
success self
func init*(
_: type Poseidon2Proof,
index: int,
nleaves: int,
nodes: openArray[Poseidon2Hash]): ?!Poseidon2Proof =
_: type Poseidon2Proof, index: int, nleaves: int, nodes: openArray[Poseidon2Hash]
): ?!Poseidon2Proof =
if nodes.len == 0:
return failure "Empty nodes"
let
compressor = proc(
x, y: Poseidon2Hash,
key: PoseidonKeysEnum): ?!Poseidon2Hash {.noSideEffect.} =
let compressor = proc(
x, y: Poseidon2Hash, key: PoseidonKeysEnum
): ?!Poseidon2Hash {.noSideEffect.} =
success compress(x, y, key.toKey)
success Poseidon2Proof(
@ -145,4 +126,5 @@ func init*(
zero: Poseidon2Zero,
index: index,
nleaves: nleaves,
path: @nodes)
path: @nodes,
)

3
codex/namespaces.nim
View File

@ -11,7 +11,8 @@ const
# Namespaces
CodexMetaNamespace* = "meta" # meta info stored here
CodexRepoNamespace* = "repo" # repository namespace, blocks and manifests are subkeys
CodexBlockTotalNamespace* = CodexMetaNamespace & "/total" # number of blocks in the repo
CodexBlockTotalNamespace* = CodexMetaNamespace & "/total"
# number of blocks in the repo
CodexBlocksNamespace* = CodexRepoNamespace & "/blocks" # blocks namespace
CodexManifestNamespace* = CodexRepoNamespace & "/manifests" # manifest namespace
CodexBlocksTtlNamespace* = # Cid TTL

138
codex/nat.nim
View File

@ -9,8 +9,10 @@
{.push raises: [].}
import
std/[options, os, strutils, times, net],stew/shims/net as stewNet,
stew/[objects,results], nat_traversal/[miniupnpc, natpmp],
std/[options, os, strutils, times, net],
stew/shims/net as stewNet,
stew/[objects, results],
nat_traversal/[miniupnpc, natpmp],
json_serialization/std/net
import pkg/chronos
@ -38,8 +40,7 @@ var
logScope:
topics = "nat"
type
PrefSrcStatus = enum
type PrefSrcStatus = enum
NoRoutingInfo
PrefSrcIsPublic
PrefSrcIsPrivate
@ -63,7 +64,7 @@ proc getExternalIP*(natStrategy: NatStrategy, quiet = false): Option[IpAddress]
var
msg: cstring
canContinue = true
case upnp.selectIGD():
case upnp.selectIGD()
of IGDNotFound:
msg = "Internet Gateway Device not found. Giving up."
canContinue = false
@ -72,9 +73,11 @@ proc getExternalIP*(natStrategy: NatStrategy, quiet = false): Option[IpAddress]
of IGDNotConnected:
msg = "Internet Gateway Device found but it's not connected. Trying anyway."
of NotAnIGD:
msg = "Some device found, but it's not recognised as an Internet Gateway Device. Trying anyway."
msg =
"Some device found, but it's not recognised as an Internet Gateway Device. Trying anyway."
of IGDIpNotRoutable:
msg = "Internet Gateway Device found and is connected, but with a reserved or non-routable IP. Trying anyway."
msg =
"Internet Gateway Device found and is connected, but with a reserved or non-routable IP. Trying anyway."
if not quiet:
debug "UPnP", msg
if canContinue:
@ -116,8 +119,7 @@ proc getExternalIP*(natStrategy: NatStrategy, quiet = false): Option[IpAddress]
# Further more, we check if the bind address (user provided, or a "0.0.0.0"
# default) is a public IP. That's a long shot, because code paths involving a
# user-provided bind address are not supposed to get here.
proc getRoutePrefSrc(
bindIp: IpAddress): (Option[IpAddress], PrefSrcStatus) =
proc getRoutePrefSrc(bindIp: IpAddress): (Option[IpAddress], PrefSrcStatus) =
let bindAddress = initTAddress(bindIp, Port(0))
if bindAddress.isAnyLocal():
@ -137,10 +139,12 @@ proc getRoutePrefSrc(
return (none(IpAddress), BindAddressIsPrivate)
# Try to detect a public IP assigned to this host, before trying NAT traversal.
proc getPublicRoutePrefSrcOrExternalIP*(natStrategy: NatStrategy, bindIp: IpAddress, quiet = true): Option[IpAddress] =
proc getPublicRoutePrefSrcOrExternalIP*(
natStrategy: NatStrategy, bindIp: IpAddress, quiet = true
): Option[IpAddress] =
let (prefSrcIp, prefSrcStatus) = getRoutePrefSrc(bindIp)
case prefSrcStatus:
case prefSrcStatus
of NoRoutingInfo, PrefSrcIsPublic, BindAddressIsPublic:
return prefSrcIp
of PrefSrcIsPrivate, BindAddressIsPrivate:
@ -148,7 +152,9 @@ proc getPublicRoutePrefSrcOrExternalIP*(natStrategy: NatStrategy, bindIp: IpAddr
if extIp.isSome:
return some(extIp.get)
proc doPortMapping(tcpPort, udpPort: Port, description: string): Option[(Port, Port)] {.gcsafe.} =
proc doPortMapping(
tcpPort, udpPort: Port, description: string
): Option[(Port, Port)] {.gcsafe.} =
var
extTcpPort: Port
extUdpPort: Port
@ -157,24 +163,28 @@ proc doPortMapping(tcpPort, udpPort: Port, description: string): Option[(Port, P
for t in [(tcpPort, UPNPProtocol.TCP), (udpPort, UPNPProtocol.UDP)]:
let
(port, protocol) = t
pmres = upnp.addPortMapping(externalPort = $port,
pmres = upnp.addPortMapping(
externalPort = $port,
protocol = protocol,
internalHost = upnp.lanAddr,
internalPort = $port,
desc = description,
leaseDuration = 0)
leaseDuration = 0,
)
if pmres.isErr:
error "UPnP port mapping", msg = pmres.error, port
return
else:
# let's check it
let cres = upnp.getSpecificPortMapping(externalPort = $port,
protocol = protocol)
let cres =
upnp.getSpecificPortMapping(externalPort = $port, protocol = protocol)
if cres.isErr:
warn "UPnP port mapping check failed. Assuming the check itself is broken and the port mapping was done.", msg = cres.error
warn "UPnP port mapping check failed. Assuming the check itself is broken and the port mapping was done.",
msg = cres.error
info "UPnP: added port mapping", externalPort = port, internalPort = port, protocol = protocol
case protocol:
info "UPnP: added port mapping",
externalPort = port, internalPort = port, protocol = protocol
case protocol
of UPNPProtocol.TCP:
extTcpPort = port
of UPNPProtocol.UDP:
@ -183,17 +193,20 @@ proc doPortMapping(tcpPort, udpPort: Port, description: string): Option[(Port, P
for t in [(tcpPort, NatPmpProtocol.TCP), (udpPort, NatPmpProtocol.UDP)]:
let
(port, protocol) = t
pmres = npmp.addPortMapping(eport = port.cushort,
pmres = npmp.addPortMapping(
eport = port.cushort,
iport = port.cushort,
protocol = protocol,
lifetime = NATPMP_LIFETIME)
lifetime = NATPMP_LIFETIME,
)
if pmres.isErr:
error "NAT-PMP port mapping", msg = pmres.error, port
return
else:
let extPort = Port(pmres.value)
info "NAT-PMP: added port mapping", externalPort = extPort, internalPort = port, protocol = protocol
case protocol:
info "NAT-PMP: added port mapping",
externalPort = extPort, internalPort = port, protocol = protocol
case protocol
of NatPmpProtocol.TCP:
extTcpPort = extPort
of NatPmpProtocol.UDP:
@ -223,8 +236,11 @@ proc repeatPortMapping(args: PortMappingArgs) {.thread, raises: [ValueError].} =
while true:
# we're being silly here with this channel polling because we can't
# select on Nim channels like on Go ones
let (dataAvailable, _) = try: natCloseChan.tryRecv()
except Exception: (false, false)
let (dataAvailable, _) =
try:
natCloseChan.tryRecv()
except Exception:
(false, false)
if dataAvailable:
return
else:
@ -255,26 +271,33 @@ proc stopNatThread() {.noconv.} =
let ipres = getExternalIP(strategy, quiet = true)
if ipres.isSome:
if strategy == NatStrategy.NatUpnp:
for t in [(externalTcpPort, internalTcpPort, UPNPProtocol.TCP), (externalUdpPort, internalUdpPort, UPNPProtocol.UDP)]:
for t in [
(externalTcpPort, internalTcpPort, UPNPProtocol.TCP),
(externalUdpPort, internalUdpPort, UPNPProtocol.UDP),
]:
let
(eport, iport, protocol) = t
pmres = upnp.deletePortMapping(externalPort = $eport,
protocol = protocol)
pmres = upnp.deletePortMapping(externalPort = $eport, protocol = protocol)
if pmres.isErr:
error "UPnP port mapping deletion", msg = pmres.error
else:
debug "UPnP: deleted port mapping", externalPort = eport, internalPort = iport, protocol = protocol
debug "UPnP: deleted port mapping",
externalPort = eport, internalPort = iport, protocol = protocol
elif strategy == NatStrategy.NatPmp:
for t in [(externalTcpPort, internalTcpPort, NatPmpProtocol.TCP), (externalUdpPort, internalUdpPort, NatPmpProtocol.UDP)]:
for t in [
(externalTcpPort, internalTcpPort, NatPmpProtocol.TCP),
(externalUdpPort, internalUdpPort, NatPmpProtocol.UDP),
]:
let
(eport, iport, protocol) = t
pmres = npmp.deletePortMapping(eport = eport.cushort,
iport = iport.cushort,
protocol = protocol)
pmres = npmp.deletePortMapping(
eport = eport.cushort, iport = iport.cushort, protocol = protocol
)
if pmres.isErr:
error "NAT-PMP port mapping deletion", msg = pmres.error
else:
debug "NAT-PMP: deleted port mapping", externalPort = eport, internalPort = iport, protocol = protocol
debug "NAT-PMP: deleted port mapping",
externalPort = eport, internalPort = iport, protocol = protocol
proc redirectPorts*(tcpPort, udpPort: Port, description: string): Option[(Port, Port)] =
result = doPortMapping(tcpPort, udpPort, description)
@ -288,15 +311,17 @@ proc redirectPorts*(tcpPort, udpPort: Port, description: string): Option[(Port,
# these mappings.
natCloseChan.open()
try:
natThread.createThread(repeatPortMapping, (externalTcpPort, externalUdpPort, description))
natThread.createThread(
repeatPortMapping, (externalTcpPort, externalUdpPort, description)
)
# atexit() in disguise
addQuitProc(stopNatThread)
except Exception as exc:
warn "Failed to create NAT port mapping renewal thread", exc = exc.msg
proc setupNat*(natStrategy: NatStrategy, tcpPort, udpPort: Port,
clientId: string):
tuple[ip: Option[IpAddress], tcpPort, udpPort: Option[Port]] =
proc setupNat*(
natStrategy: NatStrategy, tcpPort, udpPort: Port, clientId: string
): tuple[ip: Option[IpAddress], tcpPort, udpPort: Option[Port]] =
## Setup NAT port mapping and get external IP address.
## If any of this fails, we don't return any IP address but do return the
## original ports as best effort.
@ -304,10 +329,10 @@ proc setupNat*(natStrategy: NatStrategy, tcpPort, udpPort: Port,
let extIp = getExternalIP(natStrategy)
if extIp.isSome:
let ip = extIp.get
let extPorts = ({.gcsafe.}:
redirectPorts(tcpPort = tcpPort,
udpPort = udpPort,
description = clientId))
let extPorts = (
{.gcsafe.}:
redirectPorts(tcpPort = tcpPort, udpPort = udpPort, description = clientId)
)
if extPorts.isSome:
let (extTcpPort, extUdpPort) = extPorts.get()
(ip: some(ip), tcpPort: some(extTcpPort), udpPort: some(extUdpPort))
@ -318,16 +343,14 @@ proc setupNat*(natStrategy: NatStrategy, tcpPort, udpPort: Port,
warn "UPnP/NAT-PMP not available"
(ip: none(IpAddress), tcpPort: some(tcpPort), udpPort: some(udpPort))
type
NatConfig* = object
type NatConfig* = object
case hasExtIp*: bool
of true: extIp*: IpAddress
of false: nat*: NatStrategy
proc setupAddress*(natConfig: NatConfig, bindIp: IpAddress,
tcpPort, udpPort: Port, clientId: string):
tuple[ip: Option[IpAddress], tcpPort, udpPort: Option[Port]]
{.gcsafe.} =
proc setupAddress*(
natConfig: NatConfig, bindIp: IpAddress, tcpPort, udpPort: Port, clientId: string
): tuple[ip: Option[IpAddress], tcpPort, udpPort: Option[Port]] {.gcsafe.} =
## Set-up of the external address via any of the ways as configured in
## `NatConfig`. In case all fails an error is logged and the bind ports are
## selected also as external ports, as best effort and in hope that the
@ -338,11 +361,11 @@ proc setupAddress*(natConfig: NatConfig, bindIp: IpAddress,
# any required port redirection must be done by hand
return (some(natConfig.extIp), some(tcpPort), some(udpPort))
case natConfig.nat:
case natConfig.nat
of NatStrategy.NatAny:
let (prefSrcIp, prefSrcStatus) = getRoutePrefSrc(bindIp)
case prefSrcStatus:
case prefSrcStatus
of NoRoutingInfo, PrefSrcIsPublic, BindAddressIsPublic:
return (prefSrcIp, some(tcpPort), some(udpPort))
of PrefSrcIsPrivate, BindAddressIsPrivate:
@ -350,7 +373,7 @@ proc setupAddress*(natConfig: NatConfig, bindIp: IpAddress,
of NatStrategy.NatNone:
let (prefSrcIp, prefSrcStatus) = getRoutePrefSrc(bindIp)
case prefSrcStatus:
case prefSrcStatus
of NoRoutingInfo, PrefSrcIsPublic, BindAddressIsPublic:
return (prefSrcIp, some(tcpPort), some(udpPort))
of PrefSrcIsPrivate:
@ -362,20 +385,22 @@ proc setupAddress*(natConfig: NatConfig, bindIp: IpAddress,
of NatStrategy.NatUpnp, NatStrategy.NatPmp:
return setupNat(natConfig.nat, tcpPort, udpPort, clientId)
proc nattedAddress*(natConfig: NatConfig, addrs: seq[MultiAddress], udpPort: Port): tuple[libp2p, discovery: seq[MultiAddress]] =
proc nattedAddress*(
natConfig: NatConfig, addrs: seq[MultiAddress], udpPort: Port
): tuple[libp2p, discovery: seq[MultiAddress]] =
## Takes a NAT configuration, sequence of multiaddresses and UDP port and returns:
## - Modified multiaddresses with NAT-mapped addresses for libp2p
## - Discovery addresses with NAT-mapped UDP ports
var discoveryAddrs = newSeq[MultiAddress](0)
let
newAddrs = addrs.mapIt:
let newAddrs = addrs.mapIt:
block:
# Extract IP address and port from the multiaddress
let (ipPart, port) = getAddressAndPort(it)
if ipPart.isSome and port.isSome:
# Try to setup NAT mapping for the address
let (newIP, tcp, udp) = setupAddress(natConfig, ipPart.get, port.get, udpPort, "codex")
let (newIP, tcp, udp) =
setupAddress(natConfig, ipPart.get, port.get, udpPort, "codex")
if newIP.isSome:
# NAT mapping successful - add discovery address with mapped UDP port
discoveryAddrs.add(getMultiAddrWithIPAndUDPPort(newIP.get, udp.get))
@ -390,6 +415,3 @@ proc nattedAddress*(natConfig: NatConfig, addrs: seq[MultiAddress], udpPort: Por
# Invalid multiaddress format - return as is
it
(newAddrs, discoveryAddrs)

194
codex/node.nim
View File

@ -50,14 +50,15 @@ export logutils
logScope:
topics = "codex node"
const
FetchBatch = 200
const FetchBatch = 200
type
Contracts* = tuple
client: ?ClientInteractions
host: ?HostInteractions
validator: ?ValidatorInteractions
Contracts* =
tuple[
client: ?ClientInteractions,
host: ?HostInteractions,
validator: ?ValidatorInteractions,
]
CodexNode* = object
switch: Switch
@ -88,8 +89,8 @@ func discovery*(self: CodexNodeRef): Discovery =
return self.discovery
proc storeManifest*(
self: CodexNodeRef,
manifest: Manifest): Future[?!bt.Block] {.async.} =
self: CodexNodeRef, manifest: Manifest
): Future[?!bt.Block] {.async.} =
without encodedVerifiable =? manifest.encode(), err:
trace "Unable to encode manifest"
return failure(err)
@ -104,9 +105,7 @@ proc storeManifest*(
success blk
proc fetchManifest*(
self: CodexNodeRef,
cid: Cid): Future[?!Manifest] {.async.} =
proc fetchManifest*(self: CodexNodeRef, cid: Cid): Future[?!Manifest] {.async.} =
## Fetch and decode a manifest block
##
@ -129,33 +128,27 @@ proc fetchManifest*(
return manifest.success
proc findPeer*(
self: CodexNodeRef,
peerId: PeerId): Future[?PeerRecord] {.async.} =
proc findPeer*(self: CodexNodeRef, peerId: PeerId): Future[?PeerRecord] {.async.} =
## Find peer using the discovery service from the given CodexNode
##
return await self.discovery.findPeer(peerId)
proc connect*(
self: CodexNodeRef,
peerId: PeerId,
addrs: seq[MultiAddress]
self: CodexNodeRef, peerId: PeerId, addrs: seq[MultiAddress]
): Future[void] =
self.switch.connect(peerId, addrs)
proc updateExpiry*(
self: CodexNodeRef,
manifestCid: Cid,
expiry: SecondsSince1970): Future[?!void] {.async.} =
self: CodexNodeRef, manifestCid: Cid, expiry: SecondsSince1970
): Future[?!void] {.async.} =
without manifest =? await self.fetchManifest(manifestCid), error:
trace "Unable to fetch manifest for cid", manifestCid
return failure(error)
try:
let
ensuringFutures = Iter[int].new(0..<manifest.blocksCount)
.mapIt(self.networkStore.localStore.ensureExpiry( manifest.treeCid, it, expiry ))
let ensuringFutures = Iter[int].new(0 ..< manifest.blocksCount).mapIt(
self.networkStore.localStore.ensureExpiry(manifest.treeCid, it, expiry)
)
await allFuturesThrowing(ensuringFutures)
except CancelledError as exc:
raise exc
@ -169,7 +162,8 @@ proc fetchBatched*(
cid: Cid,
iter: Iter[int],
batchSize = FetchBatch,
onBatch: BatchProc = nil): Future[?!void] {.async, gcsafe.} =
onBatch: BatchProc = nil,
): Future[?!void] {.async, gcsafe.} =
## Fetch blocks in batches of `batchSize`
##
@ -198,7 +192,8 @@ proc fetchBatched*(
self: CodexNodeRef,
manifest: Manifest,
batchSize = FetchBatch,
onBatch: BatchProc = nil): Future[?!void] =
onBatch: BatchProc = nil,
): Future[?!void] =
## Fetch manifest in batches of `batchSize`
##
@ -207,16 +202,12 @@ proc fetchBatched*(
let iter = Iter[int].new(0 ..< manifest.blocksCount)
self.fetchBatched(manifest.treeCid, iter, batchSize, onBatch)
proc streamSingleBlock(
self: CodexNodeRef,
cid: Cid
): Future[?!LPStream] {.async.} =
proc streamSingleBlock(self: CodexNodeRef, cid: Cid): Future[?!LPStream] {.async.} =
## Streams the contents of a single block.
##
trace "Streaming single block", cid = cid
let
stream = BufferStream.new()
let stream = BufferStream.new()
without blk =? (await self.networkStore.getBlock(BlockAddress.init(cid))), err:
return failure(err)
@ -234,9 +225,7 @@ proc streamSingleBlock(
LPStream(stream).success
proc streamEntireDataset(
self: CodexNodeRef,
manifest: Manifest,
manifestCid: Cid,
self: CodexNodeRef, manifest: Manifest, manifestCid: Cid
): Future[?!LPStream] {.async.} =
## Streams the contents of the entire dataset described by the manifest.
##
@ -246,11 +235,8 @@ proc streamEntireDataset(
# Retrieve, decode and save to the local store all EС groups
proc erasureJob(): Future[?!void] {.async.} =
# Spawn an erasure decoding job
let
erasure = Erasure.new(
self.networkStore,
leoEncoderProvider,
leoDecoderProvider)
let erasure =
Erasure.new(self.networkStore, leoEncoderProvider, leoDecoderProvider)
without _ =? (await erasure.decode(manifest)), error:
error "Unable to erasure decode manifest", manifestCid, exc = error.msg
return failure(error)
@ -265,9 +251,8 @@ proc streamEntireDataset(
LPStream(StoreStream.new(self.networkStore, manifest, pad = false)).success
proc retrieve*(
self: CodexNodeRef,
cid: Cid,
local: bool = true): Future[?!LPStream] {.async.} =
self: CodexNodeRef, cid: Cid, local: bool = true
): Future[?!LPStream] {.async.} =
## Retrieve by Cid a single block or an entire dataset described by manifest
##
@ -287,7 +272,8 @@ proc store*(
stream: LPStream,
filename: ?string = string.none,
mimetype: ?string = string.none,
blockSize = DefaultBlockSize): Future[?!Cid] {.async.} =
blockSize = DefaultBlockSize,
): Future[?!Cid] {.async.} =
## Save stream contents as dataset with given blockSize
## to nodes's BlockStore, and return Cid of its manifest
##
@ -301,10 +287,7 @@ proc store*(
var cids: seq[Cid]
try:
while (
let chunk = await chunker.getBytes();
chunk.len > 0):
while (let chunk = await chunker.getBytes(); chunk.len > 0):
without mhash =? MultiHash.digest($hcodec, chunk).mapFailure, err:
return failure(err)
@ -335,7 +318,8 @@ proc store*(
for index, cid in cids:
without proof =? tree.getProof(index), err:
return failure(err)
if err =? (await self.networkStore.putCidAndProof(treeCid, index, cid, proof)).errorOption:
if err =?
(await self.networkStore.putCidAndProof(treeCid, index, cid, proof)).errorOption:
# TODO add log here
return failure(err)
@ -348,13 +332,15 @@ proc store*(
codec = dataCodec,
filename = filename,
mimetype = mimetype,
uploadedAt = now().utc.toTime.toUnix.some)
uploadedAt = now().utc.toTime.toUnix.some,
)
without manifestBlk =? await self.storeManifest(manifest), err:
error "Unable to store manifest"
return failure(err)
info "Stored data", manifestCid = manifestBlk.cid,
info "Stored data",
manifestCid = manifestBlk.cid,
treeCid = treeCid,
blocks = manifest.blocksCount,
datasetSize = manifest.datasetSize,
@ -389,7 +375,8 @@ proc setupRequest(
tolerance: uint,
reward: UInt256,
collateral: UInt256,
expiry: UInt256): Future[?!StorageRequest] {.async.} =
expiry: UInt256,
): Future[?!StorageRequest] {.async.} =
## Setup slots for a given dataset
##
@ -416,11 +403,8 @@ proc setupRequest(
return failure error
# Erasure code the dataset according to provided parameters
let
erasure = Erasure.new(
self.networkStore.localStore,
leoEncoderProvider,
leoDecoderProvider)
let erasure =
Erasure.new(self.networkStore.localStore, leoEncoderProvider, leoDecoderProvider)
without encoded =? (await erasure.encode(manifest, ecK, ecM)), error:
trace "Unable to erasure code dataset"
@ -453,13 +437,13 @@ proc setupRequest(
proofProbability: proofProbability,
reward: reward,
collateral: collateral,
maxSlotLoss: tolerance
maxSlotLoss: tolerance,
),
content: StorageContent(
cid: $manifestBlk.cid, # TODO: why string?
merkleRoot: verifyRoot
merkleRoot: verifyRoot,
),
expiry: expiry
expiry: expiry,
)
trace "Request created", request = $request
@ -474,7 +458,8 @@ proc requestStorage*(
tolerance: uint,
reward: UInt256,
collateral: UInt256,
expiry: UInt256): Future[?!PurchaseId] {.async.} =
expiry: UInt256,
): Future[?!PurchaseId] {.async.} =
## Initiate a request for storage sequence, this might
## be a multistep procedure.
##
@ -496,16 +481,11 @@ proc requestStorage*(
trace "Purchasing not available"
return failure "Purchasing not available"
without request =?
(await self.setupRequest(
cid,
duration,
proofProbability,
nodes,
tolerance,
reward,
collateral,
expiry)), err:
without request =? (
await self.setupRequest(
cid, duration, proofProbability, nodes, tolerance, reward, collateral, expiry
)
), err:
trace "Unable to setup request"
return failure err
@ -513,10 +493,8 @@ proc requestStorage*(
success purchase.id
proc onStore(
self: CodexNodeRef,
request: StorageRequest,
slotIdx: UInt256,
blocksCb: BlocksCb): Future[?!void] {.async.} =
self: CodexNodeRef, request: StorageRequest, slotIdx: UInt256, blocksCb: BlocksCb
): Future[?!void] {.async.} =
## store data in local storage
##
@ -534,9 +512,8 @@ proc onStore(
trace "Unable to fetch manifest for cid", cid, err = err.msg
return failure(err)
without builder =? Poseidon2Builder.new(
self.networkStore, manifest, manifest.verifiableStrategy
), err:
without builder =?
Poseidon2Builder.new(self.networkStore, manifest, manifest.verifiableStrategy), err:
trace "Unable to create slots builder", err = err.msg
return failure(err)
@ -551,7 +528,8 @@ proc onStore(
proc updateExpiry(blocks: seq[bt.Block]): Future[?!void] {.async.} =
trace "Updating expiry for blocks", blocks = blocks.len
let ensureExpiryFutures = blocks.mapIt(self.networkStore.ensureExpiry(it.cid, expiry))
let ensureExpiryFutures =
blocks.mapIt(self.networkStore.ensureExpiry(it.cid, expiry))
if updateExpiryErr =? (await allFutureResult(ensureExpiryFutures)).errorOption:
return failure(updateExpiryErr)
@ -561,8 +539,9 @@ proc onStore(
return success()
without indexer =? manifest.verifiableStrategy.init(
0, manifest.blocksCount - 1, manifest.numSlots).catch, err:
without indexer =?
manifest.verifiableStrategy.init(0, manifest.blocksCount - 1, manifest.numSlots).catch,
err:
trace "Unable to create indexing strategy from protected manifest", err = err.msg
return failure(err)
@ -570,10 +549,9 @@ proc onStore(
trace "Unable to get indicies from strategy", err = err.msg
return failure(err)
if err =? (await self.fetchBatched(
manifest.treeCid,
blksIter,
onBatch = updateExpiry)).errorOption:
if err =? (
await self.fetchBatched(manifest.treeCid, blksIter, onBatch = updateExpiry)
).errorOption:
trace "Unable to fetch blocks", err = err.msg
return failure(err)
@ -584,7 +562,8 @@ proc onStore(
trace "Slot successfully retrieved and reconstructed"
if cid =? slotRoot.toSlotCid() and cid != manifest.slotRoots[slotIdx.int]:
trace "Slot root mismatch", manifest = manifest.slotRoots[slotIdx.int], recovered = slotRoot.toSlotCid()
trace "Slot root mismatch",
manifest = manifest.slotRoots[slotIdx.int], recovered = slotRoot.toSlotCid()
return failure(newException(CodexError, "Slot root mismatch"))
trace "Slot successfully retrieved and reconstructed"
@ -592,9 +571,8 @@ proc onStore(
return success()
proc onProve(
self: CodexNodeRef,
slot: Slot,
challenge: ProofChallenge): Future[?!Groth16Proof] {.async.} =
self: CodexNodeRef, slot: Slot, challenge: ProofChallenge
): Future[?!Groth16Proof] {.async.} =
## Generats a proof for a given slot and challenge
##
@ -648,9 +626,8 @@ proc onProve(
failure "Prover not enabled"
proc onExpiryUpdate(
self: CodexNodeRef,
rootCid: string,
expiry: SecondsSince1970): Future[?!void] {.async.} =
self: CodexNodeRef, rootCid: string, expiry: SecondsSince1970
): Future[?!void] {.async.} =
without cid =? Cid.init(rootCid):
trace "Unable to parse Cid", cid
let error = newException(CodexError, "Unable to parse Cid")
@ -658,10 +635,7 @@ proc onExpiryUpdate(
return await self.updateExpiry(cid, expiry)
proc onClear(
self: CodexNodeRef,
request: StorageRequest,
slotIndex: UInt256) =
proc onClear(self: CodexNodeRef, request: StorageRequest, slotIndex: UInt256) =
# TODO: remove data from local storage
discard
@ -676,23 +650,23 @@ proc start*(self: CodexNodeRef) {.async.} =
await self.clock.start()
if hostContracts =? self.contracts.host:
hostContracts.sales.onStore =
proc(
request: StorageRequest,
slot: UInt256,
onBatch: BatchProc): Future[?!void] = self.onStore(request, slot, onBatch)
hostContracts.sales.onStore = proc(
request: StorageRequest, slot: UInt256, onBatch: BatchProc
): Future[?!void] =
self.onStore(request, slot, onBatch)
hostContracts.sales.onExpiryUpdate =
proc(rootCid: string, expiry: SecondsSince1970): Future[?!void] =
hostContracts.sales.onExpiryUpdate = proc(
rootCid: string, expiry: SecondsSince1970
): Future[?!void] =
self.onExpiryUpdate(rootCid, expiry)
hostContracts.sales.onClear =
proc(request: StorageRequest, slotIndex: UInt256) =
hostContracts.sales.onClear = proc(request: StorageRequest, slotIndex: UInt256) =
# TODO: remove data from local storage
self.onClear(request, slotIndex)
hostContracts.sales.onProve =
proc(slot: Slot, challenge: ProofChallenge): Future[?!Groth16Proof] =
hostContracts.sales.onProve = proc(
slot: Slot, challenge: ProofChallenge
): Future[?!Groth16Proof] =
# TODO: generate proof
self.onProve(slot, challenge)
@ -756,7 +730,8 @@ proc new*(
engine: BlockExcEngine,
discovery: Discovery,
prover = Prover.none,
contracts = Contracts.default): CodexNodeRef =
contracts = Contracts.default,
): CodexNodeRef =
## Create new instance of a Codex self, call `start` to run it
##
@ -766,4 +741,5 @@ proc new*(
engine: engine,
prover: prover,
discovery: discovery,
contracts: contracts)
contracts: contracts,
)

1
codex/periods.nim
View File

@ -3,6 +3,7 @@ import pkg/stint
type
Periodicity* = object
seconds*: UInt256
Period* = UInt256
Timestamp* = UInt256

16
codex/purchasing.nim
View File

@ -18,16 +18,13 @@ type
clock: Clock
purchases: Table[PurchaseId, Purchase]
proofProbability*: UInt256
PurchaseTimeout* = Timeout
const DefaultProofProbability = 100.u256
proc new*(_: type Purchasing, market: Market, clock: Clock): Purchasing =
Purchasing(
market: market,
clock: clock,
proofProbability: DefaultProofProbability,
)
Purchasing(market: market, clock: clock, proofProbability: DefaultProofProbability)
proc load*(purchasing: Purchasing) {.async.} =
let market = purchasing.market
@ -43,8 +40,8 @@ proc start*(purchasing: Purchasing) {.async.} =
proc stop*(purchasing: Purchasing) {.async.} =
discard
proc populate*(purchasing: Purchasing,
request: StorageRequest
proc populate*(
purchasing: Purchasing, request: StorageRequest
): Future[StorageRequest] {.async.} =
result = request
if result.ask.proofProbability == 0.u256:
@ -55,8 +52,8 @@ proc populate*(purchasing: Purchasing,
result.nonce = Nonce(id)
result.client = await purchasing.market.getSigner()
proc purchase*(purchasing: Purchasing,
request: StorageRequest
proc purchase*(
purchasing: Purchasing, request: StorageRequest
): Future[Purchase] {.async.} =
let request = await purchasing.populate(request)
let purchase = Purchase.new(request, purchasing.market, purchasing.clock)
@ -75,4 +72,3 @@ func getPurchaseIds*(purchasing: Purchasing): seq[PurchaseId] =
for key in purchasing.purchases.keys:
pIds.add(key)
return pIds

11
codex/purchasing/purchase.nim
View File

@ -25,10 +25,7 @@ export purchaseid
export statemachine
func new*(
_: type Purchase,
requestId: RequestId,
market: Market,
clock: Clock
_: type Purchase, requestId: RequestId, market: Market, clock: Clock
): Purchase =
## create a new instance of a Purchase
##
@ -42,10 +39,7 @@ func new*(
return purchase
func new*(
_: type Purchase,
request: StorageRequest,
market: Market,
clock: Clock
_: type Purchase, request: StorageRequest, market: Market, clock: Clock
): Purchase =
## Create a new purchase using the given market and clock
let purchase = Purchase.new(request.id, market, clock)
@ -76,4 +70,5 @@ func error*(purchase: Purchase): ?(ref CatchableError) =
func state*(purchase: Purchase): ?string =
proc description(state: State): string =
$state
purchase.query(description)

9
codex/purchasing/purchaseid.nim
View File

@ -3,9 +3,12 @@ import ../logutils
type PurchaseId* = distinct array[32, byte]
logutils.formatIt(LogFormat.textLines, PurchaseId): it.short0xHexLog
logutils.formatIt(LogFormat.json, PurchaseId): it.to0xHexLog
logutils.formatIt(LogFormat.textLines, PurchaseId):
it.short0xHexLog
logutils.formatIt(LogFormat.json, PurchaseId):
it.to0xHexLog
proc hash*(x: PurchaseId): Hash {.borrow.}
proc `==`*(x, y: PurchaseId): bool {.borrow.}
proc toHex*(x: PurchaseId): string = array[32, byte](x).toHex
proc toHex*(x: PurchaseId): string =
array[32, byte](x).toHex

1
codex/purchasing/statemachine.nim
View File

@ -14,5 +14,6 @@ type
clock*: Clock
requestId*: RequestId
request*: ?StorageRequest
PurchaseState* = ref object of State
PurchaseError* = object of CodexError

3
codex/purchasing/states/error.nim
View File

@ -18,6 +18,7 @@ method run*(state: PurchaseErrored, machine: Machine): Future[?State] {.async.}
codex_purchases_error.inc()
let purchase = Purchase(machine)
error "Purchasing error", error=state.error.msgDetail, requestId = purchase.requestId
error "Purchasing error",
error = state.error.msgDetail, requestId = purchase.requestId
purchase.future.fail(state.error)

3
codex/purchasing/states/errorhandling.nim
View File

@ -2,8 +2,7 @@ import pkg/questionable
import ../statemachine
import ./error
type
ErrorHandlingState* = ref object of PurchaseState
type ErrorHandlingState* = ref object of PurchaseState
method onError*(state: ErrorHandlingState, error: ref CatchableError): ?State =
some State(PurchaseErrored(error: error))

3
codex/purchasing/states/failed.nim
View File

@ -5,8 +5,7 @@ import ./error
declareCounter(codex_purchases_failed, "codex purchases failed")
type
PurchaseFailed* = ref object of PurchaseState
type PurchaseFailed* = ref object of PurchaseState
method `$`*(state: PurchaseFailed): string =
"failed"

1
codex/purchasing/states/started.nim
View File

@ -27,6 +27,7 @@ method run*(state: PurchaseStarted, machine: Machine): Future[?State] {.async.}
let failed = newFuture[void]()
proc callback(_: RequestId) =
failed.complete()
let subscription = await market.subscribeRequestFailed(purchase.requestId, callback)
# Ensure that we're past the request end by waiting an additional second

6
codex/purchasing/states/submitted.nim
View File

@ -23,12 +23,14 @@ method run*(state: PurchaseSubmitted, machine: Machine): Future[?State] {.async.
let market = purchase.market
let clock = purchase.clock
info "Request submitted, waiting for slots to be filled", requestId = purchase.requestId
info "Request submitted, waiting for slots to be filled",
requestId = purchase.requestId
proc wait {.async.} =
proc wait() {.async.} =
let done = newFuture[void]()
proc callback(_: RequestId) =
done.complete()
let subscription = await market.subscribeFulfillment(request.id, callback)
await done
await subscription.unsubscribe()

1
codex/purchasing/states/unknown.nim
View File

@ -19,7 +19,6 @@ method run*(state: PurchaseUnknown, machine: Machine): Future[?State] {.async.}
let purchase = Purchase(machine)
if (request =? await purchase.market.getRequest(purchase.requestId)) and
(requestState =? await purchase.market.requestState(purchase.requestId)):
purchase.request = some request
case requestState

414
codex/rest/api.nim
View File

@ -9,8 +9,8 @@
import pkg/upraises
push: {.upraises: [].}
push:
{.upraises: [].}
import std/sequtils
import mimetypes
@ -49,10 +49,7 @@ logScope:
declareCounter(codex_api_uploads, "codex API uploads")
declareCounter(codex_api_downloads, "codex API downloads")
proc validate(
pattern: string,
value: string): int
{.gcsafe, raises: [Defect].} =
proc validate(pattern: string, value: string): int {.gcsafe, raises: [Defect].} =
0
proc formatManifest(cid: Cid, manifest: Manifest): RestContent =
@ -63,21 +60,19 @@ proc formatManifestBlocks(node: CodexNodeRef): Future[JsonNode] {.async.} =
proc addManifest(cid: Cid, manifest: Manifest) =
content.add(formatManifest(cid, manifest))
await node.iterateManifests(addManifest)
return %RestContentList.init(content)
proc retrieveCid(
node: CodexNodeRef,
cid: Cid,
local: bool = true,
resp: HttpResponseRef): Future[RestApiResponse] {.async.} =
node: CodexNodeRef, cid: Cid, local: bool = true, resp: HttpResponseRef
): Future[RestApiResponse] {.async.} =
## Download a file from the node in a streaming
## manner
##
var
stream: LPStream
var stream: LPStream
var bytes = 0
try:
@ -101,7 +96,10 @@ proc retrieveCid(
resp.addHeader("Content-Type", "application/octet-stream")
if manifest.filename.isSome:
resp.setHeader("Content-Disposition", "attachment; filename=\"" & manifest.filename.get() & "\"")
resp.setHeader(
"Content-Disposition",
"attachment; filename=\"" & manifest.filename.get() & "\"",
)
else:
resp.setHeader("Content-Disposition", "attachment")
@ -130,7 +128,9 @@ proc retrieveCid(
if not stream.isNil:
await stream.close()
proc buildCorsHeaders(httpMethod: string, allowedOrigin: Option[string]): seq[(string, string)] =
proc buildCorsHeaders(
httpMethod: string, allowedOrigin: Option[string]
): seq[(string, string)] =
var headers: seq[(string, string)] = newSeq[(string, string)]()
if corsOrigin =? allowedOrigin:
@ -160,22 +160,19 @@ proc getFilenameFromContentDisposition(contentDisposition: string): ?string =
proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRouter) =
let allowedOrigin = router.allowedOrigin # prevents capture inside of api defintion
router.api(
MethodOptions,
"/api/codex/v1/data") do (
resp: HttpResponseRef) -> RestApiResponse:
router.api(MethodOptions, "/api/codex/v1/data") do(
resp: HttpResponseRef
) -> RestApiResponse:
if corsOrigin =? allowedOrigin:
resp.setCorsHeaders("POST", corsOrigin)
resp.setHeader("Access-Control-Allow-Headers", "content-type, content-disposition")
resp.setHeader(
"Access-Control-Allow-Headers", "content-type, content-disposition"
)
resp.status = Http204
await resp.sendBody("")
router.rawApi(
MethodPost,
"/api/codex/v1/data") do (
) -> RestApiResponse:
router.rawApi(MethodPost, "/api/codex/v1/data") do() -> RestApiResponse:
## Upload a file in a streaming manner
##
@ -209,12 +206,16 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute
# Here we could check if the extension matches the filename if needed
let
reader = bodyReader.get()
let reader = bodyReader.get()
try:
without cid =? (
await node.store(AsyncStreamWrapper.new(reader = AsyncStreamReader(reader)), filename = filename, mimetype = mimetype)), error:
await node.store(
AsyncStreamWrapper.new(reader = AsyncStreamReader(reader)),
filename = filename,
mimetype = mimetype,
)
), error:
error "Error uploading file", exc = error.msg
return RestApiResponse.error(Http500, error.msg)
@ -233,26 +234,19 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute
trace "Something went wrong error"
return RestApiResponse.error(Http500)
router.api(
MethodGet,
"/api/codex/v1/data") do () -> RestApiResponse:
router.api(MethodGet, "/api/codex/v1/data") do() -> RestApiResponse:
let json = await formatManifestBlocks(node)
return RestApiResponse.response($json, contentType = "application/json")
router.api(
MethodGet,
"/api/codex/v1/data/{cid}") do (
cid: Cid, resp: HttpResponseRef) -> RestApiResponse:
router.api(MethodGet, "/api/codex/v1/data/{cid}") do(
cid: Cid, resp: HttpResponseRef
) -> RestApiResponse:
var headers = buildCorsHeaders("GET", allowedOrigin)
## Download a file from the local node in a streaming
## manner
if cid.isErr:
return RestApiResponse.error(
Http400,
$cid.error(),
headers = headers)
return RestApiResponse.error(Http400, $cid.error(), headers = headers)
if corsOrigin =? allowedOrigin:
resp.setCorsHeaders("GET", corsOrigin)
@ -260,25 +254,20 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute
await node.retrieveCid(cid.get(), local = true, resp = resp)
router.api(
MethodPost,
"/api/codex/v1/data/{cid}/network") do (
cid: Cid, resp: HttpResponseRef) -> RestApiResponse:
router.api(MethodPost, "/api/codex/v1/data/{cid}/network") do(
cid: Cid, resp: HttpResponseRef
) -> RestApiResponse:
## Download a file from the network to the local node
##
var headers = buildCorsHeaders("GET", allowedOrigin)
if cid.isErr:
return RestApiResponse.error(
Http400,
$cid.error(), headers = headers)
return RestApiResponse.error(Http400, $cid.error(), headers = headers)
without manifest =? (await node.fetchManifest(cid.get())), err:
error "Failed to fetch manifest", err = err.msg
return RestApiResponse.error(
Http404,
err.msg, headers = headers)
return RestApiResponse.error(Http404, err.msg, headers = headers)
proc fetchDatasetAsync(): Future[void] {.async.} =
try:
@ -293,10 +282,9 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute
let json = %formatManifest(cid.get(), manifest)
return RestApiResponse.response($json, contentType = "application/json")
router.api(
MethodGet,
"/api/codex/v1/data/{cid}/network/stream") do (
cid: Cid, resp: HttpResponseRef) -> RestApiResponse:
router.api(MethodGet, "/api/codex/v1/data/{cid}/network/stream") do(
cid: Cid, resp: HttpResponseRef
) -> RestApiResponse:
## Download a file from the network in a streaming
## manner
##
@ -304,9 +292,7 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute
var headers = buildCorsHeaders("GET", allowedOrigin)
if cid.isErr:
return RestApiResponse.error(
Http400,
$cid.error(), headers = headers)
return RestApiResponse.error(Http400, $cid.error(), headers = headers)
if corsOrigin =? allowedOrigin:
resp.setCorsHeaders("GET", corsOrigin)
@ -314,74 +300,72 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute
await node.retrieveCid(cid.get(), local = false, resp = resp)
router.api(
MethodGet,
"/api/codex/v1/data/{cid}/network/manifest") do (
cid: Cid, resp: HttpResponseRef) -> RestApiResponse:
router.api(MethodGet, "/api/codex/v1/data/{cid}/network/manifest") do(
cid: Cid, resp: HttpResponseRef
) -> RestApiResponse:
## Download only the manifest.
##
var headers = buildCorsHeaders("GET", allowedOrigin)
if cid.isErr:
return RestApiResponse.error(
Http400,
$cid.error(), headers = headers)
return RestApiResponse.error(Http400, $cid.error(), headers = headers)
without manifest =? (await node.fetchManifest(cid.get())), err:
error "Failed to fetch manifest", err = err.msg
return RestApiResponse.error(
Http404,
err.msg, headers = headers)
return RestApiResponse.error(Http404, err.msg, headers = headers)
let json = %formatManifest(cid.get(), manifest)
return RestApiResponse.response($json, contentType = "application/json")
router.api(
MethodGet,
"/api/codex/v1/space") do () -> RestApiResponse:
let json = % RestRepoStore(
router.api(MethodGet, "/api/codex/v1/space") do() -> RestApiResponse:
let json =
%RestRepoStore(
totalBlocks: repoStore.totalBlocks,
quotaMaxBytes: repoStore.quotaMaxBytes,
quotaUsedBytes: repoStore.quotaUsedBytes,
quotaReservedBytes: repoStore.quotaReservedBytes
quotaReservedBytes: repoStore.quotaReservedBytes,
)
return RestApiResponse.response($json, contentType = "application/json")
proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
let allowedOrigin = router.allowedOrigin
router.api(
MethodGet,
"/api/codex/v1/sales/slots") do () -> RestApiResponse:
router.api(MethodGet, "/api/codex/v1/sales/slots") do() -> RestApiResponse:
var headers = buildCorsHeaders("GET", allowedOrigin)
## Returns active slots for the host
try:
without contracts =? node.contracts.host:
return RestApiResponse.error(Http503, "Persistence is not enabled", headers = headers)
return RestApiResponse.error(
Http503, "Persistence is not enabled", headers = headers
)
let json = %(await contracts.sales.mySlots())
return RestApiResponse.response($json, contentType="application/json", headers = headers)
return RestApiResponse.response(
$json, contentType = "application/json", headers = headers
)
except CatchableError as exc:
trace "Excepting processing request", exc = exc.msg
return RestApiResponse.error(Http500, headers = headers)
router.api(
MethodGet,
"/api/codex/v1/sales/slots/{slotId}") do (slotId: SlotId) -> RestApiResponse:
router.api(MethodGet, "/api/codex/v1/sales/slots/{slotId}") do(
slotId: SlotId
) -> RestApiResponse:
## Returns active slot with id {slotId} for the host. Returns 404 if the
## slot is not active for the host.
var headers = buildCorsHeaders("GET", allowedOrigin)
without contracts =? node.contracts.host:
return RestApiResponse.error(Http503, "Persistence is not enabled", headers = headers)
return
RestApiResponse.error(Http503, "Persistence is not enabled", headers = headers)
without slotId =? slotId.tryGet.catch, error:
return RestApiResponse.error(Http400, error.msg, headers = headers)
without agent =? await contracts.sales.activeSale(slotId):
return RestApiResponse.error(Http404, "Provider not filling slot", headers = headers)
return
RestApiResponse.error(Http404, "Provider not filling slot", headers = headers)
let restAgent = RestSalesAgent(
state: agent.state() |? "none",
@ -391,30 +375,33 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
reservation: agent.data.reservation,
)
return RestApiResponse.response(restAgent.toJson, contentType="application/json", headers = headers)
return RestApiResponse.response(
restAgent.toJson, contentType = "application/json", headers = headers
)
router.api(
MethodGet,
"/api/codex/v1/sales/availability") do () -> RestApiResponse:
router.api(MethodGet, "/api/codex/v1/sales/availability") do() -> RestApiResponse:
## Returns storage that is for sale
var headers = buildCorsHeaders("GET", allowedOrigin)
try:
without contracts =? node.contracts.host:
return RestApiResponse.error(Http503, "Persistence is not enabled", headers = headers)
return RestApiResponse.error(
Http503, "Persistence is not enabled", headers = headers
)
without avails =? (await contracts.sales.context.reservations.all(Availability)), err:
without avails =? (await contracts.sales.context.reservations.all(Availability)),
err:
return RestApiResponse.error(Http500, err.msg, headers = headers)
let json = %avails
return RestApiResponse.response($json, contentType="application/json", headers = headers)
return RestApiResponse.response(
$json, contentType = "application/json", headers = headers
)
except CatchableError as exc:
trace "Excepting processing request", exc = exc.msg
return RestApiResponse.error(Http500, headers = headers)
router.rawApi(
MethodPost,
"/api/codex/v1/sales/availability") do () -> RestApiResponse:
router.rawApi(MethodPost, "/api/codex/v1/sales/availability") do() -> RestApiResponse:
## Add available storage to sell.
## Every time Availability's offer finishes, its capacity is returned to the availability.
##
@ -427,7 +414,9 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
try:
without contracts =? node.contracts.host:
return RestApiResponse.error(Http503, "Persistence is not enabled", headers = headers)
return RestApiResponse.error(
Http503, "Persistence is not enabled", headers = headers
)
let body = await request.getBody()
@ -437,41 +426,43 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
let reservations = contracts.sales.context.reservations
if restAv.totalSize == 0:
return RestApiResponse.error(Http400, "Total size must be larger then zero", headers = headers)
return RestApiResponse.error(
Http400, "Total size must be larger then zero", headers = headers
)
if not reservations.hasAvailable(restAv.totalSize.truncate(uint)):
return RestApiResponse.error(Http422, "Not enough storage quota", headers = headers)
return
RestApiResponse.error(Http422, "Not enough storage quota", headers = headers)
without availability =? (
await reservations.createAvailability(
restAv.totalSize,
restAv.duration,
restAv.minPrice,
restAv.maxCollateral)
restAv.totalSize, restAv.duration, restAv.minPrice, restAv.maxCollateral
)
), error:
return RestApiResponse.error(Http500, error.msg, headers = headers)
return RestApiResponse.response(availability.toJson,
return RestApiResponse.response(
availability.toJson,
Http201,
contentType = "application/json",
headers = headers)
headers = headers,
)
except CatchableError as exc:
trace "Excepting processing request", exc = exc.msg
return RestApiResponse.error(Http500, headers = headers)
router.api(
MethodOptions,
"/api/codex/v1/sales/availability/{id}") do (id: AvailabilityId, resp: HttpResponseRef) -> RestApiResponse:
router.api(MethodOptions, "/api/codex/v1/sales/availability/{id}") do(
id: AvailabilityId, resp: HttpResponseRef
) -> RestApiResponse:
if corsOrigin =? allowedOrigin:
resp.setCorsHeaders("PATCH", corsOrigin)
resp.status = Http204
await resp.sendBody("")
router.rawApi(
MethodPatch,
"/api/codex/v1/sales/availability/{id}") do (id: AvailabilityId) -> RestApiResponse:
router.rawApi(MethodPatch, "/api/codex/v1/sales/availability/{id}") do(
id: AvailabilityId
) -> RestApiResponse:
## Updates Availability.
## The new parameters will be only considered for new requests.
## Existing Requests linked to this Availability will continue as is.
@ -509,7 +500,11 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
if size =? restAv.totalSize:
# we don't allow lowering the totalSize bellow currently utilized size
if size < (availability.totalSize - availability.freeSize):
return RestApiResponse.error(Http400, "New totalSize must be larger then current totalSize - freeSize, which is currently: " & $(availability.totalSize - availability.freeSize))
return RestApiResponse.error(
Http400,
"New totalSize must be larger then current totalSize - freeSize, which is currently: " &
$(availability.totalSize - availability.freeSize),
)
availability.freeSize += size - availability.totalSize
availability.totalSize = size
@ -531,15 +526,17 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
trace "Excepting processing request", exc = exc.msg
return RestApiResponse.error(Http500)
router.rawApi(
MethodGet,
"/api/codex/v1/sales/availability/{id}/reservations") do (id: AvailabilityId) -> RestApiResponse:
router.rawApi(MethodGet, "/api/codex/v1/sales/availability/{id}/reservations") do(
id: AvailabilityId
) -> RestApiResponse:
## Gets Availability's reservations.
var headers = buildCorsHeaders("GET", allowedOrigin)
try:
without contracts =? node.contracts.host:
return RestApiResponse.error(Http503, "Persistence is not enabled", headers = headers)
return RestApiResponse.error(
Http503, "Persistence is not enabled", headers = headers
)
without id =? id.tryGet.catch, error:
return RestApiResponse.error(Http400, error.msg, headers = headers)
@ -551,15 +548,21 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
if error =? (await reservations.get(keyId, Availability)).errorOption:
if error of NotExistsError:
return RestApiResponse.error(Http404, "Availability not found", headers = headers)
return
RestApiResponse.error(Http404, "Availability not found", headers = headers)
else:
return RestApiResponse.error(Http500, error.msg, headers = headers)
without availabilitysReservations =? (await reservations.all(Reservation, id)), err:
without availabilitysReservations =? (await reservations.all(Reservation, id)),
err:
return RestApiResponse.error(Http500, err.msg, headers = headers)
# TODO: Expand this structure with information about the linked StorageRequest not only RequestID
return RestApiResponse.response(availabilitysReservations.toJson, contentType="application/json", headers = headers)
return RestApiResponse.response(
availabilitysReservations.toJson,
contentType = "application/json",
headers = headers,
)
except CatchableError as exc:
trace "Excepting processing request", exc = exc.msg
return RestApiResponse.error(Http500, headers = headers)
@ -567,9 +570,9 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
let allowedOrigin = router.allowedOrigin
router.rawApi(
MethodPost,
"/api/codex/v1/storage/request/{cid}") do (cid: Cid) -> RestApiResponse:
router.rawApi(MethodPost, "/api/codex/v1/storage/request/{cid}") do(
cid: Cid
) -> RestApiResponse:
var headers = buildCorsHeaders("POST", allowedOrigin)
## Create a request for storage
@ -584,7 +587,9 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
## colateral - requested collateral from hosts when they fill slot
try:
without contracts =? node.contracts.client:
return RestApiResponse.error(Http503, "Persistence is not enabled", headers = headers)
return RestApiResponse.error(
Http503, "Persistence is not enabled", headers = headers
)
without cid =? cid.tryGet.catch, error:
return RestApiResponse.error(Http400, error.msg, headers = headers)
@ -598,39 +603,51 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
let tolerance = params.tolerance |? 1
if tolerance == 0:
return RestApiResponse.error(Http400, "Tolerance needs to be bigger then zero", headers = headers)
return RestApiResponse.error(
Http400, "Tolerance needs to be bigger then zero", headers = headers
)
# prevent underflow
if tolerance > nodes:
return RestApiResponse.error(Http400, "Invalid parameters: `tolerance` cannot be greater than `nodes`", headers = headers)
return RestApiResponse.error(
Http400,
"Invalid parameters: `tolerance` cannot be greater than `nodes`",
headers = headers,
)
let ecK = nodes - tolerance
let ecM = tolerance # for readability
# ensure leopard constrainst of 1 < K ≥ M
if ecK <= 1 or ecK < ecM:
return RestApiResponse.error(Http400, "Invalid parameters: parameters must satify `1 < (nodes - tolerance) ≥ tolerance`", headers = headers)
return RestApiResponse.error(
Http400,
"Invalid parameters: parameters must satify `1 < (nodes - tolerance) ≥ tolerance`",
headers = headers,
)
without expiry =? params.expiry:
return RestApiResponse.error(Http400, "Expiry required", headers = headers)
if expiry <= 0 or expiry >= params.duration:
return RestApiResponse.error(Http400, "Expiry needs value bigger then zero and smaller then the request's duration", headers = headers)
without purchaseId =? await node.requestStorage(
cid,
params.duration,
params.proofProbability,
nodes,
tolerance,
params.reward,
params.collateral,
expiry), error:
return RestApiResponse.error(
Http400,
"Expiry needs value bigger then zero and smaller then the request's duration",
headers = headers,
)
without purchaseId =?
await node.requestStorage(
cid, params.duration, params.proofProbability, nodes, tolerance,
params.reward, params.collateral, expiry,
), error:
if error of InsufficientBlocksError:
return RestApiResponse.error(Http400,
return RestApiResponse.error(
Http400,
"Dataset too small for erasure parameters, need at least " &
$(ref InsufficientBlocksError)(error).minSize.int & " bytes", headers = headers)
$(ref InsufficientBlocksError)(error).minSize.int & " bytes",
headers = headers,
)
return RestApiResponse.error(Http500, error.msg, headers = headers)
@ -639,15 +656,16 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
trace "Excepting processing request", exc = exc.msg
return RestApiResponse.error(Http500, headers = headers)
router.api(
MethodGet,
"/api/codex/v1/storage/purchases/{id}") do (
id: PurchaseId) -> RestApiResponse:
router.api(MethodGet, "/api/codex/v1/storage/purchases/{id}") do(
id: PurchaseId
) -> RestApiResponse:
var headers = buildCorsHeaders("GET", allowedOrigin)
try:
without contracts =? node.contracts.client:
return RestApiResponse.error(Http503, "Persistence is not enabled", headers = headers)
return RestApiResponse.error(
Http503, "Persistence is not enabled", headers = headers
)
without id =? id.tryGet.catch, error:
return RestApiResponse.error(Http400, error.msg, headers = headers)
@ -655,29 +673,34 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
without purchase =? contracts.purchasing.getPurchase(id):
return RestApiResponse.error(Http404, headers = headers)
let json = % RestPurchase(
let json =
%RestPurchase(
state: purchase.state |? "none",
error: purchase.error .? msg,
request: purchase.request,
requestId: purchase.requestId
requestId: purchase.requestId,
)
return RestApiResponse.response($json, contentType="application/json", headers = headers)
return RestApiResponse.response(
$json, contentType = "application/json", headers = headers
)
except CatchableError as exc:
trace "Excepting processing request", exc = exc.msg
return RestApiResponse.error(Http500, headers = headers)
router.api(
MethodGet,
"/api/codex/v1/storage/purchases") do () -> RestApiResponse:
router.api(MethodGet, "/api/codex/v1/storage/purchases") do() -> RestApiResponse:
var headers = buildCorsHeaders("GET", allowedOrigin)
try:
without contracts =? node.contracts.client:
return RestApiResponse.error(Http503, "Persistence is not enabled", headers = headers)
return RestApiResponse.error(
Http503, "Persistence is not enabled", headers = headers
)
let purchaseIds = contracts.purchasing.getPurchaseIds()
return RestApiResponse.response($ %purchaseIds, contentType="application/json", headers = headers)
return RestApiResponse.response(
$ %purchaseIds, contentType = "application/json", headers = headers
)
except CatchableError as exc:
trace "Excepting processing request", exc = exc.msg
return RestApiResponse.error(Http500, headers = headers)
@ -687,28 +710,30 @@ proc initNodeApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) =
## various node management api's
##
router.api(
MethodGet,
"/api/codex/v1/spr") do () -> RestApiResponse:
router.api(MethodGet, "/api/codex/v1/spr") do() -> RestApiResponse:
## Returns node SPR in requested format, json or text.
##
var headers = buildCorsHeaders("GET", allowedOrigin)
try:
without spr =? node.discovery.dhtRecord:
return RestApiResponse.response("", status=Http503, contentType="application/json", headers = headers)
return RestApiResponse.response(
"", status = Http503, contentType = "application/json", headers = headers
)
if $preferredContentType().get() == "text/plain":
return RestApiResponse.response(spr.toURI, contentType="text/plain", headers = headers)
return RestApiResponse.response(
spr.toURI, contentType = "text/plain", headers = headers
)
else:
return RestApiResponse.response($ %* {"spr": spr.toURI}, contentType="application/json", headers = headers)
return RestApiResponse.response(
$ %*{"spr": spr.toURI}, contentType = "application/json", headers = headers
)
except CatchableError as exc:
trace "Excepting processing request", exc = exc.msg
return RestApiResponse.error(Http500, headers = headers)
router.api(
MethodGet,
"/api/codex/v1/peerid") do () -> RestApiResponse:
router.api(MethodGet, "/api/codex/v1/peerid") do() -> RestApiResponse:
## Returns node's peerId in requested format, json or text.
##
var headers = buildCorsHeaders("GET", allowedOrigin)
@ -717,18 +742,19 @@ proc initNodeApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) =
let id = $node.switch.peerInfo.peerId
if $preferredContentType().get() == "text/plain":
return RestApiResponse.response(id, contentType="text/plain", headers = headers)
return
RestApiResponse.response(id, contentType = "text/plain", headers = headers)
else:
return RestApiResponse.response($ %* {"id": id}, contentType="application/json", headers = headers)
return RestApiResponse.response(
$ %*{"id": id}, contentType = "application/json", headers = headers
)
except CatchableError as exc:
trace "Excepting processing request", exc = exc.msg
return RestApiResponse.error(Http500, headers = headers)
router.api(
MethodGet,
"/api/codex/v1/connect/{peerId}") do (
peerId: PeerId,
addrs: seq[MultiAddress]) -> RestApiResponse:
router.api(MethodGet, "/api/codex/v1/connect/{peerId}") do(
peerId: PeerId, addrs: seq[MultiAddress]
) -> RestApiResponse:
## Connect to a peer
##
## If `addrs` param is supplied, it will be used to
@ -741,34 +767,30 @@ proc initNodeApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) =
var headers = buildCorsHeaders("GET", allowedOrigin)
if peerId.isErr:
return RestApiResponse.error(
Http400,
$peerId.error(),
headers = headers)
return RestApiResponse.error(Http400, $peerId.error(), headers = headers)
let addresses = if addrs.isOk and addrs.get().len > 0:
let addresses =
if addrs.isOk and addrs.get().len > 0:
addrs.get()
else:
without peerRecord =? (await node.findPeer(peerId.get())):
return RestApiResponse.error(
Http400,
"Unable to find Peer!",
headers = headers)
return
RestApiResponse.error(Http400, "Unable to find Peer!", headers = headers)
peerRecord.addresses.mapIt(it.address)
try:
await node.connect(peerId.get(), addresses)
return RestApiResponse.response("Successfully connected to peer", headers = headers)
return
RestApiResponse.response("Successfully connected to peer", headers = headers)
except DialFailedError:
return RestApiResponse.error(Http400, "Unable to dial peer", headers = headers)
except CatchableError:
return RestApiResponse.error(Http500, "Unknown error dialling peer", headers = headers)
return
RestApiResponse.error(Http500, "Unknown error dialling peer", headers = headers)
proc initDebugApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) =
let allowedOrigin = router.allowedOrigin
router.api(
MethodGet,
"/api/codex/v1/debug/info") do () -> RestApiResponse:
router.api(MethodGet, "/api/codex/v1/debug/info") do() -> RestApiResponse:
## Print rudimentary node information
##
var headers = buildCorsHeaders("GET", allowedOrigin)
@ -776,8 +798,8 @@ proc initDebugApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) =
try:
let table = RestRoutingTable.init(node.discovery.protocol.routingTable)
let
json = %*{
let json =
%*{
"id": $node.switch.peerInfo.peerId,
"addrs": node.switch.peerInfo.addrs.mapIt($it),
"repo": $conf.dataDir,
@ -788,22 +810,20 @@ proc initDebugApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) =
"",
"announceAddresses": node.discovery.announceAddrs,
"table": table,
"codex": {
"version": $codexVersion,
"revision": $codexRevision
}
"codex": {"version": $codexVersion, "revision": $codexRevision},
}
# return pretty json for human readability
return RestApiResponse.response(json.pretty(), contentType="application/json", headers = headers)
return RestApiResponse.response(
json.pretty(), contentType = "application/json", headers = headers
)
except CatchableError as exc:
trace "Excepting processing request", exc = exc.msg
return RestApiResponse.error(Http500, headers = headers)
router.api(
MethodPost,
"/api/codex/v1/debug/chronicles/loglevel") do (
level: Option[string]) -> RestApiResponse:
router.api(MethodPost, "/api/codex/v1/debug/chronicles/loglevel") do(
level: Option[string]
) -> RestApiResponse:
## Set log level at run time
##
## e.g. `chronicles/loglevel?level=DEBUG`
@ -828,19 +848,17 @@ proc initDebugApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) =
return RestApiResponse.error(Http500, headers = headers)
when codex_enable_api_debug_peers:
router.api(
MethodGet,
"/api/codex/v1/debug/peer/{peerId}") do (peerId: PeerId) -> RestApiResponse:
router.api(MethodGet, "/api/codex/v1/debug/peer/{peerId}") do(
peerId: PeerId
) -> RestApiResponse:
var headers = buildCorsHeaders("GET", allowedOrigin)
try:
trace "debug/peer start"
without peerRecord =? (await node.findPeer(peerId.get())):
trace "debug/peer peer not found!"
return RestApiResponse.error(
Http400,
"Unable to find Peer!",
headers = headers)
return
RestApiResponse.error(Http400, "Unable to find Peer!", headers = headers)
let json = %RestPeerRecord.init(peerRecord)
trace "debug/peer returning peer record"
@ -853,8 +871,8 @@ proc initRestApi*(
node: CodexNodeRef,
conf: CodexConf,
repoStore: RepoStore,
corsAllowedOrigin: ?string): RestRouter =
corsAllowedOrigin: ?string,
): RestRouter =
var router = RestRouter.init(validate, corsAllowedOrigin)
initDataApi(node, repoStore, router)

22
codex/rest/coders.nim
View File

@ -25,9 +25,7 @@ proc encodeString*(cid: type Cid): Result[string, cstring] =
ok($cid)
proc decodeString*(T: type Cid, value: string): Result[Cid, cstring] =
Cid
.init(value)
.mapErr do(e: CidError) -> cstring:
Cid.init(value).mapErr do(e: CidError) -> cstring:
case e
of CidError.Incorrect: "Incorrect Cid".cstring
of CidError.Unsupported: "Unsupported Cid".cstring
@ -44,9 +42,8 @@ proc encodeString*(address: MultiAddress): Result[string, cstring] =
ok($address)
proc decodeString*(T: type MultiAddress, value: string): Result[MultiAddress, cstring] =
MultiAddress
.init(value)
.mapErr do(e: string) -> cstring: cstring(e)
MultiAddress.init(value).mapErr do(e: string) -> cstring:
cstring(e)
proc decodeString*(T: type SomeUnsignedInt, value: string): Result[T, cstring] =
Base10.decode(T, value)
@ -77,19 +74,20 @@ proc decodeString*(_: type UInt256, value: string): Result[UInt256, cstring] =
except ValueError as e:
err e.msg.cstring
proc decodeString*(_: type array[32, byte],
value: string): Result[array[32, byte], cstring] =
proc decodeString*(
_: type array[32, byte], value: string
): Result[array[32, byte], cstring] =
try:
ok array[32, byte].fromHex(value)
except ValueError as e:
err e.msg.cstring
proc decodeString*[T: PurchaseId | RequestId | Nonce | SlotId | AvailabilityId](_: type T,
value: string): Result[T, cstring] =
proc decodeString*[T: PurchaseId | RequestId | Nonce | SlotId | AvailabilityId](
_: type T, value: string
): Result[T, cstring] =
array[32, byte].decodeString(value).map(id => T(id))
proc decodeString*(t: typedesc[string],
value: string): Result[string, cstring] =
proc decodeString*(t: typedesc[string], value: string): Result[string, cstring] =
ok(value)
proc encodeString*(value: string): RestResult[string] =

30
codex/rest/json.nim
View File

@ -74,15 +74,10 @@ type
quotaReservedBytes* {.serialize.}: NBytes
proc init*(_: type RestContentList, content: seq[RestContent]): RestContentList =
RestContentList(
content: content
)
RestContentList(content: content)
proc init*(_: type RestContent, cid: Cid, manifest: Manifest): RestContent =
RestContent(
cid: cid,
manifest: manifest
)
RestContent(cid: cid, manifest: manifest)
proc init*(_: type RestNode, node: dn.Node): RestNode =
RestNode(
@ -90,7 +85,7 @@ proc init*(_: type RestNode, node: dn.Node): RestNode =
peerId: node.record.data.peerId,
record: node.record,
address: node.address,
seen: node.seen > 0.5
seen: node.seen > 0.5,
)
proc init*(_: type RestRoutingTable, routingTable: rt.RoutingTable): RestRoutingTable =
@ -99,28 +94,23 @@ proc init*(_: type RestRoutingTable, routingTable: rt.RoutingTable): RestRouting
for node in bucket.nodes:
nodes.add(RestNode.init(node))
RestRoutingTable(
localNode: RestNode.init(routingTable.localNode),
nodes: nodes
)
RestRoutingTable(localNode: RestNode.init(routingTable.localNode), nodes: nodes)
proc init*(_: type RestPeerRecord, peerRecord: PeerRecord): RestPeerRecord =
RestPeerRecord(
peerId: peerRecord.peerId,
seqNo: peerRecord.seqNo,
addresses: peerRecord.addresses
peerId: peerRecord.peerId, seqNo: peerRecord.seqNo, addresses: peerRecord.addresses
)
proc init*(_: type RestNodeId, id: NodeId): RestNodeId =
RestNodeId(
id: id
)
RestNodeId(id: id)
proc `%`*(obj: StorageRequest | Slot): JsonNode =
let jsonObj = newJObject()
for k, v in obj.fieldPairs: jsonObj[k] = %v
for k, v in obj.fieldPairs:
jsonObj[k] = %v
jsonObj["id"] = %(obj.id)
return jsonObj
proc `%`*(obj: RestNodeId): JsonNode = % $obj.id
proc `%`*(obj: RestNodeId): JsonNode =
% $obj.id

10
codex/rng.nim
View File

@ -9,7 +9,8 @@
import pkg/upraises
push: {.upraises: [].}
push:
{.upraises: [].}
import pkg/libp2p/crypto/crypto
import pkg/bearssl/rand
@ -30,7 +31,8 @@ proc instance*(t: type Rng): Rng =
const randMax = 18_446_744_073_709_551_615'u64
proc rand*(rng: Rng, max: Natural): int =
if max == 0: return 0
if max == 0:
return 0
while true:
let x = rng[].generate(uint64)
@ -41,8 +43,8 @@ proc sample*[T](rng: Rng, a: openArray[T]): T =
result = a[rng.rand(a.high)]
proc sample*[T](
rng: Rng, sample, exclude: openArray[T]): T
{.raises: [Defect, RngSampleError].} =
rng: Rng, sample, exclude: openArray[T]
): T {.raises: [Defect, RngSampleError].} =
if sample == exclude:
raise newException(RngSampleError, "Sample and exclude arrays are the same!")

130
codex/sales.nim
View File

@ -45,8 +45,7 @@ export salescontext
logScope:
topics = "sales marketplace"
type
Sales* = ref object
type Sales* = ref object
context*: SalesContext
agents*: seq[SalesAgent]
running: bool
@ -68,28 +67,31 @@ proc `onProve=`*(sales: Sales, callback: OnProve) =
proc `onExpiryUpdate=`*(sales: Sales, callback: OnExpiryUpdate) =
sales.context.onExpiryUpdate = some callback
proc onStore*(sales: Sales): ?OnStore = sales.context.onStore
proc onStore*(sales: Sales): ?OnStore =
sales.context.onStore
proc onClear*(sales: Sales): ?OnClear = sales.context.onClear
proc onClear*(sales: Sales): ?OnClear =
sales.context.onClear
proc onSale*(sales: Sales): ?OnSale = sales.context.onSale
proc onSale*(sales: Sales): ?OnSale =
sales.context.onSale
proc onProve*(sales: Sales): ?OnProve = sales.context.onProve
proc onProve*(sales: Sales): ?OnProve =
sales.context.onProve
proc onExpiryUpdate*(sales: Sales): ?OnExpiryUpdate = sales.context.onExpiryUpdate
proc onExpiryUpdate*(sales: Sales): ?OnExpiryUpdate =
sales.context.onExpiryUpdate
proc new*(_: type Sales,
market: Market,
clock: Clock,
repo: RepoStore): Sales =
proc new*(_: type Sales, market: Market, clock: Clock, repo: RepoStore): Sales =
Sales.new(market, clock, repo, 0)
proc new*(_: type Sales,
proc new*(
_: type Sales,
market: Market,
clock: Clock,
repo: RepoStore,
simulateProofFailures: int): Sales =
simulateProofFailures: int,
): Sales =
let reservations = Reservations.new(repo)
Sales(
context: SalesContext(
@ -97,10 +99,10 @@ proc new*(_: type Sales,
clock: clock,
reservations: reservations,
slotQueue: SlotQueue.new(),
simulateProofFailures: simulateProofFailures
simulateProofFailures: simulateProofFailures,
),
trackedFutures: TrackedFutures.new(),
subscriptions: @[]
subscriptions: @[],
)
proc remove(sales: Sales, agent: SalesAgent) {.async.} =
@ -108,12 +110,13 @@ proc remove(sales: Sales, agent: SalesAgent) {.async.} =
if sales.running:
sales.agents.keepItIf(it != agent)
proc cleanUp(sales: Sales,
proc cleanUp(
sales: Sales,
agent: SalesAgent,
returnBytes: bool,
reprocessSlot: bool,
processing: Future[void]) {.async.} =
processing: Future[void],
) {.async.} =
let data = agent.data
logScope:
@ -129,36 +132,37 @@ proc cleanUp(sales: Sales,
# that the cleanUp was called before the sales process really started, so
# there are not really any bytes to be returned
if returnBytes and request =? data.request and reservation =? data.reservation:
if returnErr =? (await sales.context.reservations.returnBytesToAvailability(
reservation.availabilityId,
reservation.id,
request.ask.slotSize
)).errorOption:
if returnErr =? (
await sales.context.reservations.returnBytesToAvailability(
reservation.availabilityId, reservation.id, request.ask.slotSize
)
).errorOption:
error "failure returning bytes",
error = returnErr.msg,
bytes = request.ask.slotSize
error = returnErr.msg, bytes = request.ask.slotSize
# delete reservation and return reservation bytes back to the availability
if reservation =? data.reservation and
deleteErr =? (await sales.context.reservations.deleteReservation(
reservation.id,
reservation.availabilityId
)).errorOption:
deleteErr =? (
await sales.context.reservations.deleteReservation(
reservation.id, reservation.availabilityId
)
).errorOption:
error "failure deleting reservation", error = deleteErr.msg
# Re-add items back into the queue to prevent small availabilities from
# draining the queue. Seen items will be ordered last.
if reprocessSlot and request =? data.request:
let queue = sales.context.slotQueue
var seenItem = SlotQueueItem.init(data.requestId,
var seenItem = SlotQueueItem.init(
data.requestId,
data.slotIndex.truncate(uint16),
data.ask,
request.expiry,
seen = true)
seen = true,
)
trace "pushing ignored item to queue, marked as seen"
if err =? queue.push(seenItem).errorOption:
error "failed to readd slot to queue",
errorType = $(type err), error = err.msg
error "failed to readd slot to queue", errorType = $(type err), error = err.msg
await sales.remove(agent)
@ -167,11 +171,8 @@ proc cleanUp(sales: Sales,
processing.complete()
proc filled(
sales: Sales,
request: StorageRequest,
slotIndex: UInt256,
processing: Future[void]) =
sales: Sales, request: StorageRequest, slotIndex: UInt256, processing: Future[void]
) =
if onSale =? sales.context.onSale:
onSale(request, slotIndex)
@ -180,14 +181,10 @@ proc filled(
processing.complete()
proc processSlot(sales: Sales, item: SlotQueueItem, done: Future[void]) =
debug "Processing slot from queue", requestId = item.requestId,
slot = item.slotIndex
debug "Processing slot from queue", requestId = item.requestId, slot = item.slotIndex
let agent = newSalesAgent(
sales.context,
item.requestId,
item.slotIndex.u256,
none StorageRequest
sales.context, item.requestId, item.slotIndex.u256, none StorageRequest
)
agent.onCleanUp = proc(returnBytes = false, reprocessSlot = false) {.async.} =
@ -204,10 +201,12 @@ proc deleteInactiveReservations(sales: Sales, activeSlots: seq[Slot]) {.async.}
without reservs =? await reservations.all(Reservation):
return
let unused = reservs.filter(r => (
let unused = reservs.filter(
r => (
let slotId = slotId(r.requestId, r.slotIndex)
not activeSlots.any(slot => slot.id == slotId)
))
)
)
if unused.len == 0:
return
@ -215,14 +214,13 @@ proc deleteInactiveReservations(sales: Sales, activeSlots: seq[Slot]) {.async.}
info "Found unused reservations for deletion", unused = unused.len
for reservation in unused:
logScope:
reservationId = reservation.id
availabilityId = reservation.availabilityId
if err =? (await reservations.deleteReservation(
reservation.id, reservation.availabilityId
)).errorOption:
if err =? (
await reservations.deleteReservation(reservation.id, reservation.availabilityId)
).errorOption:
error "Failed to delete unused reservation", error = err.msg
else:
trace "Deleted unused reservation"
@ -252,11 +250,8 @@ proc load*(sales: Sales) {.async.} =
await sales.deleteInactiveReservations(activeSlots)
for slot in activeSlots:
let agent = newSalesAgent(
sales.context,
slot.request.id,
slot.slotIndex,
some slot.request)
let agent =
newSalesAgent(sales.context, slot.request.id, slot.slotIndex, some slot.request)
agent.onCleanUp = proc(returnBytes = false, reprocessSlot = false) {.async.} =
# since workers are not being dispatched, this future has not been created
@ -282,11 +277,9 @@ proc onAvailabilityAdded(sales: Sales, availability: Availability) {.async.} =
trace "unpausing queue after new availability added"
queue.unpause()
proc onStorageRequested(sales: Sales,
requestId: RequestId,
ask: StorageAsk,
expiry: UInt256) =
proc onStorageRequested(
sales: Sales, requestId: RequestId, ask: StorageAsk, expiry: UInt256
) =
logScope:
topics = "marketplace sales onStorageRequested"
requestId
@ -314,10 +307,7 @@ proc onStorageRequested(sales: Sales,
else:
warn "Error adding request to SlotQueue", error = err.msg
proc onSlotFreed(sales: Sales,
requestId: RequestId,
slotIndex: UInt256) =
proc onSlotFreed(sales: Sales, requestId: RequestId, slotIndex: UInt256) =
logScope:
topics = "marketplace sales onSlotFreed"
requestId
@ -331,8 +321,7 @@ proc onSlotFreed(sales: Sales,
let queue = context.slotQueue
# first attempt to populate request using existing slot metadata in queue
without var found =? queue.populateItem(requestId,
slotIndex.truncate(uint16)):
without var found =? queue.populateItem(requestId, slotIndex.truncate(uint16)):
trace "no existing request metadata, getting request info from contract"
# if there's no existing slot for that request, retrieve the request
# from the contract.
@ -359,9 +348,7 @@ proc subscribeRequested(sales: Sales) {.async.} =
let context = sales.context
let market = context.market
proc onStorageRequested(requestId: RequestId,
ask: StorageAsk,
expiry: UInt256) =
proc onStorageRequested(requestId: RequestId, ask: StorageAsk, expiry: UInt256) =
sales.onStorageRequested(requestId, ask, expiry)
try:
@ -485,8 +472,7 @@ proc startSlotQueue(sales: Sales) =
let slotQueue = sales.context.slotQueue
let reservations = sales.context.reservations
slotQueue.onProcessSlot =
proc(item: SlotQueueItem, done: Future[void]) {.async.} =
slotQueue.onProcessSlot = proc(item: SlotQueueItem, done: Future[void]) {.async.} =
trace "processing slot queue item", reqId = item.requestId, slotIdx = item.slotIndex
sales.processSlot(item, done)

245
codex/sales/reservations.nim
View File

@ -26,7 +26,8 @@
## +----------------------------------------+
import pkg/upraises
push: {.upraises: [].}
push:
{.upraises: [].}
import std/sequtils
import std/sugar
@ -54,7 +55,6 @@ export logutils
logScope:
topics = "sales reservations"
type
AvailabilityId* = distinct array[32, byte]
ReservationId* = distinct array[32, byte]
@ -65,25 +65,32 @@ type
totalSize* {.serialize.}: UInt256
freeSize* {.serialize.}: UInt256
duration* {.serialize.}: UInt256
minPrice* {.serialize.}: UInt256 # minimal price paid for the whole hosted slot for the request's duration
minPrice* {.serialize.}: UInt256
# minimal price paid for the whole hosted slot for the request's duration
maxCollateral* {.serialize.}: UInt256
Reservation* = ref object
id* {.serialize.}: ReservationId
availabilityId* {.serialize.}: AvailabilityId
size* {.serialize.}: UInt256
requestId* {.serialize.}: RequestId
slotIndex* {.serialize.}: UInt256
Reservations* = ref object of RootObj
availabilityLock: AsyncLock # Lock for protecting assertions of availability's sizes when searching for matching availability
availabilityLock: AsyncLock
# Lock for protecting assertions of availability's sizes when searching for matching availability
repo: RepoStore
onAvailabilityAdded: ?OnAvailabilityAdded
GetNext* = proc(): Future[?seq[byte]] {.upraises: [], gcsafe, closure.}
IterDispose* = proc(): Future[?!void] {.gcsafe, closure.}
OnAvailabilityAdded* = proc(availability: Availability): Future[void] {.upraises: [], gcsafe.}
OnAvailabilityAdded* =
proc(availability: Availability): Future[void] {.upraises: [], gcsafe.}
StorableIter* = ref object
finished*: bool
next*: GetNext
dispose*: IterDispose
ReservationsError* = object of CodexError
ReserveFailedError* = object of ReservationsError
ReleaseFailedError* = object of ReservationsError
@ -109,10 +116,7 @@ template withLock(lock, body) =
if lock.locked:
lock.release()
proc new*(T: type Reservations,
repo: RepoStore): Reservations =
proc new*(T: type Reservations, repo: RepoStore): Reservations =
T(availabilityLock: newAsyncLock(), repo: repo)
proc init*(
@ -121,23 +125,35 @@ proc init*(
freeSize: UInt256,
duration: UInt256,
minPrice: UInt256,
maxCollateral: UInt256): Availability =
maxCollateral: UInt256,
): Availability =
var id: array[32, byte]
doAssert randomBytes(id) == 32
Availability(id: AvailabilityId(id), totalSize:totalSize, freeSize: freeSize, duration: duration, minPrice: minPrice, maxCollateral: maxCollateral)
Availability(
id: AvailabilityId(id),
totalSize: totalSize,
freeSize: freeSize,
duration: duration,
minPrice: minPrice,
maxCollateral: maxCollateral,
)
proc init*(
_: type Reservation,
availabilityId: AvailabilityId,
size: UInt256,
requestId: RequestId,
slotIndex: UInt256
slotIndex: UInt256,
): Reservation =
var id: array[32, byte]
doAssert randomBytes(id) == 32
Reservation(id: ReservationId(id), availabilityId: availabilityId, size: size, requestId: requestId, slotIndex: slotIndex)
Reservation(
id: ReservationId(id),
availabilityId: availabilityId,
size: size,
requestId: requestId,
slotIndex: slotIndex,
)
func toArray(id: SomeStorableId): array[32, byte] =
array[32, byte](id)
@ -146,23 +162,26 @@ proc `==`*(x, y: AvailabilityId): bool {.borrow.}
proc `==`*(x, y: ReservationId): bool {.borrow.}
proc `==`*(x, y: Reservation): bool =
x.id == y.id
proc `==`*(x, y: Availability): bool =
x.id == y.id
proc `$`*(id: SomeStorableId): string = id.toArray.toHex
proc `$`*(id: SomeStorableId): string =
id.toArray.toHex
proc toErr[E1: ref CatchableError, E2: ReservationsError](
e1: E1,
_: type E2,
msg: string = e1.msg): ref E2 =
e1: E1, _: type E2, msg: string = e1.msg
): ref E2 =
return newException(E2, msg, e1)
logutils.formatIt(LogFormat.textLines, SomeStorableId): it.short0xHexLog
logutils.formatIt(LogFormat.json, SomeStorableId): it.to0xHexLog
logutils.formatIt(LogFormat.textLines, SomeStorableId):
it.short0xHexLog
logutils.formatIt(LogFormat.json, SomeStorableId):
it.to0xHexLog
proc `onAvailabilityAdded=`*(self: Reservations,
onAvailabilityAdded: OnAvailabilityAdded) =
proc `onAvailabilityAdded=`*(
self: Reservations, onAvailabilityAdded: OnAvailabilityAdded
) =
self.onAvailabilityAdded = some onAvailabilityAdded
func key*(id: AvailabilityId): ?!Key =
@ -179,24 +198,20 @@ func key*(availability: Availability): ?!Key =
func key*(reservation: Reservation): ?!Key =
return key(reservation.id, reservation.availabilityId)
func available*(self: Reservations): uint = self.repo.available.uint
func available*(self: Reservations): uint =
self.repo.available.uint
func hasAvailable*(self: Reservations, bytes: uint): bool =
self.repo.available(bytes.NBytes)
proc exists*(
self: Reservations,
key: Key): Future[bool] {.async.} =
proc exists*(self: Reservations, key: Key): Future[bool] {.async.} =
let exists = await self.repo.metaDs.ds.contains(key)
return exists
proc getImpl(
self: Reservations,
key: Key): Future[?!seq[byte]] {.async.} =
proc getImpl(self: Reservations, key: Key): Future[?!seq[byte]] {.async.} =
if not await self.exists(key):
let err = newException(NotExistsError, "object with key " & $key & " does not exist")
let err =
newException(NotExistsError, "object with key " & $key & " does not exist")
return failure(err)
without serialized =? await self.repo.metaDs.ds.get(key), error:
@ -205,10 +220,8 @@ proc getImpl(
return success serialized
proc get*(
self: Reservations,
key: Key,
T: type SomeStorableObject): Future[?!T] {.async.} =
self: Reservations, key: Key, T: type SomeStorableObject
): Future[?!T] {.async.} =
without serialized =? await self.getImpl(key), error:
return failure(error)
@ -217,27 +230,20 @@ proc get*(
return success obj
proc updateImpl(
self: Reservations,
obj: SomeStorableObject): Future[?!void] {.async.} =
proc updateImpl(self: Reservations, obj: SomeStorableObject): Future[?!void] {.async.} =
trace "updating " & $(obj.type), id = obj.id
without key =? obj.key, error:
return failure(error)
if err =? (await self.repo.metaDs.ds.put(
key,
@(obj.toJson.toBytes)
)).errorOption:
if err =? (await self.repo.metaDs.ds.put(key, @(obj.toJson.toBytes))).errorOption:
return failure(err.toErr(UpdateFailedError))
return success()
proc updateAvailability(
self: Reservations,
obj: Availability): Future[?!void] {.async.} =
self: Reservations, obj: Availability
): Future[?!void] {.async.} =
logScope:
availabilityId = obj.id
@ -269,11 +275,18 @@ proc updateAvailability(
if oldAvailability.totalSize != obj.totalSize:
trace "totalSize changed, updating repo reservation"
if oldAvailability.totalSize < obj.totalSize: # storage added
if reserveErr =? (await self.repo.reserve((obj.totalSize - oldAvailability.totalSize).truncate(uint).NBytes)).errorOption:
if reserveErr =? (
await self.repo.reserve(
(obj.totalSize - oldAvailability.totalSize).truncate(uint).NBytes
)
).errorOption:
return failure(reserveErr.toErr(ReserveFailedError))
elif oldAvailability.totalSize > obj.totalSize: # storage removed
if reserveErr =? (await self.repo.release((oldAvailability.totalSize - obj.totalSize).truncate(uint).NBytes)).errorOption:
if reserveErr =? (
await self.repo.release(
(oldAvailability.totalSize - obj.totalSize).truncate(uint).NBytes
)
).errorOption:
return failure(reserveErr.toErr(ReleaseFailedError))
let res = await self.updateImpl(obj)
@ -296,21 +309,14 @@ proc updateAvailability(
return res
proc update*(
self: Reservations,
obj: Reservation): Future[?!void] {.async.} =
proc update*(self: Reservations, obj: Reservation): Future[?!void] {.async.} =
return await self.updateImpl(obj)
proc update*(
self: Reservations,
obj: Availability): Future[?!void] {.async.} =
proc update*(self: Reservations, obj: Availability): Future[?!void] {.async.} =
withLock(self.availabilityLock):
return await self.updateAvailability(obj)
proc delete(
self: Reservations,
key: Key): Future[?!void] {.async.} =
proc delete(self: Reservations, key: Key): Future[?!void] {.async.} =
trace "deleting object", key
if not await self.exists(key):
@ -322,10 +328,8 @@ proc delete(
return success()
proc deleteReservation*(
self: Reservations,
reservationId: ReservationId,
availabilityId: AvailabilityId): Future[?!void] {.async.} =
self: Reservations, reservationId: ReservationId, availabilityId: AvailabilityId
): Future[?!void] {.async.} =
logScope:
reservationId
availabilityId
@ -369,20 +373,17 @@ proc createAvailability*(
size: UInt256,
duration: UInt256,
minPrice: UInt256,
maxCollateral: UInt256): Future[?!Availability] {.async.} =
maxCollateral: UInt256,
): Future[?!Availability] {.async.} =
trace "creating availability", size, duration, minPrice, maxCollateral
let availability = Availability.init(
size, size, duration, minPrice, maxCollateral
)
let availability = Availability.init(size, size, duration, minPrice, maxCollateral)
let bytes = availability.freeSize.truncate(uint)
if reserveErr =? (await self.repo.reserve(bytes.NBytes)).errorOption:
return failure(reserveErr.toErr(ReserveFailedError))
if updateErr =? (await self.update(availability)).errorOption:
# rollback the reserve
trace "rolling back reserve"
if rollbackErr =? (await self.repo.release(bytes.NBytes)).errorOption:
@ -398,9 +399,8 @@ method createReservation*(
availabilityId: AvailabilityId,
slotSize: UInt256,
requestId: RequestId,
slotIndex: UInt256
slotIndex: UInt256,
): Future[?!Reservation] {.async, base.} =
withLock(self.availabilityLock):
without availabilityKey =? availabilityId.key, error:
return failure(error)
@ -412,7 +412,8 @@ method createReservation*(
if availability.freeSize < slotSize:
let error = newException(
BytesOutOfBoundsError,
"trying to reserve an amount of bytes that is greater than the total size of the Availability")
"trying to reserve an amount of bytes that is greater than the total size of the Availability",
)
return failure(error)
trace "Creating reservation", availabilityId, slotSize, requestId, slotIndex
@ -449,8 +450,8 @@ proc returnBytesToAvailability*(
self: Reservations,
availabilityId: AvailabilityId,
reservationId: ReservationId,
bytes: UInt256): Future[?!void] {.async.} =
bytes: UInt256,
): Future[?!void] {.async.} =
logScope:
reservationId
availabilityId
@ -467,14 +468,17 @@ proc returnBytesToAvailability*(
let bytesToBeReturned = bytes - reservation.size
if bytesToBeReturned == 0:
trace "No bytes are returned", requestSizeBytes = bytes, returningBytes = bytesToBeReturned
trace "No bytes are returned",
requestSizeBytes = bytes, returningBytes = bytesToBeReturned
return success()
trace "Returning bytes", requestSizeBytes = bytes, returningBytes = bytesToBeReturned
trace "Returning bytes",
requestSizeBytes = bytes, returningBytes = bytesToBeReturned
# First lets see if we can re-reserve the bytes, if the Repo's quota
# is depleted then we will fail-fast as there is nothing to be done atm.
if reserveErr =? (await self.repo.reserve(bytesToBeReturned.truncate(uint).NBytes)).errorOption:
if reserveErr =?
(await self.repo.reserve(bytesToBeReturned.truncate(uint).NBytes)).errorOption:
return failure(reserveErr.toErr(ReserveFailedError))
without availabilityKey =? availabilityId.key, error:
@ -487,9 +491,9 @@ proc returnBytesToAvailability*(
# Update availability with returned size
if updateErr =? (await self.updateAvailability(availability)).errorOption:
trace "Rolling back returning bytes"
if rollbackErr =? (await self.repo.release(bytesToBeReturned.truncate(uint).NBytes)).errorOption:
if rollbackErr =?
(await self.repo.release(bytesToBeReturned.truncate(uint).NBytes)).errorOption:
rollbackErr.parent = updateErr
return failure(rollbackErr)
@ -501,8 +505,8 @@ proc release*(
self: Reservations,
reservationId: ReservationId,
availabilityId: AvailabilityId,
bytes: uint): Future[?!void] {.async.} =
bytes: uint,
): Future[?!void] {.async.} =
logScope:
topics = "release"
bytes
@ -520,7 +524,8 @@ proc release*(
if reservation.size < bytes.u256:
let error = newException(
BytesOutOfBoundsError,
"trying to release an amount of bytes that is greater than the total size of the Reservation")
"trying to release an amount of bytes that is greater than the total size of the Reservation",
)
return failure(error)
if releaseErr =? (await self.repo.release(bytes.NBytes)).errorOption:
@ -530,7 +535,6 @@ proc release*(
# persist partially used Reservation with updated size
if err =? (await self.update(reservation)).errorOption:
# rollback release if an update error encountered
trace "rolling back release"
if rollbackErr =? (await self.repo.reserve(bytes.NBytes)).errorOption:
@ -545,11 +549,8 @@ iterator items(self: StorableIter): Future[?seq[byte]] =
yield self.next()
proc storables(
self: Reservations,
T: type SomeStorableObject,
queryKey: Key = ReservationsKey
self: Reservations, T: type SomeStorableObject, queryKey: Key = ReservationsKey
): Future[?!StorableIter] {.async.} =
var iter = StorableIter()
let query = Query.init(queryKey)
when T is Availability:
@ -570,12 +571,8 @@ proc storables(
proc next(): Future[?seq[byte]] {.async.} =
await idleAsync()
iter.finished = results.finished
if not results.finished and
res =? (await results.next()) and
res.data.len > 0 and
key =? res.key and
key.namespaces.len == defaultKey.namespaces.len:
if not results.finished and res =? (await results.next()) and res.data.len > 0 and
key =? res.key and key.namespaces.len == defaultKey.namespaces.len:
return some res.data
return none seq[byte]
@ -588,11 +585,8 @@ proc storables(
return success iter
proc allImpl(
self: Reservations,
T: type SomeStorableObject,
queryKey: Key = ReservationsKey
self: Reservations, T: type SomeStorableObject, queryKey: Key = ReservationsKey
): Future[?!seq[T]] {.async.} =
var ret: seq[T] = @[]
without storables =? (await self.storables(T, queryKey)), error:
@ -604,24 +598,18 @@ proc allImpl(
without obj =? T.fromJson(bytes), error:
error "json deserialization error",
json = string.fromBytes(bytes),
error = error.msg
json = string.fromBytes(bytes), error = error.msg
continue
ret.add obj
return success(ret)
proc all*(
self: Reservations,
T: type SomeStorableObject
): Future[?!seq[T]] {.async.} =
proc all*(self: Reservations, T: type SomeStorableObject): Future[?!seq[T]] {.async.} =
return await self.allImpl(T)
proc all*(
self: Reservations,
T: type SomeStorableObject,
availabilityId: AvailabilityId
self: Reservations, T: type SomeStorableObject, availabilityId: AvailabilityId
): Future[?!seq[T]] {.async.} =
without key =? (ReservationsKey / $availabilityId):
return failure("no key")
@ -629,29 +617,26 @@ proc all*(
return await self.allImpl(T, key)
proc findAvailability*(
self: Reservations,
size, duration, minPrice, collateral: UInt256
self: Reservations, size, duration, minPrice, collateral: UInt256
): Future[?Availability] {.async.} =
without storables =? (await self.storables(Availability)), e:
error "failed to get all storables", error = e.msg
return none Availability
for item in storables.items:
if bytes =? (await item) and
availability =? Availability.fromJson(bytes):
if size <= availability.freeSize and
duration <= availability.duration and
collateral <= availability.maxCollateral and
minPrice >= availability.minPrice:
if bytes =? (await item) and availability =? Availability.fromJson(bytes):
if size <= availability.freeSize and duration <= availability.duration and
collateral <= availability.maxCollateral and minPrice >= availability.minPrice:
trace "availability matched",
id = availability.id,
size, availFreeSize = availability.freeSize,
duration, availDuration = availability.duration,
minPrice, availMinPrice = availability.minPrice,
collateral, availMaxCollateral = availability.maxCollateral
size,
availFreeSize = availability.freeSize,
duration,
availDuration = availability.duration,
minPrice,
availMinPrice = availability.minPrice,
collateral,
availMaxCollateral = availability.maxCollateral
# TODO: As soon as we're on ARC-ORC, we can use destructors
# to automatically dispose our iterators when they fall out of scope.
@ -663,7 +648,11 @@ proc findAvailability*(
trace "availability did not match",
id = availability.id,
size, availFreeSize = availability.freeSize,
duration, availDuration = availability.duration,
minPrice, availMinPrice = availability.minPrice,
collateral, availMaxCollateral = availability.maxCollateral
size,
availFreeSize = availability.freeSize,
duration,
availDuration = availability.duration,
minPrice,
availMinPrice = availability.minPrice,
collateral,
availMaxCollateral = availability.maxCollateral

46
codex/sales/salesagent.nim
View File

@ -25,27 +25,26 @@ type
onCleanUp*: OnCleanUp
onFilled*: ?OnFilled
OnCleanUp* = proc (returnBytes = false, reprocessSlot = false): Future[void] {.gcsafe, upraises: [].}
OnFilled* = proc(request: StorageRequest,
slotIndex: UInt256) {.gcsafe, upraises: [].}
OnCleanUp* = proc(returnBytes = false, reprocessSlot = false): Future[void] {.
gcsafe, upraises: []
.}
OnFilled* = proc(request: StorageRequest, slotIndex: UInt256) {.gcsafe, upraises: [].}
SalesAgentError = object of CodexError
AllSlotsFilledError* = object of SalesAgentError
func `==`*(a, b: SalesAgent): bool =
a.data.requestId == b.data.requestId and
a.data.slotIndex == b.data.slotIndex
a.data.requestId == b.data.requestId and a.data.slotIndex == b.data.slotIndex
proc newSalesAgent*(context: SalesContext,
proc newSalesAgent*(
context: SalesContext,
requestId: RequestId,
slotIndex: UInt256,
request: ?StorageRequest): SalesAgent =
request: ?StorageRequest,
): SalesAgent =
var agent = SalesAgent.new()
agent.context = context
agent.data = SalesData(
requestId: requestId,
slotIndex: slotIndex,
request: request)
agent.data = SalesData(requestId: requestId, slotIndex: slotIndex, request: request)
return agent
proc retrieveRequest*(agent: SalesAgent) {.async.} =
@ -62,6 +61,7 @@ proc retrieveRequestState*(agent: SalesAgent): Future[?RequestState] {.async.} =
func state*(agent: SalesAgent): ?string =
proc description(state: State): string =
$state
agent.query(description)
proc subscribeCancellation(agent: SalesAgent) {.async.} =
@ -93,27 +93,29 @@ proc subscribeCancellation(agent: SalesAgent) {.async.} =
of RequestState.Started, RequestState.Finished, RequestState.Failed:
break
debug "The request is not yet canceled, even though it should be. Waiting for some more time.", currentState = state, now=clock.now
debug "The request is not yet canceled, even though it should be. Waiting for some more time.",
currentState = state, now = clock.now
data.cancelled = onCancelled()
method onFulfilled*(agent: SalesAgent, requestId: RequestId) {.base, gcsafe, upraises: [].} =
if agent.data.requestId == requestId and
not agent.data.cancelled.isNil:
method onFulfilled*(
agent: SalesAgent, requestId: RequestId
) {.base, gcsafe, upraises: [].} =
if agent.data.requestId == requestId and not agent.data.cancelled.isNil:
agent.data.cancelled.cancelSoon()
method onFailed*(agent: SalesAgent, requestId: RequestId) {.base, gcsafe, upraises: [].} =
method onFailed*(
agent: SalesAgent, requestId: RequestId
) {.base, gcsafe, upraises: [].} =
without request =? agent.data.request:
return
if agent.data.requestId == requestId:
agent.schedule(failedEvent(request))
method onSlotFilled*(agent: SalesAgent,
requestId: RequestId,
slotIndex: UInt256) {.base, gcsafe, upraises: [].} =
if agent.data.requestId == requestId and
agent.data.slotIndex == slotIndex:
method onSlotFilled*(
agent: SalesAgent, requestId: RequestId, slotIndex: UInt256
) {.base, gcsafe, upraises: [].} =
if agent.data.requestId == requestId and agent.data.slotIndex == slotIndex:
agent.schedule(slotFilledEvent(requestId, slotIndex))
proc subscribe*(agent: SalesAgent) {.async.} =

20
codex/sales/salescontext.nim
View File

@ -24,12 +24,14 @@ type
simulateProofFailures*: int
BlocksCb* = proc(blocks: seq[bt.Block]): Future[?!void] {.gcsafe, raises: [].}
OnStore* = proc(request: StorageRequest,
slot: UInt256,
blocksCb: BlocksCb): Future[?!void] {.gcsafe, upraises: [].}
OnProve* = proc(slot: Slot, challenge: ProofChallenge): Future[?!Groth16Proof] {.gcsafe, upraises: [].}
OnExpiryUpdate* = proc(rootCid: string, expiry: SecondsSince1970): Future[?!void] {.gcsafe, upraises: [].}
OnClear* = proc(request: StorageRequest,
slotIndex: UInt256) {.gcsafe, upraises: [].}
OnSale* = proc(request: StorageRequest,
slotIndex: UInt256) {.gcsafe, upraises: [].}
OnStore* = proc(
request: StorageRequest, slot: UInt256, blocksCb: BlocksCb
): Future[?!void] {.gcsafe, upraises: [].}
OnProve* = proc(slot: Slot, challenge: ProofChallenge): Future[?!Groth16Proof] {.
gcsafe, upraises: []
.}
OnExpiryUpdate* = proc(rootCid: string, expiry: SecondsSince1970): Future[?!void] {.
gcsafe, upraises: []
.}
OnClear* = proc(request: StorageRequest, slotIndex: UInt256) {.gcsafe, upraises: [].}
OnSale* = proc(request: StorageRequest, slotIndex: UInt256) {.gcsafe, upraises: [].}

3
codex/sales/salesdata.nim
View File

@ -3,8 +3,7 @@ import ../contracts/requests
import ../market
import ./reservations
type
SalesData* = ref object
type SalesData* = ref object
requestId*: RequestId
ask*: StorageAsk
request*: ?StorageRequest

126
codex/sales/slotqueue.nim
View File

@ -69,10 +69,12 @@ const DefaultMaxWorkers = 3
const DefaultMaxSize = 128'u16
proc profitability(item: SlotQueueItem): UInt256 =
StorageAsk(collateral: item.collateral,
StorageAsk(
collateral: item.collateral,
duration: item.duration,
reward: item.reward,
slotSize: item.slotSize).pricePerSlot
slotSize: item.slotSize,
).pricePerSlot
proc `<`*(a, b: SlotQueueItem): bool =
# for A to have a higher priority than B (in a min queue), A must be less than
@ -102,13 +104,13 @@ proc `<`*(a, b: SlotQueueItem): bool =
return scoreA > scoreB
proc `==`*(a, b: SlotQueueItem): bool =
a.requestId == b.requestId and
a.slotIndex == b.slotIndex
a.requestId == b.requestId and a.slotIndex == b.slotIndex
proc new*(_: type SlotQueue,
proc new*(
_: type SlotQueue,
maxWorkers = DefaultMaxWorkers,
maxSize: SlotQueueSize = DefaultMaxSize): SlotQueue =
maxSize: SlotQueueSize = DefaultMaxSize,
): SlotQueue =
if maxWorkers <= 0:
raise newException(ValueError, "maxWorkers must be positive")
if maxWorkers.uint16 > maxSize:
@ -121,23 +123,22 @@ proc new*(_: type SlotQueue,
queue: newAsyncHeapQueue[SlotQueueItem](maxSize.int + 1),
running: false,
trackedFutures: TrackedFutures.new(),
unpaused: newAsyncEvent()
unpaused: newAsyncEvent(),
)
# avoid instantiating `workers` in constructor to avoid side effects in
# `newAsyncQueue` procedure
proc init(_: type SlotQueueWorker): SlotQueueWorker =
SlotQueueWorker(
doneProcessing: newFuture[void]("slotqueue.worker.processing")
)
SlotQueueWorker(doneProcessing: newFuture[void]("slotqueue.worker.processing"))
proc init*(_: type SlotQueueItem,
proc init*(
_: type SlotQueueItem,
requestId: RequestId,
slotIndex: uint16,
ask: StorageAsk,
expiry: UInt256,
seen = false): SlotQueueItem =
seen = false,
): SlotQueueItem =
SlotQueueItem(
requestId: requestId,
slotIndex: slotIndex,
@ -146,28 +147,22 @@ proc init*(_: type SlotQueueItem,
reward: ask.reward,
collateral: ask.collateral,
expiry: expiry,
seen: seen
seen: seen,
)
proc init*(_: type SlotQueueItem,
request: StorageRequest,
slotIndex: uint16): SlotQueueItem =
SlotQueueItem.init(request.id,
slotIndex,
request.ask,
request.expiry)
proc init*(_: type SlotQueueItem,
requestId: RequestId,
ask: StorageAsk,
expiry: UInt256): seq[SlotQueueItem] =
proc init*(
_: type SlotQueueItem, request: StorageRequest, slotIndex: uint16
): SlotQueueItem =
SlotQueueItem.init(request.id, slotIndex, request.ask, request.expiry)
proc init*(
_: type SlotQueueItem, requestId: RequestId, ask: StorageAsk, expiry: UInt256
): seq[SlotQueueItem] =
if not ask.slots.inRange:
raise newException(SlotsOutOfRangeError, "Too many slots")
var i = 0'u16
proc initSlotQueueItem: SlotQueueItem =
proc initSlotQueueItem(): SlotQueueItem =
let item = SlotQueueItem.init(requestId, i, ask, expiry)
inc i
return item
@ -176,37 +171,54 @@ proc init*(_: type SlotQueueItem,
Rng.instance.shuffle(items)
return items
proc init*(_: type SlotQueueItem,
request: StorageRequest): seq[SlotQueueItem] =
proc init*(_: type SlotQueueItem, request: StorageRequest): seq[SlotQueueItem] =
return SlotQueueItem.init(request.id, request.ask, request.expiry)
proc inRange*(val: SomeUnsignedInt): bool =
val.uint16 in SlotQueueSize.low .. SlotQueueSize.high
proc requestId*(self: SlotQueueItem): RequestId = self.requestId
proc slotIndex*(self: SlotQueueItem): uint16 = self.slotIndex
proc slotSize*(self: SlotQueueItem): UInt256 = self.slotSize
proc duration*(self: SlotQueueItem): UInt256 = self.duration
proc reward*(self: SlotQueueItem): UInt256 = self.reward
proc collateral*(self: SlotQueueItem): UInt256 = self.collateral
proc seen*(self: SlotQueueItem): bool = self.seen
proc requestId*(self: SlotQueueItem): RequestId =
self.requestId
proc running*(self: SlotQueue): bool = self.running
proc slotIndex*(self: SlotQueueItem): uint16 =
self.slotIndex
proc len*(self: SlotQueue): int = self.queue.len
proc slotSize*(self: SlotQueueItem): UInt256 =
self.slotSize
proc size*(self: SlotQueue): int = self.queue.size - 1
proc duration*(self: SlotQueueItem): UInt256 =
self.duration
proc paused*(self: SlotQueue): bool = not self.unpaused.isSet
proc reward*(self: SlotQueueItem): UInt256 =
self.reward
proc `$`*(self: SlotQueue): string = $self.queue
proc collateral*(self: SlotQueueItem): UInt256 =
self.collateral
proc seen*(self: SlotQueueItem): bool =
self.seen
proc running*(self: SlotQueue): bool =
self.running
proc len*(self: SlotQueue): int =
self.queue.len
proc size*(self: SlotQueue): int =
self.queue.size - 1
proc paused*(self: SlotQueue): bool =
not self.unpaused.isSet
proc `$`*(self: SlotQueue): string =
$self.queue
proc `onProcessSlot=`*(self: SlotQueue, onProcessSlot: OnProcessSlot) =
self.onProcessSlot = some onProcessSlot
proc activeWorkers*(self: SlotQueue): int =
if not self.running: return 0
if not self.running:
return 0
# active = capacity - available
self.maxWorkers - self.workers.len
@ -222,10 +234,9 @@ proc unpause*(self: SlotQueue) =
# set unpaused flag to true -- unblocks coroutines waiting on unpaused.wait()
self.unpaused.fire()
proc populateItem*(self: SlotQueue,
requestId: RequestId,
slotIndex: uint16): ?SlotQueueItem =
proc populateItem*(
self: SlotQueue, requestId: RequestId, slotIndex: uint16
): ?SlotQueueItem =
trace "populate item, items in queue", len = self.queue.len
for item in self.queue.items:
trace "populate item search", itemRequestId = item.requestId, requestId
@ -237,12 +248,11 @@ proc populateItem*(self: SlotQueue,
duration: item.duration,
reward: item.reward,
collateral: item.collateral,
expiry: item.expiry
expiry: item.expiry,
)
return none SlotQueueItem
proc push*(self: SlotQueue, item: SlotQueueItem): ?!void =
logScope:
requestId = item.requestId
slotIndex = item.slotIndex
@ -330,9 +340,9 @@ proc addWorker(self: SlotQueue): ?!void =
return success()
proc dispatch(self: SlotQueue,
worker: SlotQueueWorker,
item: SlotQueueItem) {.async: (raises: []).} =
proc dispatch(
self: SlotQueue, worker: SlotQueueWorker, item: SlotQueueItem
) {.async: (raises: []).} =
logScope:
requestId = item.requestId
slotIndex = item.slotIndex
@ -349,10 +359,8 @@ proc dispatch(self: SlotQueue,
if err =? self.addWorker().errorOption:
raise err # catch below
except QueueNotRunningError as e:
info "could not re-add worker to worker queue, queue not running",
error = e.msg
info "could not re-add worker to worker queue, queue not running", error = e.msg
except CancelledError:
# do not bubble exception up as it is called with `asyncSpawn` which would
# convert the exception into a `FutureDefect`
@ -380,7 +388,6 @@ proc clearSeenFlags*(self: SlotQueue) =
trace "all 'seen' flags cleared"
proc run(self: SlotQueue) {.async: (raises: []).} =
while self.running:
try:
if self.paused:
@ -389,7 +396,8 @@ proc run(self: SlotQueue) {.async: (raises: []).} =
# block until unpaused is true/fired, ie wait for queue to be unpaused
await self.unpaused.wait()
let worker = await self.workers.popFirst() # if workers saturated, wait here for new workers
let worker =
await self.workers.popFirst() # if workers saturated, wait here for new workers
let item = await self.queue.pop() # if queue empty, wait here for new items
logScope:

13
codex/sales/statemachine.nim
View File

@ -14,14 +14,19 @@ type
SaleState* = ref object of State
SaleError* = ref object of CodexError
method onCancelled*(state: SaleState, request: StorageRequest): ?State {.base, upraises:[].} =
method onCancelled*(
state: SaleState, request: StorageRequest
): ?State {.base, upraises: [].} =
discard
method onFailed*(state: SaleState, request: StorageRequest): ?State {.base, upraises:[].} =
method onFailed*(
state: SaleState, request: StorageRequest
): ?State {.base, upraises: [].} =
discard
method onSlotFilled*(state: SaleState, requestId: RequestId,
slotIndex: UInt256): ?State {.base, upraises:[].} =
method onSlotFilled*(
state: SaleState, requestId: RequestId, slotIndex: UInt256
): ?State {.base, upraises: [].} =
discard
proc cancelledEvent*(request: StorageRequest): Event =

15
codex/sales/states/cancelled.nim
View File

@ -6,10 +6,10 @@ import ./errorhandling
logScope:
topics = "marketplace sales cancelled"
type
SaleCancelled* = ref object of ErrorHandlingState
type SaleCancelled* = ref object of ErrorHandlingState
method `$`*(state: SaleCancelled): string = "SaleCancelled"
method `$`*(state: SaleCancelled): string =
"SaleCancelled"
method run*(state: SaleCancelled, machine: Machine): Future[?State] {.async.} =
let agent = SalesAgent(machine)
@ -20,14 +20,15 @@ method run*(state: SaleCancelled, machine: Machine): Future[?State] {.async.} =
raiseAssert "no sale request"
let slot = Slot(request: request, slotIndex: data.slotIndex)
debug "Collecting collateral and partial payout", requestId = data.requestId, slotIndex = data.slotIndex
debug "Collecting collateral and partial payout",
requestId = data.requestId, slotIndex = data.slotIndex
await market.freeSlot(slot.id)
if onClear =? agent.context.onClear and
request =? data.request:
if onClear =? agent.context.onClear and request =? data.request:
onClear(request, data.slotIndex)
if onCleanUp =? agent.onCleanUp:
await onCleanUp(returnBytes = true, reprocessSlot = false)
warn "Sale cancelled due to timeout", requestId = data.requestId, slotIndex = data.slotIndex
warn "Sale cancelled due to timeout",
requestId = data.requestId, slotIndex = data.slotIndex

19
codex/sales/states/downloading.nim
View File

@ -13,13 +13,13 @@ import ./filled
import ./initialproving
import ./errored
type
SaleDownloading* = ref object of ErrorHandlingState
type SaleDownloading* = ref object of ErrorHandlingState
logScope:
topics = "marketplace sales downloading"
method `$`*(state: SaleDownloading): string = "SaleDownloading"
method `$`*(state: SaleDownloading): string =
"SaleDownloading"
method onCancelled*(state: SaleDownloading, request: StorageRequest): ?State =
return some State(SaleCancelled())
@ -27,8 +27,9 @@ method onCancelled*(state: SaleDownloading, request: StorageRequest): ?State =
method onFailed*(state: SaleDownloading, request: StorageRequest): ?State =
return some State(SaleFailed())
method onSlotFilled*(state: SaleDownloading, requestId: RequestId,
slotIndex: UInt256): ?State =
method onSlotFilled*(
state: SaleDownloading, requestId: RequestId, slotIndex: UInt256
): ?State =
return some State(SaleFilled())
method run*(state: SaleDownloading, machine: Machine): Future[?State] {.async.} =
@ -61,14 +62,10 @@ method run*(state: SaleDownloading, machine: Machine): Future[?State] {.async.}
bytes += blk.data.len.uint
trace "Releasing batch of bytes written to disk", bytes
return await reservations.release(reservation.id,
reservation.availabilityId,
bytes)
return await reservations.release(reservation.id, reservation.availabilityId, bytes)
trace "Starting download"
if err =? (await onStore(request,
data.slotIndex,
onBlocks)).errorOption:
if err =? (await onStore(request, data.slotIndex, onBlocks)).errorOption:
return some State(SaleErrored(error: err, reprocessSlot: false))
trace "Download complete"

12
codex/sales/states/errored.nim
View File

@ -14,7 +14,8 @@ type SaleErrored* = ref object of SaleState
error*: ref CatchableError
reprocessSlot*: bool
method `$`*(state: SaleErrored): string = "SaleErrored"
method `$`*(state: SaleErrored): string =
"SaleErrored"
method onError*(state: SaleState, err: ref CatchableError): ?State {.upraises: [].} =
error "error during SaleErrored run", error = err.msg
@ -24,12 +25,13 @@ method run*(state: SaleErrored, machine: Machine): Future[?State] {.async.} =
let data = agent.data
let context = agent.context
error "Sale error", error=state.error.msgDetail, requestId = data.requestId, slotIndex = data.slotIndex
error "Sale error",
error = state.error.msgDetail,
requestId = data.requestId,
slotIndex = data.slotIndex
if onClear =? context.onClear and
request =? data.request:
if onClear =? context.onClear and request =? data.request:
onClear(request, data.slotIndex)
if onCleanUp =? agent.onCleanUp:
await onCleanUp(returnBytes = true, reprocessSlot = state.reprocessSlot)

3
codex/sales/states/errorhandling.nim
View File

@ -2,8 +2,7 @@ import pkg/questionable
import ../statemachine
import ./errored
type
ErrorHandlingState* = ref object of SaleState
type ErrorHandlingState* = ref object of SaleState
method onError*(state: ErrorHandlingState, error: ref CatchableError): ?State =
some State(SaleErrored(error: error))

6
codex/sales/states/failed.nim
View File

@ -11,7 +11,8 @@ type
SaleFailed* = ref object of ErrorHandlingState
SaleFailedError* = object of SaleError
method `$`*(state: SaleFailed): string = "SaleFailed"
method `$`*(state: SaleFailed): string =
"SaleFailed"
method run*(state: SaleFailed, machine: Machine): Future[?State] {.async.} =
let data = SalesAgent(machine).data
@ -21,7 +22,8 @@ method run*(state: SaleFailed, machine: Machine): Future[?State] {.async.} =
raiseAssert "no sale request"
let slot = Slot(request: request, slotIndex: data.slotIndex)
debug "Removing slot from mySlots", requestId = data.requestId, slotIndex = data.slotIndex
debug "Removing slot from mySlots",
requestId = data.requestId, slotIndex = data.slotIndex
await market.freeSlot(slot.id)
let error = newException(SaleFailedError, "Sale failed")

11
codex/sales/states/filled.nim
View File

@ -27,7 +27,8 @@ method onCancelled*(state: SaleFilled, request: StorageRequest): ?State =
method onFailed*(state: SaleFilled, request: StorageRequest): ?State =
return some State(SaleFailed())
method `$`*(state: SaleFilled): string = "SaleFilled"
method `$`*(state: SaleFilled): string =
"SaleFilled"
method run*(state: SaleFilled, machine: Machine): Future[?State] {.async.} =
let agent = SalesAgent(machine)
@ -39,7 +40,8 @@ method run*(state: SaleFilled, machine: Machine): Future[?State] {.async.} =
let me = await market.getSigner()
if host == me.some:
info "Slot succesfully filled", requestId = data.requestId, slotIndex = data.slotIndex
info "Slot succesfully filled",
requestId = data.requestId, slotIndex = data.slotIndex
without request =? data.request:
raiseAssert "no sale request"
@ -57,10 +59,11 @@ method run*(state: SaleFilled, machine: Machine): Future[?State] {.async.} =
when codex_enable_proof_failures:
if context.simulateProofFailures > 0:
info "Proving with failure rate", rate = context.simulateProofFailures
return some State(SaleProvingSimulated(failEveryNProofs: context.simulateProofFailures))
return some State(
SaleProvingSimulated(failEveryNProofs: context.simulateProofFailures)
)
return some State(SaleProving())
else:
let error = newException(HostMismatchError, "Slot filled by other host")
return some State(SaleErrored(error: error))

9
codex/sales/states/filling.nim
View File

@ -13,11 +13,11 @@ import ./errored
logScope:
topics = "marketplace sales filling"
type
SaleFilling* = ref object of ErrorHandlingState
type SaleFilling* = ref object of ErrorHandlingState
proof*: Groth16Proof
method `$`*(state: SaleFilling): string = "SaleFilling"
method `$`*(state: SaleFilling): string =
"SaleFilling"
method onCancelled*(state: SaleFilling, request: StorageRequest): ?State =
return some State(SaleCancelled())
@ -41,7 +41,8 @@ method run(state: SaleFilling, machine: Machine): Future[?State] {.async.} =
if slotState == SlotState.Repair:
# When repairing the node gets "discount" on the collateral that it needs to
let repairRewardPercentage = (await market.repairRewardPercentage).u256
collateral = fullCollateral - ((fullCollateral * repairRewardPercentage)).div(100.u256)
collateral =
fullCollateral - ((fullCollateral * repairRewardPercentage)).div(100.u256)
else:
collateral = fullCollateral

9
codex/sales/states/finished.nim
View File

@ -10,10 +10,10 @@ import ./failed
logScope:
topics = "marketplace sales finished"
type
SaleFinished* = ref object of ErrorHandlingState
type SaleFinished* = ref object of ErrorHandlingState
method `$`*(state: SaleFinished): string = "SaleFinished"
method `$`*(state: SaleFinished): string =
"SaleFinished"
method onCancelled*(state: SaleFinished, request: StorageRequest): ?State =
return some State(SaleCancelled())
@ -28,7 +28,8 @@ method run*(state: SaleFinished, machine: Machine): Future[?State] {.async.} =
without request =? data.request:
raiseAssert "no sale request"
info "Slot finished and paid out", requestId = data.requestId, slotIndex = data.slotIndex
info "Slot finished and paid out",
requestId = data.requestId, slotIndex = data.slotIndex
if onCleanUp =? agent.onCleanUp:
await onCleanUp()

11
codex/sales/states/ignored.nim
View File

@ -11,16 +11,17 @@ logScope:
# Ignored slots could mean there was no availability or that the slot could
# not be reserved.
type
SaleIgnored* = ref object of ErrorHandlingState
type SaleIgnored* = ref object of ErrorHandlingState
reprocessSlot*: bool # readd slot to queue with `seen` flag
returnBytes*: bool # return unreleased bytes from Reservation to Availability
method `$`*(state: SaleIgnored): string = "SaleIgnored"
method `$`*(state: SaleIgnored): string =
"SaleIgnored"
method run*(state: SaleIgnored, machine: Machine): Future[?State] {.async.} =
let agent = SalesAgent(machine)
if onCleanUp =? agent.onCleanUp:
await onCleanUp(reprocessSlot = state.reprocessSlot,
returnBytes = state.returnBytes)
await onCleanUp(
reprocessSlot = state.reprocessSlot, returnBytes = state.returnBytes
)

6
codex/sales/states/initialproving.nim
View File

@ -12,10 +12,10 @@ import ./failed
logScope:
topics = "marketplace sales initial-proving"
type
SaleInitialProving* = ref object of ErrorHandlingState
type SaleInitialProving* = ref object of ErrorHandlingState
method `$`*(state: SaleInitialProving): string = "SaleInitialProving"
method `$`*(state: SaleInitialProving): string =
"SaleInitialProving"
method onCancelled*(state: SaleInitialProving, request: StorageRequest): ?State =
return some State(SaleCancelled())

9
codex/sales/states/payout.nim
View File

@ -10,10 +10,10 @@ import ./finished
logScope:
topics = "marketplace sales payout"
type
SalePayout* = ref object of ErrorHandlingState
type SalePayout* = ref object of ErrorHandlingState
method `$`*(state: SalePayout): string = "SalePayout"
method `$`*(state: SalePayout): string =
"SalePayout"
method onCancelled*(state: SalePayout, request: StorageRequest): ?State =
return some State(SaleCancelled())
@ -29,7 +29,8 @@ method run(state: SalePayout, machine: Machine): Future[?State] {.async.} =
raiseAssert "no sale request"
let slot = Slot(request: request, slotIndex: data.slotIndex)
debug "Collecting finished slot's reward", requestId = data.requestId, slotIndex = data.slotIndex
debug "Collecting finished slot's reward",
requestId = data.requestId, slotIndex = data.slotIndex
await market.freeSlot(slot.id)
return some State(SaleFinished())

33
codex/sales/states/preparing.nim
View File

@ -14,15 +14,17 @@ import ./ignored
import ./slotreserving
import ./errored
declareCounter(codex_reservations_availability_mismatch, "codex reservations availability_mismatch")
declareCounter(
codex_reservations_availability_mismatch, "codex reservations availability_mismatch"
)
type
SalePreparing* = ref object of ErrorHandlingState
type SalePreparing* = ref object of ErrorHandlingState
logScope:
topics = "marketplace sales preparing"
method `$`*(state: SalePreparing): string = "SalePreparing"
method `$`*(state: SalePreparing): string =
"SalePreparing"
method onCancelled*(state: SalePreparing, request: StorageRequest): ?State =
return some State(SaleCancelled())
@ -30,8 +32,9 @@ method onCancelled*(state: SalePreparing, request: StorageRequest): ?State =
method onFailed*(state: SalePreparing, request: StorageRequest): ?State =
return some State(SaleFailed())
method onSlotFilled*(state: SalePreparing, requestId: RequestId,
slotIndex: UInt256): ?State =
method onSlotFilled*(
state: SalePreparing, requestId: RequestId, slotIndex: UInt256
): ?State =
return some State(SaleFilled())
method run*(state: SalePreparing, machine: Machine): Future[?State] {.async.} =
@ -64,22 +67,20 @@ method run*(state: SalePreparing, machine: Machine): Future[?State] {.async.} =
# availability was checked for this slot when it entered the queue, however
# check to the ensure that there is still availability as they may have
# changed since being added (other slots may have been processed in that time)
without availability =? await reservations.findAvailability(
request.ask.slotSize,
request.ask.duration,
request.ask.pricePerSlot,
request.ask.collateral):
without availability =?
await reservations.findAvailability(
request.ask.slotSize, request.ask.duration, request.ask.pricePerSlot,
request.ask.collateral,
):
debug "No availability found for request, ignoring"
return some State(SaleIgnored(reprocessSlot: true))
info "Availability found for request, creating reservation"
without reservation =? await reservations.createReservation(
availability.id,
request.ask.slotSize,
request.id,
data.slotIndex
without reservation =?
await reservations.createReservation(
availability.id, request.ask.slotSize, request.id, data.slotIndex
), error:
trace "Creation of reservation failed"
# Race condition:

11
codex/sales/states/proving.nim
View File

@ -27,7 +27,7 @@ method prove*(
challenge: ProofChallenge,
onProve: OnProve,
market: Market,
currentPeriod: Period
currentPeriod: Period,
) {.base, async.} =
try:
without proof =? (await onProve(slot, challenge)), err:
@ -48,9 +48,8 @@ proc proveLoop(
clock: Clock,
request: StorageRequest,
slotIndex: UInt256,
onProve: OnProve
onProve: OnProve,
) {.async.} =
let slot = Slot(request: request, slotIndex: slotIndex)
let slotId = slot.id
@ -76,7 +75,8 @@ proc proveLoop(
case slotState
of SlotState.Filled:
debug "Proving for new period", period = currentPeriod
if (await market.isProofRequired(slotId)) or (await market.willProofBeRequired(slotId)):
if (await market.isProofRequired(slotId)) or
(await market.willProofBeRequired(slotId)):
let challenge = await market.getChallenge(slotId)
debug "Proof is required", period = currentPeriod, challenge = challenge
await state.prove(slot, challenge, onProve, market, currentPeriod)
@ -100,7 +100,8 @@ proc proveLoop(
debug "waiting until next period"
await waitUntilPeriod(currentPeriod + 1)
method `$`*(state: SaleProving): string = "SaleProving"
method `$`*(state: SaleProving): string =
"SaleProving"
method onCancelled*(state: SaleProving, request: StorageRequest): ?State =
# state.loop cancellation happens automatically when run is cancelled due to

19
codex/sales/states/provingsimulated.nim
View File

@ -14,19 +14,24 @@ when codex_enable_proof_failures:
logScope:
topics = "marketplace sales simulated-proving"
type
SaleProvingSimulated* = ref object of SaleProving
type SaleProvingSimulated* = ref object of SaleProving
failEveryNProofs*: int
proofCount: int
proc onSubmitProofError(error: ref CatchableError, period: UInt256, slotId: SlotId) =
error "Submitting invalid proof failed", period, slotId, msg = error.msgDetail
method prove*(state: SaleProvingSimulated, slot: Slot, challenge: ProofChallenge, onProve: OnProve, market: Market, currentPeriod: Period) {.async.} =
method prove*(
state: SaleProvingSimulated,
slot: Slot,
challenge: ProofChallenge,
onProve: OnProve,
market: Market,
currentPeriod: Period,
) {.async.} =
trace "Processing proving in simulated mode"
state.proofCount += 1
if state.failEveryNProofs > 0 and
state.proofCount mod state.failEveryNProofs == 0:
if state.failEveryNProofs > 0 and state.proofCount mod state.failEveryNProofs == 0:
state.proofCount = 0
try:
@ -40,4 +45,6 @@ when codex_enable_proof_failures:
except CatchableError as e:
onSubmitProofError(e, currentPeriod, slot.id)
else:
await procCall SaleProving(state).prove(slot, challenge, onProve, market, currentPeriod)
await procCall SaleProving(state).prove(
slot, challenge, onProve, market, currentPeriod
)

8
codex/sales/states/slotreserving.nim
View File

@ -12,13 +12,13 @@ import ./ignored
import ./downloading
import ./errored
type
SaleSlotReserving* = ref object of ErrorHandlingState
type SaleSlotReserving* = ref object of ErrorHandlingState
logScope:
topics = "marketplace sales reserving"
method `$`*(state: SaleSlotReserving): string = "SaleSlotReserving"
method `$`*(state: SaleSlotReserving): string =
"SaleSlotReserving"
method onCancelled*(state: SaleSlotReserving, request: StorageRequest): ?State =
return some State(SaleCancelled())
@ -51,10 +51,8 @@ method run*(state: SaleSlotReserving, machine: Machine): Future[?State] {.async.
trace "Slot successfully reserved"
return some State(SaleDownloading())
else:
# do not re-add this slot to the queue, and return bytes from Reservation to
# the Availability
debug "Slot cannot be reserved, ignoring"
return some State(SaleIgnored(reprocessSlot: false, returnBytes: true))

12
codex/sales/states/unknown.nim
View File

@ -17,7 +17,8 @@ type
SaleUnknownError* = object of CatchableError
UnexpectedSlotError* = object of SaleUnknownError
method `$`*(state: SaleUnknown): string = "SaleUnknown"
method `$`*(state: SaleUnknown): string =
"SaleUnknown"
method onCancelled*(state: SaleUnknown, request: StorageRequest): ?State =
return some State(SaleCancelled())
@ -38,8 +39,8 @@ method run*(state: SaleUnknown, machine: Machine): Future[?State] {.async.} =
case slotState
of SlotState.Free:
let error = newException(UnexpectedSlotError,
"Slot state on chain should not be 'free'")
let error =
newException(UnexpectedSlotError, "Slot state on chain should not be 'free'")
return some State(SaleErrored(error: error))
of SlotState.Filled:
return some State(SaleFilled())
@ -52,6 +53,7 @@ method run*(state: SaleUnknown, machine: Machine): Future[?State] {.async.} =
of SlotState.Cancelled:
return some State(SaleCancelled())
of SlotState.Repair:
let error = newException(SlotFreedError,
"Slot was forcible freed and host was removed from its hosting")
let error = newException(
SlotFreedError, "Slot was forcible freed and host was removed from its hosting"
)
return some State(SaleErrored(error: error))

3
codex/slots/builder.nim
View File

@ -5,5 +5,4 @@ import ../merkletree
export builder, converters
type
Poseidon2Builder* = SlotsBuilder[Poseidon2Tree, Poseidon2Hash]
type Poseidon2Builder* = SlotsBuilder[Poseidon2Tree, Poseidon2Hash]

73
codex/slots/builder/builder.nim
View File

@ -34,13 +34,13 @@ export converters, asynciter
logScope:
topics = "codex slotsbuilder"
type
SlotsBuilder*[T, H] = ref object of RootObj
type SlotsBuilder*[T, H] = ref object of RootObj
store: BlockStore
manifest: Manifest # current manifest
strategy: IndexingStrategy # indexing strategy
cellSize: NBytes # cell size
numSlotBlocks: Natural # number of blocks per slot (should yield a power of two number of cells)
numSlotBlocks: Natural
# number of blocks per slot (should yield a power of two number of cells)
slotRoots: seq[H] # roots of the slots
emptyBlock: seq[byte] # empty block
verifiableTree: ?T # verification tree (dataset tree)
@ -133,9 +133,8 @@ func manifest*[T, H](self: SlotsBuilder[T, H]): Manifest =
self.manifest
proc buildBlockTree*[T, H](
self: SlotsBuilder[T, H],
blkIdx: Natural,
slotPos: Natural): Future[?!(seq[byte], T)] {.async.} =
self: SlotsBuilder[T, H], blkIdx: Natural, slotPos: Natural
): Future[?!(seq[byte], T)] {.async.} =
## Build the block digest tree and return a tuple with the
## block data and the tree.
##
@ -160,16 +159,15 @@ proc buildBlockTree*[T, H](
if blk.isEmpty:
success (self.emptyBlock, self.emptyDigestTree)
else:
without tree =?
T.digestTree(blk.data, self.cellSize.int), err:
without tree =? T.digestTree(blk.data, self.cellSize.int), err:
error "Failed to create digest for block", err = err.msg
return failure(err)
success (blk.data, tree)
proc getCellHashes*[T, H](
self: SlotsBuilder[T, H],
slotIndex: Natural): Future[?!seq[H]] {.async.} =
self: SlotsBuilder[T, H], slotIndex: Natural
): Future[?!seq[H]] {.async.} =
## Collect all the cells from a block and return
## their hashes.
##
@ -192,8 +190,8 @@ proc getCellHashes*[T, H](
pos = i
trace "Getting block CID for tree at index"
without (_, tree) =? (await self.buildBlockTree(blkIdx, i)) and
digest =? tree.root, err:
without (_, tree) =? (await self.buildBlockTree(blkIdx, i)) and digest =? tree.root,
err:
error "Failed to get block CID for tree at index", err = err.msg
return failure(err)
@ -203,8 +201,8 @@ proc getCellHashes*[T, H](
success hashes
proc buildSlotTree*[T, H](
self: SlotsBuilder[T, H],
slotIndex: Natural): Future[?!T] {.async.} =
self: SlotsBuilder[T, H], slotIndex: Natural
): Future[?!T] {.async.} =
## Build the slot tree from the block digest hashes
## and return the tree.
@ -215,8 +213,8 @@ proc buildSlotTree*[T, H](
T.init(cellHashes)
proc buildSlot*[T, H](
self: SlotsBuilder[T, H],
slotIndex: Natural): Future[?!H] {.async.} =
self: SlotsBuilder[T, H], slotIndex: Natural
): Future[?!H] {.async.} =
## Build a slot tree and store the proofs in
## the block store.
##
@ -238,13 +236,12 @@ proc buildSlot*[T, H](
error "Failed to get CID for slot cell", err = err.msg
return failure(err)
without proof =? tree.getProof(i) and
encodableProof =? proof.toEncodableProof, err:
without proof =? tree.getProof(i) and encodableProof =? proof.toEncodableProof, err:
error "Failed to get proof for slot tree", err = err.msg
return failure(err)
if err =? (await self.store.putCidAndProof(
treeCid, i, cellCid, encodableProof)).errorOption:
if err =?
(await self.store.putCidAndProof(treeCid, i, cellCid, encodableProof)).errorOption:
error "Failed to store slot tree", err = err.msg
return failure(err)
@ -293,24 +290,21 @@ proc buildManifest*[T, H](self: SlotsBuilder[T, H]): Future[?!Manifest] {.async.
return failure(err)
without rootProvingCidRes =? self.verifyRoot .? toVerifyCid() and
rootProvingCid =? rootProvingCidRes, err: # TODO: why doesn't `.?` unpack the result?
rootProvingCid =? rootProvingCidRes, err:
error "Failed to map slot roots to CIDs", err = err.msg
return failure(err)
Manifest.new(
self.manifest,
rootProvingCid,
rootCids,
self.cellSize,
self.strategy.strategyType)
self.manifest, rootProvingCid, rootCids, self.cellSize, self.strategy.strategyType
)
proc new*[T, H](
_: type SlotsBuilder[T, H],
store: BlockStore,
manifest: Manifest,
strategy = SteppedStrategy,
cellSize = DefaultCellSize): ?!SlotsBuilder[T, H] =
cellSize = DefaultCellSize,
): ?!SlotsBuilder[T, H] =
if not manifest.protected:
trace "Manifest is not protected."
return failure("Manifest is not protected.")
@ -332,11 +326,14 @@ proc new*[T, H](
let
numSlotBlocks = manifest.numSlotBlocks
numBlockCells = (manifest.blockSize div cellSize).int # number of cells per block
numSlotCells = manifest.numSlotBlocks * numBlockCells # number of uncorrected slot cells
numSlotCells = manifest.numSlotBlocks * numBlockCells
# number of uncorrected slot cells
pow2SlotCells = nextPowerOfTwo(numSlotCells) # pow2 cells per slot
numPadSlotBlocks = (pow2SlotCells div numBlockCells) - numSlotBlocks # pow2 blocks per slot
numPadSlotBlocks = (pow2SlotCells div numBlockCells) - numSlotBlocks
# pow2 blocks per slot
numSlotBlocksTotal = # pad blocks per slot
numSlotBlocksTotal =
# pad blocks per slot
if numPadSlotBlocks > 0:
numPadSlotBlocks + numSlotBlocks
else:
@ -347,10 +344,7 @@ proc new*[T, H](
emptyBlock = newSeq[byte](manifest.blockSize.int)
emptyDigestTree = ?T.digestTree(emptyBlock, cellSize.int)
strategy = ? strategy.init(
0,
numBlocksTotal - 1,
manifest.numSlots).catch
strategy = ?strategy.init(0, numBlocksTotal - 1, manifest.numSlots).catch
logScope:
numSlotBlocks = numSlotBlocks
@ -364,19 +358,18 @@ proc new*[T, H](
trace "Creating slots builder"
var
self = SlotsBuilder[T, H](
var self = SlotsBuilder[T, H](
store: store,
manifest: manifest,
strategy: strategy,
cellSize: cellSize,
emptyBlock: emptyBlock,
numSlotBlocks: numSlotBlocksTotal,
emptyDigestTree: emptyDigestTree)
emptyDigestTree: emptyDigestTree,
)
if manifest.verifiable:
if manifest.slotRoots.len == 0 or
manifest.slotRoots.len != manifest.numSlots:
if manifest.slotRoots.len == 0 or manifest.slotRoots.len != manifest.numSlots:
return failure "Manifest is verifiable but slot roots are missing or invalid."
let

32
codex/slots/converters.nim
View File

@ -27,12 +27,16 @@ func toCid(hash: Poseidon2Hash, mcodec: MultiCodec, cidCodec: MultiCodec): ?!Cid
treeCid = ?Cid.init(CIDv1, cidCodec, mhash).mapFailure
success treeCid
proc toPoseidon2Hash(cid: Cid, mcodec: MultiCodec, cidCodec: MultiCodec): ?!Poseidon2Hash =
proc toPoseidon2Hash(
cid: Cid, mcodec: MultiCodec, cidCodec: MultiCodec
): ?!Poseidon2Hash =
if cid.cidver != CIDv1:
return failure("Unexpected CID version")
if cid.mcodec != cidCodec:
return failure("Cid is not of expected codec. Was: " & $cid.mcodec & " but expected: " & $cidCodec)
return failure(
"Cid is not of expected codec. Was: " & $cid.mcodec & " but expected: " & $cidCodec
)
let
mhash = ?cid.mhash.mapFailure
@ -62,27 +66,17 @@ func toVerifyCid*(hash: Poseidon2Hash): ?!Cid =
func fromVerifyCid*(cid: Cid): ?!Poseidon2Hash =
toPoseidon2Hash(cid, multiCodec("identity"), SlotProvingRootCodec)
func toEncodableProof*(
proof: Poseidon2Proof): ?!CodexProof =
let
encodableProof = CodexProof(
func toEncodableProof*(proof: Poseidon2Proof): ?!CodexProof =
let encodableProof = CodexProof(
mcodec: multiCodec("identity"),
index: proof.index,
nleaves: proof.nleaves,
path: proof.path.mapIt( @( it.toBytes ) ))
path: proof.path.mapIt(@(it.toBytes)),
)
success encodableProof
func toVerifiableProof*(
proof: CodexProof): ?!Poseidon2Proof =
func toVerifiableProof*(proof: CodexProof): ?!Poseidon2Proof =
let nodes = proof.path.mapIt(?Poseidon2Hash.fromBytes(it.toArray32).toFailure)
let
nodes = proof.path.mapIt(
? Poseidon2Hash.fromBytes(it.toArray32).toFailure
)
Poseidon2Proof.init(
index = proof.index,
nleaves = proof.nleaves,
nodes = nodes)
Poseidon2Proof.init(index = proof.index, nleaves = proof.nleaves, nodes = nodes)

51
codex/slots/proofs/backendfactory.nim
View File

@ -11,9 +11,7 @@ import ../../conf
import ./backends
import ./backendutils
proc initializeFromConfig(
config: CodexConf,
utils: BackendUtils): ?!AnyBackend =
proc initializeFromConfig(config: CodexConf, utils: BackendUtils): ?!AnyBackend =
if not fileAccessible($config.circomR1cs, {AccessFlags.Read}) or
not endsWith($config.circomR1cs, ".r1cs"):
return failure("Circom R1CS file not accessible")
@ -27,10 +25,11 @@ proc initializeFromConfig(
return failure("Circom zkey file not accessible")
trace "Initialized prover backend from cli config"
success(utils.initializeCircomBackend(
$config.circomR1cs,
$config.circomWasm,
$config.circomZkey))
success(
utils.initializeCircomBackend(
$config.circomR1cs, $config.circomWasm, $config.circomZkey
)
)
proc r1csFilePath(config: CodexConf): string =
config.circuitDir / "proof_main.r1cs"
@ -42,42 +41,40 @@ proc zkeyFilePath(config: CodexConf): string =
config.circuitDir / "proof_main.zkey"
proc initializeFromCircuitDirFiles(
config: CodexConf,
utils: BackendUtils): ?!AnyBackend {.gcsafe.} =
if fileExists(config.r1csFilePath) and
fileExists(config.wasmFilePath) and
config: CodexConf, utils: BackendUtils
): ?!AnyBackend {.gcsafe.} =
if fileExists(config.r1csFilePath) and fileExists(config.wasmFilePath) and
fileExists(config.zkeyFilePath):
trace "Initialized prover backend from local files"
return success(utils.initializeCircomBackend(
config.r1csFilePath,
config.wasmFilePath,
config.zkeyFilePath))
return success(
utils.initializeCircomBackend(
config.r1csFilePath, config.wasmFilePath, config.zkeyFilePath
)
)
failure("Circuit files not found")
proc suggestDownloadTool(config: CodexConf) =
without address =? config.marketplaceAddress:
raise (ref Defect)(msg: "Proving backend initializing while marketplace address not set.")
raise (ref Defect)(
msg: "Proving backend initializing while marketplace address not set."
)
let
tokens = [
"cirdl",
"\"" & $config.circuitDir & "\"",
config.ethProvider,
$address
]
tokens = ["cirdl", "\"" & $config.circuitDir & "\"", config.ethProvider, $address]
instructions = "'./" & tokens.join(" ") & "'"
warn "Proving circuit files are not found. Please run the following to download them:", instructions
warn "Proving circuit files are not found. Please run the following to download them:",
instructions
proc initializeBackend*(
config: CodexConf,
utils: BackendUtils = BackendUtils()): ?!AnyBackend =
config: CodexConf, utils: BackendUtils = BackendUtils()
): ?!AnyBackend =
without backend =? initializeFromConfig(config, utils), cliErr:
info "Could not initialize prover backend from CLI options...", msg = cliErr.msg
without backend =? initializeFromCircuitDirFiles(config, utils), localErr:
info "Could not initialize prover backend from circuit dir files...", msg = localErr.msg
info "Could not initialize prover backend from circuit dir files...",
msg = localErr.msg
suggestDownloadTool(config)
return failure("CircuitFilesNotFound")
# Unexpected: value of backend does not survive leaving each scope. (definition does though...)

3
codex/slots/proofs/backends.nim
View File

@ -2,5 +2,4 @@ import ./backends/circomcompat
export circomcompat
type
AnyBackend* = CircomCompat
type AnyBackend* = CircomCompat

109
codex/slots/proofs/backends/circomcompat.nim
View File

@ -38,8 +38,9 @@ type
NormalizedProofInputs*[H] {.borrow: `.`.} = distinct ProofInputs[H]
func normalizeInput*[H](self: CircomCompat, input: ProofInputs[H]):
NormalizedProofInputs[H] =
func normalizeInput*[H](
self: CircomCompat, input: ProofInputs[H]
): NormalizedProofInputs[H] =
## Parameters in CIRCOM circuits are statically sized and must be properly
## padded before they can be passed onto the circuit. This function takes
## variable length parameters and performs that padding.
@ -52,10 +53,7 @@ func normalizeInput*[H](self: CircomCompat, input: ProofInputs[H]):
for sample in input.samples:
var merklePaths = sample.merklePaths
merklePaths.setLen(self.slotDepth)
Sample[H](
cellData: sample.cellData,
merklePaths: merklePaths
)
Sample[H](cellData: sample.cellData, merklePaths: merklePaths)
var normSlotProof = input.slotProof
normSlotProof.setLen(self.datasetDepth)
@ -68,7 +66,7 @@ func normalizeInput*[H](self: CircomCompat, input: ProofInputs[H]):
nCellsPerSlot: input.nCellsPerSlot,
nSlotsPerDataSet: input.nSlotsPerDataSet,
slotProof: normSlotProof,
samples: normSamples
samples: normSamples,
)
proc release*(self: CircomCompat) =
@ -81,32 +79,28 @@ proc release*(self: CircomCompat) =
if not isNil(self.vkp):
self.vkp.unsafeAddr.release_key()
proc prove[H](
self: CircomCompat,
input: NormalizedProofInputs[H]): ?!CircomProof =
doAssert input.samples.len == self.numSamples,
"Number of samples does not match"
proc prove[H](self: CircomCompat, input: NormalizedProofInputs[H]): ?!CircomProof =
doAssert input.samples.len == self.numSamples, "Number of samples does not match"
doAssert input.slotProof.len <= self.datasetDepth,
"Slot proof is too deep - dataset has more slots than what we can handle?"
doAssert input.samples.allIt(
block:
(it.merklePaths.len <= self.slotDepth + self.blkDepth and
it.cellData.len == self.cellElms)), "Merkle paths too deep or cells too big for circuit"
(
it.merklePaths.len <= self.slotDepth + self.blkDepth and
it.cellData.len == self.cellElms
)
), "Merkle paths too deep or cells too big for circuit"
# TODO: All parameters should match circom's static parametter
var
ctx: ptr CircomCompatCtx
var ctx: ptr CircomCompatCtx
defer:
if ctx != nil:
ctx.addr.release_circom_compat()
if init_circom_compat(
self.backendCfg,
addr ctx) != ERR_OK or ctx == nil:
if init_circom_compat(self.backendCfg, addr ctx) != ERR_OK or ctx == nil:
raiseAssert("failed to initialize CircomCompat ctx")
var
@ -114,39 +108,37 @@ proc prove[H](
dataSetRoot = input.datasetRoot.toBytes
slotRoot = input.slotRoot.toBytes
if ctx.push_input_u256_array(
"entropy".cstring, entropy[0].addr, entropy.len.uint32) != ERR_OK:
if ctx.push_input_u256_array("entropy".cstring, entropy[0].addr, entropy.len.uint32) !=
ERR_OK:
return failure("Failed to push entropy")
if ctx.push_input_u256_array(
"dataSetRoot".cstring, dataSetRoot[0].addr, dataSetRoot.len.uint32) != ERR_OK:
"dataSetRoot".cstring, dataSetRoot[0].addr, dataSetRoot.len.uint32
) != ERR_OK:
return failure("Failed to push data set root")
if ctx.push_input_u256_array(
"slotRoot".cstring, slotRoot[0].addr, slotRoot.len.uint32) != ERR_OK:
"slotRoot".cstring, slotRoot[0].addr, slotRoot.len.uint32
) != ERR_OK:
return failure("Failed to push data set root")
if ctx.push_input_u32(
"nCellsPerSlot".cstring, input.nCellsPerSlot.uint32) != ERR_OK:
if ctx.push_input_u32("nCellsPerSlot".cstring, input.nCellsPerSlot.uint32) != ERR_OK:
return failure("Failed to push nCellsPerSlot")
if ctx.push_input_u32(
"nSlotsPerDataSet".cstring, input.nSlotsPerDataSet.uint32) != ERR_OK:
if ctx.push_input_u32("nSlotsPerDataSet".cstring, input.nSlotsPerDataSet.uint32) !=
ERR_OK:
return failure("Failed to push nSlotsPerDataSet")
if ctx.push_input_u32(
"slotIndex".cstring, input.slotIndex.uint32) != ERR_OK:
if ctx.push_input_u32("slotIndex".cstring, input.slotIndex.uint32) != ERR_OK:
return failure("Failed to push slotIndex")
var
slotProof = input.slotProof.mapIt( it.toBytes ).concat
var slotProof = input.slotProof.mapIt(it.toBytes).concat
doAssert(slotProof.len == self.datasetDepth)
# arrays are always flattened
if ctx.push_input_u256_array(
"slotProof".cstring,
slotProof[0].addr,
uint (slotProof[0].len * slotProof.len)) != ERR_OK:
"slotProof".cstring, slotProof[0].addr, uint (slotProof[0].len * slotProof.len)
) != ERR_OK:
return failure("Failed to push slot proof")
for s in input.samples:
@ -157,23 +149,19 @@ proc prove[H](
if ctx.push_input_u256_array(
"merklePaths".cstring,
merklePaths[0].addr,
uint (merklePaths[0].len * merklePaths.len)) != ERR_OK:
uint (merklePaths[0].len * merklePaths.len),
) != ERR_OK:
return failure("Failed to push merkle paths")
if ctx.push_input_u256_array(
"cellData".cstring,
data[0].addr,
data.len.uint) != ERR_OK:
if ctx.push_input_u256_array("cellData".cstring, data[0].addr, data.len.uint) !=
ERR_OK:
return failure("Failed to push cell data")
var
proofPtr: ptr Proof = nil
var proofPtr: ptr Proof = nil
let proof =
try:
if (
let res = self.backendCfg.prove_circuit(ctx, proofPtr.addr);
res != ERR_OK) or
if (let res = self.backendCfg.prove_circuit(ctx, proofPtr.addr); res != ERR_OK) or
proofPtr == nil:
return failure("Failed to prove - err code: " & $res)
@ -184,16 +172,12 @@ proc prove[H](
success proof
proc prove*[H](
self: CircomCompat,
input: ProofInputs[H]): ?!CircomProof =
proc prove*[H](self: CircomCompat, input: ProofInputs[H]): ?!CircomProof =
self.prove(self.normalizeInput(input))
proc verify*[H](
self: CircomCompat,
proof: CircomProof,
inputs: ProofInputs[H]): ?!bool =
self: CircomCompat, proof: CircomProof, inputs: ProofInputs[H]
): ?!bool =
## Verify a proof using a ctx
##
@ -221,25 +205,25 @@ proc init*(
datasetDepth = DefaultMaxDatasetDepth,
blkDepth = DefaultBlockDepth,
cellElms = DefaultCellElms,
numSamples = DefaultSamplesNum): CircomCompat =
numSamples = DefaultSamplesNum,
): CircomCompat =
## Create a new ctx
##
var cfg: ptr CircomBn254Cfg
var zkey = if zkeyPath.len > 0: zkeyPath.cstring else: nil
if init_circom_config(
r1csPath.cstring,
wasmPath.cstring,
zkey, cfg.addr) != ERR_OK or cfg == nil:
if cfg != nil: cfg.addr.release_cfg()
if init_circom_config(r1csPath.cstring, wasmPath.cstring, zkey, cfg.addr) != ERR_OK or
cfg == nil:
if cfg != nil:
cfg.addr.release_cfg()
raiseAssert("failed to initialize circom compat config")
var
vkpPtr: ptr VerifyingKey = nil
var vkpPtr: ptr VerifyingKey = nil
if cfg.get_verifying_key(vkpPtr.addr) != ERR_OK or vkpPtr == nil:
if vkpPtr != nil: vkpPtr.addr.release_key()
if vkpPtr != nil:
vkpPtr.addr.release_key()
raiseAssert("Failed to get verifying key")
CircomCompat(
@ -252,4 +236,5 @@ proc init*(
cellElms: cellElms,
numSamples: numSamples,
backendCfg: cfg,
vkp : vkpPtr)
vkp: vkpPtr,
)

30
codex/slots/proofs/backends/converters.nim
View File

@ -29,18 +29,12 @@ proc toCircomInputs*(inputs: ProofInputs[Poseidon2Hash]): CircomInputs =
datasetRoot = inputs.datasetRoot.toBytes.toArray32
entropy = inputs.entropy.toBytes.toArray32
elms = [
entropy,
datasetRoot,
slotIndex
]
elms = [entropy, datasetRoot, slotIndex]
let inputsPtr = allocShared0(32 * elms.len)
copyMem(inputsPtr, addr elms[0], elms.len * 32)
CircomInputs(
elms: cast[ptr array[32, byte]](inputsPtr),
len: elms.len.uint)
CircomInputs(elms: cast[ptr array[32, byte]](inputsPtr), len: elms.len.uint)
proc releaseCircomInputs*(inputs: var CircomInputs) =
if not inputs.elms.isNil:
@ -48,23 +42,13 @@ proc releaseCircomInputs*(inputs: var CircomInputs) =
inputs.elms = nil
func toG1*(g: CircomG1): G1Point =
G1Point(
x: UInt256.fromBytesLE(g.x),
y: UInt256.fromBytesLE(g.y))
G1Point(x: UInt256.fromBytesLE(g.x), y: UInt256.fromBytesLE(g.y))
func toG2*(g: CircomG2): G2Point =
G2Point(
x: Fp2Element(
real: UInt256.fromBytesLE(g.x[0]),
imag: UInt256.fromBytesLE(g.x[1])
),
y: Fp2Element(
real: UInt256.fromBytesLE(g.y[0]),
imag: UInt256.fromBytesLE(g.y[1])
))
x: Fp2Element(real: UInt256.fromBytesLE(g.x[0]), imag: UInt256.fromBytesLE(g.x[1])),
y: Fp2Element(real: UInt256.fromBytesLE(g.y[0]), imag: UInt256.fromBytesLE(g.y[1])),
)
func toGroth16Proof*(proof: CircomProof): Groth16Proof =
Groth16Proof(
a: proof.a.toG1,
b: proof.b.toG2,
c: proof.c.toG1)
Groth16Proof(a: proof.a.toG1, b: proof.b.toG2, c: proof.c.toG1)

8
codex/slots/proofs/backendutils.nim
View File

@ -1,12 +1,8 @@
import ./backends
type
BackendUtils* = ref object of RootObj
type BackendUtils* = ref object of RootObj
method initializeCircomBackend*(
self: BackendUtils,
r1csFile: string,
wasmFile: string,
zKeyFile: string
self: BackendUtils, r1csFile: string, wasmFile: string, zKeyFile: string
): AnyBackend {.base, gcsafe.} =
CircomCompat.init(r1csFile, wasmFile, zKeyFile)

23
codex/slots/proofs/prover.nim
View File

@ -47,10 +47,8 @@ type
nSamples: int
proc prove*(
self: Prover,
slotIdx: int,
manifest: Manifest,
challenge: ProofChallenge): Future[?!(AnyProofInputs, AnyProof)] {.async.} =
self: Prover, slotIdx: int, manifest: Manifest, challenge: ProofChallenge
): Future[?!(AnyProofInputs, AnyProof)] {.async.} =
## Prove a statement using backend.
## Returns a future that resolves to a proof.
@ -81,20 +79,13 @@ proc prove*(
success (proofInput, proof)
proc verify*(
self: Prover,
proof: AnyProof,
inputs: AnyProofInputs): Future[?!bool] {.async.} =
self: Prover, proof: AnyProof, inputs: AnyProofInputs
): Future[?!bool] {.async.} =
## Prove a statement using backend.
## Returns a future that resolves to a proof.
self.backend.verify(proof, inputs)
proc new*(
_: type Prover,
store: BlockStore,
backend: AnyBackend,
nSamples: int): Prover =
Prover(
store: store,
backend: backend,
nSamples: nSamples)
_: type Prover, store: BlockStore, backend: AnyBackend, nSamples: int
): Prover =
Prover(store: store, backend: backend, nSamples: nSamples)

Some files were not shown because too many files have changed in this diff Show More