Merge branch 'master' into async-profiling
# Conflicts: # vendor/codex-contracts-eth
This commit is contained in:
Ben
commit
86e357e0b2
|
|
@ -14,7 +14,7 @@ inputs:
|
|||
default: "version-1-6"
|
||||
rust_version:
|
||||
description: "Rust version"
|
||||
default: "1.78.0"
|
||||
default: "1.79.0"
|
||||
shell:
|
||||
description: "Shell to run commands in"
|
||||
default: "bash --noprofile --norc -e -o pipefail"
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ jobs:
|
|||
run:
|
||||
shell: ${{ matrix.shell }} {0}
|
||||
|
||||
name: '${{ matrix.os }}-${{ matrix.cpu }}-${{ matrix.nim_version }}-${{ matrix.tests }}'
|
||||
name: ${{ matrix.os }}-${{ matrix.tests }}-${{ matrix.cpu }}-${{ matrix.nim_version }}
|
||||
runs-on: ${{ matrix.builder }}
|
||||
timeout-minutes: 100
|
||||
steps:
|
||||
|
|
|
|||
|
|
@ -28,8 +28,14 @@ jobs:
|
|||
uses: fabiocaccamo/create-matrix-action@v4
|
||||
with:
|
||||
matrix: |
|
||||
os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {all}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
|
||||
os {macos}, cpu {amd64}, builder {macos-13}, tests {all}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
|
||||
os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
|
||||
os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
|
||||
os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
|
||||
os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
|
||||
os {macos}, cpu {amd64}, builder {macos-13}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
|
||||
os {macos}, cpu {amd64}, builder {macos-13}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
|
||||
os {macos}, cpu {amd64}, builder {macos-13}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
|
||||
os {macos}, cpu {amd64}, builder {macos-13}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
|
||||
os {windows}, cpu {amd64}, builder {windows-latest}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {msys2}
|
||||
os {windows}, cpu {amd64}, builder {windows-latest}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {msys2}
|
||||
os {windows}, cpu {amd64}, builder {windows-latest}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {msys2}
|
||||
|
|
|
|||
|
|
@ -28,14 +28,13 @@ jobs:
|
|||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: '0'
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 18
|
||||
|
||||
- name: Lint OpenAPI
|
||||
shell: bash
|
||||
run: npx @redocly/cli lint openapi.yaml
|
||||
|
||||
deploy:
|
||||
|
|
@ -46,20 +45,22 @@ jobs:
|
|||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: '0'
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 18
|
||||
|
||||
- name: Build OpenAPI
|
||||
shell: bash
|
||||
run: npx @redocly/cli build-docs openapi.yaml --output "openapi/index.html" --title "Codex API"
|
||||
run: npx @redocly/cli build-docs openapi.yaml --output openapi/index.html --title "Codex API"
|
||||
|
||||
- name: Build Postman Collection
|
||||
run: npx -y openapi-to-postmanv2 -s openapi.yaml -o openapi/postman.json -p -O folderStrategy=Tags,includeAuthInfoInExample=false
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-pages-artifact@v3
|
||||
with:
|
||||
path: './openapi'
|
||||
path: openapi
|
||||
|
||||
- name: Deploy to GitHub Pages
|
||||
uses: actions/deploy-pages@v4
|
||||
|
|
|
|||
24
Makefile
24
Makefile
|
|
@ -40,6 +40,30 @@ DOCKER_IMAGE_NIM_PARAMS ?= -d:chronicles_colors:none -d:insecure
|
|||
|
||||
LINK_PCRE := 0
|
||||
|
||||
ifeq ($(OS),Windows_NT)
|
||||
ifeq ($(PROCESSOR_ARCHITECTURE), AMD64)
|
||||
ARCH = x86_64
|
||||
endif
|
||||
ifeq ($(PROCESSOR_ARCHITECTURE), ARM64)
|
||||
ARCH = arm64
|
||||
endif
|
||||
else
|
||||
UNAME_P := $(shell uname -p)
|
||||
ifneq ($(filter $(UNAME_P), i686 i386 x86_64),)
|
||||
ARCH = x86_64
|
||||
endif
|
||||
ifneq ($(filter $(UNAME_P), aarch64 arm),)
|
||||
ARCH = arm64
|
||||
endif
|
||||
endif
|
||||
|
||||
ifeq ($(ARCH), x86_64)
|
||||
CXXFLAGS ?= -std=c++17 -mssse3
|
||||
else
|
||||
CXXFLAGS ?= -std=c++17
|
||||
endif
|
||||
export CXXFLAGS
|
||||
|
||||
# we don't want an error here, so we can handle things later, in the ".DEFAULT" target
|
||||
-include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk
|
||||
|
||||
|
|
|
|||
|
|
@ -20,9 +20,9 @@ const knownAddresses = {
|
|||
"167005": {
|
||||
"Marketplace": Address.init("0x948CF9291b77Bd7ad84781b9047129Addf1b894F")
|
||||
}.toTable,
|
||||
# Codex Testnet - Oct 21 2024 07:31:50 AM (+00:00 UTC)
|
||||
# Codex Testnet - Nov 03 2024 07:30:30 AM (+00:00 UTC)
|
||||
"789987": {
|
||||
"Marketplace": Address.init("0x3F9Cf3F40F0e87d804B776D8403e3d29F85211f4")
|
||||
"Marketplace": Address.init("0x5Bd66fA15Eb0E546cd26808248867a572cFF5706")
|
||||
}.toTable
|
||||
}.toTable
|
||||
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ proc approveFunds(market: OnChainMarket, amount: UInt256) {.async.} =
|
|||
discard await token.increaseAllowance(market.contract.address(), amount).confirm(0)
|
||||
|
||||
method getZkeyHash*(market: OnChainMarket): Future[?string] {.async.} =
|
||||
let config = await market.contract.config()
|
||||
let config = await market.contract.configuration()
|
||||
return some config.proofs.zkeyHash
|
||||
|
||||
method getSigner*(market: OnChainMarket): Future[Address] {.async.} =
|
||||
|
|
@ -65,18 +65,18 @@ method getSigner*(market: OnChainMarket): Future[Address] {.async.} =
|
|||
|
||||
method periodicity*(market: OnChainMarket): Future[Periodicity] {.async.} =
|
||||
convertEthersError:
|
||||
let config = await market.contract.config()
|
||||
let config = await market.contract.configuration()
|
||||
let period = config.proofs.period
|
||||
return Periodicity(seconds: period)
|
||||
|
||||
method proofTimeout*(market: OnChainMarket): Future[UInt256] {.async.} =
|
||||
convertEthersError:
|
||||
let config = await market.contract.config()
|
||||
let config = await market.contract.configuration()
|
||||
return config.proofs.timeout
|
||||
|
||||
method proofDowntime*(market: OnChainMarket): Future[uint8] {.async.} =
|
||||
convertEthersError:
|
||||
let config = await market.contract.config()
|
||||
let config = await market.contract.configuration()
|
||||
return config.proofs.downtime
|
||||
|
||||
method getPointer*(market: OnChainMarket, slotId: SlotId): Future[uint8] {.async.} =
|
||||
|
|
@ -165,12 +165,18 @@ method fillSlot(market: OnChainMarket,
|
|||
proof: Groth16Proof,
|
||||
collateral: UInt256) {.async.} =
|
||||
convertEthersError:
|
||||
logScope:
|
||||
requestId
|
||||
slotIndex
|
||||
|
||||
await market.approveFunds(collateral)
|
||||
trace "calling fillSlot on contract"
|
||||
discard await market.contract.fillSlot(requestId, slotIndex, proof).confirm(0)
|
||||
trace "fillSlot transaction completed"
|
||||
|
||||
method freeSlot*(market: OnChainMarket, slotId: SlotId) {.async.} =
|
||||
convertEthersError:
|
||||
var freeSlot: Future[?TransactionResponse]
|
||||
var freeSlot: Future[Confirmable]
|
||||
if rewardRecipient =? market.rewardRecipient:
|
||||
# If --reward-recipient specified, use it as the reward recipient, and use
|
||||
# the SP's address as the collateral recipient
|
||||
|
|
@ -253,7 +259,12 @@ method reserveSlot*(
|
|||
slotIndex: UInt256) {.async.} =
|
||||
|
||||
convertEthersError:
|
||||
discard await market.contract.reserveSlot(requestId, slotIndex).confirm(0)
|
||||
discard await market.contract.reserveSlot(
|
||||
requestId,
|
||||
slotIndex,
|
||||
# reserveSlot runs out of gas for unknown reason, but 100k gas covers it
|
||||
TransactionOverrides(gasLimit: some 100000.u256)
|
||||
).confirm(0)
|
||||
|
||||
method canReserveSlot*(
|
||||
market: OnChainMarket,
|
||||
|
|
|
|||
|
|
@ -17,18 +17,18 @@ export requests
|
|||
type
|
||||
Marketplace* = ref object of Contract
|
||||
|
||||
proc config*(marketplace: Marketplace): MarketplaceConfig {.contract, view.}
|
||||
proc configuration*(marketplace: Marketplace): MarketplaceConfig {.contract, view.}
|
||||
proc token*(marketplace: Marketplace): Address {.contract, view.}
|
||||
proc slashMisses*(marketplace: Marketplace): UInt256 {.contract, view.}
|
||||
proc slashPercentage*(marketplace: Marketplace): UInt256 {.contract, view.}
|
||||
proc minCollateralThreshold*(marketplace: Marketplace): UInt256 {.contract, view.}
|
||||
|
||||
proc requestStorage*(marketplace: Marketplace, request: StorageRequest): ?TransactionResponse {.contract.}
|
||||
proc fillSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256, proof: Groth16Proof): ?TransactionResponse {.contract.}
|
||||
proc withdrawFunds*(marketplace: Marketplace, requestId: RequestId): ?TransactionResponse {.contract.}
|
||||
proc withdrawFunds*(marketplace: Marketplace, requestId: RequestId, withdrawAddress: Address): ?TransactionResponse {.contract.}
|
||||
proc freeSlot*(marketplace: Marketplace, id: SlotId): ?TransactionResponse {.contract.}
|
||||
proc freeSlot*(marketplace: Marketplace, id: SlotId, rewardRecipient: Address, collateralRecipient: Address): ?TransactionResponse {.contract.}
|
||||
proc requestStorage*(marketplace: Marketplace, request: StorageRequest): Confirmable {.contract.}
|
||||
proc fillSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256, proof: Groth16Proof): Confirmable {.contract.}
|
||||
proc withdrawFunds*(marketplace: Marketplace, requestId: RequestId): Confirmable {.contract.}
|
||||
proc withdrawFunds*(marketplace: Marketplace, requestId: RequestId, withdrawAddress: Address): Confirmable {.contract.}
|
||||
proc freeSlot*(marketplace: Marketplace, id: SlotId): Confirmable {.contract.}
|
||||
proc freeSlot*(marketplace: Marketplace, id: SlotId, rewardRecipient: Address, collateralRecipient: Address): Confirmable {.contract.}
|
||||
proc getRequest*(marketplace: Marketplace, id: RequestId): StorageRequest {.contract, view.}
|
||||
proc getHost*(marketplace: Marketplace, id: SlotId): Address {.contract, view.}
|
||||
proc getActiveSlot*(marketplace: Marketplace, id: SlotId): Slot {.contract, view.}
|
||||
|
|
@ -49,8 +49,8 @@ proc willProofBeRequired*(marketplace: Marketplace, id: SlotId): bool {.contract
|
|||
proc getChallenge*(marketplace: Marketplace, id: SlotId): array[32, byte] {.contract, view.}
|
||||
proc getPointer*(marketplace: Marketplace, id: SlotId): uint8 {.contract, view.}
|
||||
|
||||
proc submitProof*(marketplace: Marketplace, id: SlotId, proof: Groth16Proof): ?TransactionResponse {.contract.}
|
||||
proc markProofAsMissing*(marketplace: Marketplace, id: SlotId, period: UInt256): ?TransactionResponse {.contract.}
|
||||
proc submitProof*(marketplace: Marketplace, id: SlotId, proof: Groth16Proof): Confirmable {.contract.}
|
||||
proc markProofAsMissing*(marketplace: Marketplace, id: SlotId, period: UInt256): Confirmable {.contract.}
|
||||
|
||||
proc reserveSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256): ?TransactionResponse {.contract.}
|
||||
proc reserveSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256): Confirmable {.contract.}
|
||||
proc canReserveSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256): bool {.contract, view.}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
# This module implements serialization and deserialization of Manifest
|
||||
|
||||
import pkg/upraises
|
||||
import times
|
||||
|
||||
push: {.upraises: [].}
|
||||
|
||||
|
|
@ -59,6 +60,9 @@ proc encode*(manifest: Manifest): ?!seq[byte] =
|
|||
# optional hcodec: MultiCodec = 5 # Multihash codec
|
||||
# optional version: CidVersion = 6; # Cid version
|
||||
# optional ErasureInfo erasure = 7; # erasure coding info
|
||||
# optional filename: ?string = 8; # original filename
|
||||
# optional mimetype: ?string = 9; # original mimetype
|
||||
# optional uploadedAt: ?int64 = 10; # original uploadedAt
|
||||
# }
|
||||
# ```
|
||||
#
|
||||
|
|
@ -70,6 +74,7 @@ proc encode*(manifest: Manifest): ?!seq[byte] =
|
|||
header.write(4, manifest.codec.uint32)
|
||||
header.write(5, manifest.hcodec.uint32)
|
||||
header.write(6, manifest.version.uint32)
|
||||
|
||||
if manifest.protected:
|
||||
var erasureInfo = initProtoBuffer()
|
||||
erasureInfo.write(1, manifest.ecK.uint32)
|
||||
|
|
@ -90,6 +95,15 @@ proc encode*(manifest: Manifest): ?!seq[byte] =
|
|||
erasureInfo.finish()
|
||||
header.write(7, erasureInfo)
|
||||
|
||||
if manifest.filename.isSome:
|
||||
header.write(8, manifest.filename.get())
|
||||
|
||||
if manifest.mimetype.isSome:
|
||||
header.write(9, manifest.mimetype.get())
|
||||
|
||||
if manifest.uploadedAt.isSome:
|
||||
header.write(10, manifest.uploadedAt.get().uint64)
|
||||
|
||||
pbNode.write(1, header) # set the treeCid as the data field
|
||||
pbNode.finish()
|
||||
|
||||
|
|
@ -118,6 +132,9 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
|
|||
slotRoots: seq[seq[byte]]
|
||||
cellSize: uint32
|
||||
verifiableStrategy: uint32
|
||||
filename: string
|
||||
mimetype: string
|
||||
uploadedAt: uint64
|
||||
|
||||
# Decode `Header` message
|
||||
if pbNode.getField(1, pbHeader).isErr:
|
||||
|
|
@ -145,6 +162,15 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
|
|||
if pbHeader.getField(7, pbErasureInfo).isErr:
|
||||
return failure("Unable to decode `erasureInfo` from manifest!")
|
||||
|
||||
if pbHeader.getField(8, filename).isErr:
|
||||
return failure("Unable to decode `filename` from manifest!")
|
||||
|
||||
if pbHeader.getField(9, mimetype).isErr:
|
||||
return failure("Unable to decode `mimetype` from manifest!")
|
||||
|
||||
if pbHeader.getField(10, uploadedAt).isErr:
|
||||
return failure("Unable to decode `uploadedAt` from manifest!")
|
||||
|
||||
let protected = pbErasureInfo.buffer.len > 0
|
||||
var verifiable = false
|
||||
if protected:
|
||||
|
|
@ -183,6 +209,10 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
|
|||
let
|
||||
treeCid = ? Cid.init(treeCidBuf).mapFailure
|
||||
|
||||
var filenameOption = if filename.len == 0: string.none else: filename.some
|
||||
var mimetypeOption = if mimetype.len == 0: string.none else: mimetype.some
|
||||
var uploadedAtOption = if uploadedAt == 0: int64.none else: uploadedAt.int64.some
|
||||
|
||||
let
|
||||
self = if protected:
|
||||
Manifest.new(
|
||||
|
|
@ -196,7 +226,10 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
|
|||
ecM = ecM.int,
|
||||
originalTreeCid = ? Cid.init(originalTreeCid).mapFailure,
|
||||
originalDatasetSize = originalDatasetSize.NBytes,
|
||||
strategy = StrategyType(protectedStrategy))
|
||||
strategy = StrategyType(protectedStrategy),
|
||||
filename = filenameOption,
|
||||
mimetype = mimetypeOption,
|
||||
uploadedAt = uploadedAtOption)
|
||||
else:
|
||||
Manifest.new(
|
||||
treeCid = treeCid,
|
||||
|
|
@ -204,7 +237,10 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
|
|||
blockSize = blockSize.NBytes,
|
||||
version = CidVersion(version),
|
||||
hcodec = hcodec.MultiCodec,
|
||||
codec = codec.MultiCodec)
|
||||
codec = codec.MultiCodec,
|
||||
filename = filenameOption,
|
||||
mimetype = mimetypeOption,
|
||||
uploadedAt = uploadedAtOption)
|
||||
|
||||
? self.verify()
|
||||
|
||||
|
|
|
|||
|
|
@ -36,6 +36,9 @@ type
|
|||
codec: MultiCodec # Dataset codec
|
||||
hcodec: MultiCodec # Multihash codec
|
||||
version: CidVersion # Cid version
|
||||
filename {.serialize.}: ?string # The filename of the content uploaded (optional)
|
||||
mimetype {.serialize.}: ?string # The mimetype of the content uploaded (optional)
|
||||
uploadedAt {.serialize.}: ?int64 # The UTC creation timestamp in seconds
|
||||
case protected {.serialize.}: bool # Protected datasets have erasure coded info
|
||||
of true:
|
||||
ecK: int # Number of blocks to encode
|
||||
|
|
@ -121,6 +124,14 @@ func verifiableStrategy*(self: Manifest): StrategyType =
|
|||
func numSlotBlocks*(self: Manifest): int =
|
||||
divUp(self.blocksCount, self.numSlots)
|
||||
|
||||
func filename*(self: Manifest): ?string =
|
||||
self.filename
|
||||
|
||||
func mimetype*(self: Manifest): ?string =
|
||||
self.mimetype
|
||||
|
||||
func uploadedAt*(self: Manifest): ?int64 =
|
||||
self.uploadedAt
|
||||
############################################################
|
||||
# Operations on block list
|
||||
############################################################
|
||||
|
|
@ -163,6 +174,9 @@ func `==`*(a, b: Manifest): bool =
|
|||
(a.hcodec == b.hcodec) and
|
||||
(a.codec == b.codec) and
|
||||
(a.protected == b.protected) and
|
||||
(a.filename == b.filename) and
|
||||
(a.mimetype == b.mimetype) and
|
||||
(a.uploadedAt == b.uploadedAt) and
|
||||
(if a.protected:
|
||||
(a.ecK == b.ecK) and
|
||||
(a.ecM == b.ecM) and
|
||||
|
|
@ -181,26 +195,38 @@ func `==`*(a, b: Manifest): bool =
|
|||
true)
|
||||
|
||||
func `$`*(self: Manifest): string =
|
||||
"treeCid: " & $self.treeCid &
|
||||
result = "treeCid: " & $self.treeCid &
|
||||
", datasetSize: " & $self.datasetSize &
|
||||
", blockSize: " & $self.blockSize &
|
||||
", version: " & $self.version &
|
||||
", hcodec: " & $self.hcodec &
|
||||
", codec: " & $self.codec &
|
||||
", protected: " & $self.protected &
|
||||
(if self.protected:
|
||||
", ecK: " & $self.ecK &
|
||||
", ecM: " & $self.ecM &
|
||||
", originalTreeCid: " & $self.originalTreeCid &
|
||||
", originalDatasetSize: " & $self.originalDatasetSize &
|
||||
", verifiable: " & $self.verifiable &
|
||||
(if self.verifiable:
|
||||
", verifyRoot: " & $self.verifyRoot &
|
||||
", slotRoots: " & $self.slotRoots
|
||||
else:
|
||||
"")
|
||||
", protected: " & $self.protected
|
||||
|
||||
if self.filename.isSome:
|
||||
result &= ", filename: " & $self.filename
|
||||
|
||||
if self.mimetype.isSome:
|
||||
result &= ", mimetype: " & $self.mimetype
|
||||
|
||||
if self.uploadedAt.isSome:
|
||||
result &= ", uploadedAt: " & $self.uploadedAt
|
||||
|
||||
result &= (if self.protected:
|
||||
", ecK: " & $self.ecK &
|
||||
", ecM: " & $self.ecM &
|
||||
", originalTreeCid: " & $self.originalTreeCid &
|
||||
", originalDatasetSize: " & $self.originalDatasetSize &
|
||||
", verifiable: " & $self.verifiable &
|
||||
(if self.verifiable:
|
||||
", verifyRoot: " & $self.verifyRoot &
|
||||
", slotRoots: " & $self.slotRoots
|
||||
else:
|
||||
"")
|
||||
else:
|
||||
"")
|
||||
|
||||
return result
|
||||
|
||||
############################################################
|
||||
# Constructors
|
||||
|
|
@ -214,7 +240,10 @@ func new*(
|
|||
version: CidVersion = CIDv1,
|
||||
hcodec = Sha256HashCodec,
|
||||
codec = BlockCodec,
|
||||
protected = false): Manifest =
|
||||
protected = false,
|
||||
filename: ?string = string.none,
|
||||
mimetype: ?string = string.none,
|
||||
uploadedAt: ?int64 = int64.none): Manifest =
|
||||
|
||||
T(
|
||||
treeCid: treeCid,
|
||||
|
|
@ -223,7 +252,10 @@ func new*(
|
|||
version: version,
|
||||
codec: codec,
|
||||
hcodec: hcodec,
|
||||
protected: protected)
|
||||
protected: protected,
|
||||
filename: filename,
|
||||
mimetype: mimetype,
|
||||
uploadedAt: uploadedAt)
|
||||
|
||||
func new*(
|
||||
T: type Manifest,
|
||||
|
|
@ -247,7 +279,11 @@ func new*(
|
|||
ecK: ecK, ecM: ecM,
|
||||
originalTreeCid: manifest.treeCid,
|
||||
originalDatasetSize: manifest.datasetSize,
|
||||
protectedStrategy: strategy)
|
||||
protectedStrategy: strategy,
|
||||
filename: manifest.filename,
|
||||
mimetype: manifest.mimetype,
|
||||
uploadedAt: manifest.uploadedAt
|
||||
)
|
||||
|
||||
func new*(
|
||||
T: type Manifest,
|
||||
|
|
@ -263,7 +299,10 @@ func new*(
|
|||
codec: manifest.codec,
|
||||
hcodec: manifest.hcodec,
|
||||
blockSize: manifest.blockSize,
|
||||
protected: false)
|
||||
protected: false,
|
||||
filename: manifest.filename,
|
||||
mimetype: manifest.mimetype,
|
||||
uploadedAt: manifest.uploadedAt)
|
||||
|
||||
func new*(
|
||||
T: type Manifest,
|
||||
|
|
@ -277,7 +316,10 @@ func new*(
|
|||
ecM: int,
|
||||
originalTreeCid: Cid,
|
||||
originalDatasetSize: NBytes,
|
||||
strategy = SteppedStrategy): Manifest =
|
||||
strategy = SteppedStrategy,
|
||||
filename: ?string = string.none,
|
||||
mimetype: ?string = string.none,
|
||||
uploadedAt: ?int64 = int64.none): Manifest =
|
||||
|
||||
Manifest(
|
||||
treeCid: treeCid,
|
||||
|
|
@ -291,7 +333,10 @@ func new*(
|
|||
ecM: ecM,
|
||||
originalTreeCid: originalTreeCid,
|
||||
originalDatasetSize: originalDatasetSize,
|
||||
protectedStrategy: strategy)
|
||||
protectedStrategy: strategy,
|
||||
filename: filename,
|
||||
mimetype: mimetype,
|
||||
uploadedAt: uploadedAt)
|
||||
|
||||
func new*(
|
||||
T: type Manifest,
|
||||
|
|
@ -329,7 +374,11 @@ func new*(
|
|||
verifyRoot: verifyRoot,
|
||||
slotRoots: @slotRoots,
|
||||
cellSize: cellSize,
|
||||
verifiableStrategy: strategy)
|
||||
verifiableStrategy: strategy,
|
||||
filename: manifest.filename,
|
||||
mimetype: manifest.mimetype,
|
||||
uploadedAt: manifest.uploadedAt
|
||||
)
|
||||
|
||||
func new*(
|
||||
T: type Manifest,
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import std/sequtils
|
|||
import std/strformat
|
||||
import std/sugar
|
||||
import std/cpuinfo
|
||||
import times
|
||||
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
|
|
@ -297,6 +298,8 @@ proc retrieve*(
|
|||
proc store*(
|
||||
self: CodexNodeRef,
|
||||
stream: LPStream,
|
||||
filename: ?string = string.none,
|
||||
mimetype: ?string = string.none,
|
||||
blockSize = DefaultBlockSize): Future[?!Cid] {.async.} =
|
||||
## Save stream contents as dataset with given blockSize
|
||||
## to nodes's BlockStore, and return Cid of its manifest
|
||||
|
|
@ -355,7 +358,10 @@ proc store*(
|
|||
datasetSize = NBytes(chunker.offset),
|
||||
version = CIDv1,
|
||||
hcodec = hcodec,
|
||||
codec = dataCodec)
|
||||
codec = dataCodec,
|
||||
filename = filename,
|
||||
mimetype = mimetype,
|
||||
uploadedAt = now().utc.toTime.toUnix.some)
|
||||
|
||||
without manifestBlk =? await self.storeManifest(manifest), err:
|
||||
error "Unable to store manifest"
|
||||
|
|
@ -364,7 +370,9 @@ proc store*(
|
|||
info "Stored data", manifestCid = manifestBlk.cid,
|
||||
treeCid = treeCid,
|
||||
blocks = manifest.blocksCount,
|
||||
datasetSize = manifest.datasetSize
|
||||
datasetSize = manifest.datasetSize,
|
||||
filename = manifest.filename,
|
||||
mimetype = manifest.mimetype
|
||||
|
||||
return manifestBlk.cid.success
|
||||
|
||||
|
|
@ -749,15 +757,15 @@ proc stop*(self: CodexNodeRef) {.async.} =
|
|||
if not self.discovery.isNil:
|
||||
await self.discovery.stop()
|
||||
|
||||
if not self.clock.isNil:
|
||||
await self.clock.stop()
|
||||
|
||||
if clientContracts =? self.contracts.client:
|
||||
await clientContracts.stop()
|
||||
|
||||
if hostContracts =? self.contracts.host:
|
||||
await hostContracts.stop()
|
||||
|
||||
if not self.clock.isNil:
|
||||
await self.clock.stop()
|
||||
|
||||
if validatorContracts =? self.contracts.validator:
|
||||
await validatorContracts.stop()
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@ push: {.upraises: [].}
|
|||
|
||||
|
||||
import std/sequtils
|
||||
import mimetypes
|
||||
import os
|
||||
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
|
|
@ -82,11 +84,27 @@ proc retrieveCid(
|
|||
try:
|
||||
without stream =? (await node.retrieve(cid, local)), error:
|
||||
if error of BlockNotFoundError:
|
||||
return RestApiResponse.error(Http404, error.msg)
|
||||
resp.status = Http404
|
||||
return await resp.sendBody("")
|
||||
else:
|
||||
return RestApiResponse.error(Http500, error.msg)
|
||||
resp.status = Http500
|
||||
return await resp.sendBody(error.msg)
|
||||
|
||||
# It is ok to fetch again the manifest because it will hit the cache
|
||||
without manifest =? (await node.fetchManifest(cid)), err:
|
||||
error "Failed to fetch manifest", err = err.msg
|
||||
resp.status = Http404
|
||||
return await resp.sendBody(err.msg)
|
||||
|
||||
if manifest.mimetype.isSome:
|
||||
resp.setHeader("Content-Type", manifest.mimetype.get())
|
||||
else:
|
||||
resp.addHeader("Content-Type", "application/octet-stream")
|
||||
|
||||
if manifest.filename.isSome:
|
||||
resp.setHeader("Content-Disposition", "attachment; filename=\"" & manifest.filename.get() & "\"")
|
||||
|
||||
|
||||
resp.addHeader("Content-Type", "application/octet-stream")
|
||||
await resp.prepareChunked()
|
||||
|
||||
while not stream.atEof:
|
||||
|
|
@ -99,12 +117,14 @@ proc retrieveCid(
|
|||
break
|
||||
|
||||
bytes += buff.len
|
||||
|
||||
await resp.sendChunk(addr buff[0], buff.len)
|
||||
await resp.finish()
|
||||
codex_api_downloads.inc()
|
||||
except CatchableError as exc:
|
||||
warn "Excepting streaming blocks", exc = exc.msg
|
||||
return RestApiResponse.error(Http500)
|
||||
resp.status = Http500
|
||||
return await resp.sendBody("")
|
||||
finally:
|
||||
info "Sent bytes", cid = cid, bytes
|
||||
if not stream.isNil:
|
||||
|
|
@ -125,6 +145,18 @@ proc setCorsHeaders(resp: HttpResponseRef, httpMethod: string, origin: string) =
|
|||
resp.setHeader("Access-Control-Allow-Methods", httpMethod & ", OPTIONS")
|
||||
resp.setHeader("Access-Control-Max-Age", "86400")
|
||||
|
||||
proc getFilenameFromContentDisposition(contentDisposition: string): ?string =
|
||||
if not("filename=" in contentDisposition):
|
||||
return string.none
|
||||
|
||||
let parts = contentDisposition.split("filename=\"")
|
||||
|
||||
if parts.len < 2:
|
||||
return string.none
|
||||
|
||||
let filename = parts[1].strip()
|
||||
return filename[0..^2].some
|
||||
|
||||
proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRouter) =
|
||||
let allowedOrigin = router.allowedOrigin # prevents capture inside of api defintion
|
||||
|
||||
|
|
@ -135,7 +167,7 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute
|
|||
|
||||
if corsOrigin =? allowedOrigin:
|
||||
resp.setCorsHeaders("POST", corsOrigin)
|
||||
resp.setHeader("Access-Control-Allow-Headers", "content-type")
|
||||
resp.setHeader("Access-Control-Allow-Headers", "content-type, content-disposition")
|
||||
|
||||
resp.status = Http204
|
||||
await resp.sendBody("")
|
||||
|
|
@ -158,12 +190,31 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute
|
|||
#
|
||||
await request.handleExpect()
|
||||
|
||||
var mimetype = request.headers.getString(ContentTypeHeader).some
|
||||
|
||||
if mimetype.get() != "":
|
||||
var m = newMimetypes()
|
||||
let extension = m.getExt(mimetype.get(), "")
|
||||
if extension == "":
|
||||
return RestApiResponse.error(Http422, "The MIME type is not valid.")
|
||||
else:
|
||||
mimetype = string.none
|
||||
|
||||
const ContentDispositionHeader = "Content-Disposition"
|
||||
let contentDisposition = request.headers.getString(ContentDispositionHeader)
|
||||
let filename = getFilenameFromContentDisposition(contentDisposition)
|
||||
|
||||
if filename.isSome and not isValidFilename(filename.get()):
|
||||
return RestApiResponse.error(Http422, "The filename is not valid.")
|
||||
|
||||
# Here we could check if the extension matches the filename if needed
|
||||
|
||||
let
|
||||
reader = bodyReader.get()
|
||||
|
||||
try:
|
||||
without cid =? (
|
||||
await node.store(AsyncStreamWrapper.new(reader = AsyncStreamReader(reader)))), error:
|
||||
await node.store(AsyncStreamWrapper.new(reader = AsyncStreamReader(reader)), filename = filename, mimetype = mimetype)), error:
|
||||
error "Error uploading file", exc = error.msg
|
||||
return RestApiResponse.error(Http500, error.msg)
|
||||
|
||||
|
|
@ -281,10 +332,6 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute
|
|||
Http400,
|
||||
$cid.error(), headers = headers)
|
||||
|
||||
if corsOrigin =? allowedOrigin:
|
||||
resp.setCorsHeaders("GET", corsOrigin)
|
||||
resp.setHeader("Access-Control-Headers", "X-Requested-With")
|
||||
|
||||
without manifest =? (await node.fetchManifest(cid.get())), err:
|
||||
error "Failed to fetch manifest", err = err.msg
|
||||
return RestApiResponse.error(
|
||||
|
|
@ -542,7 +589,7 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
|
|||
try:
|
||||
without contracts =? node.contracts.client:
|
||||
return RestApiResponse.error(Http503, "Persistence is not enabled", headers = headers)
|
||||
|
||||
|
||||
without cid =? cid.tryGet.catch, error:
|
||||
return RestApiResponse.error(Http400, error.msg, headers = headers)
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ import ./errorhandling
|
|||
import ./filled
|
||||
import ./cancelled
|
||||
import ./failed
|
||||
import ./ignored
|
||||
import ./errored
|
||||
|
||||
logScope:
|
||||
topics = "marketplace sales filling"
|
||||
|
|
@ -22,16 +24,25 @@ method onCancelled*(state: SaleFilling, request: StorageRequest): ?State =
|
|||
method onFailed*(state: SaleFilling, request: StorageRequest): ?State =
|
||||
return some State(SaleFailed())
|
||||
|
||||
method onSlotFilled*(state: SaleFilling, requestId: RequestId,
|
||||
slotIndex: UInt256): ?State =
|
||||
return some State(SaleFilled())
|
||||
|
||||
method run(state: SaleFilling, machine: Machine): Future[?State] {.async.} =
|
||||
let data = SalesAgent(machine).data
|
||||
let market = SalesAgent(machine).context.market
|
||||
without (collateral =? data.request.?ask.?collateral):
|
||||
raiseAssert "Request not set"
|
||||
|
||||
debug "Filling slot", requestId = data.requestId, slotIndex = data.slotIndex
|
||||
await market.fillSlot(data.requestId, data.slotIndex, state.proof, collateral)
|
||||
debug "Waiting for slot filled event...", requestId = $data.requestId, slotIndex = $data.slotIndex
|
||||
logScope:
|
||||
requestId = data.requestId
|
||||
slotIndex = data.slotIndex
|
||||
|
||||
debug "Filling slot"
|
||||
try:
|
||||
await market.fillSlot(data.requestId, data.slotIndex, state.proof, collateral)
|
||||
except MarketError as e:
|
||||
if e.msg.contains "Slot is not free":
|
||||
debug "Slot is already filled, ignoring slot"
|
||||
return some State( SaleIgnored(reprocessSlot: false, returnBytes: true) )
|
||||
else:
|
||||
return some State( SaleErrored(error: e) )
|
||||
# other CatchableErrors are handled "automatically" by the ErrorHandlingState
|
||||
|
||||
return some State(SaleFilled())
|
||||
|
|
|
|||
|
|
@ -28,10 +28,6 @@ method onCancelled*(state: SaleSlotReserving, request: StorageRequest): ?State =
|
|||
method onFailed*(state: SaleSlotReserving, request: StorageRequest): ?State =
|
||||
return some State(SaleFailed())
|
||||
|
||||
method onSlotFilled*(state: SaleSlotReserving, requestId: RequestId,
|
||||
slotIndex: UInt256): ?State =
|
||||
return some State(SaleFilled())
|
||||
|
||||
method run*(state: SaleSlotReserving, machine: Machine): Future[?State] {.async.} =
|
||||
let agent = SalesAgent(machine)
|
||||
let data = agent.data
|
||||
|
|
@ -48,7 +44,12 @@ method run*(state: SaleSlotReserving, machine: Machine): Future[?State] {.async.
|
|||
trace "Reserving slot"
|
||||
await market.reserveSlot(data.requestId, data.slotIndex)
|
||||
except MarketError as e:
|
||||
return some State( SaleErrored(error: e) )
|
||||
if e.msg.contains "Reservation not allowed":
|
||||
debug "Slot cannot be reserved, ignoring", error = e.msg
|
||||
return some State( SaleIgnored(reprocessSlot: false, returnBytes: true) )
|
||||
else:
|
||||
return some State( SaleErrored(error: e) )
|
||||
# other CatchableErrors are handled "automatically" by the ErrorHandlingState
|
||||
|
||||
trace "Slot successfully reserved"
|
||||
return some State( SaleDownloading() )
|
||||
|
|
|
|||
|
|
@ -59,31 +59,28 @@ proc onError(machine: Machine, error: ref CatchableError): Event =
|
|||
state.onError(error)
|
||||
|
||||
proc run(machine: Machine, state: State) {.async.} =
|
||||
try:
|
||||
if next =? await state.run(machine):
|
||||
machine.schedule(Event.transition(state, next))
|
||||
except CancelledError:
|
||||
discard
|
||||
if next =? await state.run(machine):
|
||||
machine.schedule(Event.transition(state, next))
|
||||
|
||||
proc scheduler(machine: Machine) {.async.} =
|
||||
var running: Future[void]
|
||||
try:
|
||||
while machine.started:
|
||||
let event = await machine.scheduled.get().track(machine)
|
||||
if next =? event(machine.state):
|
||||
if not running.isNil and not running.finished:
|
||||
await running.cancelAndWait()
|
||||
let fromState = if machine.state.isNil: "<none>" else: $machine.state
|
||||
machine.state = next
|
||||
debug "enter state", state = machine.state, fromState
|
||||
running = machine.run(machine.state)
|
||||
running
|
||||
.track(machine)
|
||||
.catch((err: ref CatchableError) =>
|
||||
machine.schedule(machine.onError(err))
|
||||
)
|
||||
except CancelledError:
|
||||
discard
|
||||
while machine.started:
|
||||
let event = await machine.scheduled.get().track(machine)
|
||||
if next =? event(machine.state):
|
||||
if not running.isNil and not running.finished:
|
||||
trace "cancelling current state", state = $machine.state
|
||||
await running.cancelAndWait()
|
||||
let fromState = if machine.state.isNil: "<none>" else: $machine.state
|
||||
machine.state = next
|
||||
debug "enter state", state = fromState & " => " & $machine.state
|
||||
running = machine.run(machine.state)
|
||||
running
|
||||
.track(machine)
|
||||
.cancelled(proc() = trace "state.run cancelled, swallowing", state = $machine.state)
|
||||
.catch(proc(err: ref CatchableError) =
|
||||
trace "error caught in state.run, calling state.onError", state = $machine.state
|
||||
machine.schedule(machine.onError(err))
|
||||
)
|
||||
|
||||
proc start*(machine: Machine, initialState: State) =
|
||||
if machine.started:
|
||||
|
|
@ -93,12 +90,13 @@ proc start*(machine: Machine, initialState: State) =
|
|||
machine.scheduled = newAsyncQueue[Event]()
|
||||
|
||||
machine.started = true
|
||||
machine.scheduler()
|
||||
.track(machine)
|
||||
.catch((err: ref CatchableError) =>
|
||||
error("Error in scheduler", error = err.msg)
|
||||
)
|
||||
machine.schedule(Event.transition(machine.state, initialState))
|
||||
try:
|
||||
discard machine.scheduler().track(machine)
|
||||
machine.schedule(Event.transition(machine.state, initialState))
|
||||
except CancelledError as e:
|
||||
discard
|
||||
except CatchableError as e:
|
||||
error("Error in scheduler", error = e.msg)
|
||||
|
||||
proc stop*(machine: Machine) {.async.} =
|
||||
if not machine.started:
|
||||
|
|
|
|||
|
|
@ -45,12 +45,16 @@ fi
|
|||
|
||||
# If marketplace is enabled from the testing environment,
|
||||
# The file has to be written before Codex starts.
|
||||
if [ -n "${PRIV_KEY}" ]; then
|
||||
echo ${PRIV_KEY} > "private.key"
|
||||
chmod 600 "private.key"
|
||||
export CODEX_ETH_PRIVATE_KEY="private.key"
|
||||
echo "Private key set"
|
||||
fi
|
||||
for key in PRIV_KEY ETH_PRIVATE_KEY; do
|
||||
keyfile="private.key"
|
||||
if [[ -n "${!key}" ]]; then
|
||||
[[ "${key}" == "PRIV_KEY" ]] && echo "PRIV_KEY variable is deprecated and will be removed in the next releases, please use ETH_PRIVATE_KEY instead!"
|
||||
echo "${!key}" > "${keyfile}"
|
||||
chmod 600 "${keyfile}"
|
||||
export CODEX_ETH_PRIVATE_KEY="${keyfile}"
|
||||
echo "Private key set"
|
||||
fi
|
||||
done
|
||||
|
||||
# Circuit downloader
|
||||
# cirdl [circuitPath] [rpcEndpoint] [marketplaceAddress]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
{
|
||||
"nodes": {
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1729449015,
|
||||
"narHash": "sha256-Gf04dXB0n4q0A9G5nTGH3zuMGr6jtJppqdeljxua1fo=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "89172919243df199fe237ba0f776c3e3e3d72367",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixos-24.05",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"nixpkgs": "nixpkgs"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
"version": 7
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
{
|
||||
description = "Codex build flake";
|
||||
|
||||
inputs = {
|
||||
nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05";
|
||||
};
|
||||
|
||||
outputs = { self, nixpkgs }:
|
||||
let
|
||||
supportedSystems = [
|
||||
"x86_64-linux" "aarch64-linux"
|
||||
"x86_64-darwin" "aarch64-darwin"
|
||||
];
|
||||
forAllSystems = f: nixpkgs.lib.genAttrs supportedSystems (system: f system);
|
||||
pkgsFor = forAllSystems (system: import nixpkgs { inherit system; });
|
||||
in rec {
|
||||
devShells = forAllSystems (system: let
|
||||
pkgs = pkgsFor.${system};
|
||||
inherit (pkgs) lib stdenv mkShell;
|
||||
in {
|
||||
default = mkShell.override { stdenv = pkgs.gcc11Stdenv; } {
|
||||
buildInputs = with pkgs; [
|
||||
# General
|
||||
git pkg-config openssl lsb-release
|
||||
# Build
|
||||
rustc cargo nimble gcc11 cmake nim-unwrapped-1
|
||||
# Libraries
|
||||
gmp llvmPackages.openmp
|
||||
# Tests
|
||||
nodejs_18
|
||||
];
|
||||
};
|
||||
});
|
||||
};
|
||||
}
|
||||
77
openapi.yaml
77
openapi.yaml
|
|
@ -90,6 +90,40 @@ components:
|
|||
cid:
|
||||
$ref: "#/components/schemas/Cid"
|
||||
|
||||
Node:
|
||||
type: object
|
||||
properties:
|
||||
nodeId:
|
||||
type: string
|
||||
peerId:
|
||||
type: string
|
||||
record:
|
||||
type: string
|
||||
address:
|
||||
type: string
|
||||
seen:
|
||||
type: boolean
|
||||
|
||||
CodexVersion:
|
||||
type: object
|
||||
properties:
|
||||
version:
|
||||
type: string
|
||||
example: v0.1.7
|
||||
revision:
|
||||
type: string
|
||||
example: 0c647d8
|
||||
|
||||
PeersTable:
|
||||
type: object
|
||||
properties:
|
||||
localNode:
|
||||
$ref: "#/components/schemas/Node"
|
||||
nodes:
|
||||
type: array
|
||||
items:
|
||||
$ref: "#/components/schemas/Node"
|
||||
|
||||
DebugInfo:
|
||||
type: object
|
||||
properties:
|
||||
|
|
@ -104,6 +138,10 @@ components:
|
|||
description: Path of the data repository where all nodes data are stored
|
||||
spr:
|
||||
$ref: "#/components/schemas/SPR"
|
||||
table:
|
||||
$ref: "#/components/schemas/PeersTable"
|
||||
codex:
|
||||
$ref: "#/components/schemas/CodexVersion"
|
||||
|
||||
SalesAvailability:
|
||||
type: object
|
||||
|
|
@ -306,10 +344,10 @@ components:
|
|||
ManifestItem:
|
||||
type: object
|
||||
properties:
|
||||
rootHash:
|
||||
treeCid:
|
||||
$ref: "#/components/schemas/Cid"
|
||||
description: "Root hash of the content"
|
||||
originalBytes:
|
||||
description: "Unique data identifier"
|
||||
datasetSize:
|
||||
type: integer
|
||||
format: int64
|
||||
description: "Length of original content in bytes"
|
||||
|
|
@ -319,6 +357,22 @@ components:
|
|||
protected:
|
||||
type: boolean
|
||||
description: "Indicates if content is protected by erasure-coding"
|
||||
filename:
|
||||
type: string
|
||||
nullable: true
|
||||
description: "The original name of the uploaded content (optional)"
|
||||
example: codex.png
|
||||
mimetype:
|
||||
type: string
|
||||
nullable: true
|
||||
description: "The original mimetype of the uploaded content (optional)"
|
||||
example: image/png
|
||||
uploadedAt:
|
||||
type: integer
|
||||
format: int64
|
||||
nullable: true
|
||||
description: "The UTC upload timestamp in seconds"
|
||||
example: 1729244192
|
||||
|
||||
Space:
|
||||
type: object
|
||||
|
|
@ -404,12 +458,29 @@ paths:
|
|||
description: Invalid CID is specified
|
||||
"404":
|
||||
description: Content specified by the CID is not found
|
||||
"422":
|
||||
description: The content type is not a valid content type or the filename is not valid
|
||||
"500":
|
||||
description: Well it was bad-bad
|
||||
post:
|
||||
summary: "Upload a file in a streaming manner. Once finished, the file is stored in the node and can be retrieved by any node in the network using the returned CID."
|
||||
tags: [ Data ]
|
||||
operationId: upload
|
||||
parameters:
|
||||
- name: content-type
|
||||
in: header
|
||||
required: false
|
||||
description: The content type of the file. Must be valid.
|
||||
schema:
|
||||
type: string
|
||||
example: "image/png"
|
||||
- name: content-disposition
|
||||
in: header
|
||||
required: false
|
||||
description: The content disposition used to send the filename.
|
||||
schema:
|
||||
type: string
|
||||
example: "attachment; filename=\"codex.png\""
|
||||
requestBody:
|
||||
content:
|
||||
application/octet-stream:
|
||||
|
|
|
|||
|
|
@ -24,7 +24,3 @@ checksuite "sales state 'filling'":
|
|||
test "switches to failed state when request fails":
|
||||
let next = state.onFailed(request)
|
||||
check !next of SaleFailed
|
||||
|
||||
test "switches to filled state when slot is filled":
|
||||
let next = state.onSlotFilled(request.id, slotIndex)
|
||||
check !next of SaleFilled
|
||||
|
|
|
|||
|
|
@ -51,10 +51,6 @@ asyncchecksuite "sales state 'SlotReserving'":
|
|||
let next = state.onFailed(request)
|
||||
check !next of SaleFailed
|
||||
|
||||
test "switches to filled state when slot is filled":
|
||||
let next = state.onSlotFilled(request.id, slotIndex)
|
||||
check !next of SaleFilled
|
||||
|
||||
test "run switches to downloading when slot successfully reserved":
|
||||
let next = await state.run(agent)
|
||||
check !next of SaleDownloading
|
||||
|
|
|
|||
|
|
@ -83,6 +83,8 @@ suite "Manifest - Attribute Inheritance":
|
|||
treeCid = Cid.example,
|
||||
blockSize = 1.MiBs,
|
||||
datasetSize = 100.MiBs,
|
||||
filename = "codex.png".some,
|
||||
mimetype = "image/png".some
|
||||
),
|
||||
treeCid = Cid.example,
|
||||
datasetSize = 200.MiBs,
|
||||
|
|
@ -107,3 +109,15 @@ suite "Manifest - Attribute Inheritance":
|
|||
).tryGet()
|
||||
|
||||
check verifiable.protectedStrategy == LinearStrategy
|
||||
|
||||
test "Should preserve metadata for manifest in verifiable manifest":
|
||||
var verifiable = Manifest.new(
|
||||
manifest = makeProtectedManifest(SteppedStrategy),
|
||||
verifyRoot = Cid.example,
|
||||
slotRoots = @[Cid.example, Cid.example]
|
||||
).tryGet()
|
||||
|
||||
check verifiable.filename.isSome == true
|
||||
check verifiable.filename.get() == "codex.png"
|
||||
check verifiable.mimetype.isSome == true
|
||||
check verifiable.mimetype.get() == "image/png"
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ ethersuite "Marketplace contracts":
|
|||
let tokenAddress = await marketplace.token()
|
||||
token = Erc20Token.new(tokenAddress, ethProvider.getSigner())
|
||||
|
||||
let config = await marketplace.config()
|
||||
let config = await marketplace.configuration()
|
||||
periodicity = Periodicity(seconds: config.proofs.period)
|
||||
|
||||
request = StorageRequest.example
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import ../checktest
|
|||
type MockProvider = ref object of Provider
|
||||
chainId*: UInt256
|
||||
|
||||
method getChainId*(provider: MockProvider): Future[UInt256] {.async.} =
|
||||
method getChainId*(provider: MockProvider): Future[UInt256] {.async: (raises:[ProviderError]).} =
|
||||
return provider.chainId
|
||||
|
||||
proc configFactory(): CodexConf =
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ ethersuite "On-Chain Market":
|
|||
setup:
|
||||
let address = Marketplace.address(dummyVerifier = true)
|
||||
marketplace = Marketplace.new(address, ethProvider.getSigner())
|
||||
let config = await marketplace.config()
|
||||
let config = await marketplace.configuration()
|
||||
hostRewardRecipient = accounts[2]
|
||||
|
||||
market = OnChainMarket.new(marketplace)
|
||||
|
|
@ -76,13 +76,13 @@ ethersuite "On-Chain Market":
|
|||
|
||||
test "can retrieve proof periodicity":
|
||||
let periodicity = await market.periodicity()
|
||||
let config = await marketplace.config()
|
||||
let config = await marketplace.configuration()
|
||||
let periodLength = config.proofs.period
|
||||
check periodicity.seconds == periodLength
|
||||
|
||||
test "can retrieve proof timeout":
|
||||
let proofTimeout = await market.proofTimeout()
|
||||
let config = await marketplace.config()
|
||||
let config = await marketplace.configuration()
|
||||
check proofTimeout == config.proofs.timeout
|
||||
|
||||
test "supports marketplace requests":
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import pkg/ethers
|
||||
import pkg/serde/json
|
||||
|
||||
proc currentTime*(provider: Provider): Future[UInt256] {.async.} =
|
||||
return (!await provider.getBlock(BlockTag.pending)).timestamp
|
||||
|
|
|
|||
|
|
@ -257,3 +257,13 @@ proc saleStateIs*(client: CodexClient, id: SlotId, state: string): bool =
|
|||
|
||||
proc requestId*(client: CodexClient, id: PurchaseId): ?RequestId =
|
||||
return client.getPurchase(id).option.?requestId
|
||||
|
||||
proc uploadRaw*(client: CodexClient, contents: string, headers = newHttpHeaders()): Response =
|
||||
return client.http.request(client.baseurl & "/data", body = contents, httpMethod=HttpPost, headers = headers)
|
||||
|
||||
proc listRaw*(client: CodexClient): Response =
|
||||
return client.http.request(client.baseurl & "/data", httpMethod=HttpGet)
|
||||
|
||||
proc downloadRaw*(client: CodexClient, cid: string, local = false): Response =
|
||||
return client.http.request(client.baseurl & "/data/" & cid &
|
||||
(if local: "" else: "/network/stream"), httpMethod=HttpGet)
|
||||
|
|
@ -115,7 +115,7 @@ method onOutputLineCaptured(node: HardhatProcess, line: string) =
|
|||
return
|
||||
|
||||
if error =? logFile.writeFile(line & "\n").errorOption:
|
||||
error "failed to write to hardhat file", errorCode = error
|
||||
error "failed to write to hardhat file", errorCode = $error
|
||||
discard logFile.closeFile()
|
||||
node.logFile = none IoHandle
|
||||
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ template marketplacesuite*(name: string, body: untyped) =
|
|||
marketplace = Marketplace.new(Marketplace.address, ethProvider.getSigner())
|
||||
let tokenAddress = await marketplace.token()
|
||||
token = Erc20Token.new(tokenAddress, ethProvider.getSigner())
|
||||
let config = await mp.config(marketplace)
|
||||
let config = await marketplace.configuration()
|
||||
period = config.proofs.period.truncate(uint64)
|
||||
periodicity = Periodicity(seconds: period.u256)
|
||||
|
||||
|
|
|
|||
|
|
@ -128,7 +128,7 @@ method stop*(node: NodeProcess) {.base, async.} =
|
|||
try:
|
||||
trace "terminating node process..."
|
||||
if errCode =? node.process.terminate().errorOption:
|
||||
error "failed to terminate process", errCode
|
||||
error "failed to terminate process", errCode = $errCode
|
||||
|
||||
trace "waiting for node process to exit"
|
||||
let exitCode = await node.process.waitForExit(3.seconds)
|
||||
|
|
|
|||
|
|
@ -4,9 +4,9 @@ from pkg/libp2p import `==`
|
|||
import pkg/codex/units
|
||||
import ./twonodes
|
||||
import ../examples
|
||||
import json
|
||||
|
||||
twonodessuite "REST API", debug1 = false, debug2 = false:
|
||||
|
||||
test "nodes can print their peer information":
|
||||
check !client1.info() != !client2.info()
|
||||
|
||||
|
|
@ -16,6 +16,7 @@ twonodessuite "REST API", debug1 = false, debug2 = false:
|
|||
test "node accepts file uploads":
|
||||
let cid1 = client1.upload("some file contents").get
|
||||
let cid2 = client1.upload("some other contents").get
|
||||
|
||||
check cid1 != cid2
|
||||
|
||||
test "node shows used and available space":
|
||||
|
|
@ -25,7 +26,7 @@ twonodessuite "REST API", debug1 = false, debug2 = false:
|
|||
check:
|
||||
space.totalBlocks == 2
|
||||
space.quotaMaxBytes == 8589934592.NBytes
|
||||
space.quotaUsedBytes == 65592.NBytes
|
||||
space.quotaUsedBytes == 65598.NBytes
|
||||
space.quotaReservedBytes == 12.NBytes
|
||||
|
||||
test "node lists local files":
|
||||
|
|
@ -151,3 +152,89 @@ twonodessuite "REST API", debug1 = false, debug2 = false:
|
|||
tolerance.uint)
|
||||
|
||||
check responseBefore.status == "200 OK"
|
||||
|
||||
test "node accepts file uploads with content type":
|
||||
let headers = newHttpHeaders({"Content-Type": "text/plain"})
|
||||
let response = client1.uploadRaw("some file contents", headers)
|
||||
|
||||
check response.status == "200 OK"
|
||||
check response.body != ""
|
||||
|
||||
test "node accepts file uploads with content disposition":
|
||||
let headers = newHttpHeaders({"Content-Disposition": "attachment; filename=\"example.txt\""})
|
||||
let response = client1.uploadRaw("some file contents", headers)
|
||||
|
||||
check response.status == "200 OK"
|
||||
check response.body != ""
|
||||
|
||||
test "node accepts file uploads with content disposition without filename":
|
||||
let headers = newHttpHeaders({"Content-Disposition": "attachment"})
|
||||
let response = client1.uploadRaw("some file contents", headers)
|
||||
|
||||
check response.status == "200 OK"
|
||||
check response.body != ""
|
||||
|
||||
test "upload fails if content disposition contains bad filename":
|
||||
let headers = newHttpHeaders({"Content-Disposition": "attachment; filename=\"exam*ple.txt\""})
|
||||
let response = client1.uploadRaw("some file contents", headers)
|
||||
|
||||
check response.status == "422 Unprocessable Entity"
|
||||
check response.body == "The filename is not valid."
|
||||
|
||||
test "upload fails if content type is invalid":
|
||||
let headers = newHttpHeaders({"Content-Type": "hello/world"})
|
||||
let response = client1.uploadRaw("some file contents", headers)
|
||||
|
||||
check response.status == "422 Unprocessable Entity"
|
||||
check response.body == "The MIME type is not valid."
|
||||
|
||||
test "node retrieve the metadata":
|
||||
let headers = newHttpHeaders({"Content-Type": "text/plain", "Content-Disposition": "attachment; filename=\"example.txt\""})
|
||||
let uploadResponse = client1.uploadRaw("some file contents", headers)
|
||||
let cid = uploadResponse.body
|
||||
let listResponse = client1.listRaw()
|
||||
|
||||
let jsonData = parseJson(listResponse.body)
|
||||
|
||||
check jsonData.hasKey("content") == true
|
||||
|
||||
let content = jsonData["content"][0]
|
||||
|
||||
check content.hasKey("manifest") == true
|
||||
|
||||
let manifest = content["manifest"]
|
||||
|
||||
check manifest.hasKey("filename") == true
|
||||
check manifest["filename"].getStr() == "example.txt"
|
||||
check manifest.hasKey("mimetype") == true
|
||||
check manifest["mimetype"].getStr() == "text/plain"
|
||||
check manifest.hasKey("uploadedAt") == true
|
||||
check manifest["uploadedAt"].getInt() > 0
|
||||
|
||||
test "node set the headers when for download":
|
||||
let headers = newHttpHeaders({
|
||||
"Content-Disposition": "attachment; filename=\"example.txt\"",
|
||||
"Content-Type": "text/plain"
|
||||
})
|
||||
|
||||
let uploadResponse = client1.uploadRaw("some file contents", headers)
|
||||
let cid = uploadResponse.body
|
||||
|
||||
check uploadResponse.status == "200 OK"
|
||||
|
||||
let response = client1.downloadRaw(cid)
|
||||
|
||||
check response.status == "200 OK"
|
||||
check response.headers.hasKey("Content-Type") == true
|
||||
check response.headers["Content-Type"] == "text/plain"
|
||||
check response.headers.hasKey("Content-Disposition") == true
|
||||
check response.headers["Content-Disposition"] == "attachment; filename=\"example.txt\""
|
||||
|
||||
let local = true
|
||||
let localResponse = client1.downloadRaw(cid, local)
|
||||
|
||||
check localResponse.status == "200 OK"
|
||||
check localResponse.headers.hasKey("Content-Type") == true
|
||||
check localResponse.headers["Content-Type"] == "text/plain"
|
||||
check localResponse.headers.hasKey("Content-Disposition") == true
|
||||
check localResponse.headers["Content-Disposition"] == "attachment; filename=\"example.txt\""
|
||||
|
|
@ -1,5 +1,7 @@
|
|||
import pkg/codex/rest/json
|
||||
import ./twonodes
|
||||
import json
|
||||
from pkg/libp2p import Cid, `$`
|
||||
|
||||
twonodessuite "Uploads and downloads", debug1 = false, debug2 = false:
|
||||
|
||||
|
|
@ -39,24 +41,42 @@ twonodessuite "Uploads and downloads", debug1 = false, debug2 = false:
|
|||
check:
|
||||
resp2.error.msg == "404 Not Found"
|
||||
|
||||
proc checkRestContent(content: ?!string) =
|
||||
proc checkRestContent(cid: Cid, content: ?!string) =
|
||||
let c = content.tryGet()
|
||||
|
||||
# tried to JSON (very easy) and checking the resulting object (would be much nicer)
|
||||
# spent an hour to try and make it work.
|
||||
check:
|
||||
c == "{\"cid\":\"zDvZRwzm1ePSzKSXt57D5YxHwcSDmsCyYN65wW4HT7fuX9HrzFXy\",\"manifest\":{\"treeCid\":\"zDzSvJTezk7bJNQqFq8k1iHXY84psNuUfZVusA5bBQQUSuyzDSVL\",\"datasetSize\":18,\"blockSize\":65536,\"protected\":false}}"
|
||||
let jsonData = parseJson(c)
|
||||
|
||||
check jsonData.hasKey("cid") == true
|
||||
|
||||
check jsonData["cid"].getStr() == $cid
|
||||
check jsonData.hasKey("manifest") == true
|
||||
|
||||
let manifest = jsonData["manifest"]
|
||||
|
||||
check manifest.hasKey("treeCid") == true
|
||||
check manifest["treeCid"].getStr() == "zDzSvJTezk7bJNQqFq8k1iHXY84psNuUfZVusA5bBQQUSuyzDSVL"
|
||||
check manifest.hasKey("datasetSize") == true
|
||||
check manifest["datasetSize"].getInt() == 18
|
||||
check manifest.hasKey("blockSize") == true
|
||||
check manifest["blockSize"].getInt() == 65536
|
||||
check manifest.hasKey("protected") == true
|
||||
check manifest["protected"].getBool() == false
|
||||
|
||||
test "node allows downloading only manifest":
|
||||
let content1 = "some file contents"
|
||||
let cid1 = client1.upload(content1).get
|
||||
let resp2 = client2.downloadManifestOnly(cid1)
|
||||
checkRestContent(resp2)
|
||||
|
||||
let resp2 = client1.downloadManifestOnly(cid1)
|
||||
checkRestContent(cid1, resp2)
|
||||
|
||||
test "node allows downloading content without stream":
|
||||
let content1 = "some file contents"
|
||||
let cid1 = client1.upload(content1).get
|
||||
|
||||
let resp1 = client2.downloadNoStream(cid1)
|
||||
checkRestContent(resp1)
|
||||
checkRestContent(cid1, resp1)
|
||||
let resp2 = client2.download(cid1, local = true).get
|
||||
check:
|
||||
content1 == resp2
|
||||
|
|
|
|||
|
|
@ -2,10 +2,11 @@ import std/os
|
|||
import std/osproc
|
||||
import std/options
|
||||
import pkg/chronos
|
||||
import codex/contracts
|
||||
import ../../integration/marketplacesuite
|
||||
import pkg/codex/contracts
|
||||
import ../../asynctest
|
||||
import ../../contracts/deployment
|
||||
|
||||
marketplacesuite "tools/cirdl":
|
||||
suite "tools/cirdl":
|
||||
const
|
||||
cirdl = "build" / "cirdl"
|
||||
workdir = "."
|
||||
|
|
@ -14,11 +15,11 @@ marketplacesuite "tools/cirdl":
|
|||
let
|
||||
circuitPath = "testcircuitpath"
|
||||
rpcEndpoint = "ws://localhost:8545"
|
||||
marketplaceAddress = $marketplace.address
|
||||
marketplaceAddress = Marketplace.address
|
||||
|
||||
discard existsOrCreateDir(circuitPath)
|
||||
|
||||
let args = [circuitPath, rpcEndpoint, marketplaceAddress]
|
||||
let args = [circuitPath, rpcEndpoint, $marketplaceAddress]
|
||||
|
||||
let process = osproc.startProcess(
|
||||
cirdl,
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ proc getCircuitHash(rpcEndpoint: string, marketplaceAddress: string): Future[?!s
|
|||
return failure("Invalid address: " & marketplaceAddress)
|
||||
|
||||
let marketplace = Marketplace.new(address, provider)
|
||||
let config = await marketplace.config()
|
||||
let config = await marketplace.configuration()
|
||||
return success config.proofs.zkeyHash
|
||||
|
||||
proc formatUrl(hash: string): string =
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
Subproject commit 807fc973c875b5bde8f517c71c818ba8f2f720dd
|
||||
Subproject commit 11ccefd720f1932608b67db95af5b72d73d1257b
|
||||
|
|
@ -1 +1 @@
|
|||
Subproject commit 5b170adcb1ffb1dbb273d1b7679bf3d9a08adb76
|
||||
Subproject commit 0ce6abf0fe942fe7cb1d14b8e4485621be9c3fe8
|
||||
|
|
@ -1 +1 @@
|
|||
Subproject commit 0bf2bcbe74a18a3c7a709d57108bb7b51e748a92
|
||||
Subproject commit 0408795be95c00d75e96eaef6eae8a9c734014f5
|
||||
|
|
@ -1 +1 @@
|
|||
Subproject commit bb53d49caf2a6c6cf1df365ba84af93cdcfa7aa3
|
||||
Subproject commit 5127b26ee58076e9369e7c126c196793c2b12e73
|
||||
|
|
@ -1 +1 @@
|
|||
Subproject commit b1e5e5d39a99ea56b750f6d9272dd319f4ad4291
|
||||
Subproject commit 83e4a2ccf621d3040c6e7e0267393ca2d205988e
|
||||
|
|
@ -1 +1 @@
|
|||
Subproject commit 384eb2561ee755446cff512a8e057325848b86a7
|
||||
Subproject commit f709bd9e16b1b6870fe3e4401196479e014a2ef6
|
||||
Loading…
Reference in New Issue