diff --git a/.github/actions/nimbus-build-system/action.yml b/.github/actions/nimbus-build-system/action.yml index e4f58209..b80c05c7 100644 --- a/.github/actions/nimbus-build-system/action.yml +++ b/.github/actions/nimbus-build-system/action.yml @@ -14,7 +14,7 @@ inputs: default: "version-1-6" rust_version: description: "Rust version" - default: "1.78.0" + default: "1.79.0" shell: description: "Shell to run commands in" default: "bash --noprofile --norc -e -o pipefail" diff --git a/.github/workflows/ci-reusable.yml b/.github/workflows/ci-reusable.yml index 591e2003..605edeed 100644 --- a/.github/workflows/ci-reusable.yml +++ b/.github/workflows/ci-reusable.yml @@ -24,7 +24,7 @@ jobs: run: shell: ${{ matrix.shell }} {0} - name: '${{ matrix.os }}-${{ matrix.cpu }}-${{ matrix.nim_version }}-${{ matrix.tests }}' + name: ${{ matrix.os }}-${{ matrix.tests }}-${{ matrix.cpu }}-${{ matrix.nim_version }} runs-on: ${{ matrix.builder }} timeout-minutes: 100 steps: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53f6b392..dbe18e64 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,8 +28,14 @@ jobs: uses: fabiocaccamo/create-matrix-action@v4 with: matrix: | - os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {all}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {macos}, cpu {amd64}, builder {macos-13}, tests {all}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {macos}, cpu {amd64}, builder {macos-13}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {macos}, cpu {amd64}, builder {macos-13}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {macos}, cpu {amd64}, builder {macos-13}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {macos}, cpu {amd64}, builder {macos-13}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} os {windows}, cpu {amd64}, builder {windows-latest}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {msys2} os {windows}, cpu {amd64}, builder {windows-latest}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {msys2} os {windows}, cpu {amd64}, builder {windows-latest}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {msys2} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 6ec42ebe..50b14d05 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -28,14 +28,13 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - fetch-depth: '0' + fetch-depth: 0 - uses: actions/setup-node@v4 with: node-version: 18 - name: Lint OpenAPI - shell: bash run: npx @redocly/cli lint openapi.yaml deploy: @@ -46,20 +45,22 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - fetch-depth: '0' + fetch-depth: 0 - uses: actions/setup-node@v4 with: node-version: 18 - name: Build OpenAPI - shell: bash - run: npx @redocly/cli build-docs openapi.yaml --output "openapi/index.html" --title "Codex API" + run: npx @redocly/cli build-docs openapi.yaml --output openapi/index.html --title "Codex API" + + - name: Build Postman Collection + run: npx -y openapi-to-postmanv2 -s openapi.yaml -o openapi/postman.json -p -O folderStrategy=Tags,includeAuthInfoInExample=false - name: Upload artifact uses: actions/upload-pages-artifact@v3 with: - path: './openapi' + path: openapi - name: Deploy to GitHub Pages uses: actions/deploy-pages@v4 diff --git a/Makefile b/Makefile index 5e6054c1..7c8f2b1a 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,30 @@ DOCKER_IMAGE_NIM_PARAMS ?= -d:chronicles_colors:none -d:insecure LINK_PCRE := 0 +ifeq ($(OS),Windows_NT) + ifeq ($(PROCESSOR_ARCHITECTURE), AMD64) + ARCH = x86_64 + endif + ifeq ($(PROCESSOR_ARCHITECTURE), ARM64) + ARCH = arm64 + endif +else + UNAME_P := $(shell uname -p) + ifneq ($(filter $(UNAME_P), i686 i386 x86_64),) + ARCH = x86_64 + endif + ifneq ($(filter $(UNAME_P), aarch64 arm),) + ARCH = arm64 + endif +endif + +ifeq ($(ARCH), x86_64) + CXXFLAGS ?= -std=c++17 -mssse3 +else + CXXFLAGS ?= -std=c++17 +endif +export CXXFLAGS + # we don't want an error here, so we can handle things later, in the ".DEFAULT" target -include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk diff --git a/codex/contracts/deployment.nim b/codex/contracts/deployment.nim index 1a4223c1..565b68fd 100644 --- a/codex/contracts/deployment.nim +++ b/codex/contracts/deployment.nim @@ -20,9 +20,9 @@ const knownAddresses = { "167005": { "Marketplace": Address.init("0x948CF9291b77Bd7ad84781b9047129Addf1b894F") }.toTable, - # Codex Testnet - Oct 21 2024 07:31:50 AM (+00:00 UTC) + # Codex Testnet - Nov 03 2024 07:30:30 AM (+00:00 UTC) "789987": { - "Marketplace": Address.init("0x3F9Cf3F40F0e87d804B776D8403e3d29F85211f4") + "Marketplace": Address.init("0x5Bd66fA15Eb0E546cd26808248867a572cFF5706") }.toTable }.toTable diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index 7cb639c8..b8d1da68 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -56,7 +56,7 @@ proc approveFunds(market: OnChainMarket, amount: UInt256) {.async.} = discard await token.increaseAllowance(market.contract.address(), amount).confirm(0) method getZkeyHash*(market: OnChainMarket): Future[?string] {.async.} = - let config = await market.contract.config() + let config = await market.contract.configuration() return some config.proofs.zkeyHash method getSigner*(market: OnChainMarket): Future[Address] {.async.} = @@ -65,18 +65,18 @@ method getSigner*(market: OnChainMarket): Future[Address] {.async.} = method periodicity*(market: OnChainMarket): Future[Periodicity] {.async.} = convertEthersError: - let config = await market.contract.config() + let config = await market.contract.configuration() let period = config.proofs.period return Periodicity(seconds: period) method proofTimeout*(market: OnChainMarket): Future[UInt256] {.async.} = convertEthersError: - let config = await market.contract.config() + let config = await market.contract.configuration() return config.proofs.timeout method proofDowntime*(market: OnChainMarket): Future[uint8] {.async.} = convertEthersError: - let config = await market.contract.config() + let config = await market.contract.configuration() return config.proofs.downtime method getPointer*(market: OnChainMarket, slotId: SlotId): Future[uint8] {.async.} = @@ -165,12 +165,18 @@ method fillSlot(market: OnChainMarket, proof: Groth16Proof, collateral: UInt256) {.async.} = convertEthersError: + logScope: + requestId + slotIndex + await market.approveFunds(collateral) + trace "calling fillSlot on contract" discard await market.contract.fillSlot(requestId, slotIndex, proof).confirm(0) + trace "fillSlot transaction completed" method freeSlot*(market: OnChainMarket, slotId: SlotId) {.async.} = convertEthersError: - var freeSlot: Future[?TransactionResponse] + var freeSlot: Future[Confirmable] if rewardRecipient =? market.rewardRecipient: # If --reward-recipient specified, use it as the reward recipient, and use # the SP's address as the collateral recipient @@ -253,7 +259,12 @@ method reserveSlot*( slotIndex: UInt256) {.async.} = convertEthersError: - discard await market.contract.reserveSlot(requestId, slotIndex).confirm(0) + discard await market.contract.reserveSlot( + requestId, + slotIndex, + # reserveSlot runs out of gas for unknown reason, but 100k gas covers it + TransactionOverrides(gasLimit: some 100000.u256) + ).confirm(0) method canReserveSlot*( market: OnChainMarket, diff --git a/codex/contracts/marketplace.nim b/codex/contracts/marketplace.nim index 6425bfa0..020f501e 100644 --- a/codex/contracts/marketplace.nim +++ b/codex/contracts/marketplace.nim @@ -17,18 +17,18 @@ export requests type Marketplace* = ref object of Contract -proc config*(marketplace: Marketplace): MarketplaceConfig {.contract, view.} +proc configuration*(marketplace: Marketplace): MarketplaceConfig {.contract, view.} proc token*(marketplace: Marketplace): Address {.contract, view.} proc slashMisses*(marketplace: Marketplace): UInt256 {.contract, view.} proc slashPercentage*(marketplace: Marketplace): UInt256 {.contract, view.} proc minCollateralThreshold*(marketplace: Marketplace): UInt256 {.contract, view.} -proc requestStorage*(marketplace: Marketplace, request: StorageRequest): ?TransactionResponse {.contract.} -proc fillSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256, proof: Groth16Proof): ?TransactionResponse {.contract.} -proc withdrawFunds*(marketplace: Marketplace, requestId: RequestId): ?TransactionResponse {.contract.} -proc withdrawFunds*(marketplace: Marketplace, requestId: RequestId, withdrawAddress: Address): ?TransactionResponse {.contract.} -proc freeSlot*(marketplace: Marketplace, id: SlotId): ?TransactionResponse {.contract.} -proc freeSlot*(marketplace: Marketplace, id: SlotId, rewardRecipient: Address, collateralRecipient: Address): ?TransactionResponse {.contract.} +proc requestStorage*(marketplace: Marketplace, request: StorageRequest): Confirmable {.contract.} +proc fillSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256, proof: Groth16Proof): Confirmable {.contract.} +proc withdrawFunds*(marketplace: Marketplace, requestId: RequestId): Confirmable {.contract.} +proc withdrawFunds*(marketplace: Marketplace, requestId: RequestId, withdrawAddress: Address): Confirmable {.contract.} +proc freeSlot*(marketplace: Marketplace, id: SlotId): Confirmable {.contract.} +proc freeSlot*(marketplace: Marketplace, id: SlotId, rewardRecipient: Address, collateralRecipient: Address): Confirmable {.contract.} proc getRequest*(marketplace: Marketplace, id: RequestId): StorageRequest {.contract, view.} proc getHost*(marketplace: Marketplace, id: SlotId): Address {.contract, view.} proc getActiveSlot*(marketplace: Marketplace, id: SlotId): Slot {.contract, view.} @@ -49,8 +49,8 @@ proc willProofBeRequired*(marketplace: Marketplace, id: SlotId): bool {.contract proc getChallenge*(marketplace: Marketplace, id: SlotId): array[32, byte] {.contract, view.} proc getPointer*(marketplace: Marketplace, id: SlotId): uint8 {.contract, view.} -proc submitProof*(marketplace: Marketplace, id: SlotId, proof: Groth16Proof): ?TransactionResponse {.contract.} -proc markProofAsMissing*(marketplace: Marketplace, id: SlotId, period: UInt256): ?TransactionResponse {.contract.} +proc submitProof*(marketplace: Marketplace, id: SlotId, proof: Groth16Proof): Confirmable {.contract.} +proc markProofAsMissing*(marketplace: Marketplace, id: SlotId, period: UInt256): Confirmable {.contract.} -proc reserveSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256): ?TransactionResponse {.contract.} +proc reserveSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256): Confirmable {.contract.} proc canReserveSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256): bool {.contract, view.} diff --git a/codex/manifest/coders.nim b/codex/manifest/coders.nim index e36039c7..4eed4299 100644 --- a/codex/manifest/coders.nim +++ b/codex/manifest/coders.nim @@ -10,6 +10,7 @@ # This module implements serialization and deserialization of Manifest import pkg/upraises +import times push: {.upraises: [].} @@ -59,6 +60,9 @@ proc encode*(manifest: Manifest): ?!seq[byte] = # optional hcodec: MultiCodec = 5 # Multihash codec # optional version: CidVersion = 6; # Cid version # optional ErasureInfo erasure = 7; # erasure coding info + # optional filename: ?string = 8; # original filename + # optional mimetype: ?string = 9; # original mimetype + # optional uploadedAt: ?int64 = 10; # original uploadedAt # } # ``` # @@ -70,6 +74,7 @@ proc encode*(manifest: Manifest): ?!seq[byte] = header.write(4, manifest.codec.uint32) header.write(5, manifest.hcodec.uint32) header.write(6, manifest.version.uint32) + if manifest.protected: var erasureInfo = initProtoBuffer() erasureInfo.write(1, manifest.ecK.uint32) @@ -90,6 +95,15 @@ proc encode*(manifest: Manifest): ?!seq[byte] = erasureInfo.finish() header.write(7, erasureInfo) + if manifest.filename.isSome: + header.write(8, manifest.filename.get()) + + if manifest.mimetype.isSome: + header.write(9, manifest.mimetype.get()) + + if manifest.uploadedAt.isSome: + header.write(10, manifest.uploadedAt.get().uint64) + pbNode.write(1, header) # set the treeCid as the data field pbNode.finish() @@ -118,6 +132,9 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest = slotRoots: seq[seq[byte]] cellSize: uint32 verifiableStrategy: uint32 + filename: string + mimetype: string + uploadedAt: uint64 # Decode `Header` message if pbNode.getField(1, pbHeader).isErr: @@ -145,6 +162,15 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest = if pbHeader.getField(7, pbErasureInfo).isErr: return failure("Unable to decode `erasureInfo` from manifest!") + if pbHeader.getField(8, filename).isErr: + return failure("Unable to decode `filename` from manifest!") + + if pbHeader.getField(9, mimetype).isErr: + return failure("Unable to decode `mimetype` from manifest!") + + if pbHeader.getField(10, uploadedAt).isErr: + return failure("Unable to decode `uploadedAt` from manifest!") + let protected = pbErasureInfo.buffer.len > 0 var verifiable = false if protected: @@ -183,6 +209,10 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest = let treeCid = ? Cid.init(treeCidBuf).mapFailure + var filenameOption = if filename.len == 0: string.none else: filename.some + var mimetypeOption = if mimetype.len == 0: string.none else: mimetype.some + var uploadedAtOption = if uploadedAt == 0: int64.none else: uploadedAt.int64.some + let self = if protected: Manifest.new( @@ -196,7 +226,10 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest = ecM = ecM.int, originalTreeCid = ? Cid.init(originalTreeCid).mapFailure, originalDatasetSize = originalDatasetSize.NBytes, - strategy = StrategyType(protectedStrategy)) + strategy = StrategyType(protectedStrategy), + filename = filenameOption, + mimetype = mimetypeOption, + uploadedAt = uploadedAtOption) else: Manifest.new( treeCid = treeCid, @@ -204,7 +237,10 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest = blockSize = blockSize.NBytes, version = CidVersion(version), hcodec = hcodec.MultiCodec, - codec = codec.MultiCodec) + codec = codec.MultiCodec, + filename = filenameOption, + mimetype = mimetypeOption, + uploadedAt = uploadedAtOption) ? self.verify() diff --git a/codex/manifest/manifest.nim b/codex/manifest/manifest.nim index 93aa5ba5..73644dd2 100644 --- a/codex/manifest/manifest.nim +++ b/codex/manifest/manifest.nim @@ -36,6 +36,9 @@ type codec: MultiCodec # Dataset codec hcodec: MultiCodec # Multihash codec version: CidVersion # Cid version + filename {.serialize.}: ?string # The filename of the content uploaded (optional) + mimetype {.serialize.}: ?string # The mimetype of the content uploaded (optional) + uploadedAt {.serialize.}: ?int64 # The UTC creation timestamp in seconds case protected {.serialize.}: bool # Protected datasets have erasure coded info of true: ecK: int # Number of blocks to encode @@ -121,6 +124,14 @@ func verifiableStrategy*(self: Manifest): StrategyType = func numSlotBlocks*(self: Manifest): int = divUp(self.blocksCount, self.numSlots) +func filename*(self: Manifest): ?string = + self.filename + +func mimetype*(self: Manifest): ?string = + self.mimetype + +func uploadedAt*(self: Manifest): ?int64 = + self.uploadedAt ############################################################ # Operations on block list ############################################################ @@ -163,6 +174,9 @@ func `==`*(a, b: Manifest): bool = (a.hcodec == b.hcodec) and (a.codec == b.codec) and (a.protected == b.protected) and + (a.filename == b.filename) and + (a.mimetype == b.mimetype) and + (a.uploadedAt == b.uploadedAt) and (if a.protected: (a.ecK == b.ecK) and (a.ecM == b.ecM) and @@ -181,26 +195,38 @@ func `==`*(a, b: Manifest): bool = true) func `$`*(self: Manifest): string = - "treeCid: " & $self.treeCid & + result = "treeCid: " & $self.treeCid & ", datasetSize: " & $self.datasetSize & ", blockSize: " & $self.blockSize & ", version: " & $self.version & ", hcodec: " & $self.hcodec & ", codec: " & $self.codec & - ", protected: " & $self.protected & - (if self.protected: - ", ecK: " & $self.ecK & - ", ecM: " & $self.ecM & - ", originalTreeCid: " & $self.originalTreeCid & - ", originalDatasetSize: " & $self.originalDatasetSize & - ", verifiable: " & $self.verifiable & - (if self.verifiable: - ", verifyRoot: " & $self.verifyRoot & - ", slotRoots: " & $self.slotRoots - else: - "") + ", protected: " & $self.protected + + if self.filename.isSome: + result &= ", filename: " & $self.filename + + if self.mimetype.isSome: + result &= ", mimetype: " & $self.mimetype + + if self.uploadedAt.isSome: + result &= ", uploadedAt: " & $self.uploadedAt + + result &= (if self.protected: + ", ecK: " & $self.ecK & + ", ecM: " & $self.ecM & + ", originalTreeCid: " & $self.originalTreeCid & + ", originalDatasetSize: " & $self.originalDatasetSize & + ", verifiable: " & $self.verifiable & + (if self.verifiable: + ", verifyRoot: " & $self.verifyRoot & + ", slotRoots: " & $self.slotRoots else: "") + else: + "") + + return result ############################################################ # Constructors @@ -214,7 +240,10 @@ func new*( version: CidVersion = CIDv1, hcodec = Sha256HashCodec, codec = BlockCodec, - protected = false): Manifest = + protected = false, + filename: ?string = string.none, + mimetype: ?string = string.none, + uploadedAt: ?int64 = int64.none): Manifest = T( treeCid: treeCid, @@ -223,7 +252,10 @@ func new*( version: version, codec: codec, hcodec: hcodec, - protected: protected) + protected: protected, + filename: filename, + mimetype: mimetype, + uploadedAt: uploadedAt) func new*( T: type Manifest, @@ -247,7 +279,11 @@ func new*( ecK: ecK, ecM: ecM, originalTreeCid: manifest.treeCid, originalDatasetSize: manifest.datasetSize, - protectedStrategy: strategy) + protectedStrategy: strategy, + filename: manifest.filename, + mimetype: manifest.mimetype, + uploadedAt: manifest.uploadedAt + ) func new*( T: type Manifest, @@ -263,7 +299,10 @@ func new*( codec: manifest.codec, hcodec: manifest.hcodec, blockSize: manifest.blockSize, - protected: false) + protected: false, + filename: manifest.filename, + mimetype: manifest.mimetype, + uploadedAt: manifest.uploadedAt) func new*( T: type Manifest, @@ -277,7 +316,10 @@ func new*( ecM: int, originalTreeCid: Cid, originalDatasetSize: NBytes, - strategy = SteppedStrategy): Manifest = + strategy = SteppedStrategy, + filename: ?string = string.none, + mimetype: ?string = string.none, + uploadedAt: ?int64 = int64.none): Manifest = Manifest( treeCid: treeCid, @@ -291,7 +333,10 @@ func new*( ecM: ecM, originalTreeCid: originalTreeCid, originalDatasetSize: originalDatasetSize, - protectedStrategy: strategy) + protectedStrategy: strategy, + filename: filename, + mimetype: mimetype, + uploadedAt: uploadedAt) func new*( T: type Manifest, @@ -329,7 +374,11 @@ func new*( verifyRoot: verifyRoot, slotRoots: @slotRoots, cellSize: cellSize, - verifiableStrategy: strategy) + verifiableStrategy: strategy, + filename: manifest.filename, + mimetype: manifest.mimetype, + uploadedAt: manifest.uploadedAt + ) func new*( T: type Manifest, diff --git a/codex/node.nim b/codex/node.nim index 88305a08..f180fd62 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -14,6 +14,7 @@ import std/sequtils import std/strformat import std/sugar import std/cpuinfo +import times import pkg/questionable import pkg/questionable/results @@ -297,6 +298,8 @@ proc retrieve*( proc store*( self: CodexNodeRef, stream: LPStream, + filename: ?string = string.none, + mimetype: ?string = string.none, blockSize = DefaultBlockSize): Future[?!Cid] {.async.} = ## Save stream contents as dataset with given blockSize ## to nodes's BlockStore, and return Cid of its manifest @@ -355,7 +358,10 @@ proc store*( datasetSize = NBytes(chunker.offset), version = CIDv1, hcodec = hcodec, - codec = dataCodec) + codec = dataCodec, + filename = filename, + mimetype = mimetype, + uploadedAt = now().utc.toTime.toUnix.some) without manifestBlk =? await self.storeManifest(manifest), err: error "Unable to store manifest" @@ -364,7 +370,9 @@ proc store*( info "Stored data", manifestCid = manifestBlk.cid, treeCid = treeCid, blocks = manifest.blocksCount, - datasetSize = manifest.datasetSize + datasetSize = manifest.datasetSize, + filename = manifest.filename, + mimetype = manifest.mimetype return manifestBlk.cid.success @@ -749,15 +757,15 @@ proc stop*(self: CodexNodeRef) {.async.} = if not self.discovery.isNil: await self.discovery.stop() - if not self.clock.isNil: - await self.clock.stop() - if clientContracts =? self.contracts.client: await clientContracts.stop() if hostContracts =? self.contracts.host: await hostContracts.stop() + if not self.clock.isNil: + await self.clock.stop() + if validatorContracts =? self.contracts.validator: await validatorContracts.stop() diff --git a/codex/rest/api.nim b/codex/rest/api.nim index a0531fde..d888276b 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -13,6 +13,8 @@ push: {.upraises: [].} import std/sequtils +import mimetypes +import os import pkg/questionable import pkg/questionable/results @@ -82,11 +84,27 @@ proc retrieveCid( try: without stream =? (await node.retrieve(cid, local)), error: if error of BlockNotFoundError: - return RestApiResponse.error(Http404, error.msg) + resp.status = Http404 + return await resp.sendBody("") else: - return RestApiResponse.error(Http500, error.msg) + resp.status = Http500 + return await resp.sendBody(error.msg) + + # It is ok to fetch again the manifest because it will hit the cache + without manifest =? (await node.fetchManifest(cid)), err: + error "Failed to fetch manifest", err = err.msg + resp.status = Http404 + return await resp.sendBody(err.msg) + + if manifest.mimetype.isSome: + resp.setHeader("Content-Type", manifest.mimetype.get()) + else: + resp.addHeader("Content-Type", "application/octet-stream") + + if manifest.filename.isSome: + resp.setHeader("Content-Disposition", "attachment; filename=\"" & manifest.filename.get() & "\"") + - resp.addHeader("Content-Type", "application/octet-stream") await resp.prepareChunked() while not stream.atEof: @@ -99,12 +117,14 @@ proc retrieveCid( break bytes += buff.len + await resp.sendChunk(addr buff[0], buff.len) await resp.finish() codex_api_downloads.inc() except CatchableError as exc: warn "Excepting streaming blocks", exc = exc.msg - return RestApiResponse.error(Http500) + resp.status = Http500 + return await resp.sendBody("") finally: info "Sent bytes", cid = cid, bytes if not stream.isNil: @@ -125,6 +145,18 @@ proc setCorsHeaders(resp: HttpResponseRef, httpMethod: string, origin: string) = resp.setHeader("Access-Control-Allow-Methods", httpMethod & ", OPTIONS") resp.setHeader("Access-Control-Max-Age", "86400") +proc getFilenameFromContentDisposition(contentDisposition: string): ?string = + if not("filename=" in contentDisposition): + return string.none + + let parts = contentDisposition.split("filename=\"") + + if parts.len < 2: + return string.none + + let filename = parts[1].strip() + return filename[0..^2].some + proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRouter) = let allowedOrigin = router.allowedOrigin # prevents capture inside of api defintion @@ -135,7 +167,7 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute if corsOrigin =? allowedOrigin: resp.setCorsHeaders("POST", corsOrigin) - resp.setHeader("Access-Control-Allow-Headers", "content-type") + resp.setHeader("Access-Control-Allow-Headers", "content-type, content-disposition") resp.status = Http204 await resp.sendBody("") @@ -158,12 +190,31 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute # await request.handleExpect() + var mimetype = request.headers.getString(ContentTypeHeader).some + + if mimetype.get() != "": + var m = newMimetypes() + let extension = m.getExt(mimetype.get(), "") + if extension == "": + return RestApiResponse.error(Http422, "The MIME type is not valid.") + else: + mimetype = string.none + + const ContentDispositionHeader = "Content-Disposition" + let contentDisposition = request.headers.getString(ContentDispositionHeader) + let filename = getFilenameFromContentDisposition(contentDisposition) + + if filename.isSome and not isValidFilename(filename.get()): + return RestApiResponse.error(Http422, "The filename is not valid.") + + # Here we could check if the extension matches the filename if needed + let reader = bodyReader.get() try: without cid =? ( - await node.store(AsyncStreamWrapper.new(reader = AsyncStreamReader(reader)))), error: + await node.store(AsyncStreamWrapper.new(reader = AsyncStreamReader(reader)), filename = filename, mimetype = mimetype)), error: error "Error uploading file", exc = error.msg return RestApiResponse.error(Http500, error.msg) @@ -281,10 +332,6 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute Http400, $cid.error(), headers = headers) - if corsOrigin =? allowedOrigin: - resp.setCorsHeaders("GET", corsOrigin) - resp.setHeader("Access-Control-Headers", "X-Requested-With") - without manifest =? (await node.fetchManifest(cid.get())), err: error "Failed to fetch manifest", err = err.msg return RestApiResponse.error( @@ -542,7 +589,7 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) = try: without contracts =? node.contracts.client: return RestApiResponse.error(Http503, "Persistence is not enabled", headers = headers) - + without cid =? cid.tryGet.catch, error: return RestApiResponse.error(Http400, error.msg, headers = headers) diff --git a/codex/sales/states/filling.nim b/codex/sales/states/filling.nim index c96dd0b9..678e2a21 100644 --- a/codex/sales/states/filling.nim +++ b/codex/sales/states/filling.nim @@ -6,6 +6,8 @@ import ./errorhandling import ./filled import ./cancelled import ./failed +import ./ignored +import ./errored logScope: topics = "marketplace sales filling" @@ -22,16 +24,25 @@ method onCancelled*(state: SaleFilling, request: StorageRequest): ?State = method onFailed*(state: SaleFilling, request: StorageRequest): ?State = return some State(SaleFailed()) -method onSlotFilled*(state: SaleFilling, requestId: RequestId, - slotIndex: UInt256): ?State = - return some State(SaleFilled()) - method run(state: SaleFilling, machine: Machine): Future[?State] {.async.} = let data = SalesAgent(machine).data let market = SalesAgent(machine).context.market without (collateral =? data.request.?ask.?collateral): raiseAssert "Request not set" - debug "Filling slot", requestId = data.requestId, slotIndex = data.slotIndex - await market.fillSlot(data.requestId, data.slotIndex, state.proof, collateral) - debug "Waiting for slot filled event...", requestId = $data.requestId, slotIndex = $data.slotIndex + logScope: + requestId = data.requestId + slotIndex = data.slotIndex + + debug "Filling slot" + try: + await market.fillSlot(data.requestId, data.slotIndex, state.proof, collateral) + except MarketError as e: + if e.msg.contains "Slot is not free": + debug "Slot is already filled, ignoring slot" + return some State( SaleIgnored(reprocessSlot: false, returnBytes: true) ) + else: + return some State( SaleErrored(error: e) ) + # other CatchableErrors are handled "automatically" by the ErrorHandlingState + + return some State(SaleFilled()) diff --git a/codex/sales/states/slotreserving.nim b/codex/sales/states/slotreserving.nim index d4343fd3..7e143173 100644 --- a/codex/sales/states/slotreserving.nim +++ b/codex/sales/states/slotreserving.nim @@ -28,10 +28,6 @@ method onCancelled*(state: SaleSlotReserving, request: StorageRequest): ?State = method onFailed*(state: SaleSlotReserving, request: StorageRequest): ?State = return some State(SaleFailed()) -method onSlotFilled*(state: SaleSlotReserving, requestId: RequestId, - slotIndex: UInt256): ?State = - return some State(SaleFilled()) - method run*(state: SaleSlotReserving, machine: Machine): Future[?State] {.async.} = let agent = SalesAgent(machine) let data = agent.data @@ -48,7 +44,12 @@ method run*(state: SaleSlotReserving, machine: Machine): Future[?State] {.async. trace "Reserving slot" await market.reserveSlot(data.requestId, data.slotIndex) except MarketError as e: - return some State( SaleErrored(error: e) ) + if e.msg.contains "Reservation not allowed": + debug "Slot cannot be reserved, ignoring", error = e.msg + return some State( SaleIgnored(reprocessSlot: false, returnBytes: true) ) + else: + return some State( SaleErrored(error: e) ) + # other CatchableErrors are handled "automatically" by the ErrorHandlingState trace "Slot successfully reserved" return some State( SaleDownloading() ) diff --git a/codex/utils/asyncstatemachine.nim b/codex/utils/asyncstatemachine.nim index 9aeb3eab..3f15af31 100644 --- a/codex/utils/asyncstatemachine.nim +++ b/codex/utils/asyncstatemachine.nim @@ -59,31 +59,28 @@ proc onError(machine: Machine, error: ref CatchableError): Event = state.onError(error) proc run(machine: Machine, state: State) {.async.} = - try: - if next =? await state.run(machine): - machine.schedule(Event.transition(state, next)) - except CancelledError: - discard + if next =? await state.run(machine): + machine.schedule(Event.transition(state, next)) proc scheduler(machine: Machine) {.async.} = var running: Future[void] - try: - while machine.started: - let event = await machine.scheduled.get().track(machine) - if next =? event(machine.state): - if not running.isNil and not running.finished: - await running.cancelAndWait() - let fromState = if machine.state.isNil: "" else: $machine.state - machine.state = next - debug "enter state", state = machine.state, fromState - running = machine.run(machine.state) - running - .track(machine) - .catch((err: ref CatchableError) => - machine.schedule(machine.onError(err)) - ) - except CancelledError: - discard + while machine.started: + let event = await machine.scheduled.get().track(machine) + if next =? event(machine.state): + if not running.isNil and not running.finished: + trace "cancelling current state", state = $machine.state + await running.cancelAndWait() + let fromState = if machine.state.isNil: "" else: $machine.state + machine.state = next + debug "enter state", state = fromState & " => " & $machine.state + running = machine.run(machine.state) + running + .track(machine) + .cancelled(proc() = trace "state.run cancelled, swallowing", state = $machine.state) + .catch(proc(err: ref CatchableError) = + trace "error caught in state.run, calling state.onError", state = $machine.state + machine.schedule(machine.onError(err)) + ) proc start*(machine: Machine, initialState: State) = if machine.started: @@ -93,12 +90,13 @@ proc start*(machine: Machine, initialState: State) = machine.scheduled = newAsyncQueue[Event]() machine.started = true - machine.scheduler() - .track(machine) - .catch((err: ref CatchableError) => - error("Error in scheduler", error = err.msg) - ) - machine.schedule(Event.transition(machine.state, initialState)) + try: + discard machine.scheduler().track(machine) + machine.schedule(Event.transition(machine.state, initialState)) + except CancelledError as e: + discard + except CatchableError as e: + error("Error in scheduler", error = e.msg) proc stop*(machine: Machine) {.async.} = if not machine.started: diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 325025c9..c824c7c6 100644 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -45,12 +45,16 @@ fi # If marketplace is enabled from the testing environment, # The file has to be written before Codex starts. -if [ -n "${PRIV_KEY}" ]; then - echo ${PRIV_KEY} > "private.key" - chmod 600 "private.key" - export CODEX_ETH_PRIVATE_KEY="private.key" - echo "Private key set" -fi +for key in PRIV_KEY ETH_PRIVATE_KEY; do + keyfile="private.key" + if [[ -n "${!key}" ]]; then + [[ "${key}" == "PRIV_KEY" ]] && echo "PRIV_KEY variable is deprecated and will be removed in the next releases, please use ETH_PRIVATE_KEY instead!" + echo "${!key}" > "${keyfile}" + chmod 600 "${keyfile}" + export CODEX_ETH_PRIVATE_KEY="${keyfile}" + echo "Private key set" + fi +done # Circuit downloader # cirdl [circuitPath] [rpcEndpoint] [marketplaceAddress] diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000..2ea7b7ee --- /dev/null +++ b/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1729449015, + "narHash": "sha256-Gf04dXB0n4q0A9G5nTGH3zuMGr6jtJppqdeljxua1fo=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "89172919243df199fe237ba0f776c3e3e3d72367", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-24.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000..8778ca4a --- /dev/null +++ b/flake.nix @@ -0,0 +1,35 @@ +{ + description = "Codex build flake"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; + }; + + outputs = { self, nixpkgs }: + let + supportedSystems = [ + "x86_64-linux" "aarch64-linux" + "x86_64-darwin" "aarch64-darwin" + ]; + forAllSystems = f: nixpkgs.lib.genAttrs supportedSystems (system: f system); + pkgsFor = forAllSystems (system: import nixpkgs { inherit system; }); + in rec { + devShells = forAllSystems (system: let + pkgs = pkgsFor.${system}; + inherit (pkgs) lib stdenv mkShell; + in { + default = mkShell.override { stdenv = pkgs.gcc11Stdenv; } { + buildInputs = with pkgs; [ + # General + git pkg-config openssl lsb-release + # Build + rustc cargo nimble gcc11 cmake nim-unwrapped-1 + # Libraries + gmp llvmPackages.openmp + # Tests + nodejs_18 + ]; + }; + }); + }; +} \ No newline at end of file diff --git a/openapi.yaml b/openapi.yaml index c84072a7..931ad7b1 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -90,6 +90,40 @@ components: cid: $ref: "#/components/schemas/Cid" + Node: + type: object + properties: + nodeId: + type: string + peerId: + type: string + record: + type: string + address: + type: string + seen: + type: boolean + + CodexVersion: + type: object + properties: + version: + type: string + example: v0.1.7 + revision: + type: string + example: 0c647d8 + + PeersTable: + type: object + properties: + localNode: + $ref: "#/components/schemas/Node" + nodes: + type: array + items: + $ref: "#/components/schemas/Node" + DebugInfo: type: object properties: @@ -104,6 +138,10 @@ components: description: Path of the data repository where all nodes data are stored spr: $ref: "#/components/schemas/SPR" + table: + $ref: "#/components/schemas/PeersTable" + codex: + $ref: "#/components/schemas/CodexVersion" SalesAvailability: type: object @@ -306,10 +344,10 @@ components: ManifestItem: type: object properties: - rootHash: + treeCid: $ref: "#/components/schemas/Cid" - description: "Root hash of the content" - originalBytes: + description: "Unique data identifier" + datasetSize: type: integer format: int64 description: "Length of original content in bytes" @@ -319,6 +357,22 @@ components: protected: type: boolean description: "Indicates if content is protected by erasure-coding" + filename: + type: string + nullable: true + description: "The original name of the uploaded content (optional)" + example: codex.png + mimetype: + type: string + nullable: true + description: "The original mimetype of the uploaded content (optional)" + example: image/png + uploadedAt: + type: integer + format: int64 + nullable: true + description: "The UTC upload timestamp in seconds" + example: 1729244192 Space: type: object @@ -404,12 +458,29 @@ paths: description: Invalid CID is specified "404": description: Content specified by the CID is not found + "422": + description: The content type is not a valid content type or the filename is not valid "500": description: Well it was bad-bad post: summary: "Upload a file in a streaming manner. Once finished, the file is stored in the node and can be retrieved by any node in the network using the returned CID." tags: [ Data ] operationId: upload + parameters: + - name: content-type + in: header + required: false + description: The content type of the file. Must be valid. + schema: + type: string + example: "image/png" + - name: content-disposition + in: header + required: false + description: The content disposition used to send the filename. + schema: + type: string + example: "attachment; filename=\"codex.png\"" requestBody: content: application/octet-stream: diff --git a/tests/codex/sales/states/testfilling.nim b/tests/codex/sales/states/testfilling.nim index ef2e2aa4..e68e2e70 100644 --- a/tests/codex/sales/states/testfilling.nim +++ b/tests/codex/sales/states/testfilling.nim @@ -24,7 +24,3 @@ checksuite "sales state 'filling'": test "switches to failed state when request fails": let next = state.onFailed(request) check !next of SaleFailed - - test "switches to filled state when slot is filled": - let next = state.onSlotFilled(request.id, slotIndex) - check !next of SaleFilled diff --git a/tests/codex/sales/states/testslotreserving.nim b/tests/codex/sales/states/testslotreserving.nim index c54fb2aa..39419adc 100644 --- a/tests/codex/sales/states/testslotreserving.nim +++ b/tests/codex/sales/states/testslotreserving.nim @@ -51,10 +51,6 @@ asyncchecksuite "sales state 'SlotReserving'": let next = state.onFailed(request) check !next of SaleFailed - test "switches to filled state when slot is filled": - let next = state.onSlotFilled(request.id, slotIndex) - check !next of SaleFilled - test "run switches to downloading when slot successfully reserved": let next = await state.run(agent) check !next of SaleDownloading diff --git a/tests/codex/testmanifest.nim b/tests/codex/testmanifest.nim index 3393fa08..cacf5f7a 100644 --- a/tests/codex/testmanifest.nim +++ b/tests/codex/testmanifest.nim @@ -83,6 +83,8 @@ suite "Manifest - Attribute Inheritance": treeCid = Cid.example, blockSize = 1.MiBs, datasetSize = 100.MiBs, + filename = "codex.png".some, + mimetype = "image/png".some ), treeCid = Cid.example, datasetSize = 200.MiBs, @@ -107,3 +109,15 @@ suite "Manifest - Attribute Inheritance": ).tryGet() check verifiable.protectedStrategy == LinearStrategy + + test "Should preserve metadata for manifest in verifiable manifest": + var verifiable = Manifest.new( + manifest = makeProtectedManifest(SteppedStrategy), + verifyRoot = Cid.example, + slotRoots = @[Cid.example, Cid.example] + ).tryGet() + + check verifiable.filename.isSome == true + check verifiable.filename.get() == "codex.png" + check verifiable.mimetype.isSome == true + check verifiable.mimetype.get() == "image/png" diff --git a/tests/contracts/testContracts.nim b/tests/contracts/testContracts.nim index b098b80f..bbbf41aa 100644 --- a/tests/contracts/testContracts.nim +++ b/tests/contracts/testContracts.nim @@ -38,7 +38,7 @@ ethersuite "Marketplace contracts": let tokenAddress = await marketplace.token() token = Erc20Token.new(tokenAddress, ethProvider.getSigner()) - let config = await marketplace.config() + let config = await marketplace.configuration() periodicity = Periodicity(seconds: config.proofs.period) request = StorageRequest.example diff --git a/tests/contracts/testDeployment.nim b/tests/contracts/testDeployment.nim index 4101b71a..f89e28a8 100644 --- a/tests/contracts/testDeployment.nim +++ b/tests/contracts/testDeployment.nim @@ -9,7 +9,7 @@ import ../checktest type MockProvider = ref object of Provider chainId*: UInt256 -method getChainId*(provider: MockProvider): Future[UInt256] {.async.} = +method getChainId*(provider: MockProvider): Future[UInt256] {.async: (raises:[ProviderError]).} = return provider.chainId proc configFactory(): CodexConf = diff --git a/tests/contracts/testMarket.nim b/tests/contracts/testMarket.nim index 66088b71..a836628c 100644 --- a/tests/contracts/testMarket.nim +++ b/tests/contracts/testMarket.nim @@ -34,7 +34,7 @@ ethersuite "On-Chain Market": setup: let address = Marketplace.address(dummyVerifier = true) marketplace = Marketplace.new(address, ethProvider.getSigner()) - let config = await marketplace.config() + let config = await marketplace.configuration() hostRewardRecipient = accounts[2] market = OnChainMarket.new(marketplace) @@ -76,13 +76,13 @@ ethersuite "On-Chain Market": test "can retrieve proof periodicity": let periodicity = await market.periodicity() - let config = await marketplace.config() + let config = await marketplace.configuration() let periodLength = config.proofs.period check periodicity.seconds == periodLength test "can retrieve proof timeout": let proofTimeout = await market.proofTimeout() - let config = await marketplace.config() + let config = await marketplace.configuration() check proofTimeout == config.proofs.timeout test "supports marketplace requests": diff --git a/tests/contracts/time.nim b/tests/contracts/time.nim index cd6aac1b..ae448789 100644 --- a/tests/contracts/time.nim +++ b/tests/contracts/time.nim @@ -1,4 +1,5 @@ import pkg/ethers +import pkg/serde/json proc currentTime*(provider: Provider): Future[UInt256] {.async.} = return (!await provider.getBlock(BlockTag.pending)).timestamp diff --git a/tests/integration/codexclient.nim b/tests/integration/codexclient.nim index 69958cb2..0e415a86 100644 --- a/tests/integration/codexclient.nim +++ b/tests/integration/codexclient.nim @@ -257,3 +257,13 @@ proc saleStateIs*(client: CodexClient, id: SlotId, state: string): bool = proc requestId*(client: CodexClient, id: PurchaseId): ?RequestId = return client.getPurchase(id).option.?requestId + +proc uploadRaw*(client: CodexClient, contents: string, headers = newHttpHeaders()): Response = + return client.http.request(client.baseurl & "/data", body = contents, httpMethod=HttpPost, headers = headers) + +proc listRaw*(client: CodexClient): Response = + return client.http.request(client.baseurl & "/data", httpMethod=HttpGet) + +proc downloadRaw*(client: CodexClient, cid: string, local = false): Response = + return client.http.request(client.baseurl & "/data/" & cid & + (if local: "" else: "/network/stream"), httpMethod=HttpGet) \ No newline at end of file diff --git a/tests/integration/hardhatprocess.nim b/tests/integration/hardhatprocess.nim index 6cfab47d..0e88aa78 100644 --- a/tests/integration/hardhatprocess.nim +++ b/tests/integration/hardhatprocess.nim @@ -115,7 +115,7 @@ method onOutputLineCaptured(node: HardhatProcess, line: string) = return if error =? logFile.writeFile(line & "\n").errorOption: - error "failed to write to hardhat file", errorCode = error + error "failed to write to hardhat file", errorCode = $error discard logFile.closeFile() node.logFile = none IoHandle diff --git a/tests/integration/marketplacesuite.nim b/tests/integration/marketplacesuite.nim index 2b81bdd8..d3b1ef57 100644 --- a/tests/integration/marketplacesuite.nim +++ b/tests/integration/marketplacesuite.nim @@ -85,7 +85,7 @@ template marketplacesuite*(name: string, body: untyped) = marketplace = Marketplace.new(Marketplace.address, ethProvider.getSigner()) let tokenAddress = await marketplace.token() token = Erc20Token.new(tokenAddress, ethProvider.getSigner()) - let config = await mp.config(marketplace) + let config = await marketplace.configuration() period = config.proofs.period.truncate(uint64) periodicity = Periodicity(seconds: period.u256) diff --git a/tests/integration/nodeprocess.nim b/tests/integration/nodeprocess.nim index 97f4507f..61947c20 100644 --- a/tests/integration/nodeprocess.nim +++ b/tests/integration/nodeprocess.nim @@ -128,7 +128,7 @@ method stop*(node: NodeProcess) {.base, async.} = try: trace "terminating node process..." if errCode =? node.process.terminate().errorOption: - error "failed to terminate process", errCode + error "failed to terminate process", errCode = $errCode trace "waiting for node process to exit" let exitCode = await node.process.waitForExit(3.seconds) diff --git a/tests/integration/testrestapi.nim b/tests/integration/testrestapi.nim index 1848ba0e..361e616f 100644 --- a/tests/integration/testrestapi.nim +++ b/tests/integration/testrestapi.nim @@ -4,9 +4,9 @@ from pkg/libp2p import `==` import pkg/codex/units import ./twonodes import ../examples +import json twonodessuite "REST API", debug1 = false, debug2 = false: - test "nodes can print their peer information": check !client1.info() != !client2.info() @@ -16,6 +16,7 @@ twonodessuite "REST API", debug1 = false, debug2 = false: test "node accepts file uploads": let cid1 = client1.upload("some file contents").get let cid2 = client1.upload("some other contents").get + check cid1 != cid2 test "node shows used and available space": @@ -25,7 +26,7 @@ twonodessuite "REST API", debug1 = false, debug2 = false: check: space.totalBlocks == 2 space.quotaMaxBytes == 8589934592.NBytes - space.quotaUsedBytes == 65592.NBytes + space.quotaUsedBytes == 65598.NBytes space.quotaReservedBytes == 12.NBytes test "node lists local files": @@ -151,3 +152,89 @@ twonodessuite "REST API", debug1 = false, debug2 = false: tolerance.uint) check responseBefore.status == "200 OK" + + test "node accepts file uploads with content type": + let headers = newHttpHeaders({"Content-Type": "text/plain"}) + let response = client1.uploadRaw("some file contents", headers) + + check response.status == "200 OK" + check response.body != "" + + test "node accepts file uploads with content disposition": + let headers = newHttpHeaders({"Content-Disposition": "attachment; filename=\"example.txt\""}) + let response = client1.uploadRaw("some file contents", headers) + + check response.status == "200 OK" + check response.body != "" + + test "node accepts file uploads with content disposition without filename": + let headers = newHttpHeaders({"Content-Disposition": "attachment"}) + let response = client1.uploadRaw("some file contents", headers) + + check response.status == "200 OK" + check response.body != "" + + test "upload fails if content disposition contains bad filename": + let headers = newHttpHeaders({"Content-Disposition": "attachment; filename=\"exam*ple.txt\""}) + let response = client1.uploadRaw("some file contents", headers) + + check response.status == "422 Unprocessable Entity" + check response.body == "The filename is not valid." + + test "upload fails if content type is invalid": + let headers = newHttpHeaders({"Content-Type": "hello/world"}) + let response = client1.uploadRaw("some file contents", headers) + + check response.status == "422 Unprocessable Entity" + check response.body == "The MIME type is not valid." + + test "node retrieve the metadata": + let headers = newHttpHeaders({"Content-Type": "text/plain", "Content-Disposition": "attachment; filename=\"example.txt\""}) + let uploadResponse = client1.uploadRaw("some file contents", headers) + let cid = uploadResponse.body + let listResponse = client1.listRaw() + + let jsonData = parseJson(listResponse.body) + + check jsonData.hasKey("content") == true + + let content = jsonData["content"][0] + + check content.hasKey("manifest") == true + + let manifest = content["manifest"] + + check manifest.hasKey("filename") == true + check manifest["filename"].getStr() == "example.txt" + check manifest.hasKey("mimetype") == true + check manifest["mimetype"].getStr() == "text/plain" + check manifest.hasKey("uploadedAt") == true + check manifest["uploadedAt"].getInt() > 0 + + test "node set the headers when for download": + let headers = newHttpHeaders({ + "Content-Disposition": "attachment; filename=\"example.txt\"", + "Content-Type": "text/plain" + }) + + let uploadResponse = client1.uploadRaw("some file contents", headers) + let cid = uploadResponse.body + + check uploadResponse.status == "200 OK" + + let response = client1.downloadRaw(cid) + + check response.status == "200 OK" + check response.headers.hasKey("Content-Type") == true + check response.headers["Content-Type"] == "text/plain" + check response.headers.hasKey("Content-Disposition") == true + check response.headers["Content-Disposition"] == "attachment; filename=\"example.txt\"" + + let local = true + let localResponse = client1.downloadRaw(cid, local) + + check localResponse.status == "200 OK" + check localResponse.headers.hasKey("Content-Type") == true + check localResponse.headers["Content-Type"] == "text/plain" + check localResponse.headers.hasKey("Content-Disposition") == true + check localResponse.headers["Content-Disposition"] == "attachment; filename=\"example.txt\"" \ No newline at end of file diff --git a/tests/integration/testupdownload.nim b/tests/integration/testupdownload.nim index 242a868e..8077b1af 100644 --- a/tests/integration/testupdownload.nim +++ b/tests/integration/testupdownload.nim @@ -1,5 +1,7 @@ import pkg/codex/rest/json import ./twonodes +import json +from pkg/libp2p import Cid, `$` twonodessuite "Uploads and downloads", debug1 = false, debug2 = false: @@ -39,24 +41,42 @@ twonodessuite "Uploads and downloads", debug1 = false, debug2 = false: check: resp2.error.msg == "404 Not Found" - proc checkRestContent(content: ?!string) = + proc checkRestContent(cid: Cid, content: ?!string) = let c = content.tryGet() + # tried to JSON (very easy) and checking the resulting object (would be much nicer) # spent an hour to try and make it work. - check: - c == "{\"cid\":\"zDvZRwzm1ePSzKSXt57D5YxHwcSDmsCyYN65wW4HT7fuX9HrzFXy\",\"manifest\":{\"treeCid\":\"zDzSvJTezk7bJNQqFq8k1iHXY84psNuUfZVusA5bBQQUSuyzDSVL\",\"datasetSize\":18,\"blockSize\":65536,\"protected\":false}}" + let jsonData = parseJson(c) + + check jsonData.hasKey("cid") == true + + check jsonData["cid"].getStr() == $cid + check jsonData.hasKey("manifest") == true + + let manifest = jsonData["manifest"] + + check manifest.hasKey("treeCid") == true + check manifest["treeCid"].getStr() == "zDzSvJTezk7bJNQqFq8k1iHXY84psNuUfZVusA5bBQQUSuyzDSVL" + check manifest.hasKey("datasetSize") == true + check manifest["datasetSize"].getInt() == 18 + check manifest.hasKey("blockSize") == true + check manifest["blockSize"].getInt() == 65536 + check manifest.hasKey("protected") == true + check manifest["protected"].getBool() == false test "node allows downloading only manifest": let content1 = "some file contents" let cid1 = client1.upload(content1).get - let resp2 = client2.downloadManifestOnly(cid1) - checkRestContent(resp2) + + let resp2 = client1.downloadManifestOnly(cid1) + checkRestContent(cid1, resp2) test "node allows downloading content without stream": let content1 = "some file contents" let cid1 = client1.upload(content1).get + let resp1 = client2.downloadNoStream(cid1) - checkRestContent(resp1) + checkRestContent(cid1, resp1) let resp2 = client2.download(cid1, local = true).get check: content1 == resp2 diff --git a/tests/tools/cirdl/testcirdl.nim b/tests/tools/cirdl/testcirdl.nim index d8e71e79..b639be28 100644 --- a/tests/tools/cirdl/testcirdl.nim +++ b/tests/tools/cirdl/testcirdl.nim @@ -2,10 +2,11 @@ import std/os import std/osproc import std/options import pkg/chronos -import codex/contracts -import ../../integration/marketplacesuite +import pkg/codex/contracts +import ../../asynctest +import ../../contracts/deployment -marketplacesuite "tools/cirdl": +suite "tools/cirdl": const cirdl = "build" / "cirdl" workdir = "." @@ -14,11 +15,11 @@ marketplacesuite "tools/cirdl": let circuitPath = "testcircuitpath" rpcEndpoint = "ws://localhost:8545" - marketplaceAddress = $marketplace.address + marketplaceAddress = Marketplace.address discard existsOrCreateDir(circuitPath) - let args = [circuitPath, rpcEndpoint, marketplaceAddress] + let args = [circuitPath, rpcEndpoint, $marketplaceAddress] let process = osproc.startProcess( cirdl, diff --git a/tools/cirdl/cirdl.nim b/tools/cirdl/cirdl.nim index d1533ceb..11051672 100644 --- a/tools/cirdl/cirdl.nim +++ b/tools/cirdl/cirdl.nim @@ -34,7 +34,7 @@ proc getCircuitHash(rpcEndpoint: string, marketplaceAddress: string): Future[?!s return failure("Invalid address: " & marketplaceAddress) let marketplace = Marketplace.new(address, provider) - let config = await marketplace.config() + let config = await marketplace.configuration() return success config.proofs.zkeyHash proc formatUrl(hash: string): string = diff --git a/vendor/codex-contracts-eth b/vendor/codex-contracts-eth index 807fc973..11ccefd7 160000 --- a/vendor/codex-contracts-eth +++ b/vendor/codex-contracts-eth @@ -1 +1 @@ -Subproject commit 807fc973c875b5bde8f517c71c818ba8f2f720dd +Subproject commit 11ccefd720f1932608b67db95af5b72d73d1257b diff --git a/vendor/nim-ethers b/vendor/nim-ethers index 5b170adc..0ce6abf0 160000 --- a/vendor/nim-ethers +++ b/vendor/nim-ethers @@ -1 +1 @@ -Subproject commit 5b170adcb1ffb1dbb273d1b7679bf3d9a08adb76 +Subproject commit 0ce6abf0fe942fe7cb1d14b8e4485621be9c3fe8 diff --git a/vendor/nim-json-rpc b/vendor/nim-json-rpc index 0bf2bcbe..0408795b 160000 --- a/vendor/nim-json-rpc +++ b/vendor/nim-json-rpc @@ -1 +1 @@ -Subproject commit 0bf2bcbe74a18a3c7a709d57108bb7b51e748a92 +Subproject commit 0408795be95c00d75e96eaef6eae8a9c734014f5 diff --git a/vendor/nim-json-serialization b/vendor/nim-json-serialization index bb53d49c..5127b26e 160000 --- a/vendor/nim-json-serialization +++ b/vendor/nim-json-serialization @@ -1 +1 @@ -Subproject commit bb53d49caf2a6c6cf1df365ba84af93cdcfa7aa3 +Subproject commit 5127b26ee58076e9369e7c126c196793c2b12e73 diff --git a/vendor/nim-serde b/vendor/nim-serde index b1e5e5d3..83e4a2cc 160000 --- a/vendor/nim-serde +++ b/vendor/nim-serde @@ -1 +1 @@ -Subproject commit b1e5e5d39a99ea56b750f6d9272dd319f4ad4291 +Subproject commit 83e4a2ccf621d3040c6e7e0267393ca2d205988e diff --git a/vendor/nim-serialization b/vendor/nim-serialization index 384eb256..f709bd9e 160000 --- a/vendor/nim-serialization +++ b/vendor/nim-serialization @@ -1 +1 @@ -Subproject commit 384eb2561ee755446cff512a8e057325848b86a7 +Subproject commit f709bd9e16b1b6870fe3e4401196479e014a2ef6