Merge branch 'master' into async-profiling

# Conflicts:
#	vendor/codex-contracts-eth

.github/actions/nimbus-build-system/action.yml

@@ -14,7 +14,7 @@ inputs:
     default: "version-1-6"
   rust_version:
     description: "Rust version"
-    default: "1.78.0"
+    default: "1.79.0"
   shell:
     description: "Shell to run commands in"
     default: "bash --noprofile --norc -e -o pipefail"

.github/workflows/ci-reusable.yml

@@ -24,7 +24,7 @@ jobs:
       run:
         shell: ${{ matrix.shell }} {0}
 
-    name: '${{ matrix.os }}-${{ matrix.cpu }}-${{ matrix.nim_version }}-${{ matrix.tests }}'
+    name: ${{ matrix.os }}-${{ matrix.tests }}-${{ matrix.cpu }}-${{ matrix.nim_version }}
     runs-on: ${{ matrix.builder }}
     timeout-minutes: 100
     steps:

.github/workflows/ci.yml

@@ -28,8 +28,14 @@ jobs:
       uses: fabiocaccamo/create-matrix-action@v4
       with:
         matrix: |
-          os {linux},   cpu {amd64}, builder {ubuntu-20.04},   tests {all},         nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+          os {linux},   cpu {amd64}, builder {ubuntu-20.04},   tests {unittest},    nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
-          os {macos},   cpu {amd64}, builder {macos-13},       tests {all},         nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+          os {linux},   cpu {amd64}, builder {ubuntu-20.04},   tests {contract},    nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+          os {linux},   cpu {amd64}, builder {ubuntu-20.04},   tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+          os {linux},   cpu {amd64}, builder {ubuntu-20.04},   tests {tools},       nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+          os {macos},   cpu {amd64}, builder {macos-13},       tests {unittest},    nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+          os {macos},   cpu {amd64}, builder {macos-13},       tests {contract},    nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+          os {macos},   cpu {amd64}, builder {macos-13},       tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+          os {macos},   cpu {amd64}, builder {macos-13},       tests {tools},       nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
           os {windows}, cpu {amd64}, builder {windows-latest}, tests {unittest},    nim_version {${{ env.nim_version }}}, shell {msys2}
           os {windows}, cpu {amd64}, builder {windows-latest}, tests {contract},    nim_version {${{ env.nim_version }}}, shell {msys2}
           os {windows}, cpu {amd64}, builder {windows-latest}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {msys2}

.github/workflows/docs.yml

@@ -28,14 +28,13 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          fetch-depth: '0'
+          fetch-depth: 0
 
       - uses: actions/setup-node@v4
         with:
           node-version: 18
 
       - name: Lint OpenAPI
-        shell: bash
         run: npx @redocly/cli lint openapi.yaml
 
   deploy:
@@ -46,20 +45,22 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          fetch-depth: '0'
+          fetch-depth: 0
 
       - uses: actions/setup-node@v4
         with:
           node-version: 18
 
       - name: Build OpenAPI
-        shell: bash
+        run: npx @redocly/cli build-docs openapi.yaml --output openapi/index.html --title "Codex API"
-        run: npx @redocly/cli build-docs openapi.yaml --output "openapi/index.html" --title "Codex API"
+
+      - name: Build Postman Collection
+        run: npx -y openapi-to-postmanv2 -s openapi.yaml -o openapi/postman.json -p -O folderStrategy=Tags,includeAuthInfoInExample=false
 
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v3
         with:
-          path: './openapi'
+          path: openapi
 
       - name: Deploy to GitHub Pages
         uses: actions/deploy-pages@v4

Makefile

@@ -40,6 +40,30 @@ DOCKER_IMAGE_NIM_PARAMS ?= -d:chronicles_colors:none -d:insecure
 
 LINK_PCRE := 0
 
+ifeq ($(OS),Windows_NT)
+    ifeq ($(PROCESSOR_ARCHITECTURE), AMD64)
+        ARCH = x86_64
+    endif
+    ifeq ($(PROCESSOR_ARCHITECTURE), ARM64)
+        ARCH = arm64
+    endif
+else
+    UNAME_P := $(shell uname -p)
+    ifneq ($(filter $(UNAME_P), i686 i386 x86_64),)
+        ARCH = x86_64
+    endif
+    ifneq ($(filter $(UNAME_P), aarch64 arm),)
+        ARCH = arm64
+    endif
+endif
+
+ifeq ($(ARCH), x86_64)
+    CXXFLAGS ?= -std=c++17 -mssse3
+else
+    CXXFLAGS ?= -std=c++17
+endif
+export CXXFLAGS
+
 # we don't want an error here, so we can handle things later, in the ".DEFAULT" target
 -include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk
 

codex/contracts/deployment.nim

@@ -20,9 +20,9 @@ const knownAddresses = {
  "167005": {
   "Marketplace": Address.init("0x948CF9291b77Bd7ad84781b9047129Addf1b894F")
  }.toTable,
- # Codex Testnet - Oct 21 2024 07:31:50 AM (+00:00 UTC)
+ # Codex Testnet - Nov 03 2024 07:30:30 AM (+00:00 UTC)
  "789987": {
-  "Marketplace": Address.init("0x3F9Cf3F40F0e87d804B776D8403e3d29F85211f4")
+  "Marketplace": Address.init("0x5Bd66fA15Eb0E546cd26808248867a572cFF5706")
  }.toTable
 }.toTable
 

codex/contracts/market.nim

@@ -56,7 +56,7 @@ proc approveFunds(market: OnChainMarket, amount: UInt256) {.async.} =
     discard await token.increaseAllowance(market.contract.address(), amount).confirm(0)
 
 method getZkeyHash*(market: OnChainMarket): Future[?string] {.async.} =
-  let config = await market.contract.config()
+  let config = await market.contract.configuration()
   return some config.proofs.zkeyHash
 
 method getSigner*(market: OnChainMarket): Future[Address] {.async.} =
@@ -65,18 +65,18 @@ method getSigner*(market: OnChainMarket): Future[Address] {.async.} =
 
 method periodicity*(market: OnChainMarket): Future[Periodicity] {.async.} =
   convertEthersError:
-    let config = await market.contract.config()
+    let config = await market.contract.configuration()
     let period = config.proofs.period
     return Periodicity(seconds: period)
 
 method proofTimeout*(market: OnChainMarket): Future[UInt256] {.async.} =
   convertEthersError:
-    let config = await market.contract.config()
+    let config = await market.contract.configuration()
     return config.proofs.timeout
 
 method proofDowntime*(market: OnChainMarket): Future[uint8] {.async.} =
   convertEthersError:
-    let config = await market.contract.config()
+    let config = await market.contract.configuration()
     return config.proofs.downtime
 
 method getPointer*(market: OnChainMarket, slotId: SlotId): Future[uint8] {.async.} =
@@ -165,12 +165,18 @@ method fillSlot(market: OnChainMarket,
                 proof: Groth16Proof,
                 collateral: UInt256) {.async.} =
   convertEthersError:
+    logScope:
+      requestId
+      slotIndex
+
     await market.approveFunds(collateral)
+    trace "calling fillSlot on contract"
     discard await market.contract.fillSlot(requestId, slotIndex, proof).confirm(0)
+    trace "fillSlot transaction completed"
 
 method freeSlot*(market: OnChainMarket, slotId: SlotId) {.async.} =
   convertEthersError:
-    var freeSlot: Future[?TransactionResponse]
+    var freeSlot: Future[Confirmable]
     if rewardRecipient =? market.rewardRecipient:
       # If --reward-recipient specified, use it as the reward recipient, and use
       # the SP's address as the collateral recipient
@@ -253,7 +259,12 @@ method reserveSlot*(
   slotIndex: UInt256) {.async.} =
 
   convertEthersError:
-    discard await market.contract.reserveSlot(requestId, slotIndex).confirm(0)
+    discard await market.contract.reserveSlot(
+      requestId,
+      slotIndex,
+      # reserveSlot runs out of gas for unknown reason, but 100k gas covers it
+      TransactionOverrides(gasLimit: some 100000.u256)
+    ).confirm(0)
 
 method canReserveSlot*(
   market: OnChainMarket,

codex/contracts/marketplace.nim

@@ -17,18 +17,18 @@ export requests
 type
   Marketplace* = ref object of Contract
 
-proc config*(marketplace: Marketplace): MarketplaceConfig {.contract, view.}
+proc configuration*(marketplace: Marketplace): MarketplaceConfig {.contract, view.}
 proc token*(marketplace: Marketplace): Address {.contract, view.}
 proc slashMisses*(marketplace: Marketplace): UInt256 {.contract, view.}
 proc slashPercentage*(marketplace: Marketplace): UInt256 {.contract, view.}
 proc minCollateralThreshold*(marketplace: Marketplace): UInt256 {.contract, view.}
 
-proc requestStorage*(marketplace: Marketplace, request: StorageRequest): ?TransactionResponse {.contract.}
+proc requestStorage*(marketplace: Marketplace, request: StorageRequest): Confirmable {.contract.}
-proc fillSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256, proof: Groth16Proof): ?TransactionResponse {.contract.}
+proc fillSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256, proof: Groth16Proof): Confirmable {.contract.}
-proc withdrawFunds*(marketplace: Marketplace, requestId: RequestId): ?TransactionResponse {.contract.}
+proc withdrawFunds*(marketplace: Marketplace, requestId: RequestId): Confirmable {.contract.}
-proc withdrawFunds*(marketplace: Marketplace, requestId: RequestId, withdrawAddress: Address): ?TransactionResponse {.contract.}
+proc withdrawFunds*(marketplace: Marketplace, requestId: RequestId, withdrawAddress: Address): Confirmable {.contract.}
-proc freeSlot*(marketplace: Marketplace, id: SlotId): ?TransactionResponse {.contract.}
+proc freeSlot*(marketplace: Marketplace, id: SlotId): Confirmable {.contract.}
-proc freeSlot*(marketplace: Marketplace, id: SlotId, rewardRecipient: Address, collateralRecipient: Address): ?TransactionResponse {.contract.}
+proc freeSlot*(marketplace: Marketplace, id: SlotId, rewardRecipient: Address, collateralRecipient: Address): Confirmable {.contract.}
 proc getRequest*(marketplace: Marketplace, id: RequestId): StorageRequest {.contract, view.}
 proc getHost*(marketplace: Marketplace, id: SlotId): Address {.contract, view.}
 proc getActiveSlot*(marketplace: Marketplace, id: SlotId): Slot {.contract, view.}
@@ -49,8 +49,8 @@ proc willProofBeRequired*(marketplace: Marketplace, id: SlotId): bool {.contract
 proc getChallenge*(marketplace: Marketplace, id: SlotId): array[32, byte] {.contract, view.}
 proc getPointer*(marketplace: Marketplace, id: SlotId): uint8 {.contract, view.}
 
-proc submitProof*(marketplace: Marketplace, id: SlotId, proof: Groth16Proof): ?TransactionResponse {.contract.}
+proc submitProof*(marketplace: Marketplace, id: SlotId, proof: Groth16Proof): Confirmable {.contract.}
-proc markProofAsMissing*(marketplace: Marketplace, id: SlotId, period: UInt256): ?TransactionResponse {.contract.}
+proc markProofAsMissing*(marketplace: Marketplace, id: SlotId, period: UInt256): Confirmable {.contract.}
 
-proc reserveSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256): ?TransactionResponse {.contract.}
+proc reserveSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256): Confirmable {.contract.}
 proc canReserveSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256): bool {.contract, view.}

codex/manifest/coders.nim

@@ -10,6 +10,7 @@
 # This module implements serialization and deserialization of Manifest
 
 import pkg/upraises
+import times 
 
 push: {.upraises: [].}
 
@@ -59,6 +60,9 @@ proc encode*(manifest: Manifest): ?!seq[byte] =
   #     optional hcodec: MultiCodec = 5   # Multihash codec
   #     optional version: CidVersion = 6; # Cid version
   #     optional ErasureInfo erasure = 7; # erasure coding info
+  #     optional filename: ?string = 8;    # original filename
+  #     optional mimetype: ?string = 9;    # original mimetype
+  #     optional uploadedAt: ?int64 = 10;  # original uploadedAt
   #   }
   # ```
   #
@@ -70,6 +74,7 @@ proc encode*(manifest: Manifest): ?!seq[byte] =
   header.write(4, manifest.codec.uint32)
   header.write(5, manifest.hcodec.uint32)
   header.write(6, manifest.version.uint32)
+
   if manifest.protected:
     var erasureInfo = initProtoBuffer()
     erasureInfo.write(1, manifest.ecK.uint32)
@@ -90,6 +95,15 @@ proc encode*(manifest: Manifest): ?!seq[byte] =
     erasureInfo.finish()
     header.write(7, erasureInfo)
 
+  if manifest.filename.isSome:
+    header.write(8, manifest.filename.get())
+
+  if manifest.mimetype.isSome: 
+    header.write(9, manifest.mimetype.get())
+
+  if manifest.uploadedAt.isSome:
+    header.write(10, manifest.uploadedAt.get().uint64)
+
   pbNode.write(1, header) # set the treeCid as the data field
   pbNode.finish()
 
@@ -118,6 +132,9 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
     slotRoots: seq[seq[byte]]
     cellSize: uint32
     verifiableStrategy: uint32
+    filename: string
+    mimetype: string
+    uploadedAt: uint64
 
   # Decode `Header` message
   if pbNode.getField(1, pbHeader).isErr:
@@ -145,6 +162,15 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
   if pbHeader.getField(7, pbErasureInfo).isErr:
     return failure("Unable to decode `erasureInfo` from manifest!")
 
+  if pbHeader.getField(8, filename).isErr:
+    return failure("Unable to decode `filename` from manifest!")
+
+  if pbHeader.getField(9, mimetype).isErr:
+    return failure("Unable to decode `mimetype` from manifest!")
+
+  if pbHeader.getField(10, uploadedAt).isErr:
+    return failure("Unable to decode `uploadedAt` from manifest!")
+
   let protected = pbErasureInfo.buffer.len > 0
   var verifiable = false
   if protected:
@@ -183,6 +209,10 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
   let
     treeCid = ? Cid.init(treeCidBuf).mapFailure
 
+  var filenameOption = if filename.len == 0: string.none else: filename.some
+  var mimetypeOption = if mimetype.len == 0: string.none else: mimetype.some
+  var uploadedAtOption = if uploadedAt == 0: int64.none else: uploadedAt.int64.some
+
   let
     self = if protected:
       Manifest.new(
@@ -196,7 +226,10 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
         ecM = ecM.int,
         originalTreeCid = ? Cid.init(originalTreeCid).mapFailure,
         originalDatasetSize = originalDatasetSize.NBytes,
-        strategy = StrategyType(protectedStrategy))
+        strategy = StrategyType(protectedStrategy),
+        filename = filenameOption,
+        mimetype = mimetypeOption,
+        uploadedAt = uploadedAtOption)
       else:
         Manifest.new(
           treeCid = treeCid,
@@ -204,7 +237,10 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
           blockSize = blockSize.NBytes,
           version = CidVersion(version),
           hcodec = hcodec.MultiCodec,
-          codec = codec.MultiCodec)
+          codec = codec.MultiCodec,
+          filename = filenameOption,
+          mimetype = mimetypeOption,
+          uploadedAt = uploadedAtOption)
 
   ? self.verify()
 

codex/manifest/manifest.nim

@@ -36,6 +36,9 @@ type
     codec: MultiCodec                       # Dataset codec
     hcodec: MultiCodec                      # Multihash codec
     version: CidVersion                     # Cid version
+    filename {.serialize.}: ?string          # The filename of the content uploaded (optional)
+    mimetype {.serialize.}: ?string          # The mimetype of the content uploaded (optional)
+    uploadedAt {.serialize.}: ?int64          # The UTC creation timestamp in seconds
     case protected {.serialize.}: bool      # Protected datasets have erasure coded info
     of true:
       ecK: int                              # Number of blocks to encode
@@ -121,6 +124,14 @@ func verifiableStrategy*(self: Manifest): StrategyType =
 func numSlotBlocks*(self: Manifest): int =
   divUp(self.blocksCount, self.numSlots)
 
+func filename*(self: Manifest): ?string =
+  self.filename
+
+func mimetype*(self: Manifest): ?string =
+  self.mimetype
+
+func uploadedAt*(self: Manifest): ?int64 =
+  self.uploadedAt
 ############################################################
 # Operations on block list
 ############################################################
@@ -163,6 +174,9 @@ func `==`*(a, b: Manifest): bool =
   (a.hcodec == b.hcodec) and
   (a.codec == b.codec) and
   (a.protected == b.protected) and
+  (a.filename == b.filename) and
+  (a.mimetype == b.mimetype) and
+  (a.uploadedAt == b.uploadedAt) and
     (if a.protected:
       (a.ecK == b.ecK) and
       (a.ecM == b.ecM) and
@@ -181,26 +195,38 @@ func `==`*(a, b: Manifest): bool =
       true)
 
 func `$`*(self: Manifest): string =
-  "treeCid: " & $self.treeCid &
+  result = "treeCid: " & $self.treeCid &
     ", datasetSize: " & $self.datasetSize &
     ", blockSize: " & $self.blockSize &
     ", version: " & $self.version &
     ", hcodec: " & $self.hcodec &
     ", codec: " & $self.codec &
-    ", protected: " & $self.protected &
+    ", protected: " & $self.protected
-    (if self.protected:
+
-      ", ecK: " & $self.ecK &
+  if self.filename.isSome:
-      ", ecM: " & $self.ecM &
+    result &= ", filename: " & $self.filename
-      ", originalTreeCid: " & $self.originalTreeCid &
+
-      ", originalDatasetSize: " & $self.originalDatasetSize &
+  if self.mimetype.isSome:
-      ", verifiable: " & $self.verifiable &
+    result &= ", mimetype: " & $self.mimetype
-      (if self.verifiable:
+
-        ", verifyRoot: " & $self.verifyRoot &
+  if self.uploadedAt.isSome:
-        ", slotRoots: " & $self.slotRoots
+    result &= ", uploadedAt: " & $self.uploadedAt
-      else:
+
-        "")
+  result &= (if self.protected:
+    ", ecK: " & $self.ecK &
+    ", ecM: " & $self.ecM &
+    ", originalTreeCid: " & $self.originalTreeCid &
+    ", originalDatasetSize: " & $self.originalDatasetSize &
+    ", verifiable: " & $self.verifiable &
+    (if self.verifiable:
+      ", verifyRoot: " & $self.verifyRoot &
+      ", slotRoots: " & $self.slotRoots
     else:
       "")
+  else:
+    "")
+
+  return result
 
 ############################################################
 # Constructors
@@ -214,7 +240,10 @@ func new*(
   version: CidVersion = CIDv1,
   hcodec = Sha256HashCodec,
   codec = BlockCodec,
-  protected = false): Manifest =
+  protected = false,
+  filename: ?string = string.none,
+  mimetype: ?string = string.none,
+  uploadedAt: ?int64 = int64.none): Manifest =
 
   T(
     treeCid: treeCid,
@@ -223,7 +252,10 @@ func new*(
     version: version,
     codec: codec,
     hcodec: hcodec,
-    protected: protected)
+    protected: protected,
+    filename: filename,
+    mimetype: mimetype,
+    uploadedAt: uploadedAt)
 
 func new*(
   T: type Manifest,
@@ -247,7 +279,11 @@ func new*(
     ecK: ecK, ecM: ecM,
     originalTreeCid: manifest.treeCid,
     originalDatasetSize: manifest.datasetSize,
-    protectedStrategy: strategy)
+    protectedStrategy: strategy,
+    filename: manifest.filename,
+    mimetype: manifest.mimetype,
+    uploadedAt: manifest.uploadedAt
+    )
 
 func new*(
   T: type Manifest,
@@ -263,7 +299,10 @@ func new*(
     codec: manifest.codec,
     hcodec: manifest.hcodec,
     blockSize: manifest.blockSize,
-    protected: false)
+    protected: false,
+    filename: manifest.filename,
+    mimetype: manifest.mimetype,
+    uploadedAt: manifest.uploadedAt)
 
 func new*(
   T: type Manifest,
@@ -277,7 +316,10 @@ func new*(
   ecM: int,
   originalTreeCid: Cid,
   originalDatasetSize: NBytes,
-  strategy = SteppedStrategy): Manifest =
+  strategy = SteppedStrategy,
+  filename: ?string = string.none,
+  mimetype: ?string = string.none,
+  uploadedAt: ?int64 = int64.none): Manifest =
 
   Manifest(
     treeCid: treeCid,
@@ -291,7 +333,10 @@ func new*(
     ecM: ecM,
     originalTreeCid: originalTreeCid,
     originalDatasetSize: originalDatasetSize,
-    protectedStrategy: strategy)
+    protectedStrategy: strategy,
+    filename: filename, 
+    mimetype: mimetype,
+    uploadedAt: uploadedAt)
 
 func new*(
   T: type Manifest,
@@ -329,7 +374,11 @@ func new*(
     verifyRoot: verifyRoot,
     slotRoots: @slotRoots,
     cellSize: cellSize,
-    verifiableStrategy: strategy)
+    verifiableStrategy: strategy,
+    filename: manifest.filename,
+    mimetype: manifest.mimetype,
+    uploadedAt: manifest.uploadedAt
+    )
 
 func new*(
   T: type Manifest,

codex/node.nim

@@ -14,6 +14,7 @@ import std/sequtils
 import std/strformat
 import std/sugar
 import std/cpuinfo
+import times
 
 import pkg/questionable
 import pkg/questionable/results
@@ -297,6 +298,8 @@ proc retrieve*(
 proc store*(
   self: CodexNodeRef,
   stream: LPStream,
+  filename: ?string = string.none,
+  mimetype: ?string = string.none,
   blockSize = DefaultBlockSize): Future[?!Cid] {.async.} =
   ## Save stream contents as dataset with given blockSize
   ## to nodes's BlockStore, and return Cid of its manifest
@@ -355,7 +358,10 @@ proc store*(
     datasetSize = NBytes(chunker.offset),
     version = CIDv1,
     hcodec = hcodec,
-    codec = dataCodec)
+    codec = dataCodec,
+    filename = filename,
+    mimetype = mimetype,
+    uploadedAt = now().utc.toTime.toUnix.some)
 
   without manifestBlk =? await self.storeManifest(manifest), err:
     error "Unable to store manifest"
@@ -364,7 +370,9 @@ proc store*(
   info "Stored data", manifestCid = manifestBlk.cid,
                       treeCid = treeCid,
                       blocks = manifest.blocksCount,
-                      datasetSize = manifest.datasetSize
+                      datasetSize = manifest.datasetSize,
+                      filename = manifest.filename,
+                      mimetype = manifest.mimetype
 
   return manifestBlk.cid.success
 
@@ -749,15 +757,15 @@ proc stop*(self: CodexNodeRef) {.async.} =
   if not self.discovery.isNil:
     await self.discovery.stop()
 
-  if not self.clock.isNil:
-    await self.clock.stop()
-
   if clientContracts =? self.contracts.client:
     await clientContracts.stop()
 
   if hostContracts =? self.contracts.host:
     await hostContracts.stop()
 
+  if not self.clock.isNil:
+    await self.clock.stop()
+
   if validatorContracts =? self.contracts.validator:
     await validatorContracts.stop()
 

codex/rest/api.nim

@@ -13,6 +13,8 @@ push: {.upraises: [].}
 
 
 import std/sequtils
+import mimetypes
+import os
 
 import pkg/questionable
 import pkg/questionable/results
@@ -82,11 +84,27 @@ proc retrieveCid(
   try:
     without stream =? (await node.retrieve(cid, local)), error:
       if error of BlockNotFoundError:
-        return RestApiResponse.error(Http404, error.msg)
+        resp.status = Http404
+        return await resp.sendBody("")
       else:
-        return RestApiResponse.error(Http500, error.msg)
+        resp.status = Http500
+        return await resp.sendBody(error.msg)
+
+    # It is ok to fetch again the manifest because it will hit the cache
+    without manifest =? (await node.fetchManifest(cid)), err:
+      error "Failed to fetch manifest", err = err.msg
+      resp.status = Http404
+      return await resp.sendBody(err.msg)
+
+    if manifest.mimetype.isSome:
+      resp.setHeader("Content-Type", manifest.mimetype.get())
+    else:
+      resp.addHeader("Content-Type", "application/octet-stream")
+
+    if manifest.filename.isSome:
+      resp.setHeader("Content-Disposition", "attachment; filename=\"" & manifest.filename.get() & "\"")
+
 
-    resp.addHeader("Content-Type", "application/octet-stream")
     await resp.prepareChunked()
 
     while not stream.atEof:
@@ -99,12 +117,14 @@ proc retrieveCid(
         break
 
       bytes += buff.len
+
       await resp.sendChunk(addr buff[0], buff.len)
     await resp.finish()
     codex_api_downloads.inc()
   except CatchableError as exc:
     warn "Excepting streaming blocks", exc = exc.msg
-    return RestApiResponse.error(Http500)
+    resp.status = Http500
+    return await resp.sendBody("")
   finally:
     info "Sent bytes", cid = cid, bytes
     if not stream.isNil:
@@ -125,6 +145,18 @@ proc setCorsHeaders(resp: HttpResponseRef, httpMethod: string, origin: string) =
   resp.setHeader("Access-Control-Allow-Methods", httpMethod & ", OPTIONS")
   resp.setHeader("Access-Control-Max-Age", "86400")
 
+proc getFilenameFromContentDisposition(contentDisposition: string): ?string =
+  if not("filename=" in contentDisposition):
+    return string.none
+
+  let parts = contentDisposition.split("filename=\"")
+
+  if parts.len < 2:
+    return string.none
+
+  let filename = parts[1].strip()
+  return filename[0..^2].some
+
 proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRouter) =
   let allowedOrigin = router.allowedOrigin # prevents capture inside of api defintion
    
@@ -135,7 +167,7 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute
       
       if corsOrigin =? allowedOrigin:
         resp.setCorsHeaders("POST", corsOrigin)
-        resp.setHeader("Access-Control-Allow-Headers", "content-type")
+        resp.setHeader("Access-Control-Allow-Headers", "content-type, content-disposition")
 
       resp.status = Http204
       await resp.sendBody("")
@@ -158,12 +190,31 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute
       #
       await request.handleExpect()
 
+      var mimetype = request.headers.getString(ContentTypeHeader).some
+
+      if mimetype.get() != "":
+        var m = newMimetypes()
+        let extension = m.getExt(mimetype.get(), "")
+        if extension == "":
+            return RestApiResponse.error(Http422, "The MIME type is not valid.")
+      else:
+        mimetype = string.none
+
+      const ContentDispositionHeader = "Content-Disposition"
+      let contentDisposition = request.headers.getString(ContentDispositionHeader)
+      let filename = getFilenameFromContentDisposition(contentDisposition)
+
+      if filename.isSome and not isValidFilename(filename.get()):
+          return RestApiResponse.error(Http422, "The filename is not valid.")
+
+      # Here we could check if the extension matches the filename if needed
+
       let
         reader = bodyReader.get()
 
       try:
         without cid =? (
-          await node.store(AsyncStreamWrapper.new(reader = AsyncStreamReader(reader)))), error:
+          await node.store(AsyncStreamWrapper.new(reader = AsyncStreamReader(reader)), filename = filename, mimetype = mimetype)), error:
           error "Error uploading file", exc = error.msg
           return RestApiResponse.error(Http500, error.msg)
 
@@ -281,10 +332,6 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute
           Http400,
           $cid.error(), headers = headers)
 
-      if corsOrigin =? allowedOrigin:
-        resp.setCorsHeaders("GET", corsOrigin)
-        resp.setHeader("Access-Control-Headers", "X-Requested-With")
-
       without manifest =? (await node.fetchManifest(cid.get())), err:
         error "Failed to fetch manifest", err = err.msg
         return RestApiResponse.error(

codex/sales/states/filling.nim

@@ -6,6 +6,8 @@ import ./errorhandling
 import ./filled
 import ./cancelled
 import ./failed
+import ./ignored
+import ./errored
 
 logScope:
   topics = "marketplace sales filling"
@@ -22,16 +24,25 @@ method onCancelled*(state: SaleFilling, request: StorageRequest): ?State =
 method onFailed*(state: SaleFilling, request: StorageRequest): ?State =
   return some State(SaleFailed())
 
-method onSlotFilled*(state: SaleFilling, requestId: RequestId,
-                     slotIndex: UInt256): ?State =
-  return some State(SaleFilled())
-
 method run(state: SaleFilling, machine: Machine): Future[?State] {.async.} =
   let data = SalesAgent(machine).data
   let market = SalesAgent(machine).context.market
   without (collateral =? data.request.?ask.?collateral):
     raiseAssert "Request not set"
 
-  debug "Filling slot", requestId = data.requestId, slotIndex = data.slotIndex
+  logScope:
-  await market.fillSlot(data.requestId, data.slotIndex, state.proof, collateral)
+    requestId = data.requestId
-  debug "Waiting for slot filled event...", requestId = $data.requestId, slotIndex = $data.slotIndex
+    slotIndex = data.slotIndex
+
+  debug "Filling slot"
+  try:
+    await market.fillSlot(data.requestId, data.slotIndex, state.proof, collateral)
+  except MarketError as e:
+    if e.msg.contains "Slot is not free":
+      debug "Slot is already filled, ignoring slot"
+      return some State( SaleIgnored(reprocessSlot: false, returnBytes: true) )
+    else:
+      return some State( SaleErrored(error: e) )
+  # other CatchableErrors are handled "automatically" by the ErrorHandlingState
+
+  return some State(SaleFilled())

codex/sales/states/slotreserving.nim

@@ -28,10 +28,6 @@ method onCancelled*(state: SaleSlotReserving, request: StorageRequest): ?State =
 method onFailed*(state: SaleSlotReserving, request: StorageRequest): ?State =
   return some State(SaleFailed())
 
-method onSlotFilled*(state: SaleSlotReserving, requestId: RequestId,
-                     slotIndex: UInt256): ?State =
-  return some State(SaleFilled())
-
 method run*(state: SaleSlotReserving, machine: Machine): Future[?State] {.async.} =
   let agent = SalesAgent(machine)
   let data = agent.data
@@ -48,7 +44,12 @@ method run*(state: SaleSlotReserving, machine: Machine): Future[?State] {.async.
       trace "Reserving slot"
       await market.reserveSlot(data.requestId, data.slotIndex)
     except MarketError as e:
-      return some State( SaleErrored(error: e) )
+      if e.msg.contains "Reservation not allowed":
+        debug "Slot cannot be reserved, ignoring", error = e.msg
+        return some State( SaleIgnored(reprocessSlot: false, returnBytes: true) )
+      else:
+        return some State( SaleErrored(error: e) )
+    # other CatchableErrors are handled "automatically" by the ErrorHandlingState
 
     trace "Slot successfully reserved"
     return some State( SaleDownloading() )

codex/utils/asyncstatemachine.nim

@@ -59,31 +59,28 @@ proc onError(machine: Machine, error: ref CatchableError): Event =
     state.onError(error)
 
 proc run(machine: Machine, state: State) {.async.} =
-  try:
+  if next =? await state.run(machine):
-    if next =? await state.run(machine):
+    machine.schedule(Event.transition(state, next))
-      machine.schedule(Event.transition(state, next))
-  except CancelledError:
-    discard
 
 proc scheduler(machine: Machine) {.async.} =
   var running: Future[void]
-  try:
+  while machine.started:
-    while machine.started:
+    let event = await machine.scheduled.get().track(machine)
-      let event = await machine.scheduled.get().track(machine)
+    if next =? event(machine.state):
-      if next =? event(machine.state):
+      if not running.isNil and not running.finished:
-        if not running.isNil and not running.finished:
+        trace "cancelling current state", state = $machine.state
-          await running.cancelAndWait()
+        await running.cancelAndWait()
-        let fromState = if machine.state.isNil: "<none>" else: $machine.state
+      let fromState = if machine.state.isNil: "<none>" else: $machine.state
-        machine.state = next
+      machine.state = next
-        debug "enter state", state = machine.state, fromState
+      debug "enter state", state = fromState & " => " & $machine.state
-        running = machine.run(machine.state)
+      running = machine.run(machine.state)
-        running
+      running
-          .track(machine)
+        .track(machine)
-          .catch((err: ref CatchableError) =>
+        .cancelled(proc() = trace "state.run cancelled, swallowing", state = $machine.state)
-            machine.schedule(machine.onError(err))
+        .catch(proc(err: ref CatchableError) =
-          )
+          trace "error caught in state.run, calling state.onError", state = $machine.state
-  except CancelledError:
+          machine.schedule(machine.onError(err))
-    discard
+        )
 
 proc start*(machine: Machine, initialState: State) =
   if machine.started:
@@ -93,12 +90,13 @@ proc start*(machine: Machine, initialState: State) =
     machine.scheduled = newAsyncQueue[Event]()
 
   machine.started = true
-  machine.scheduler()
+  try:
-    .track(machine)
+    discard machine.scheduler().track(machine)
-    .catch((err: ref CatchableError) =>
+    machine.schedule(Event.transition(machine.state, initialState))
-      error("Error in scheduler", error = err.msg)
+  except CancelledError as e:
-    )
+    discard
-  machine.schedule(Event.transition(machine.state, initialState))
+  except CatchableError as e:
+    error("Error in scheduler", error = e.msg)
 
 proc stop*(machine: Machine) {.async.} =
   if not machine.started:

docker/docker-entrypoint.sh

@@ -45,12 +45,16 @@ fi
 
 # If marketplace is enabled from the testing environment,
 # The file has to be written before Codex starts.
-if [ -n "${PRIV_KEY}" ]; then
+for key in PRIV_KEY ETH_PRIVATE_KEY; do
-  echo ${PRIV_KEY} > "private.key"
+  keyfile="private.key"
-  chmod 600 "private.key"
+  if [[ -n "${!key}" ]]; then
-  export CODEX_ETH_PRIVATE_KEY="private.key"
+    [[ "${key}" == "PRIV_KEY" ]] && echo "PRIV_KEY variable is deprecated and will be removed in the next releases, please use ETH_PRIVATE_KEY instead!"
-  echo "Private key set"
+    echo "${!key}" > "${keyfile}"
-fi
+    chmod 600 "${keyfile}"
+    export CODEX_ETH_PRIVATE_KEY="${keyfile}"
+    echo "Private key set"
+  fi
+done
 
 # Circuit downloader
 # cirdl [circuitPath] [rpcEndpoint] [marketplaceAddress]

flake.lock

@@ -0,0 +1,27 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1729449015,
+        "narHash": "sha256-Gf04dXB0n4q0A9G5nTGH3zuMGr6jtJppqdeljxua1fo=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "89172919243df199fe237ba0f776c3e3e3d72367",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-24.05",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,35 @@
+{
+  description = "Codex build flake";
+  
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05";
+  };
+
+  outputs = { self, nixpkgs }:
+    let
+      supportedSystems = [
+        "x86_64-linux" "aarch64-linux"
+        "x86_64-darwin" "aarch64-darwin"
+      ];
+      forAllSystems = f: nixpkgs.lib.genAttrs supportedSystems (system: f system);
+      pkgsFor = forAllSystems (system: import nixpkgs { inherit system; });
+    in rec {
+      devShells = forAllSystems (system: let
+        pkgs = pkgsFor.${system};
+        inherit (pkgs) lib stdenv mkShell;
+      in {
+        default = mkShell.override { stdenv = pkgs.gcc11Stdenv; } {
+          buildInputs = with pkgs; [
+              # General
+              git pkg-config openssl lsb-release
+              # Build
+              rustc cargo nimble gcc11 cmake nim-unwrapped-1
+              # Libraries
+              gmp llvmPackages.openmp
+              # Tests
+              nodejs_18
+          ];
+        };
+      });
+    };
+}

openapi.yaml

@@ -90,6 +90,40 @@ components:
         cid:
           $ref: "#/components/schemas/Cid"
 
+    Node:
+      type: object
+      properties:
+        nodeId:
+          type: string
+        peerId:
+          type: string
+        record:
+          type: string
+        address:
+          type: string
+        seen:
+          type: boolean
+
+    CodexVersion:
+      type: object
+      properties:
+        version:
+          type: string
+          example: v0.1.7
+        revision:
+          type: string
+          example: 0c647d8
+
+    PeersTable:
+      type: object
+      properties:
+        localNode:
+          $ref: "#/components/schemas/Node"
+        nodes:
+          type: array
+          items:
+            $ref: "#/components/schemas/Node"
+
     DebugInfo:
       type: object
       properties:
@@ -104,6 +138,10 @@ components:
           description: Path of the data repository where all nodes data are stored
         spr:
           $ref: "#/components/schemas/SPR"
+        table:
+          $ref: "#/components/schemas/PeersTable"
+        codex:
+          $ref: "#/components/schemas/CodexVersion"
 
     SalesAvailability:
       type: object
@@ -306,10 +344,10 @@ components:
     ManifestItem:
       type: object
       properties:
-        rootHash:
+        treeCid:
           $ref: "#/components/schemas/Cid"
-          description: "Root hash of the content"
+          description: "Unique data identifier"
-        originalBytes:
+        datasetSize:
           type: integer
           format: int64
           description: "Length of original content in bytes"
@@ -319,6 +357,22 @@ components:
         protected:
           type: boolean
           description: "Indicates if content is protected by erasure-coding"
+        filename:
+          type: string
+          nullable: true
+          description: "The original name of the uploaded content (optional)"
+          example: codex.png
+        mimetype:
+          type: string
+          nullable: true
+          description: "The original mimetype of the uploaded content (optional)"
+          example: image/png
+        uploadedAt:
+          type: integer
+          format: int64
+          nullable: true
+          description: "The UTC upload timestamp in seconds"
+          example: 1729244192
 
     Space:
       type: object
@@ -404,12 +458,29 @@ paths:
           description: Invalid CID is specified
         "404":
           description: Content specified by the CID is not found
+        "422":
+          description: The content type is not a valid content type or the filename is not valid
         "500":
           description: Well it was bad-bad
     post:
       summary: "Upload a file in a streaming manner. Once finished, the file is stored in the node and can be retrieved by any node in the network using the returned CID."
       tags: [ Data ]
       operationId: upload
+      parameters:
+        - name: content-type
+          in: header
+          required: false
+          description: The content type of the file. Must be valid.
+          schema:
+            type: string
+            example: "image/png"
+        - name: content-disposition
+          in: header
+          required: false
+          description: The content disposition used to send the filename.
+          schema:
+            type: string
+            example: "attachment; filename=\"codex.png\""
       requestBody:
         content:
           application/octet-stream:

tests/codex/sales/states/testfilling.nim

@@ -24,7 +24,3 @@ checksuite "sales state 'filling'":
   test "switches to failed state when request fails":
     let next = state.onFailed(request)
     check !next of SaleFailed
-
-  test "switches to filled state when slot is filled":
-    let next = state.onSlotFilled(request.id, slotIndex)
-    check !next of SaleFilled

tests/codex/sales/states/testslotreserving.nim

@@ -51,10 +51,6 @@ asyncchecksuite "sales state 'SlotReserving'":
     let next = state.onFailed(request)
     check !next of SaleFailed
 
-  test "switches to filled state when slot is filled":
-    let next = state.onSlotFilled(request.id, slotIndex)
-    check !next of SaleFilled
-
   test "run switches to downloading when slot successfully reserved":
     let next = await state.run(agent)
     check !next of SaleDownloading

tests/codex/testmanifest.nim

@@ -83,6 +83,8 @@ suite "Manifest - Attribute Inheritance":
         treeCid = Cid.example,
         blockSize = 1.MiBs,
         datasetSize = 100.MiBs,
+        filename = "codex.png".some,
+        mimetype = "image/png".some
       ),
       treeCid = Cid.example,
       datasetSize = 200.MiBs,
@@ -107,3 +109,15 @@ suite "Manifest - Attribute Inheritance":
     ).tryGet()
 
     check verifiable.protectedStrategy == LinearStrategy
+
+  test "Should preserve metadata for manifest in verifiable manifest":
+    var verifiable = Manifest.new(
+      manifest = makeProtectedManifest(SteppedStrategy),
+      verifyRoot = Cid.example,
+      slotRoots = @[Cid.example, Cid.example]
+    ).tryGet()
+
+    check verifiable.filename.isSome == true
+    check verifiable.filename.get() == "codex.png"
+    check verifiable.mimetype.isSome == true
+    check verifiable.mimetype.get() == "image/png"

tests/contracts/testContracts.nim

@@ -38,7 +38,7 @@ ethersuite "Marketplace contracts":
     let tokenAddress = await marketplace.token()
     token = Erc20Token.new(tokenAddress, ethProvider.getSigner())
 
-    let config = await marketplace.config()
+    let config = await marketplace.configuration()
     periodicity = Periodicity(seconds: config.proofs.period)
 
     request = StorageRequest.example

tests/contracts/testDeployment.nim

@@ -9,7 +9,7 @@ import ../checktest
 type MockProvider = ref object of Provider
   chainId*: UInt256
 
-method getChainId*(provider: MockProvider): Future[UInt256] {.async.} =
+method getChainId*(provider: MockProvider): Future[UInt256] {.async: (raises:[ProviderError]).} =
   return provider.chainId
 
 proc configFactory(): CodexConf =

tests/contracts/testMarket.nim

@@ -34,7 +34,7 @@ ethersuite "On-Chain Market":
   setup:
     let address = Marketplace.address(dummyVerifier = true)
     marketplace = Marketplace.new(address, ethProvider.getSigner())
-    let config = await marketplace.config()
+    let config = await marketplace.configuration()
     hostRewardRecipient = accounts[2]
 
     market = OnChainMarket.new(marketplace)
@@ -76,13 +76,13 @@ ethersuite "On-Chain Market":
 
   test "can retrieve proof periodicity":
     let periodicity = await market.periodicity()
-    let config = await marketplace.config()
+    let config = await marketplace.configuration()
     let periodLength = config.proofs.period
     check periodicity.seconds == periodLength
 
   test "can retrieve proof timeout":
     let proofTimeout = await market.proofTimeout()
-    let config = await marketplace.config()
+    let config = await marketplace.configuration()
     check proofTimeout == config.proofs.timeout
 
   test "supports marketplace requests":

tests/contracts/time.nim

@@ -1,4 +1,5 @@
 import pkg/ethers
+import pkg/serde/json
 
 proc currentTime*(provider: Provider): Future[UInt256] {.async.} =
   return (!await provider.getBlock(BlockTag.pending)).timestamp

tests/integration/codexclient.nim

@@ -257,3 +257,13 @@ proc saleStateIs*(client: CodexClient, id: SlotId, state: string): bool =
 
 proc requestId*(client: CodexClient, id: PurchaseId): ?RequestId =
   return client.getPurchase(id).option.?requestId
+
+proc uploadRaw*(client: CodexClient, contents: string, headers = newHttpHeaders()): Response =
+  return client.http.request(client.baseurl & "/data", body = contents, httpMethod=HttpPost, headers = headers)
+
+proc listRaw*(client: CodexClient): Response =
+  return client.http.request(client.baseurl & "/data", httpMethod=HttpGet)
+
+proc downloadRaw*(client: CodexClient, cid: string, local = false): Response =
+  return client.http.request(client.baseurl & "/data/" & cid &
+      (if local: "" else: "/network/stream"), httpMethod=HttpGet)

tests/integration/hardhatprocess.nim

@@ -115,7 +115,7 @@ method onOutputLineCaptured(node: HardhatProcess, line: string) =
     return
 
   if error =? logFile.writeFile(line & "\n").errorOption:
-    error "failed to write to hardhat file", errorCode = error
+    error "failed to write to hardhat file", errorCode = $error
     discard logFile.closeFile()
     node.logFile = none IoHandle
 

tests/integration/marketplacesuite.nim

@@ -85,7 +85,7 @@ template marketplacesuite*(name: string, body: untyped) =
       marketplace = Marketplace.new(Marketplace.address, ethProvider.getSigner())
       let tokenAddress = await marketplace.token()
       token = Erc20Token.new(tokenAddress, ethProvider.getSigner())
-      let config = await mp.config(marketplace)
+      let config = await marketplace.configuration()
       period = config.proofs.period.truncate(uint64)
       periodicity = Periodicity(seconds: period.u256)
 

tests/integration/nodeprocess.nim

@@ -128,7 +128,7 @@ method stop*(node: NodeProcess) {.base, async.} =
     try:
       trace "terminating node process..."
       if errCode =? node.process.terminate().errorOption:
-        error "failed to terminate process", errCode
+        error "failed to terminate process", errCode = $errCode
 
       trace "waiting for node process to exit"
       let exitCode = await node.process.waitForExit(3.seconds)

tests/integration/testrestapi.nim

@@ -4,9 +4,9 @@ from pkg/libp2p import `==`
 import pkg/codex/units
 import ./twonodes
 import ../examples
+import json
 
 twonodessuite "REST API", debug1 = false, debug2 = false:
-
   test "nodes can print their peer information":
     check !client1.info() != !client2.info()
 
@@ -16,6 +16,7 @@ twonodessuite "REST API", debug1 = false, debug2 = false:
   test "node accepts file uploads":
     let cid1 = client1.upload("some file contents").get
     let cid2 = client1.upload("some other contents").get
+
     check cid1 != cid2
 
   test "node shows used and available space":
@@ -25,7 +26,7 @@ twonodessuite "REST API", debug1 = false, debug2 = false:
     check:
       space.totalBlocks == 2
       space.quotaMaxBytes == 8589934592.NBytes
-      space.quotaUsedBytes == 65592.NBytes
+      space.quotaUsedBytes == 65598.NBytes
       space.quotaReservedBytes == 12.NBytes
 
   test "node lists local files":
@@ -151,3 +152,89 @@ twonodessuite "REST API", debug1 = false, debug2 = false:
         tolerance.uint)
 
       check responseBefore.status == "200 OK"
+
+  test "node accepts file uploads with content type":
+    let headers = newHttpHeaders({"Content-Type": "text/plain"})
+    let response = client1.uploadRaw("some file contents", headers)
+
+    check response.status == "200 OK"
+    check response.body != ""
+
+  test "node accepts file uploads with content disposition":
+    let headers = newHttpHeaders({"Content-Disposition": "attachment; filename=\"example.txt\""})
+    let response = client1.uploadRaw("some file contents", headers)
+
+    check response.status == "200 OK"
+    check response.body != ""
+
+  test "node accepts file uploads with content disposition without filename":
+    let headers = newHttpHeaders({"Content-Disposition": "attachment"})
+    let response = client1.uploadRaw("some file contents", headers)
+
+    check response.status == "200 OK"
+    check response.body != ""
+
+  test "upload fails if content disposition contains bad filename":
+    let headers = newHttpHeaders({"Content-Disposition": "attachment; filename=\"exam*ple.txt\""})
+    let response = client1.uploadRaw("some file contents", headers)
+
+    check response.status == "422 Unprocessable Entity"
+    check response.body == "The filename is not valid."
+
+  test "upload fails if content type is invalid":
+    let headers = newHttpHeaders({"Content-Type": "hello/world"})
+    let response = client1.uploadRaw("some file contents", headers)
+
+    check response.status == "422 Unprocessable Entity"
+    check response.body == "The MIME type is not valid."
+
+  test "node retrieve the metadata":
+    let headers = newHttpHeaders({"Content-Type": "text/plain", "Content-Disposition": "attachment; filename=\"example.txt\""})
+    let uploadResponse = client1.uploadRaw("some file contents", headers)
+    let cid = uploadResponse.body
+    let listResponse = client1.listRaw()
+
+    let jsonData = parseJson(listResponse.body)
+
+    check jsonData.hasKey("content") == true
+
+    let content = jsonData["content"][0]
+
+    check content.hasKey("manifest") == true
+
+    let manifest = content["manifest"]
+
+    check manifest.hasKey("filename") == true
+    check manifest["filename"].getStr() == "example.txt"
+    check manifest.hasKey("mimetype") == true
+    check manifest["mimetype"].getStr() == "text/plain"
+    check manifest.hasKey("uploadedAt") == true
+    check manifest["uploadedAt"].getInt() > 0
+
+  test "node set the headers when for download":
+    let headers = newHttpHeaders({
+      "Content-Disposition": "attachment; filename=\"example.txt\"",
+      "Content-Type": "text/plain"
+    })
+
+    let uploadResponse = client1.uploadRaw("some file contents", headers)
+    let cid = uploadResponse.body
+
+    check uploadResponse.status == "200 OK"
+
+    let response = client1.downloadRaw(cid)
+
+    check response.status == "200 OK"
+    check response.headers.hasKey("Content-Type") == true
+    check response.headers["Content-Type"] == "text/plain"
+    check response.headers.hasKey("Content-Disposition") == true
+    check response.headers["Content-Disposition"] == "attachment; filename=\"example.txt\""
+
+    let local = true
+    let localResponse = client1.downloadRaw(cid, local)
+
+    check localResponse.status == "200 OK"
+    check localResponse.headers.hasKey("Content-Type") == true
+    check localResponse.headers["Content-Type"] == "text/plain"
+    check localResponse.headers.hasKey("Content-Disposition") == true
+    check localResponse.headers["Content-Disposition"] == "attachment; filename=\"example.txt\""

tests/integration/testupdownload.nim

@@ -1,5 +1,7 @@
 import pkg/codex/rest/json
 import ./twonodes
+import json
+from pkg/libp2p import Cid, `$`
 
 twonodessuite "Uploads and downloads", debug1 = false, debug2 = false:
 
@@ -39,24 +41,42 @@ twonodessuite "Uploads and downloads", debug1 = false, debug2 = false:
     check:
       resp2.error.msg == "404 Not Found"
 
-  proc checkRestContent(content: ?!string) =
+  proc checkRestContent(cid: Cid, content: ?!string) =
     let c = content.tryGet()
+
     # tried to JSON (very easy) and checking the resulting object (would be much nicer)
     # spent an hour to try and make it work.
-    check:
+    let jsonData = parseJson(c)
-      c == "{\"cid\":\"zDvZRwzm1ePSzKSXt57D5YxHwcSDmsCyYN65wW4HT7fuX9HrzFXy\",\"manifest\":{\"treeCid\":\"zDzSvJTezk7bJNQqFq8k1iHXY84psNuUfZVusA5bBQQUSuyzDSVL\",\"datasetSize\":18,\"blockSize\":65536,\"protected\":false}}"
+
+    check jsonData.hasKey("cid") == true
+
+    check jsonData["cid"].getStr() == $cid
+    check jsonData.hasKey("manifest") == true
+
+    let manifest = jsonData["manifest"]
+
+    check manifest.hasKey("treeCid") == true
+    check manifest["treeCid"].getStr() == "zDzSvJTezk7bJNQqFq8k1iHXY84psNuUfZVusA5bBQQUSuyzDSVL"
+    check manifest.hasKey("datasetSize") == true
+    check manifest["datasetSize"].getInt() == 18
+    check manifest.hasKey("blockSize") == true
+    check manifest["blockSize"].getInt() == 65536
+    check manifest.hasKey("protected") == true
+    check manifest["protected"].getBool() == false
 
   test "node allows downloading only manifest":
     let content1 = "some file contents"
     let cid1 = client1.upload(content1).get
-    let resp2 = client2.downloadManifestOnly(cid1)
+
-    checkRestContent(resp2)
+    let resp2 = client1.downloadManifestOnly(cid1)
+    checkRestContent(cid1, resp2)
 
   test "node allows downloading content without stream":
     let content1 = "some file contents"
     let cid1 = client1.upload(content1).get
+
     let resp1 = client2.downloadNoStream(cid1)
-    checkRestContent(resp1)
+    checkRestContent(cid1, resp1)
     let resp2 = client2.download(cid1, local = true).get
     check:
       content1 == resp2

tests/tools/cirdl/testcirdl.nim

@@ -2,10 +2,11 @@ import std/os
 import std/osproc
 import std/options
 import pkg/chronos
-import codex/contracts
+import pkg/codex/contracts
-import ../../integration/marketplacesuite
+import ../../asynctest
+import ../../contracts/deployment
 
-marketplacesuite "tools/cirdl":
+suite "tools/cirdl":
   const
     cirdl = "build" / "cirdl"
     workdir = "."
@@ -14,11 +15,11 @@ marketplacesuite "tools/cirdl":
     let
       circuitPath = "testcircuitpath"
       rpcEndpoint = "ws://localhost:8545"
-      marketplaceAddress = $marketplace.address
+      marketplaceAddress = Marketplace.address
 
     discard existsOrCreateDir(circuitPath)
 
-    let args = [circuitPath, rpcEndpoint, marketplaceAddress]
+    let args = [circuitPath, rpcEndpoint, $marketplaceAddress]
 
     let process = osproc.startProcess(
       cirdl,

tools/cirdl/cirdl.nim

@@ -34,7 +34,7 @@ proc getCircuitHash(rpcEndpoint: string, marketplaceAddress: string): Future[?!s
     return failure("Invalid address: " & marketplaceAddress)
 
   let marketplace = Marketplace.new(address, provider)
-  let config = await marketplace.config()
+  let config = await marketplace.configuration()
   return success config.proofs.zkeyHash
 
 proc formatUrl(hash: string): string =

vendor/codex-contracts-eth

@@ -1 +1 @@
-Subproject commit 807fc973c875b5bde8f517c71c818ba8f2f720dd
+Subproject commit 11ccefd720f1932608b67db95af5b72d73d1257b

vendor/nim-ethers

@@ -1 +1 @@
-Subproject commit 5b170adcb1ffb1dbb273d1b7679bf3d9a08adb76
+Subproject commit 0ce6abf0fe942fe7cb1d14b8e4485621be9c3fe8

vendor/nim-json-rpc

@@ -1 +1 @@
-Subproject commit 0bf2bcbe74a18a3c7a709d57108bb7b51e748a92
+Subproject commit 0408795be95c00d75e96eaef6eae8a9c734014f5

vendor/nim-json-serialization

@@ -1 +1 @@
-Subproject commit bb53d49caf2a6c6cf1df365ba84af93cdcfa7aa3
+Subproject commit 5127b26ee58076e9369e7c126c196793c2b12e73

vendor/nim-serde

@@ -1 +1 @@
-Subproject commit b1e5e5d39a99ea56b750f6d9272dd319f4ad4291
+Subproject commit 83e4a2ccf621d3040c6e7e0267393ca2d205988e

vendor/nim-serialization

@@ -1 +1 @@
-Subproject commit 384eb2561ee755446cff512a8e057325848b86a7
+Subproject commit f709bd9e16b1b6870fe3e4401196479e014a2ef6