Merge branch 'master' into bumpdep

.github/workflows/docker.yml

@@ -9,35 +9,117 @@ on:
   workflow_dispatch:
 
 jobs:
-  docker:
+  docker-amd64:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v4
         with:
-          images: thatbenbierens/nim-codex
+          images: thatbenbierens/nim-codex-amd64
           tags: |
             type=semver,pattern={{version}}
             type=sha
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+
       - name: Login to Docker Hub
         if: github.event_name != 'pull_request'
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and push
+
+      - name: Build and export to Docker
+        id: build
         uses: docker/build-push-action@v4
         with:
           context: .
           file: docker/codex.Dockerfile
-          platforms: linux/amd64,linux/arm64,linux/arm/v7
-          push: ${{ github.event_name != 'pull_request' }}
+          platforms: linux/amd64
+          load: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      - name: Minify docker image
+        uses: kitabisa/docker-slim-action@v1
+        env:
+          DSLIM_HTTP_PROBE: false
+        with:
+          target: ${{ steps.meta.outputs.tags }}
+          overwrite: true
+
+      - name: Show slim report
+        run: echo "${{ steps.slim.outputs.report }}"
+
+      - name: Push to Docker registry
+        if: github.event_name != 'pull_request'
+        id: push
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: docker/codex.Dockerfile
+          platforms: linux/amd64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+  docker-arm64:
+    runs-on: buildjet-4vcpu-ubuntu-2204-arm
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: thatbenbierens/nim-codex-arm64
+          tags: |
+            type=semver,pattern={{version}}
+            type=sha
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and export to Docker
+        id: build
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: docker/codex.Dockerfile
+          platforms: linux/arm64
+          load: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      - name: Minify docker image
+        uses: kitabisa/docker-slim-action@v1
+        env:
+          DSLIM_HTTP_PROBE: false
+        with:
+          target: ${{ steps.meta.outputs.tags }}
+          overwrite: true
+
+      - name: Show slim report
+        run: echo "${{ steps.slim.outputs.report }}"
+
+      - name: Push to Docker registry
+        if: github.event_name != 'pull_request'
+        id: push
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: docker/codex.Dockerfile
+          platforms: linux/arm64
+          push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

.github/workflows/docs.yml

@@ -3,7 +3,7 @@ name: OpenAPI
 on:
   push:
     branches:
-      - 'main'
+      - 'master'
     paths:
       - 'openapi.yaml'
       - '.github/workflows/docs.yml'
@@ -41,7 +41,7 @@ jobs:
   deploy:
     name: Deploy
     runs-on: ubuntu-latest
-    if: github.ref == 'refs/heads/main'
+    if: github.ref == 'refs/heads/master'
     steps:
       - name: Checkout
         uses: actions/checkout@v3

README.md

@@ -54,7 +54,7 @@ For example, to configure `--log-level`, use `CODEX_LOG_LEVEL` as the environmen
 A [TOML](https://toml.io/en/) configuration file can also be used to set configuration values. Configuration option names and corresponding values are placed in the file, separated by `=`. Configuration option names can be obtained from the `codex --help` command, and should not include the `--` prefix. For example, a node's log level (`--log-level`) can be configured using TOML as follows:
 
 ```toml
-log-level = "TRACE"
+log-level = "trace"
 ```
 
 The Codex node can then read the configuration from this file using the `--config-file` CLI parameter, like `codex --config-file=/path/to/your/config.toml`.
@@ -70,7 +70,7 @@ codex [OPTIONS]... command
 The following options are available:
 
      --config-file          Loads the configuration from a TOML file [=none].
-     --log-level            Sets the log level [=INFO].
+     --log-level            Sets the log level [=info].
      --metrics              Enable the metrics server [=false].
      --metrics-address      Listening address of the metrics server [=127.0.0.1].
      --metrics-port         Listening HTTP port of the metrics server [=8008].
@@ -111,9 +111,9 @@ codex initNode
 Codex uses [Chronicles](https://github.com/status-im/nim-chronicles) logging library, which allows great flexibility in working with logs.
 Chronicles has the concept of topics, which categorize log entries into semantic groups.
 
-Using the `log-level` parameter, you can set the top-level log level like `--log-level="TRACE"`, but more importantly,
-you can set log levels for specific topics like `--log-level="INFO; TRACE: marketplace,node; ERROR: blockexchange"`,
-which sets the top-level log level to `INFO` and then for topics `marketplace` and `node` sets the level to `TRACE` and so on.
+Using the `log-level` parameter, you can set the top-level log level like `--log-level="trace"`, but more importantly,
+you can set log levels for specific topics like `--log-level="info; trace: marketplace,node; error: blockexchange"`,
+which sets the top-level log level to `info` and then for topics `marketplace` and `node` sets the level to `trace` and so on.
 
 ### Example: running two Codex clients
 

codex.nimble

@@ -7,29 +7,30 @@ license = "MIT"
 binDir = "build"
 srcDir = "."
 
-requires "nim >= 1.2.0",
-         "asynctest >= 0.3.2 & < 0.4.0",
-         "bearssl >= 0.1.4",
-         "chronicles >= 0.7.2",
-         "chronos >= 2.5.2",
-         "confutils",
-         "ethers >= 0.2.4 & < 0.3.0",
-         "libbacktrace",
-         "libp2p",
-         "metrics",
-         "nimcrypto >= 0.4.1",
-         "nitro >= 0.5.1 & < 0.6.0",
-         "presto",
-         "protobuf_serialization >= 0.2.0 & < 0.3.0",
-         "questionable >= 0.10.6 & < 0.11.0",
-         "secp256k1",
-         "stew",
-         "upraises >= 0.1.0 & < 0.2.0",
-         "https://github.com/status-im/lrucache.nim#1.2.2",
-         "leopard >= 0.1.0 & < 0.2.0",
-         "blscurve",
-         "libp2pdht",
-         "eth"
+requires "nim >= 1.2.0"
+requires "asynctest >= 0.3.2 & < 0.4.0"
+requires "bearssl >= 0.1.4"
+requires "chronicles >= 0.7.2"
+requires "chronos >= 2.5.2"
+requires "confutils"
+requires "ethers >= 0.2.4 & < 0.3.0"
+requires "libbacktrace"
+requires "libp2p"
+requires "metrics"
+requires "nimcrypto >= 0.4.1"
+requires "nitro >= 0.5.1 & < 0.6.0"
+requires "presto"
+requires "protobuf_serialization >= 0.2.0 & < 0.3.0"
+requires "questionable >= 0.10.6 & < 0.11.0"
+requires "secp256k1"
+requires "stew"
+requires "upraises >= 0.1.0 & < 0.2.0"
+requires "toml_serialization"
+requires "https://github.com/status-im/lrucache.nim#1.2.2"
+requires "leopard >= 0.1.0 & < 0.2.0"
+requires "blscurve"
+requires "libp2pdht"
+requires "eth"
 
 when declared(namedBin):
   namedBin = {

codex/codex.nim

@@ -141,8 +141,11 @@ proc stop*(s: CodexServer) {.async.} =
 
   s.runHandle.complete()
 
-proc new*(T: type CodexServer, config: CodexConf, privateKey: CodexPrivateKey): T =
-
+proc new*(
+    T: type CodexServer,
+    config: CodexConf,
+    privateKey: CodexPrivateKey): CodexServer =
+  ## create CodexServer including setting up datastore, repostore, etc
   let
     switch = SwitchBuilder
     .new()
@@ -221,6 +224,7 @@ proc new*(T: type CodexServer, config: CodexConf, privateKey: CodexPrivateKey):
       .expect("Should start rest server!")
 
   switch.mount(network)
+
   T(
     config: config,
     codexNode: codexNode,

codex/conf.nim

@@ -35,6 +35,9 @@ import ./stores
 
 export DefaultCacheSizeMiB, net, DefaultQuotaBytes, DefaultBlockTtl, DefaultBlockMaintenanceInterval, DefaultNumberOfBlocksToMaintainPerInterval
 
+const
+  codex_enable_api_debug_peers* {.booldefine.} = false
+
 type
   StartUpCommand* {.pure.} = enum
     noCommand,
@@ -59,7 +62,7 @@ type
       name: "config-file" }: Option[InputFile]
 
     logLevel* {.
-      defaultValue: "INFO"
+      defaultValue: "info"
       desc: "Sets the log level",
       name: "log-level" }: string
 
@@ -249,7 +252,10 @@ proc getCodexVersion(): string =
   return tag
 
 proc getCodexRevision(): string =
-  strip(staticExec("git rev-parse --short HEAD"))[0..5]
+  # using a slice in a static context breaks nimsuggest for some reason
+  var res = strip(staticExec("git rev-parse --short HEAD"))
+  res.setLen(6)
+  return res
 
 proc getNimBanner(): string =
   staticExec("nim --version | grep Version")
@@ -347,9 +353,9 @@ proc updateLogLevel*(logLevel: string) {.upraises: [ValueError].} =
   # Updates log levels (without clearing old ones)
   let directives = logLevel.split(";")
   try:
-    setLogLevel(parseEnum[LogLevel](directives[0]))
+    setLogLevel(parseEnum[LogLevel](directives[0].toUpperAscii))
   except ValueError:
-    raise (ref ValueError)(msg: "Please specify one of TRACE, DEBUG, INFO, NOTICE, WARN, ERROR or FATAL")
+    raise (ref ValueError)(msg: "Please specify one of: trace, debug, info, notice, warn, error or fatal")
 
   if directives.len > 1:
     for topicName, settings in parseTopicDirectives(directives[1..^1]):

codex/contracts/market.nim

@@ -1,4 +1,5 @@
 import std/strutils
+import std/strformat
 import pkg/chronicles
 import pkg/ethers
 import pkg/ethers/testing
@@ -52,7 +53,10 @@ method myRequests*(market: OnChainMarket): Future[seq[RequestId]] {.async.} =
   return await market.contract.myRequests
 
 method mySlots*(market: OnChainMarket): Future[seq[SlotId]] {.async.} =
-  return await market.contract.mySlots()
+  let slots = await market.contract.mySlots()
+  debug "Fetched my slots", numSlots=len(slots)
+
+  return slots
 
 method requestStorage(market: OnChainMarket, request: StorageRequest){.async.} =
   debug "Requesting storage"

codex/discovery.nim

@@ -35,7 +35,7 @@ logScope:
 
 type
   Discovery* = ref object of RootObj
-    protocol: discv5.Protocol           # dht protocol
+    protocol*: discv5.Protocol           # dht protocol
     key: PrivateKey                     # private key
     peerId: PeerId                      # the peer id of the local node
     announceAddrs: seq[MultiAddress]    # addresses announced as part of the provider records
@@ -58,13 +58,16 @@ proc toNodeId*(host: ca.Address): NodeId =
 proc findPeer*(
   d: Discovery,
   peerId: PeerId): Future[?PeerRecord] {.async.} =
+  trace "protocol.resolve..."
   let
     node = await d.protocol.resolve(toNodeId(peerId))
 
   return
     if node.isSome():
+      trace "protocol.resolve some data"
       node.get().record.data.some
     else:
+      trace "protocol.resolve none"
       PeerRecord.none
 
 method find*(
@@ -127,16 +130,10 @@ method provide*(d: Discovery, host: ca.Address) {.async, base.} =
     trace "Provided to nodes", nodes = nodes.len
 
 method removeProvider*(d: Discovery, peerId: PeerId): Future[void] {.base.} =
-  ## Remove provider from providers table
-  ##
-
   trace "Removing provider", peerId
   d.protocol.removeProvidersLocal(peerId)
 
 proc updateAnnounceRecord*(d: Discovery, addrs: openArray[MultiAddress]) =
-  ## Update providers record
-  ##
-
   d.announceAddrs = @addrs
 
   trace "Updating announce record", addrs = d.announceAddrs
@@ -149,9 +146,6 @@ proc updateAnnounceRecord*(d: Discovery, addrs: openArray[MultiAddress]) =
       .expect("Should update SPR")
 
 proc updateDhtRecord*(d: Discovery, ip: ValidIpAddress, port: Port) =
-  ## Update providers record
-  ##
-
   trace "Updating Dht record", ip, port = $port
   d.dhtRecord = SignedPeerRecord.init(
     d.key, PeerRecord.init(d.peerId, @[
@@ -160,6 +154,10 @@ proc updateDhtRecord*(d: Discovery, ip: ValidIpAddress, port: Port) =
         IpTransportProtocol.udpProtocol,
         port)])).expect("Should construct signed record").some
 
+  if not d.protocol.isNil:
+    d.protocol.updateRecord(d.dhtRecord)
+      .expect("Should update SPR")
+
 proc start*(d: Discovery) {.async.} =
   d.protocol.open()
   await d.protocol.start()

codex/proving.nim

@@ -56,6 +56,7 @@ proc prove(proving: Proving, slot: Slot) {.async.} =
   without onProve =? proving.onProve:
     raiseAssert "onProve callback not set"
   try:
+    debug "Proving slot"
     let proof = await onProve(slot)
     await proving.market.submitProof(slot.id, proof)
   except CatchableError as e:

codex/purchasing/states/pending.nim

@@ -1,7 +1,6 @@
 import ../statemachine
 import ./errorhandling
 import ./submitted
-import ./error
 
 type PurchasePending* = ref object of ErrorHandlingState
 

codex/purchasing/states/started.nim

@@ -1,6 +1,5 @@
 import ../statemachine
 import ./errorhandling
-import ./error
 import ./finished
 import ./failed
 

codex/purchasing/states/submitted.nim

@@ -1,6 +1,5 @@
 import ../statemachine
 import ./errorhandling
-import ./error
 import ./started
 import ./cancelled
 

codex/purchasing/states/unknown.nim

@@ -5,7 +5,6 @@ import ./started
 import ./cancelled
 import ./finished
 import ./failed
-import ./error
 
 type PurchaseUnknown* = ref object of ErrorHandlingState
 

codex/rest/api.nim

@@ -24,8 +24,11 @@ import pkg/stew/base10
 import pkg/stew/byteutils
 import pkg/confutils
 
+import pkg/libp2p
 import pkg/libp2p/routing_record
 import pkg/libp2pdht/discv5/spr as spr
+import pkg/libp2pdht/discv5/routing_table as rt
+import pkg/libp2pdht/discv5/node as dn
 
 import ../node
 import ../blocktype
@@ -45,6 +48,47 @@ proc validate(
   {.gcsafe, raises: [Defect].} =
   0
 
+proc formatAddress(address: Option[dn.Address]): string =
+  if address.isSome():
+    return $address.get()
+  return "<none>"
+
+proc formatNode(node: dn.Node): JsonNode =
+  let jobj = %*{
+    "nodeId": $node.id,
+    "peerId": $node.record.data.peerId,
+    "record": $node.record,
+    "address": formatAddress(node.address),
+    "seen": $node.seen
+  }
+  return jobj
+
+proc formatTable(routingTable: rt.RoutingTable): JsonNode =
+  let jarray = newJArray()
+  for bucket in routingTable.buckets:
+    for node in bucket.nodes:
+      jarray.add(formatNode(node))
+
+  let jobj = %*{
+    "localNode": formatNode(routingTable.localNode),
+    "nodes": jarray
+  }
+  return jobj
+
+proc formatPeerRecord(peerRecord: PeerRecord): JsonNode =
+  let jarray = newJArray()
+  for maddr in peerRecord.addresses:
+    jarray.add(%*{
+      "address": $maddr.address
+    })
+
+  let jobj = %*{
+    "peerId": $peerRecord.peerId,
+    "seqNo": $peerRecord.seqNo,
+    "addresses": jarray
+  }
+  return jobj
+
 proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter =
   var router = RestRouter.init(validate)
   router.api(
@@ -244,13 +288,41 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter =
               node.discovery.dhtRecord.get.toURI
             else:
               "",
+          "table": formatTable(node.discovery.protocol.routingTable),
           "codex": {
             "version": $codexVersion,
             "revision": $codexRevision
           }
         }
 
-      return RestApiResponse.response($json)
+      return RestApiResponse.response($json, contentType="application/json")
+
+  when codex_enable_api_debug_peers:
+    router.api(
+      MethodGet,
+      "/api/codex/v1/debug/peer/{peerId}") do (peerId: PeerId) -> RestApiResponse:
+
+        trace "debug/peer start"
+        without peerRecord =? (await node.findPeer(peerId.get())):
+          trace "debug/peer peer not found!"
+          return RestApiResponse.error(
+            Http400,
+            "Unable to find Peer!")
+
+        let json = formatPeerRecord(peerRecord)
+        trace "debug/peer returning peer record"
+        return RestApiResponse.response($json)
+
+  router.api(
+    MethodGet,
+    "/api/codex/v1/sales/slots") do () -> RestApiResponse:
+      ## Returns active slots for the host
+
+      without contracts =? node.contracts.host:
+        return RestApiResponse.error(Http503, "Sales unavailable")
+
+      let json = %(await contracts.sales.mySlots())
+      return RestApiResponse.response($json, contentType="application/json")
 
   router.api(
     MethodGet,
@@ -264,7 +336,7 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter =
         return RestApiResponse.error(Http500, err.msg)
 
       let json = %unused
-      return RestApiResponse.response($json)
+      return RestApiResponse.response($json, contentType="application/json")
 
   router.rawApi(
     MethodPost,
@@ -293,7 +365,7 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter =
         return RestApiResponse.error(Http500, err.msg)
 
       let json = %availability
-      return RestApiResponse.response($json)
+      return RestApiResponse.response($json, contentType="application/json")
 
   router.api(
     MethodGet,
@@ -311,7 +383,6 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter =
 
       let json = %purchase
 
-      return RestApiResponse.response($json)
-
+      return RestApiResponse.response($json, contentType="application/json")
 
   return router

codex/rest/coders.nim

@@ -19,6 +19,7 @@ import pkg/stint
 
 import ../sales
 import ../purchasing
+import ../utils/stintutils
 
 proc encodeString*(cid: type Cid): Result[string, cstring] =
   ok($cid)
@@ -72,7 +73,7 @@ proc encodeString*(value: bool): Result[string, cstring] =
 
 proc decodeString*(_: type UInt256, value: string): Result[UInt256, cstring] =
   try:
-    ok UInt256.fromHex(value)
+    ok UInt256.fromDecimal(value)
   except ValueError as e:
     err e.msg.cstring
 

codex/rest/json.nim

@@ -4,6 +4,9 @@ import pkg/stew/byteutils
 import pkg/questionable/results
 import ../sales
 import ../purchasing
+import ../utils/stintutils
+
+export json
 
 type
   StorageRequestParams* = object
@@ -17,22 +20,22 @@ type
 
 proc fromJson*(_: type Availability, bytes: seq[byte]): ?!Availability =
   let json = ?catch parseJson(string.fromBytes(bytes))
-  let size = ?catch UInt256.fromHex(json["size"].getStr)
-  let duration = ?catch UInt256.fromHex(json["duration"].getStr)
-  let minPrice = ?catch UInt256.fromHex(json["minPrice"].getStr)
-  let maxCollateral = ?catch UInt256.fromHex(json["maxCollateral"].getStr)
+  let size = ?catch UInt256.fromDecimal(json["size"].getStr)
+  let duration = ?catch UInt256.fromDecimal(json["duration"].getStr)
+  let minPrice = ?catch UInt256.fromDecimal(json["minPrice"].getStr)
+  let maxCollateral = ?catch UInt256.fromDecimal(json["maxCollateral"].getStr)
   success Availability.init(size, duration, minPrice, maxCollateral)
 
 proc fromJson*(_: type StorageRequestParams,
                bytes: seq[byte]): ?! StorageRequestParams =
   let json = ?catch parseJson(string.fromBytes(bytes))
-  let duration = ?catch UInt256.fromHex(json["duration"].getStr)
-  let proofProbability = ?catch UInt256.fromHex(json["proofProbability"].getStr)
-  let reward = ?catch UInt256.fromHex(json["reward"].getStr)
-  let collateral = ?catch UInt256.fromHex(json["collateral"].getStr)
-  let expiry = UInt256.fromHex(json["expiry"].getStr).catch.option
-  let nodes = strutils.fromHex[uint](json["nodes"].getStr).catch.option
-  let tolerance = strutils.fromHex[uint](json["tolerance"].getStr).catch.option
+  let duration = ?catch UInt256.fromDecimal(json["duration"].getStr)
+  let proofProbability = ?catch UInt256.fromDecimal(json["proofProbability"].getStr)
+  let reward = ?catch UInt256.fromDecimal(json["reward"].getStr)
+  let collateral = ?catch UInt256.fromDecimal(json["collateral"].getStr)
+  let expiry = UInt256.fromDecimal(json["expiry"].getStr).catch.option
+  let nodes = parseUInt(json["nodes"].getStr).catch.option
+  let tolerance = parseUInt(json["tolerance"].getStr).catch.option
   success StorageRequestParams(
     duration: duration,
     proofProbability: proofProbability,
@@ -46,8 +49,8 @@ proc fromJson*(_: type StorageRequestParams,
 func `%`*(address: Address): JsonNode =
   % $address
 
-func `%`*(stint: StInt|StUint): JsonNode =
-  %("0x" & stint.toHex)
+func `%`*(stint: StInt|StUint): JsonNode=
+  %(stint.toString)
 
 func `%`*(arr: openArray[byte]): JsonNode =
   %("0x" & arr.toHex)
@@ -55,6 +58,13 @@ func `%`*(arr: openArray[byte]): JsonNode =
 func `%`*(id: RequestId | SlotId | Nonce | AvailabilityId): JsonNode =
   % id.toArray
 
+func `%`*(obj: StorageRequest | Slot): JsonNode =
+  let jsonObj = newJObject()
+  for k, v in obj.fieldPairs: jsonObj[k] = %v
+  jsonObj["id"] = %(obj.id)
+
+  return jsonObj
+
 func `%`*(purchase: Purchase): JsonNode =
   %*{
     "state": purchase.state |? "none",

codex/sales.nim

@@ -100,20 +100,28 @@ proc handleRequest(sales: Sales,
   agent.start(SaleDownloading())
   sales.agents.add agent
 
-proc load*(sales: Sales) {.async.} =
+proc mySlots*(sales: Sales): Future[seq[Slot]] {.async.} =
   let market = sales.context.market
-
   let slotIds = await market.mySlots()
+  var slots: seq[Slot] = @[]
 
   for slotId in slotIds:
     if slot =? (await market.getActiveSlot(slotId)):
-      let agent = newSalesAgent(
-        sales.context,
-        slot.request.id,
-        slot.slotIndex,
-        some slot.request)
-      agent.start(SaleUnknown())
-      sales.agents.add agent
+      slots.add slot
+
+  return slots
+
+proc load*(sales: Sales) {.async.} =
+  let slots = await sales.mySlots()
+
+  for slot in slots:
+    let agent = newSalesAgent(
+      sales.context,
+      slot.request.id,
+      slot.slotIndex,
+      some slot.request)
+    agent.start(SaleUnknown())
+    sales.agents.add agent
 
 proc start*(sales: Sales) {.async.} =
   doAssert sales.subscription.isNone, "Sales already started"

codex/utils/stintutils.nim

@@ -0,0 +1,4 @@
+import pkg/stint
+
+func fromDecimal*(T: typedesc[StUint|StInt], s: string): T {.inline.} =
+  parse(s, type result, radix = 10)

docker/README.md

@@ -32,9 +32,15 @@ Codex docker image supports the following environment variables:
 - ETH_PROVIDER
 - ETH_ACCOUNT
 - ETH_DEPLOYMENT
+- SIMULATE_PROOF_FAILURES
+- VALIDATOR
+- PERSISTENCE
+- CODEX_NODENAME(†)
 
 (*) These variables have default values in the docker image that are different from Codex's standard default values.
 
+(†) CODEX_NODENAME is used for logging purposes only in the docker image
+
 All environment variables are optional and will default to Codex's CLI default values.
 
 # Constants
@@ -46,3 +52,13 @@ To get the IP address of a container within a network:
 Find container Id: `docker ps`
 Open terminal in container: `docker exec -it <CONTAINER ID> sh`
 Get IP addresses: `ifconfig`
+
+# Slim
+1. Build the image using `docker build -t status-im/codexsetup:latest -f codex.Dockerfile ..`
+2. The docker image can then be minifed using [slim](https://github.com/slimtoolkit/slim). Install slim on your path and then run:
+```shell
+slim # brings up interactive prompt
+>>> build --target status-im/codexsetup --http-probe-off true
+```
+3. This should output an image with name `status-im/codexsetup.slim`
+4. We can then bring up the image using `docker-compose up -d`.

docker/codex.Dockerfile

@@ -1,14 +1,18 @@
-FROM nimlang/nim:1.6.10-alpine AS builder
+FROM ubuntu:lunar-20230415 AS builder
+RUN apt-get update && apt-get install -y git cmake curl make bash lcov build-essential nim
+RUN echo 'export NIMBLE_DIR="${HOME}/.nimble"' >> "${HOME}/.bash_env"
+RUN echo 'export PATH="${NIMBLE_DIR}/bin:${PATH}"' >> "${HOME}/.bash_env"
+
 WORKDIR /src
-RUN apk update && apk add git cmake curl make git bash linux-headers
 COPY . .
 RUN make clean
-RUN make update
-RUN make NIM_PARAMS="-d:disableMarchNative"
+RUN make -j4 update
+RUN make -j4 NIM_PARAMS="-d:disableMarchNative -d:codex_enable_api_debug_peers=true"
 
-FROM alpine:3.17.2
-WORKDIR /root/
-RUN apk add --no-cache openssl libstdc++ libgcc libgomp
+FROM ubuntu:lunar-20230415
+WORKDIR /root
+RUN apt-get update && apt-get install -y libgomp1 bash net-tools
 COPY --from=builder /src/build/codex ./
 COPY --from=builder /src/docker/startCodex.sh ./
-CMD ["sh", "startCodex.sh"]
+RUN chmod +x ./startCodex.sh
+CMD ["/bin/bash", "-l", "-c", "./startCodex.sh"]

docker/docker-compose.yaml

@@ -1,77 +1,27 @@
 services:
   codex-node1:
-    image: clustertest-nim-codex
-    build:
-      context: ../.
-      dockerfile: docker/codex.Dockerfile
+    image: status-im/codexsetup.slim:latest
     ports:
       - 8080:8080
     # Available environment variables:
-    # environment:
-      # - LOG_LEVEL=TRACE
-      # - METRICS_ADDR=0.0.0.0
-      # - METRICS_PORT=9090
-      # - NAT_IP=2.3.4.5
-      # - API_PORT=8080
-      # - DISC_IP=3.4.5.6
-      # - DISC_PORT=8765
-      # - NET_PRIVKEY=privkey
-      # - BOOTSTRAP_SPR=bootstrap_record
-      # - MAX_PEERS=123
-      # - AGENT_STRING=agent_string
-      # - STORAGE_QUOTA=123456789
-      # - BLOCK_TTL=23456
-      # - CACHE_SIZE=6543
-      # - ETH_PROVIDER=eth
-      # - ETH_ACCOUNT=account
-      # - ETH_DEPLOYMENT=deploy
-    volumes:
-      - ./hostdatadir/node1:/datadir
-    networks:
-      - primary
-
-  # Example with metrics enabled.
-  codex-node2:
-    image: clustertest-nim-codex
-    ports:
-      - 8081:8080
-      - 9090:9090
     environment:
+      - LOG_LEVEL=TRACE
       - METRICS_ADDR=0.0.0.0
       - METRICS_PORT=9090
-    volumes:
-      - ./hostdatadir/node2:/datadir
-    depends_on:
-      - codex-node1
-    networks:
-      - primary
-      - secondary
+      - NAT_IP=2.3.4.5
+      - API_PORT=8080
+      - DISC_IP=3.4.5.6
+      - DISC_PORT=8765
+      - NET_PRIVKEY=privkey
+      - BOOTSTRAP_SPR=bootstrap_record
+      - MAX_PEERS=123
+      - AGENT_STRING=agent_string
+      - STORAGE_QUOTA=123456789
+      - BLOCK_TTL=23456
+      - CACHE_SIZE=6543
+      - ETH_PROVIDER=eth
+      - ETH_ACCOUNT=account
+      - ETH_MARKETPLACE_ADDRESS=0x59b670e9fA9D0A427751Af201D676719a970857b
+      - SIMULATE_PROOF_FAILURES=2
 
-  codex-node3:
-    image: clustertest-nim-codex
-    ports:
-      - 8082:8080
-    volumes:
-      - ./hostdatadir/node3:/datadir
-    depends_on:
-      - codex-node1
-    networks:
-      - secondary
 
-  prometheus:
-    image: prom/prometheus:v2.30.3
-    ports:
-      - 9000:9090
-    volumes:
-      - ./prometheus:/etc/prometheus
-      - ./prometheus-data:/prometheus
-    command: --web.enable-lifecycle --config.file=/etc/prometheus/prometheus.yml
-    networks:
-      - primary
-      - secondary
-
-networks:
-  primary:
-    name: primary
-  secondary:
-    name: secondary

docker/prometheus/alert.yml

@@ -1,6 +0,0 @@
-groups:
-  - name: DemoAlerts
-    rules:
-      - alert: InstanceDown
-        expr: up{job="services"} < 1
-        for: 5m

docker/prometheus/prometheus.yml

@@ -1,15 +0,0 @@
-global:
-  scrape_interval: 30s
-  scrape_timeout: 10s
-
-rule_files:
-  - alert.yml
-
-scrape_configs:
-  - job_name: services
-    metrics_path: /metrics
-    static_configs:
-      - targets:
-          - 'prometheus:9090'
-          - 'codex-node1:9090'
-          - 'codex-node2:9090'

docker/startCodex.sh

@@ -1,7 +1,15 @@
-echo "Starting Codex..."
+NAME=""
+if [ -n "$CODEX_NODENAME" ]; then
+  NAME=" '$CODEX_NODENAME'"
+fi
+echo "Starting Codex node$NAME"
 
 args=""
 
+## Using local ip as NAT
+nat_addr=$(ifconfig eth0 | awk '/inet/ {gsub("addr:", "", $2); print $2}')
+echo "Local IP: $nat_addr"
+
 # Required arguments
 if [ -n "$LISTEN_ADDRS" ]; then
   echo "Listen address: $LISTEN_ADDRS"
@@ -28,7 +36,7 @@ fi
 # Log level
 if [ -n "$LOG_LEVEL" ]; then
   echo "Log level: $LOG_LEVEL"
-  args="$args --log-level=$LOG_LEVEL"
+  args="$args --log-level=\"$LOG_LEVEL\""
 fi
 
 # Metrics
@@ -40,10 +48,8 @@ if [ -n "$METRICS_ADDR" ] && [ -n "$METRICS_PORT" ]; then
 fi
 
 # NAT
-if [ -n "$NAT_IP" ]; then
-  echo "NAT: $NAT_IP"
-  args="$args --nat=$NAT_IP"
-fi
+echo "NAT: $nat_addr"
+args="$args --nat=$nat_addr"
 
 # Discovery IP
 if [ -n "$DISC_IP" ]; then
@@ -106,17 +112,39 @@ if [ -n "$CACHE_SIZE" ]; then
 fi
 
 # Ethereum persistence
-if [ -n "$ETH_PROVIDER" ] && [ -n "$ETH_ACCOUNT" ] && [ -n "$ETH_MARKETPLACE_ADDRESS" ]; then
-    echo "Persistence enabled"
-    args="$args --persistence=true"
+if [ -n "$ETH_PROVIDER" ]; then
+    echo "Provider: $ETH_PROVIDER"
     args="$args --eth-provider=$ETH_PROVIDER"
-    args="$args --eth-account=$ETH_ACCOUNT"
-    # args="$args --validator"
-
-    # Remove this as soon as CLI option is available:
-    echo "{\"contracts\": { \"Marketplace\": { \"address\": \""$ACCOUNTSTR"\" } } }" > /root/marketplace_address.json
-    args="$args --eth-deployment=/root/marketplace_address.json"
 fi
 
-echo "./root/codex $args"
-sh -c "/root/codex $args"
+if [ -n "$ETH_ACCOUNT" ]; then
+  echo "Ethereum account: $ETH_ACCOUNT"
+  args="$args --eth-account=$ETH_ACCOUNT"
+fi
+
+if [ -n "$ETH_MARKETPLACE_ADDRESS" ]; then
+  echo "Marketplace address: $ETH_MARKETPLACE_ADDRESS"
+  args="$args --marketplace-address=$ETH_MARKETPLACE_ADDRESS"
+fi
+
+if [ -n "$SIMULATE_PROOF_FAILURES" ]; then
+  echo "Simulate proof failures: $SIMULATE_PROOF_FAILURES"
+  args="$args --simulate-proof-failures=$SIMULATE_PROOF_FAILURES"
+fi
+
+if [ "$PERSISTENCE" = "true" ] || [ "$PERSISTENCE" = "1" ]; then
+  echo "Persistence enabled"
+  args="$args --persistence"
+else
+  echo "Persistence disabled"
+fi
+
+if [ "$VALIDATOR" = "true" ] || [ "$VALIDATOR" = "1" ]; then
+  echo "Validator enabled"
+  args="$args --validator"
+else
+  echo "Validator disabled"
+fi
+
+echo "./codex $args"
+/bin/bash -l -c "./codex $args"

openapi.yaml

@@ -40,11 +40,11 @@ components:
 
     Duration:
       type: string
-      description: The duration of the request in seconds as hexadecimal string
+      description: The duration of the request in seconds as decimal string
 
     ProofProbability:
       type: string
-      description: How often storage proofs are required as hexadecimal string
+      description: How often storage proofs are required as decimal string
 
     Expiry:
       type: string
@@ -106,15 +106,27 @@ components:
           description: Hexadecimal identifier of the availability
         size:
           type: string
-          description: Size of available storage in bytes as hexadecimal string
+          description: Size of available storage in bytes as decimal string
         duration:
           $ref: "#/components/schemas/Duration"
         minPrice:
           type: string
-          description: Minimum price to be paid (in amount of tokens) as hexadecimal string
+          description: Minimum price to be paid (in amount of tokens) as decimal string
         maxCollateral:
           type: string
-          description: Maximum collateral user is willing to pay per filled Slot (in amount of tokens)
+          description: Maximum collateral user is willing to pay per filled Slot (in amount of tokens) as decimal string
+
+    Slot:
+      type: object
+      properties:
+        id:
+          type: string
+          description: Slot ID
+        request:
+          $ref: "#/components/schemas/StorageRequest"
+        slotIndex:
+          type: string
+          description: Slot Index as hexadecimal string
 
     StorageRequestCreation:
       type: object
@@ -152,7 +164,7 @@ components:
           description: Number of slots (eq. hosts) that the Request want to have the content spread over
         slotSize:
           type: string
-          description: Amount of storage per slot (in bytes) as hexadecimal string
+          description: Amount of storage per slot (in bytes) as decimal string
         duration:
           $ref: "#/components/schemas/Duration"
         proofProbability:
@@ -166,6 +178,9 @@ components:
     StorageRequest:
       type: object
       properties:
+        id:
+          type: string
+          description: Request ID
         client:
           $ref: "#/components/schemas/EthereumAddress"
         ask:
@@ -286,14 +301,32 @@ paths:
         "500":
           description: Well it was bad-bad and the upload did not work out
 
+  "/sales/slots":
+    get:
+      summary: "Returns active slots"
+      tags: [ Marketplace ]
+      operationId: getActiveSlots
+      responses:
+        "200":
+          description: Retrieved active slots
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/Slot"
+
+        "503":
+          description: Sales are unavailable
+
   "/sales/availability":
     get:
       summary: "Returns storage that is for sale"
       tags: [ Marketplace ]
-      operationId: getsOfferedStorage
+      operationId: getOfferedStorage
       responses:
         "200":
-          description: Retrieved content specified by CID
+          description: Retrieved storage availabilities of the node
           content:
             application/json:
               schema:
@@ -373,7 +406,7 @@ paths:
               $ref: "#/components/schemas/StorageRequestCreation"
       responses:
         "200":
-          description: Returns the Request ID as hexadecimal string
+          description: Returns the Request ID as decimal string
         "400":
           description: Invalid or missing Request ID
         "404":

tests/contracts/deployment.nim

@@ -1,4 +1,5 @@
 import std/os
+import std/options
 import pkg/ethers
 import pkg/codex/contracts/marketplace
 

tests/contracts/time.nim

@@ -4,10 +4,10 @@ proc currentTime*(provider: Provider): Future[UInt256] {.async.} =
   return (!await provider.getBlock(BlockTag.latest)).timestamp
 
 proc advanceTime*(provider: JsonRpcProvider, seconds: UInt256) {.async.} =
-  discard await provider.send("evm_increaseTime", @[%seconds])
+  discard await provider.send("evm_increaseTime", @[%("0x" & seconds.toHex)])
   discard await provider.send("evm_mine")
 
 proc advanceTimeTo*(provider: JsonRpcProvider, timestamp: UInt256) {.async.} =
   if (await provider.currentTime()) != timestamp:
-    discard await provider.send("evm_setNextBlockTimestamp", @[%timestamp])
+    discard await provider.send("evm_setNextBlockTimestamp", @[%("0x" & timestamp.toHex)])
     discard await provider.send("evm_mine")

tests/integration/codexclient.nim

@@ -35,11 +35,11 @@ proc requestStorage*(client: CodexClient,
                      collateral: uint64): string =
   let url = client.baseurl & "/storage/request/" & cid
   let json = %*{
-    "duration": "0x" & duration.toHex,
-    "reward": "0x" & reward.toHex,
-    "proofProbability": "0x" & proofProbability.toHex,
-    "expiry": "0x" & expiry.toHex,
-    "collateral": "0x" & collateral.toHex,
+    "duration": $duration,
+    "reward": $reward,
+    "proofProbability": $proofProbability,
+    "expiry": $expiry,
+    "collateral": $collateral,
   }
   let response = client.http.post(url, $json)
   assert response.status == "200 OK"
@@ -50,14 +50,19 @@ proc getPurchase*(client: CodexClient, purchase: string): JsonNode =
   let body = client.http.getContent(url)
   parseJson(body).catch |? nil
 
+proc getSlots*(client: CodexClient): JsonNode =
+  let url = client.baseurl & "/sales/slots"
+  let body = client.http.getContent(url)
+  parseJson(body).catch |? nil
+
 proc postAvailability*(client: CodexClient,
                        size, duration, minPrice: uint64, maxCollateral: uint64): JsonNode =
   let url = client.baseurl & "/sales/availability"
   let json = %*{
-    "size": "0x" & size.toHex,
-    "duration": "0x" & duration.toHex,
-    "minPrice": "0x" & minPrice.toHex,
-    "maxCollateral": "0x" & maxCollateral.toHex
+    "size": $size,
+    "duration": $duration,
+    "minPrice": $minPrice,
+    "maxCollateral": $maxCollateral,
   }
   let response = client.http.post(url, $json)
   assert response.status == "200 OK"

tests/integration/testIntegration.nim

@@ -2,7 +2,8 @@ import std/json
 import pkg/chronos
 import pkg/stint
 import pkg/ethers/erc20
-import codex/contracts
+import pkg/codex/contracts
+import pkg/codex/utils/stintutils
 import ../contracts/time
 import ../contracts/deployment
 import ../codex/helpers/eventually
@@ -52,9 +53,9 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
     let cid = client1.upload("some file contents")
     let id = client1.requestStorage(cid, duration=1, reward=2, proofProbability=3, expiry=expiry, collateral=200)
     let purchase = client1.getPurchase(id)
-    check purchase{"request"}{"ask"}{"duration"} == %"0x1"
-    check purchase{"request"}{"ask"}{"reward"} == %"0x2"
-    check purchase{"request"}{"ask"}{"proofProbability"} == %"0x3"
+    check purchase{"request"}{"ask"}{"duration"} == %"1"
+    check purchase{"request"}{"ask"}{"reward"} == %"2"
+    check purchase{"request"}{"ask"}{"proofProbability"} == %"3"
 
   test "node remembers purchase status after restart":
     let expiry = (await provider.currentTime()) + 30
@@ -66,8 +67,8 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
     client1.restart()
 
     check eventually (not isNil client1.getPurchase(id){"request"}{"ask"})
-    check client1.getPurchase(id){"request"}{"ask"}{"duration"} == %"0x1"
-    check client1.getPurchase(id){"request"}{"ask"}{"reward"} == %"0x2"
+    check client1.getPurchase(id){"request"}{"ask"}{"duration"} == %"1"
+    check client1.getPurchase(id){"request"}{"ask"}{"reward"} == %"2"
 
   test "nodes negotiate contracts on the marketplace":
     let size: uint64 = 0xFFFFF
@@ -83,7 +84,7 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
     check client1.getPurchase(purchase){"error"} == newJNull()
     let availabilities = client2.getAvailabilities()
     check availabilities.len == 1
-    let newSize = UInt256.fromHex(availabilities[0]{"size"}.getStr)
+    let newSize = UInt256.fromDecimal(availabilities[0]{"size"}.getStr)
     check newSize > 0 and newSize < size.u256
 
   test "node slots gets paid out":

vendor/nim-ethers

@@ -1 +1 @@
-Subproject commit 5a4f786757124c903ab46499689db8273ee5ac80
+Subproject commit 0321e6d7bd9c703c9e9bf31ee8664adac1d6cbe7

vendor/nim-libp2p-dht

@@ -1 +1 @@
-Subproject commit 4375b9229815c332a3b1a9d0091d5cf5a74adb2e
+Subproject commit bd517f0e8da38a1b5da15f7deb2d5c652ca389f1