diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d73b81ea..ad31c3c6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -9,35 +9,117 @@ on: workflow_dispatch: jobs: - docker: + docker-amd64: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 + - name: Docker meta id: meta uses: docker/metadata-action@v4 with: - images: thatbenbierens/nim-codex + images: thatbenbierens/nim-codex-amd64 tags: | type=semver,pattern={{version}} type=sha - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub if: github.event_name != 'pull_request' uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push + + - name: Build and export to Docker + id: build uses: docker/build-push-action@v4 with: context: . file: docker/codex.Dockerfile - platforms: linux/amd64,linux/arm64,linux/arm/v7 - push: ${{ github.event_name != 'pull_request' }} + platforms: linux/amd64 + load: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Minify docker image + uses: kitabisa/docker-slim-action@v1 + env: + DSLIM_HTTP_PROBE: false + with: + target: ${{ steps.meta.outputs.tags }} + overwrite: true + + - name: Show slim report + run: echo "${{ steps.slim.outputs.report }}" + + - name: Push to Docker registry + if: github.event_name != 'pull_request' + id: push + uses: docker/build-push-action@v4 + with: + context: . + file: docker/codex.Dockerfile + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + docker-arm64: + runs-on: buildjet-4vcpu-ubuntu-2204-arm + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: thatbenbierens/nim-codex-arm64 + tags: | + type=semver,pattern={{version}} + type=sha + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and export to Docker + id: build + uses: docker/build-push-action@v4 + with: + context: . + file: docker/codex.Dockerfile + platforms: linux/arm64 + load: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Minify docker image + uses: kitabisa/docker-slim-action@v1 + env: + DSLIM_HTTP_PROBE: false + with: + target: ${{ steps.meta.outputs.tags }} + overwrite: true + + - name: Show slim report + run: echo "${{ steps.slim.outputs.report }}" + + - name: Push to Docker registry + if: github.event_name != 'pull_request' + id: push + uses: docker/build-push-action@v4 + with: + context: . + file: docker/codex.Dockerfile + platforms: linux/arm64 + push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 4a3bf03b..1ffcef8a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -3,7 +3,7 @@ name: OpenAPI on: push: branches: - - 'main' + - 'master' paths: - 'openapi.yaml' - '.github/workflows/docs.yml' @@ -41,7 +41,7 @@ jobs: deploy: name: Deploy runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/master' steps: - name: Checkout uses: actions/checkout@v3 diff --git a/README.md b/README.md index 34458e76..07253dad 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ For example, to configure `--log-level`, use `CODEX_LOG_LEVEL` as the environmen A [TOML](https://toml.io/en/) configuration file can also be used to set configuration values. Configuration option names and corresponding values are placed in the file, separated by `=`. Configuration option names can be obtained from the `codex --help` command, and should not include the `--` prefix. For example, a node's log level (`--log-level`) can be configured using TOML as follows: ```toml -log-level = "TRACE" +log-level = "trace" ``` The Codex node can then read the configuration from this file using the `--config-file` CLI parameter, like `codex --config-file=/path/to/your/config.toml`. @@ -70,7 +70,7 @@ codex [OPTIONS]... command The following options are available: --config-file Loads the configuration from a TOML file [=none]. - --log-level Sets the log level [=INFO]. + --log-level Sets the log level [=info]. --metrics Enable the metrics server [=false]. --metrics-address Listening address of the metrics server [=127.0.0.1]. --metrics-port Listening HTTP port of the metrics server [=8008]. @@ -111,9 +111,9 @@ codex initNode Codex uses [Chronicles](https://github.com/status-im/nim-chronicles) logging library, which allows great flexibility in working with logs. Chronicles has the concept of topics, which categorize log entries into semantic groups. -Using the `log-level` parameter, you can set the top-level log level like `--log-level="TRACE"`, but more importantly, -you can set log levels for specific topics like `--log-level="INFO; TRACE: marketplace,node; ERROR: blockexchange"`, -which sets the top-level log level to `INFO` and then for topics `marketplace` and `node` sets the level to `TRACE` and so on. +Using the `log-level` parameter, you can set the top-level log level like `--log-level="trace"`, but more importantly, +you can set log levels for specific topics like `--log-level="info; trace: marketplace,node; error: blockexchange"`, +which sets the top-level log level to `info` and then for topics `marketplace` and `node` sets the level to `trace` and so on. ### Example: running two Codex clients diff --git a/codex.nimble b/codex.nimble index c137a222..1bfe4cc4 100644 --- a/codex.nimble +++ b/codex.nimble @@ -7,29 +7,30 @@ license = "MIT" binDir = "build" srcDir = "." -requires "nim >= 1.2.0", - "asynctest >= 0.3.2 & < 0.4.0", - "bearssl >= 0.1.4", - "chronicles >= 0.7.2", - "chronos >= 2.5.2", - "confutils", - "ethers >= 0.2.4 & < 0.3.0", - "libbacktrace", - "libp2p", - "metrics", - "nimcrypto >= 0.4.1", - "nitro >= 0.5.1 & < 0.6.0", - "presto", - "protobuf_serialization >= 0.2.0 & < 0.3.0", - "questionable >= 0.10.6 & < 0.11.0", - "secp256k1", - "stew", - "upraises >= 0.1.0 & < 0.2.0", - "https://github.com/status-im/lrucache.nim#1.2.2", - "leopard >= 0.1.0 & < 0.2.0", - "blscurve", - "libp2pdht", - "eth" +requires "nim >= 1.2.0" +requires "asynctest >= 0.3.2 & < 0.4.0" +requires "bearssl >= 0.1.4" +requires "chronicles >= 0.7.2" +requires "chronos >= 2.5.2" +requires "confutils" +requires "ethers >= 0.2.4 & < 0.3.0" +requires "libbacktrace" +requires "libp2p" +requires "metrics" +requires "nimcrypto >= 0.4.1" +requires "nitro >= 0.5.1 & < 0.6.0" +requires "presto" +requires "protobuf_serialization >= 0.2.0 & < 0.3.0" +requires "questionable >= 0.10.6 & < 0.11.0" +requires "secp256k1" +requires "stew" +requires "upraises >= 0.1.0 & < 0.2.0" +requires "toml_serialization" +requires "https://github.com/status-im/lrucache.nim#1.2.2" +requires "leopard >= 0.1.0 & < 0.2.0" +requires "blscurve" +requires "libp2pdht" +requires "eth" when declared(namedBin): namedBin = { diff --git a/codex/codex.nim b/codex/codex.nim index 9926811c..4617bfe3 100644 --- a/codex/codex.nim +++ b/codex/codex.nim @@ -141,8 +141,11 @@ proc stop*(s: CodexServer) {.async.} = s.runHandle.complete() -proc new*(T: type CodexServer, config: CodexConf, privateKey: CodexPrivateKey): T = - +proc new*( + T: type CodexServer, + config: CodexConf, + privateKey: CodexPrivateKey): CodexServer = + ## create CodexServer including setting up datastore, repostore, etc let switch = SwitchBuilder .new() @@ -221,6 +224,7 @@ proc new*(T: type CodexServer, config: CodexConf, privateKey: CodexPrivateKey): .expect("Should start rest server!") switch.mount(network) + T( config: config, codexNode: codexNode, diff --git a/codex/conf.nim b/codex/conf.nim index 0fb365bb..57ae29b2 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -35,6 +35,9 @@ import ./stores export DefaultCacheSizeMiB, net, DefaultQuotaBytes, DefaultBlockTtl, DefaultBlockMaintenanceInterval, DefaultNumberOfBlocksToMaintainPerInterval +const + codex_enable_api_debug_peers* {.booldefine.} = false + type StartUpCommand* {.pure.} = enum noCommand, @@ -59,7 +62,7 @@ type name: "config-file" }: Option[InputFile] logLevel* {. - defaultValue: "INFO" + defaultValue: "info" desc: "Sets the log level", name: "log-level" }: string @@ -249,7 +252,10 @@ proc getCodexVersion(): string = return tag proc getCodexRevision(): string = - strip(staticExec("git rev-parse --short HEAD"))[0..5] + # using a slice in a static context breaks nimsuggest for some reason + var res = strip(staticExec("git rev-parse --short HEAD")) + res.setLen(6) + return res proc getNimBanner(): string = staticExec("nim --version | grep Version") @@ -347,9 +353,9 @@ proc updateLogLevel*(logLevel: string) {.upraises: [ValueError].} = # Updates log levels (without clearing old ones) let directives = logLevel.split(";") try: - setLogLevel(parseEnum[LogLevel](directives[0])) + setLogLevel(parseEnum[LogLevel](directives[0].toUpperAscii)) except ValueError: - raise (ref ValueError)(msg: "Please specify one of TRACE, DEBUG, INFO, NOTICE, WARN, ERROR or FATAL") + raise (ref ValueError)(msg: "Please specify one of: trace, debug, info, notice, warn, error or fatal") if directives.len > 1: for topicName, settings in parseTopicDirectives(directives[1..^1]): diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index 841e4138..9650df24 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -1,4 +1,5 @@ import std/strutils +import std/strformat import pkg/chronicles import pkg/ethers import pkg/ethers/testing @@ -52,7 +53,10 @@ method myRequests*(market: OnChainMarket): Future[seq[RequestId]] {.async.} = return await market.contract.myRequests method mySlots*(market: OnChainMarket): Future[seq[SlotId]] {.async.} = - return await market.contract.mySlots() + let slots = await market.contract.mySlots() + debug "Fetched my slots", numSlots=len(slots) + + return slots method requestStorage(market: OnChainMarket, request: StorageRequest){.async.} = debug "Requesting storage" diff --git a/codex/discovery.nim b/codex/discovery.nim index 9f3b81ac..836ddb88 100644 --- a/codex/discovery.nim +++ b/codex/discovery.nim @@ -35,7 +35,7 @@ logScope: type Discovery* = ref object of RootObj - protocol: discv5.Protocol # dht protocol + protocol*: discv5.Protocol # dht protocol key: PrivateKey # private key peerId: PeerId # the peer id of the local node announceAddrs: seq[MultiAddress] # addresses announced as part of the provider records @@ -58,13 +58,16 @@ proc toNodeId*(host: ca.Address): NodeId = proc findPeer*( d: Discovery, peerId: PeerId): Future[?PeerRecord] {.async.} = + trace "protocol.resolve..." let node = await d.protocol.resolve(toNodeId(peerId)) return if node.isSome(): + trace "protocol.resolve some data" node.get().record.data.some else: + trace "protocol.resolve none" PeerRecord.none method find*( @@ -127,16 +130,10 @@ method provide*(d: Discovery, host: ca.Address) {.async, base.} = trace "Provided to nodes", nodes = nodes.len method removeProvider*(d: Discovery, peerId: PeerId): Future[void] {.base.} = - ## Remove provider from providers table - ## - trace "Removing provider", peerId d.protocol.removeProvidersLocal(peerId) proc updateAnnounceRecord*(d: Discovery, addrs: openArray[MultiAddress]) = - ## Update providers record - ## - d.announceAddrs = @addrs trace "Updating announce record", addrs = d.announceAddrs @@ -149,9 +146,6 @@ proc updateAnnounceRecord*(d: Discovery, addrs: openArray[MultiAddress]) = .expect("Should update SPR") proc updateDhtRecord*(d: Discovery, ip: ValidIpAddress, port: Port) = - ## Update providers record - ## - trace "Updating Dht record", ip, port = $port d.dhtRecord = SignedPeerRecord.init( d.key, PeerRecord.init(d.peerId, @[ @@ -160,6 +154,10 @@ proc updateDhtRecord*(d: Discovery, ip: ValidIpAddress, port: Port) = IpTransportProtocol.udpProtocol, port)])).expect("Should construct signed record").some + if not d.protocol.isNil: + d.protocol.updateRecord(d.dhtRecord) + .expect("Should update SPR") + proc start*(d: Discovery) {.async.} = d.protocol.open() await d.protocol.start() diff --git a/codex/proving.nim b/codex/proving.nim index 101baae2..6ba5dc1b 100644 --- a/codex/proving.nim +++ b/codex/proving.nim @@ -56,6 +56,7 @@ proc prove(proving: Proving, slot: Slot) {.async.} = without onProve =? proving.onProve: raiseAssert "onProve callback not set" try: + debug "Proving slot" let proof = await onProve(slot) await proving.market.submitProof(slot.id, proof) except CatchableError as e: diff --git a/codex/purchasing/states/pending.nim b/codex/purchasing/states/pending.nim index 92822ac7..64a7fdd5 100644 --- a/codex/purchasing/states/pending.nim +++ b/codex/purchasing/states/pending.nim @@ -1,7 +1,6 @@ import ../statemachine import ./errorhandling import ./submitted -import ./error type PurchasePending* = ref object of ErrorHandlingState diff --git a/codex/purchasing/states/started.nim b/codex/purchasing/states/started.nim index 148ccc38..27d28ddf 100644 --- a/codex/purchasing/states/started.nim +++ b/codex/purchasing/states/started.nim @@ -1,6 +1,5 @@ import ../statemachine import ./errorhandling -import ./error import ./finished import ./failed diff --git a/codex/purchasing/states/submitted.nim b/codex/purchasing/states/submitted.nim index c051d282..5e6dd892 100644 --- a/codex/purchasing/states/submitted.nim +++ b/codex/purchasing/states/submitted.nim @@ -1,6 +1,5 @@ import ../statemachine import ./errorhandling -import ./error import ./started import ./cancelled diff --git a/codex/purchasing/states/unknown.nim b/codex/purchasing/states/unknown.nim index cde4217a..38628334 100644 --- a/codex/purchasing/states/unknown.nim +++ b/codex/purchasing/states/unknown.nim @@ -5,7 +5,6 @@ import ./started import ./cancelled import ./finished import ./failed -import ./error type PurchaseUnknown* = ref object of ErrorHandlingState diff --git a/codex/rest/api.nim b/codex/rest/api.nim index 26cbd76f..5a7f69d1 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -24,8 +24,11 @@ import pkg/stew/base10 import pkg/stew/byteutils import pkg/confutils +import pkg/libp2p import pkg/libp2p/routing_record import pkg/libp2pdht/discv5/spr as spr +import pkg/libp2pdht/discv5/routing_table as rt +import pkg/libp2pdht/discv5/node as dn import ../node import ../blocktype @@ -45,6 +48,47 @@ proc validate( {.gcsafe, raises: [Defect].} = 0 +proc formatAddress(address: Option[dn.Address]): string = + if address.isSome(): + return $address.get() + return "" + +proc formatNode(node: dn.Node): JsonNode = + let jobj = %*{ + "nodeId": $node.id, + "peerId": $node.record.data.peerId, + "record": $node.record, + "address": formatAddress(node.address), + "seen": $node.seen + } + return jobj + +proc formatTable(routingTable: rt.RoutingTable): JsonNode = + let jarray = newJArray() + for bucket in routingTable.buckets: + for node in bucket.nodes: + jarray.add(formatNode(node)) + + let jobj = %*{ + "localNode": formatNode(routingTable.localNode), + "nodes": jarray + } + return jobj + +proc formatPeerRecord(peerRecord: PeerRecord): JsonNode = + let jarray = newJArray() + for maddr in peerRecord.addresses: + jarray.add(%*{ + "address": $maddr.address + }) + + let jobj = %*{ + "peerId": $peerRecord.peerId, + "seqNo": $peerRecord.seqNo, + "addresses": jarray + } + return jobj + proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter = var router = RestRouter.init(validate) router.api( @@ -244,13 +288,41 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter = node.discovery.dhtRecord.get.toURI else: "", + "table": formatTable(node.discovery.protocol.routingTable), "codex": { "version": $codexVersion, "revision": $codexRevision } } - return RestApiResponse.response($json) + return RestApiResponse.response($json, contentType="application/json") + + when codex_enable_api_debug_peers: + router.api( + MethodGet, + "/api/codex/v1/debug/peer/{peerId}") do (peerId: PeerId) -> RestApiResponse: + + trace "debug/peer start" + without peerRecord =? (await node.findPeer(peerId.get())): + trace "debug/peer peer not found!" + return RestApiResponse.error( + Http400, + "Unable to find Peer!") + + let json = formatPeerRecord(peerRecord) + trace "debug/peer returning peer record" + return RestApiResponse.response($json) + + router.api( + MethodGet, + "/api/codex/v1/sales/slots") do () -> RestApiResponse: + ## Returns active slots for the host + + without contracts =? node.contracts.host: + return RestApiResponse.error(Http503, "Sales unavailable") + + let json = %(await contracts.sales.mySlots()) + return RestApiResponse.response($json, contentType="application/json") router.api( MethodGet, @@ -264,7 +336,7 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter = return RestApiResponse.error(Http500, err.msg) let json = %unused - return RestApiResponse.response($json) + return RestApiResponse.response($json, contentType="application/json") router.rawApi( MethodPost, @@ -293,7 +365,7 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter = return RestApiResponse.error(Http500, err.msg) let json = %availability - return RestApiResponse.response($json) + return RestApiResponse.response($json, contentType="application/json") router.api( MethodGet, @@ -311,7 +383,6 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter = let json = %purchase - return RestApiResponse.response($json) - + return RestApiResponse.response($json, contentType="application/json") return router diff --git a/codex/rest/coders.nim b/codex/rest/coders.nim index 8a7c144d..66b41ee3 100644 --- a/codex/rest/coders.nim +++ b/codex/rest/coders.nim @@ -19,6 +19,7 @@ import pkg/stint import ../sales import ../purchasing +import ../utils/stintutils proc encodeString*(cid: type Cid): Result[string, cstring] = ok($cid) @@ -72,7 +73,7 @@ proc encodeString*(value: bool): Result[string, cstring] = proc decodeString*(_: type UInt256, value: string): Result[UInt256, cstring] = try: - ok UInt256.fromHex(value) + ok UInt256.fromDecimal(value) except ValueError as e: err e.msg.cstring diff --git a/codex/rest/json.nim b/codex/rest/json.nim index ff087467..33e89a7c 100644 --- a/codex/rest/json.nim +++ b/codex/rest/json.nim @@ -4,6 +4,9 @@ import pkg/stew/byteutils import pkg/questionable/results import ../sales import ../purchasing +import ../utils/stintutils + +export json type StorageRequestParams* = object @@ -17,22 +20,22 @@ type proc fromJson*(_: type Availability, bytes: seq[byte]): ?!Availability = let json = ?catch parseJson(string.fromBytes(bytes)) - let size = ?catch UInt256.fromHex(json["size"].getStr) - let duration = ?catch UInt256.fromHex(json["duration"].getStr) - let minPrice = ?catch UInt256.fromHex(json["minPrice"].getStr) - let maxCollateral = ?catch UInt256.fromHex(json["maxCollateral"].getStr) + let size = ?catch UInt256.fromDecimal(json["size"].getStr) + let duration = ?catch UInt256.fromDecimal(json["duration"].getStr) + let minPrice = ?catch UInt256.fromDecimal(json["minPrice"].getStr) + let maxCollateral = ?catch UInt256.fromDecimal(json["maxCollateral"].getStr) success Availability.init(size, duration, minPrice, maxCollateral) proc fromJson*(_: type StorageRequestParams, bytes: seq[byte]): ?! StorageRequestParams = let json = ?catch parseJson(string.fromBytes(bytes)) - let duration = ?catch UInt256.fromHex(json["duration"].getStr) - let proofProbability = ?catch UInt256.fromHex(json["proofProbability"].getStr) - let reward = ?catch UInt256.fromHex(json["reward"].getStr) - let collateral = ?catch UInt256.fromHex(json["collateral"].getStr) - let expiry = UInt256.fromHex(json["expiry"].getStr).catch.option - let nodes = strutils.fromHex[uint](json["nodes"].getStr).catch.option - let tolerance = strutils.fromHex[uint](json["tolerance"].getStr).catch.option + let duration = ?catch UInt256.fromDecimal(json["duration"].getStr) + let proofProbability = ?catch UInt256.fromDecimal(json["proofProbability"].getStr) + let reward = ?catch UInt256.fromDecimal(json["reward"].getStr) + let collateral = ?catch UInt256.fromDecimal(json["collateral"].getStr) + let expiry = UInt256.fromDecimal(json["expiry"].getStr).catch.option + let nodes = parseUInt(json["nodes"].getStr).catch.option + let tolerance = parseUInt(json["tolerance"].getStr).catch.option success StorageRequestParams( duration: duration, proofProbability: proofProbability, @@ -46,8 +49,8 @@ proc fromJson*(_: type StorageRequestParams, func `%`*(address: Address): JsonNode = % $address -func `%`*(stint: StInt|StUint): JsonNode = - %("0x" & stint.toHex) +func `%`*(stint: StInt|StUint): JsonNode= + %(stint.toString) func `%`*(arr: openArray[byte]): JsonNode = %("0x" & arr.toHex) @@ -55,6 +58,13 @@ func `%`*(arr: openArray[byte]): JsonNode = func `%`*(id: RequestId | SlotId | Nonce | AvailabilityId): JsonNode = % id.toArray +func `%`*(obj: StorageRequest | Slot): JsonNode = + let jsonObj = newJObject() + for k, v in obj.fieldPairs: jsonObj[k] = %v + jsonObj["id"] = %(obj.id) + + return jsonObj + func `%`*(purchase: Purchase): JsonNode = %*{ "state": purchase.state |? "none", diff --git a/codex/sales.nim b/codex/sales.nim index 01f3da77..8efeaf61 100644 --- a/codex/sales.nim +++ b/codex/sales.nim @@ -100,20 +100,28 @@ proc handleRequest(sales: Sales, agent.start(SaleDownloading()) sales.agents.add agent -proc load*(sales: Sales) {.async.} = +proc mySlots*(sales: Sales): Future[seq[Slot]] {.async.} = let market = sales.context.market - let slotIds = await market.mySlots() + var slots: seq[Slot] = @[] for slotId in slotIds: if slot =? (await market.getActiveSlot(slotId)): - let agent = newSalesAgent( - sales.context, - slot.request.id, - slot.slotIndex, - some slot.request) - agent.start(SaleUnknown()) - sales.agents.add agent + slots.add slot + + return slots + +proc load*(sales: Sales) {.async.} = + let slots = await sales.mySlots() + + for slot in slots: + let agent = newSalesAgent( + sales.context, + slot.request.id, + slot.slotIndex, + some slot.request) + agent.start(SaleUnknown()) + sales.agents.add agent proc start*(sales: Sales) {.async.} = doAssert sales.subscription.isNone, "Sales already started" diff --git a/codex/utils/stintutils.nim b/codex/utils/stintutils.nim new file mode 100644 index 00000000..125ff8b6 --- /dev/null +++ b/codex/utils/stintutils.nim @@ -0,0 +1,4 @@ +import pkg/stint + +func fromDecimal*(T: typedesc[StUint|StInt], s: string): T {.inline.} = + parse(s, type result, radix = 10) diff --git a/docker/README.md b/docker/README.md index da37a9ec..603c6e39 100644 --- a/docker/README.md +++ b/docker/README.md @@ -32,9 +32,15 @@ Codex docker image supports the following environment variables: - ETH_PROVIDER - ETH_ACCOUNT - ETH_DEPLOYMENT +- SIMULATE_PROOF_FAILURES +- VALIDATOR +- PERSISTENCE +- CODEX_NODENAME(†) (*) These variables have default values in the docker image that are different from Codex's standard default values. +(†) CODEX_NODENAME is used for logging purposes only in the docker image + All environment variables are optional and will default to Codex's CLI default values. # Constants @@ -46,3 +52,13 @@ To get the IP address of a container within a network: Find container Id: `docker ps` Open terminal in container: `docker exec -it sh` Get IP addresses: `ifconfig` + +# Slim +1. Build the image using `docker build -t status-im/codexsetup:latest -f codex.Dockerfile ..` +2. The docker image can then be minifed using [slim](https://github.com/slimtoolkit/slim). Install slim on your path and then run: +```shell +slim # brings up interactive prompt +>>> build --target status-im/codexsetup --http-probe-off true +``` +3. This should output an image with name `status-im/codexsetup.slim` +4. We can then bring up the image using `docker-compose up -d`. \ No newline at end of file diff --git a/docker/codex.Dockerfile b/docker/codex.Dockerfile index 2587cb5e..1c93d615 100644 --- a/docker/codex.Dockerfile +++ b/docker/codex.Dockerfile @@ -1,14 +1,18 @@ -FROM nimlang/nim:1.6.10-alpine AS builder +FROM ubuntu:lunar-20230415 AS builder +RUN apt-get update && apt-get install -y git cmake curl make bash lcov build-essential nim +RUN echo 'export NIMBLE_DIR="${HOME}/.nimble"' >> "${HOME}/.bash_env" +RUN echo 'export PATH="${NIMBLE_DIR}/bin:${PATH}"' >> "${HOME}/.bash_env" + WORKDIR /src -RUN apk update && apk add git cmake curl make git bash linux-headers COPY . . RUN make clean -RUN make update -RUN make NIM_PARAMS="-d:disableMarchNative" +RUN make -j4 update +RUN make -j4 NIM_PARAMS="-d:disableMarchNative -d:codex_enable_api_debug_peers=true" -FROM alpine:3.17.2 -WORKDIR /root/ -RUN apk add --no-cache openssl libstdc++ libgcc libgomp +FROM ubuntu:lunar-20230415 +WORKDIR /root +RUN apt-get update && apt-get install -y libgomp1 bash net-tools COPY --from=builder /src/build/codex ./ COPY --from=builder /src/docker/startCodex.sh ./ -CMD ["sh", "startCodex.sh"] +RUN chmod +x ./startCodex.sh +CMD ["/bin/bash", "-l", "-c", "./startCodex.sh"] diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 76c74e8c..643682a2 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,77 +1,27 @@ services: codex-node1: - image: clustertest-nim-codex - build: - context: ../. - dockerfile: docker/codex.Dockerfile + image: status-im/codexsetup.slim:latest ports: - 8080:8080 # Available environment variables: - # environment: - # - LOG_LEVEL=TRACE - # - METRICS_ADDR=0.0.0.0 - # - METRICS_PORT=9090 - # - NAT_IP=2.3.4.5 - # - API_PORT=8080 - # - DISC_IP=3.4.5.6 - # - DISC_PORT=8765 - # - NET_PRIVKEY=privkey - # - BOOTSTRAP_SPR=bootstrap_record - # - MAX_PEERS=123 - # - AGENT_STRING=agent_string - # - STORAGE_QUOTA=123456789 - # - BLOCK_TTL=23456 - # - CACHE_SIZE=6543 - # - ETH_PROVIDER=eth - # - ETH_ACCOUNT=account - # - ETH_DEPLOYMENT=deploy - volumes: - - ./hostdatadir/node1:/datadir - networks: - - primary - - # Example with metrics enabled. - codex-node2: - image: clustertest-nim-codex - ports: - - 8081:8080 - - 9090:9090 environment: + - LOG_LEVEL=TRACE - METRICS_ADDR=0.0.0.0 - METRICS_PORT=9090 - volumes: - - ./hostdatadir/node2:/datadir - depends_on: - - codex-node1 - networks: - - primary - - secondary + - NAT_IP=2.3.4.5 + - API_PORT=8080 + - DISC_IP=3.4.5.6 + - DISC_PORT=8765 + - NET_PRIVKEY=privkey + - BOOTSTRAP_SPR=bootstrap_record + - MAX_PEERS=123 + - AGENT_STRING=agent_string + - STORAGE_QUOTA=123456789 + - BLOCK_TTL=23456 + - CACHE_SIZE=6543 + - ETH_PROVIDER=eth + - ETH_ACCOUNT=account + - ETH_MARKETPLACE_ADDRESS=0x59b670e9fA9D0A427751Af201D676719a970857b + - SIMULATE_PROOF_FAILURES=2 - codex-node3: - image: clustertest-nim-codex - ports: - - 8082:8080 - volumes: - - ./hostdatadir/node3:/datadir - depends_on: - - codex-node1 - networks: - - secondary - prometheus: - image: prom/prometheus:v2.30.3 - ports: - - 9000:9090 - volumes: - - ./prometheus:/etc/prometheus - - ./prometheus-data:/prometheus - command: --web.enable-lifecycle --config.file=/etc/prometheus/prometheus.yml - networks: - - primary - - secondary - -networks: - primary: - name: primary - secondary: - name: secondary diff --git a/docker/prometheus/alert.yml b/docker/prometheus/alert.yml deleted file mode 100644 index b415a9bc..00000000 --- a/docker/prometheus/alert.yml +++ /dev/null @@ -1,6 +0,0 @@ -groups: - - name: DemoAlerts - rules: - - alert: InstanceDown - expr: up{job="services"} < 1 - for: 5m diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml deleted file mode 100644 index 84b40afd..00000000 --- a/docker/prometheus/prometheus.yml +++ /dev/null @@ -1,15 +0,0 @@ -global: - scrape_interval: 30s - scrape_timeout: 10s - -rule_files: - - alert.yml - -scrape_configs: - - job_name: services - metrics_path: /metrics - static_configs: - - targets: - - 'prometheus:9090' - - 'codex-node1:9090' - - 'codex-node2:9090' diff --git a/docker/startCodex.sh b/docker/startCodex.sh index 4f0a08a3..b3b7b472 100644 --- a/docker/startCodex.sh +++ b/docker/startCodex.sh @@ -1,7 +1,15 @@ -echo "Starting Codex..." +NAME="" +if [ -n "$CODEX_NODENAME" ]; then + NAME=" '$CODEX_NODENAME'" +fi +echo "Starting Codex node$NAME" args="" +## Using local ip as NAT +nat_addr=$(ifconfig eth0 | awk '/inet/ {gsub("addr:", "", $2); print $2}') +echo "Local IP: $nat_addr" + # Required arguments if [ -n "$LISTEN_ADDRS" ]; then echo "Listen address: $LISTEN_ADDRS" @@ -28,7 +36,7 @@ fi # Log level if [ -n "$LOG_LEVEL" ]; then echo "Log level: $LOG_LEVEL" - args="$args --log-level=$LOG_LEVEL" + args="$args --log-level=\"$LOG_LEVEL\"" fi # Metrics @@ -40,10 +48,8 @@ if [ -n "$METRICS_ADDR" ] && [ -n "$METRICS_PORT" ]; then fi # NAT -if [ -n "$NAT_IP" ]; then - echo "NAT: $NAT_IP" - args="$args --nat=$NAT_IP" -fi +echo "NAT: $nat_addr" +args="$args --nat=$nat_addr" # Discovery IP if [ -n "$DISC_IP" ]; then @@ -106,17 +112,39 @@ if [ -n "$CACHE_SIZE" ]; then fi # Ethereum persistence -if [ -n "$ETH_PROVIDER" ] && [ -n "$ETH_ACCOUNT" ] && [ -n "$ETH_MARKETPLACE_ADDRESS" ]; then - echo "Persistence enabled" - args="$args --persistence=true" +if [ -n "$ETH_PROVIDER" ]; then + echo "Provider: $ETH_PROVIDER" args="$args --eth-provider=$ETH_PROVIDER" - args="$args --eth-account=$ETH_ACCOUNT" - # args="$args --validator" - - # Remove this as soon as CLI option is available: - echo "{\"contracts\": { \"Marketplace\": { \"address\": \""$ACCOUNTSTR"\" } } }" > /root/marketplace_address.json - args="$args --eth-deployment=/root/marketplace_address.json" fi -echo "./root/codex $args" -sh -c "/root/codex $args" +if [ -n "$ETH_ACCOUNT" ]; then + echo "Ethereum account: $ETH_ACCOUNT" + args="$args --eth-account=$ETH_ACCOUNT" +fi + +if [ -n "$ETH_MARKETPLACE_ADDRESS" ]; then + echo "Marketplace address: $ETH_MARKETPLACE_ADDRESS" + args="$args --marketplace-address=$ETH_MARKETPLACE_ADDRESS" +fi + +if [ -n "$SIMULATE_PROOF_FAILURES" ]; then + echo "Simulate proof failures: $SIMULATE_PROOF_FAILURES" + args="$args --simulate-proof-failures=$SIMULATE_PROOF_FAILURES" +fi + +if [ "$PERSISTENCE" = "true" ] || [ "$PERSISTENCE" = "1" ]; then + echo "Persistence enabled" + args="$args --persistence" +else + echo "Persistence disabled" +fi + +if [ "$VALIDATOR" = "true" ] || [ "$VALIDATOR" = "1" ]; then + echo "Validator enabled" + args="$args --validator" +else + echo "Validator disabled" +fi + +echo "./codex $args" +/bin/bash -l -c "./codex $args" diff --git a/openapi.yaml b/openapi.yaml index 7576aa49..bbffee1b 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -40,11 +40,11 @@ components: Duration: type: string - description: The duration of the request in seconds as hexadecimal string + description: The duration of the request in seconds as decimal string ProofProbability: type: string - description: How often storage proofs are required as hexadecimal string + description: How often storage proofs are required as decimal string Expiry: type: string @@ -106,15 +106,27 @@ components: description: Hexadecimal identifier of the availability size: type: string - description: Size of available storage in bytes as hexadecimal string + description: Size of available storage in bytes as decimal string duration: $ref: "#/components/schemas/Duration" minPrice: type: string - description: Minimum price to be paid (in amount of tokens) as hexadecimal string + description: Minimum price to be paid (in amount of tokens) as decimal string maxCollateral: type: string - description: Maximum collateral user is willing to pay per filled Slot (in amount of tokens) + description: Maximum collateral user is willing to pay per filled Slot (in amount of tokens) as decimal string + + Slot: + type: object + properties: + id: + type: string + description: Slot ID + request: + $ref: "#/components/schemas/StorageRequest" + slotIndex: + type: string + description: Slot Index as hexadecimal string StorageRequestCreation: type: object @@ -152,7 +164,7 @@ components: description: Number of slots (eq. hosts) that the Request want to have the content spread over slotSize: type: string - description: Amount of storage per slot (in bytes) as hexadecimal string + description: Amount of storage per slot (in bytes) as decimal string duration: $ref: "#/components/schemas/Duration" proofProbability: @@ -166,6 +178,9 @@ components: StorageRequest: type: object properties: + id: + type: string + description: Request ID client: $ref: "#/components/schemas/EthereumAddress" ask: @@ -286,14 +301,32 @@ paths: "500": description: Well it was bad-bad and the upload did not work out + "/sales/slots": + get: + summary: "Returns active slots" + tags: [ Marketplace ] + operationId: getActiveSlots + responses: + "200": + description: Retrieved active slots + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Slot" + + "503": + description: Sales are unavailable + "/sales/availability": get: summary: "Returns storage that is for sale" tags: [ Marketplace ] - operationId: getsOfferedStorage + operationId: getOfferedStorage responses: "200": - description: Retrieved content specified by CID + description: Retrieved storage availabilities of the node content: application/json: schema: @@ -373,7 +406,7 @@ paths: $ref: "#/components/schemas/StorageRequestCreation" responses: "200": - description: Returns the Request ID as hexadecimal string + description: Returns the Request ID as decimal string "400": description: Invalid or missing Request ID "404": diff --git a/tests/contracts/deployment.nim b/tests/contracts/deployment.nim index 7689a302..73e0d424 100644 --- a/tests/contracts/deployment.nim +++ b/tests/contracts/deployment.nim @@ -1,4 +1,5 @@ import std/os +import std/options import pkg/ethers import pkg/codex/contracts/marketplace diff --git a/tests/contracts/time.nim b/tests/contracts/time.nim index 418e3fd9..05eba0b3 100644 --- a/tests/contracts/time.nim +++ b/tests/contracts/time.nim @@ -4,10 +4,10 @@ proc currentTime*(provider: Provider): Future[UInt256] {.async.} = return (!await provider.getBlock(BlockTag.latest)).timestamp proc advanceTime*(provider: JsonRpcProvider, seconds: UInt256) {.async.} = - discard await provider.send("evm_increaseTime", @[%seconds]) + discard await provider.send("evm_increaseTime", @[%("0x" & seconds.toHex)]) discard await provider.send("evm_mine") proc advanceTimeTo*(provider: JsonRpcProvider, timestamp: UInt256) {.async.} = if (await provider.currentTime()) != timestamp: - discard await provider.send("evm_setNextBlockTimestamp", @[%timestamp]) + discard await provider.send("evm_setNextBlockTimestamp", @[%("0x" & timestamp.toHex)]) discard await provider.send("evm_mine") diff --git a/tests/integration/codexclient.nim b/tests/integration/codexclient.nim index cc3e5eef..dc120de4 100644 --- a/tests/integration/codexclient.nim +++ b/tests/integration/codexclient.nim @@ -35,11 +35,11 @@ proc requestStorage*(client: CodexClient, collateral: uint64): string = let url = client.baseurl & "/storage/request/" & cid let json = %*{ - "duration": "0x" & duration.toHex, - "reward": "0x" & reward.toHex, - "proofProbability": "0x" & proofProbability.toHex, - "expiry": "0x" & expiry.toHex, - "collateral": "0x" & collateral.toHex, + "duration": $duration, + "reward": $reward, + "proofProbability": $proofProbability, + "expiry": $expiry, + "collateral": $collateral, } let response = client.http.post(url, $json) assert response.status == "200 OK" @@ -50,14 +50,19 @@ proc getPurchase*(client: CodexClient, purchase: string): JsonNode = let body = client.http.getContent(url) parseJson(body).catch |? nil +proc getSlots*(client: CodexClient): JsonNode = + let url = client.baseurl & "/sales/slots" + let body = client.http.getContent(url) + parseJson(body).catch |? nil + proc postAvailability*(client: CodexClient, size, duration, minPrice: uint64, maxCollateral: uint64): JsonNode = let url = client.baseurl & "/sales/availability" let json = %*{ - "size": "0x" & size.toHex, - "duration": "0x" & duration.toHex, - "minPrice": "0x" & minPrice.toHex, - "maxCollateral": "0x" & maxCollateral.toHex + "size": $size, + "duration": $duration, + "minPrice": $minPrice, + "maxCollateral": $maxCollateral, } let response = client.http.post(url, $json) assert response.status == "200 OK" diff --git a/tests/integration/testIntegration.nim b/tests/integration/testIntegration.nim index 796c95bf..24fc4c2d 100644 --- a/tests/integration/testIntegration.nim +++ b/tests/integration/testIntegration.nim @@ -2,7 +2,8 @@ import std/json import pkg/chronos import pkg/stint import pkg/ethers/erc20 -import codex/contracts +import pkg/codex/contracts +import pkg/codex/utils/stintutils import ../contracts/time import ../contracts/deployment import ../codex/helpers/eventually @@ -52,9 +53,9 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false: let cid = client1.upload("some file contents") let id = client1.requestStorage(cid, duration=1, reward=2, proofProbability=3, expiry=expiry, collateral=200) let purchase = client1.getPurchase(id) - check purchase{"request"}{"ask"}{"duration"} == %"0x1" - check purchase{"request"}{"ask"}{"reward"} == %"0x2" - check purchase{"request"}{"ask"}{"proofProbability"} == %"0x3" + check purchase{"request"}{"ask"}{"duration"} == %"1" + check purchase{"request"}{"ask"}{"reward"} == %"2" + check purchase{"request"}{"ask"}{"proofProbability"} == %"3" test "node remembers purchase status after restart": let expiry = (await provider.currentTime()) + 30 @@ -66,8 +67,8 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false: client1.restart() check eventually (not isNil client1.getPurchase(id){"request"}{"ask"}) - check client1.getPurchase(id){"request"}{"ask"}{"duration"} == %"0x1" - check client1.getPurchase(id){"request"}{"ask"}{"reward"} == %"0x2" + check client1.getPurchase(id){"request"}{"ask"}{"duration"} == %"1" + check client1.getPurchase(id){"request"}{"ask"}{"reward"} == %"2" test "nodes negotiate contracts on the marketplace": let size: uint64 = 0xFFFFF @@ -83,7 +84,7 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false: check client1.getPurchase(purchase){"error"} == newJNull() let availabilities = client2.getAvailabilities() check availabilities.len == 1 - let newSize = UInt256.fromHex(availabilities[0]{"size"}.getStr) + let newSize = UInt256.fromDecimal(availabilities[0]{"size"}.getStr) check newSize > 0 and newSize < size.u256 test "node slots gets paid out": diff --git a/vendor/nim-ethers b/vendor/nim-ethers index 5a4f7867..0321e6d7 160000 --- a/vendor/nim-ethers +++ b/vendor/nim-ethers @@ -1 +1 @@ -Subproject commit 5a4f786757124c903ab46499689db8273ee5ac80 +Subproject commit 0321e6d7bd9c703c9e9bf31ee8664adac1d6cbe7 diff --git a/vendor/nim-libp2p-dht b/vendor/nim-libp2p-dht index 4375b922..bd517f0e 160000 --- a/vendor/nim-libp2p-dht +++ b/vendor/nim-libp2p-dht @@ -1 +1 @@ -Subproject commit 4375b9229815c332a3b1a9d0091d5cf5a74adb2e +Subproject commit bd517f0e8da38a1b5da15f7deb2d5c652ca389f1