docs: openapi specification (#355)

Co-authored-by: Eric Mastro <github@egonat.me>

.github/workflows/docs.yml

@@ -0,0 +1,51 @@
+name: OpenAPI
+
+on:
+  push:
+    branches:
+      - 'main'
+    paths:
+      - 'openapi.yaml'
+      - '.github/workflows/docs.yml'
+#  pull_request:
+#    branches:
+#      - '**'
+#    paths:
+#      - 'openapi.yaml'
+#      - '.github/workflows/docs.yml'
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+jobs:
+  build:
+    name: Preview
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: '0'
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+
+      - name: Lint OpenAPI
+        shell: bash
+        run: npx @redocly/cli lint openapi.yaml
+
+      - name: Build OpenAPI
+        shell: bash
+        run: npx @redocly/cli build-docs openapi.yaml --output "openapi/index.html" --title "Codex API"
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v1
+        with:
+          path: './openapi'
+
+      - name: Deploy to GitHub Pages
+        uses: actions/deploy-pages@v1

.gitignore

@@ -23,6 +23,9 @@ nimble.paths
 # vscode
 .vscode
 
+# JetBrain's IDEs
+.idea
+
 # Each developer can create a personal .env file with
 # local settings overrides (e.g. WEB3_URL)
 .env

codex/rest/api.nim

@@ -59,6 +59,8 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter =
       ## to invoke peer discovery, if it succeeds
       ## the returned addresses will be used to dial
       ##
+      ## `addrs` the listening addresses of the peers to dial, eg the one specified with `--listen-addrs`
+      ##
 
       if peerId.isErr:
         return RestApiResponse.error(
@@ -132,8 +134,11 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter =
       ## Create a request for storage
       ##
       ## cid       - the cid of a previously uploaded dataset
-      ## duration - the duration of the contract
+      ## duration  - the duration of the request in seconds
-      ## reward   - the maximum price the client is willing to pay
+      ## reward    - the maximum amount of tokens paid per second per slot to hosts the client is willing to pay
+      ## expiry    - timestamp, in seconds, when the request expires if the Request does not find requested amount of nodes to host the data
+      ## nodes     - minimal number of nodes the content should be stored on
+      ## tolerance - allowed number of nodes that can be lost before pronouncing the content lost
 
       without cid =? cid.tryGet.catch, error:
         return RestApiResponse.error(Http400, error.msg)
@@ -162,7 +167,7 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter =
     MethodPost,
     "/api/codex/v1/upload") do (
     ) -> RestApiResponse:
-      ## Upload a file in a streamming manner
+      ## Upload a file in a streaming manner
       ##
 
       trace "Handling file upload"

docs/TWOCLIENTTEST.md

@@ -1,6 +1,6 @@
 # Codex Two-Client Test
 
-The two-client test is a manual test you can perform to check your setup and familiarize yourself with the Codex API. These steps will guide you through running and connecting two nodes, in order to upload a file to one and then download that file from the other. For the purpose of this test we will be running Codex disconnected from any Ethereum nodes, so no currency is required. Additionally, the contracts/sales/marketplace APIs will be unavailable for this reason.
+The two-client test is a manual test you can perform to check your setup and familiarize yourself with the Codex API. These steps will guide you through running and connecting two nodes, in order to upload a file to one and then download that file from the other. This test also includes running a local blockchain node in order to have the Marketplace functionality available. However, running a local blockchain node is not strictly necessary, and you can skip steps marked as optional if you choose not start a local blockchain node.
 
 ## Prerequisite
 
@@ -8,20 +8,41 @@ Make sure you have built the client, and can run it as explained in the [README]
 
 ## Steps
 
+### 0. Setup blockchain node (optional)
+
+You need to have installed NodeJS and npm in order to spinup a local blockchain node.
+
+Go to directory `vendor/codex-contracts-eth` and run these commands:
+```
+$ npm ci
+$ npm start
+```
+
+This will launch a local Ganache blockchain.
+
 ### 1. Launch Node #1
 
 Open a terminal and run:
-- Mac/Unx: `"build/codex" --data-dir="$(pwd)\Data1" --listen-addrs=/ip4/127.0.0.1/tcp/8070 --api-port=8080  --disc-port=8090`
+- Mac/Unx: `"build/codex" --data-dir="$(pwd)/Data1" --listen-addrs="/ip4/127.0.0.1/tcp/8070" --api-port=8080  --disc-port=8090`
-- Windows: `"build/codex.exe" --data-dir="Data1" --listen-addrs=/ip4/127.0.0.1/tcp/8070 --api-port=8080 --disc-port=8090`
+- Windows: `"build/codex.exe" --data-dir="Data1" --listen-addrs="/ip4/127.0.0.1/tcp/8070" --api-port=8080 --disc-port=8090`
 
-(Hint: If your terminal interprets the '/' in the listen-address as a reference to your root path, try running the command from a shell-script!)
+Optionally, if you want to use the Marketplace blockchain functionality, you need to also include these flags: `--persistence --eth-account=<account>`, where `account` can be one following:
+
+  - `0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266`
+  - `0x70997970C51812dc3A010C7d01b50e0d17dc79C8`
+  - `0x3C44CdDdB6a900fa2b585dd299e03d12FA4293BC`
+  - `0x90F79bf6EB2c4f870365E785982E1f101E93b906`
+
+**For each node use a different account!**
 
 | Argument       | Description                                                           |
-| -------------- | --------------------------------------------------------------------- |
+|----------------|-----------------------------------------------------------------------|
 | `data-dir`     | We specify a relative path where the node will store its data.        |
 | `listen-addrs` | Multiaddress where the node will accept connections from other nodes. |
 | `api-port`     | Port on localhost where the node will expose its API.                 |
 | `disc-port`    | Port the node will use for its discovery service.                     |
+| `persistence`  | Enables Marketplace functionality. Requires a blockchain connection.    |
+| `eth-account`  | Defines which blockchain account the node should use.                     |
 
 Codex uses sane defaults for most of its arguments. Here we specify some explicitly for the purpose of this walk-through.
 
@@ -110,3 +131,35 @@ Notice we are connecting to the second node in order to download the file. The C
 
 If your file is downloaded and identical to the file you uploaded, then this manual test has passed. Rejoice! If on the other hand that didn't happen or you were unable to complete any of these steps, please leave us a message detailing your troubles.
 
+### 8. Offer your storage for sale (optional)
+
+```bash
+curl --location 'http://localhost:8081/api/codex/v1/sales/availability' \
+--header 'Content-Type: application/json' \
+--data '{
+    "size": "0xF4240",
+    "duration": "0xE10",
+    "minPrice": "0x3E8"
+}'
+```
+
+This informs your node that you are available to store 1MB of data for a duration of one hour (3600 seconds) at a minimum price of 1,000 tokens, automatically matching any storage requests announced on the network.
+
+### 9. Create storage Request (optional)
+
+```bash
+curl --location 'http://localhost:8080/api/codex/v1/storage/request/<CID>' \
+--header 'Content-Type: application/json' \
+--data '{
+    "reward": "0x400",
+    "duration": "0x78"
+}'
+```
+
+This creates a storage Request for `<CID>` (that you have to fill in) for duration of 2 minutes and with reward of 1024 tokens.
+
+It returns Request ID which you can then use to query for the Request's state as follows:
+
+```bash
+curl --location 'http://localhost:8080/api/codex/v1/storage/purchases/<RequestID>'
+```

openapi.yaml

@@ -0,0 +1,399 @@
+openapi: 3.0.3
+
+info:
+  version: 0.0.1
+  title: Codex API
+  description: "List of endpoints and interfaces available to Codex API users"
+
+security:
+  - { }
+
+components:
+  schemas:
+    MultiAddress:
+      type: string
+      description: Address of node as specified by the multi-address specification https://multiformats.io/multiaddr/
+      example: /ip4/127.0.0.1/tcp/8080
+
+    PeerId:
+      type: string
+      description: Peer Identity reference as specified at https://docs.libp2p.io/concepts/fundamentals/peers/
+      example: QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N
+
+    Cid:
+      type: string
+      description: Content Identifier as specified at https://github.com/multiformats/cid
+      example: QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N
+
+    LogLevel:
+      type: string
+      description: "One of the log levels: TRACE, DEBUG, INFO, NOTICE, WARN, ERROR or FATAL"
+      example: DEBUG
+
+    EthereumAddress:
+      type: string
+      description: Address of Ethereum address
+
+    Reward:
+      type: string
+      description: The maximum amount of tokens paid per second per slot to hosts the client is willing to pay
+
+    Duration:
+      type: string
+      description: The duration of the request in seconds as hexadecimal string
+
+    Expiry:
+      type: string
+      description: A timestamp as seconds since unix epoch at which this request expires if the Request does not find requested amount of nodes to host the data.
+      default: 10 minutes
+
+    ErasureParameters:
+      type: object
+      properties:
+        totalChunks:
+          type: number
+
+    PoRParameters:
+      description: Parameters for Proof of Retrievability
+      type: object
+      properties:
+        u:
+          type: string
+        publicKey:
+          type: string
+        name:
+          type: string
+
+    Content:
+      type: object
+      description: Parameters specifying the content
+      properties:
+        cid:
+          $ref: "#/components/schemas/Cid"
+        erasure:
+          $ref: "#/components/schemas/ErasureParameters"
+        por:
+          $ref: "#/components/schemas/PoRParameters"
+
+    DebugInfo:
+      type: object
+      properties:
+        id:
+          $ref: "#/components/schemas/PeerId"
+        addrs:
+          type: array
+          items:
+            $ref: "#/components/schemas/MultiAddress"
+        repo:
+          type: string
+          description: Path of the data repository where all nodes data are stored
+        spr:
+          type: string
+          description: Signed Peer Record to advertise DHT connection information
+
+    SalesAvailability:
+      type: object
+      required:
+        - size
+        - minPrice
+      properties:
+        id:
+          type: string
+          description: Hexadecimal identifier of the availability
+        size:
+          type: string
+          description: Size of available storage in bytes as hexadecimal string
+        duration:
+          $ref: "#/components/schemas/Duration"
+        minPrice:
+          type: string
+          description: Minimum price to be paid (in amount of tokens) as hexadecimal string
+
+    StorageRequestCreation:
+      type: object
+      required:
+        - reward
+        - duration
+      properties:
+        duration:
+          $ref: "#/components/schemas/Duration"
+        reward:
+          $ref: "#/components/schemas/Reward"
+        nodes:
+          type: number
+          description: Minimal number of nodes the content should be stored on
+          default: 1 node
+        tolerance:
+          type: number
+          description: Additional number of nodes on top of the `nodes` property that can be lost before pronouncing the content lost
+          default: 0 nodes
+
+    StorageAsk:
+      type: object
+      required:
+        - reward
+      properties:
+        slots:
+          type: number
+          description: Number of slots (eq. hosts) that the Request want to have the content spread over
+        slotSize:
+          type: string
+          description: Amount of storage per slot (in bytes) as hexadecimal string
+        duration:
+          $ref: "#/components/schemas/Duration"
+        proofProbability:
+          type: string
+          description: How often storage proofs are required as hexadecimal string
+        reward:
+          $ref: "#/components/schemas/Reward"
+        maxSlotLoss:
+          type: number
+          description: Max slots that can be lost without data considered to be lost
+
+    StorageRequest:
+      type: object
+      properties:
+        client:
+          $ref: "#/components/schemas/EthereumAddress"
+        ask:
+          $ref: "#/components/schemas/StorageAsk"
+        content:
+          $ref: "#/components/schemas/Content"
+        expiry:
+          $ref: "#/components/schemas/Expiry"
+        nonce:
+          type: string
+          description: Random data
+
+    StorageRequestState:
+      type: object
+      properties:
+        state:
+          type: string
+          description: Description of the Request's state
+        error:
+          type: string
+          description: If Request failed, then here is presented the error message
+        request:
+          $ref: "#/components/schemas/StorageRequest"
+
+servers:
+  - url: "http://localhost:8080/api/codex/v1"
+
+tags:
+  - name: Marketplace
+    description: Marketplace information and operations
+  - name: Data
+    description: Data operations
+  - name: Node
+    description: Node management
+  - name: Debug
+    description: Debugging configuration
+
+paths:
+  "/connect/{peerId}":
+    get:
+      summary: "Connect to a peer"
+      description: |
+        If `addrs` param is supplied, it will be used to dial the peer, otherwise the `peerId` is used
+        to invoke peer discovery, if it succeeds the returned addresses will be used to dial.
+      tags: [ Node ]
+      operationId: connectPeer
+      parameters:
+        - in: path
+          name: peerId
+          required: true
+          schema:
+              $ref: "#/components/schemas/PeerId"
+          description: Peer that should be dialed.
+        - in: query
+          name: addrs
+          schema:
+            type: array
+            nullable: true
+            items:
+              $ref: "#/components/schemas/MultiAddress"
+          description: |
+            If supplied, it will be used to dial the peer.
+            The address has to target the listening address of the peer,
+            which is specified with the `--listen-addrs` CLI flag.
+
+      responses:
+        "200":
+          description: Successfully connected to peer
+        "400":
+          description: Peer either not found or was not possible to dial
+
+  "/download/{cid}":
+    get:
+      summary: "Download a file from the node in a streaming manner"
+      tags: [ Data ]
+      operationId: download
+      parameters:
+        - in: path
+          name: cid
+          required: true
+          schema:
+              $ref: "#/components/schemas/Cid"
+          description: File to be downloaded.
+
+      responses:
+        "200":
+          description: Retrieved content specified by CID
+          content:
+            application/octet-stream:
+              schema:
+                type: string
+                format: binary
+        "400":
+          description: Invalid CID is specified
+        "404":
+          description: Content specified by the CID is not found
+        "500":
+          description: Well it was bad-bad
+
+  "/upload":
+    post:
+      summary: "Upload a file in a streaming manner"
+      tags: [ Data ]
+      operationId: upload
+      requestBody:
+        content:
+          application/octet-stream:
+            schema:
+              type: string
+              format: binary
+      responses:
+        "200":
+          description: CID of uploaded file
+          content:
+            text/plain:
+              schema:
+                type: string
+        "500":
+          description: Well it was bad-bad and the upload did not work out
+
+  "/sales/availability":
+    get:
+      summary: "Returns storage that is for sale"
+      tags: [ Marketplace ]
+      operationId: getsOfferedStorage
+      responses:
+        "200":
+          description: Retrieved content specified by CID
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/SalesAvailability"
+        "503":
+          description: Sales are unavailable
+
+    post:
+      summary: "Offers storage for sale"
+      operationId: offerStorage
+      tags: [ Marketplace ]
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/SalesAvailability"
+      responses:
+        "200":
+          description: Created storage availability
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/SalesAvailability"
+        "503":
+          description: Sales are unavailable
+
+  "/storage/purchases/{id}":
+    get:
+      summary: "Returns information about node's Requests"
+      tags: [ Marketplace ]
+      operationId: getStorageRequestState
+      parameters:
+        - in: path
+          name: id
+          required: true
+          schema:
+            type: string
+          description: Hexadecimal ID of a Request
+      responses:
+        "200":
+          description: Information about the Request state and specification
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/StorageRequestState"
+        "400":
+          description: Invalid or missing Request ID
+        "404":
+          description: Request ID not found
+        "503":
+          description: Purchasing is unavailable
+
+  "/storage/request/{cid}":
+    post:
+      summary: "Creates a new Request for storage"
+      tags: [ Marketplace ]
+      operationId: createStorageRequest
+      parameters:
+        - in: path
+          name: cid
+          required: true
+          schema:
+            $ref: "#/components/schemas/Cid"
+          description: CID of the uploaded data that should be stored
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/StorageRequestCreation"
+      responses:
+        "200":
+          description: Returns the Request ID as hexadecimal string
+        "400":
+          description: Invalid or missing Request ID
+        "404":
+          description: Request ID not found
+        "503":
+          description: Purchasing is unavailable
+
+  "/debug/chronicles/loglevel":
+    post:
+      summary: "Set log level at run time"
+      tags: [ Debug ]
+      operationId: setDebugLogLevel
+
+      parameters:
+        - in: query
+          name: level
+          required: true
+          schema:
+            $ref: "#/components/schemas/LogLevel"
+
+      responses:
+        "200":
+          description: Successfully log level set
+        "400":
+          description: Invalid or missing log level
+        "500":
+          description: Well it was bad-bad
+
+  "/debug/info":
+    get:
+      summary: "Gets node information"
+      operationId: getDebugInfo
+      tags: [ Debug ]
+      responses:
+        "200":
+          description: Node's information
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/DebugInfo"