status-im
/
nim-codex
mirror of https://github.com/status-im/nim-codex.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into feature/ceremony-files

This commit is contained in:
benbierens 2024-04-02 20:12:00 +02:00
parent f2f1dd51ef 53809129a5
commit 1524803fb0
No known key found for this signature in database
GPG Key ID: 877D2C2E09A22F3A
4 changed files with 201 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
codex/discovery.nim
View File

@ -16,7 +16,7 @@ import pkg/questionable
import pkg/questionable/results import pkg/questionable/results
import pkg/stew/shims/net import pkg/stew/shims/net
import pkg/contractabi/address as ca import pkg/contractabi/address as ca
import pkg/codexdht/discv5/protocol as discv5 import pkg/codexdht/discv5/[routing_table, protocol as discv5]
import ./rng import ./rng
import ./errors import ./errors
@ -193,6 +193,18 @@ proc new*(
self.updateAnnounceRecord(announceAddrs) self.updateAnnounceRecord(announceAddrs)
# --------------------------------------------------------------------------
# FIXME disable IP limits temporarily so we can run our workshop. Re-enable
# and figure out proper solution.
let discoveryConfig = DiscoveryConfig(
tableIpLimits: TableIpLimits(
tableIpLimit: high(uint),
bucketIpLimit:high(uint)
),
bitsPerHop: DefaultBitsPerHop
)
# --------------------------------------------------------------------------
self.protocol = newProtocol( self.protocol = newProtocol(
key, key,
bindIp = bindIp.toNormalIp, bindIp = bindIp.toNormalIp,
@ -200,6 +212,7 @@ proc new*(
record = self.providerRecord.get, record = self.providerRecord.get,
bootstrapRecords = bootstrapNodes, bootstrapRecords = bootstrapNodes,
rng = Rng.instance(), rng = Rng.instance(),
providers = ProvidersManager.new(store)) providers = ProvidersManager.new(store),
config = discoveryConfig)
self self

112
codex/rest/api.nim
View File

@ -492,6 +492,78 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
trace "Excepting processing request", exc = exc.msg trace "Excepting processing request", exc = exc.msg
return RestApiResponse.error(Http500) return RestApiResponse.error(Http500)
proc initNodeApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) =
## various node management api's
##
router.api(
MethodGet,
"/api/codex/v1/spr") do () -> RestApiResponse:
## Returns node SPR in requested format, json or text.
##
try:
without spr =? node.discovery.dhtRecord:
return RestApiResponse.response("", status=Http503, contentType="application/json")
if $preferredContentType().get() == "text/plain":
return RestApiResponse.response(spr.toURI, contentType="text/plain")
else:
return RestApiResponse.response($ %* {"spr": spr.toURI}, contentType="application/json")
except CatchableError as exc:
trace "Excepting processing request", exc = exc.msg
return RestApiResponse.error(Http500)
router.api(
MethodGet,
"/api/codex/v1/peerid") do () -> RestApiResponse:
## Returns node's peerId in requested format, json or text.
##
try:
let id = $node.switch.peerInfo.peerId
if $preferredContentType().get() == "text/plain":
return RestApiResponse.response(id, contentType="text/plain")
else:
return RestApiResponse.response($ %* {"id": id}, contentType="application/json")
except CatchableError as exc:
trace "Excepting processing request", exc = exc.msg
return RestApiResponse.error(Http500)
router.api(
MethodGet,
"/api/codex/v1/connect/{peerId}") do (
peerId: PeerId,
addrs: seq[MultiAddress]) -> RestApiResponse:
## Connect to a peer
##
## If `addrs` param is supplied, it will be used to
## dial the peer, otherwise the `peerId` is used
## to invoke peer discovery, if it succeeds
## the returned addresses will be used to dial
##
## `addrs` the listening addresses of the peers to dial, eg the one specified with `--listen-addrs`
##
if peerId.isErr:
return RestApiResponse.error(
Http400,
$peerId.error())
let addresses = if addrs.isOk and addrs.get().len > 0:
addrs.get()
else:
without peerRecord =? (await node.findPeer(peerId.get())):
return RestApiResponse.error(
Http400,
"Unable to find Peer!")
peerRecord.addresses.mapIt(it.address)
try:
await node.connect(peerId.get(), addresses)
return RestApiResponse.response("Successfully connected to peer")
except DialFailedError:
return RestApiResponse.error(Http400, "Unable to dial peer")
except CatchableError:
return RestApiResponse.error(Http500, "Unknown error dialling peer")
proc initDebugApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) = proc initDebugApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) =
router.api( router.api(
MethodGet, MethodGet,
@ -520,7 +592,8 @@ proc initDebugApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) =
} }
} }
return RestApiResponse.response($json, contentType="application/json") # return pretty json for human readability
return RestApiResponse.response(json.pretty(), contentType="application/json")
except CatchableError as exc: except CatchableError as exc:
trace "Excepting processing request", exc = exc.msg trace "Excepting processing request", exc = exc.msg
return RestApiResponse.error(Http500) return RestApiResponse.error(Http500)
@ -577,42 +650,7 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf, repoStore: RepoStore): Re
initDataApi(node, repoStore, router) initDataApi(node, repoStore, router)
initSalesApi(node, router) initSalesApi(node, router)
initPurchasingApi(node, router) initPurchasingApi(node, router)
initNodeApi(node, conf, router)
initDebugApi(node, conf, router) initDebugApi(node, conf, router)
router.api(
MethodGet,
"/api/codex/v1/connect/{peerId}") do (
peerId: PeerId,
addrs: seq[MultiAddress]) -> RestApiResponse:
## Connect to a peer
##
## If `addrs` param is supplied, it will be used to
## dial the peer, otherwise the `peerId` is used
## to invoke peer discovery, if it succeeds
## the returned addresses will be used to dial
##
## `addrs` the listening addresses of the peers to dial, eg the one specified with `--listen-addrs`
##
if peerId.isErr:
return RestApiResponse.error(
Http400,
$peerId.error())
let addresses = if addrs.isOk and addrs.get().len > 0:
addrs.get()
else:
without peerRecord =? (await node.findPeer(peerId.get())):
return RestApiResponse.error(
Http400,
"Unable to find Peer!")
peerRecord.addresses.mapIt(it.address)
try:
await node.connect(peerId.get(), addresses)
return RestApiResponse.response("Successfully connected to peer")
except DialFailedError:
return RestApiResponse.error(Http400, "Unable to dial peer")
except CatchableError:
return RestApiResponse.error(Http500, "Unknown error dialling peer")
return router return router

123
docs/TWOCLIENTTEST.md
View File

@ -12,10 +12,10 @@ Make sure you have built the client, and can run it as explained in the [README]
You need to have installed NodeJS and npm in order to spinup a local blockchain node. You need to have installed NodeJS and npm in order to spinup a local blockchain node.
Go to directory `vendor/codex-contracts-eth` and run these commands: Go to directory `vendor/codex-contracts-eth` and run these two commands:
``` ```
$ npm ci npm ci
$ npm start npm start
``` ```
This will launch a local Ganache blockchain. This will launch a local Ganache blockchain.
@ -48,21 +48,39 @@ Codex uses sane defaults for most of its arguments. Here we specify some explici
### 2. Sign of life ### 2. Sign of life
Run the command :
```bash ```bash
curl --request GET \ curl -X GET http://127.0.0.1:8080/api/codex/v1/debug/info
--url http://127.0.0.1:8080/api/codex/v1/debug/info
``` ```
This GET request will return the node's debug information. The response JSON should look like: This GET request will return the node's debug information. The response will be in JSON and should look like:
```json ```json
{ {
"id": "16Uiu2HAmGzJQEvNRYVVJNxHSb1Gd5xzxTK8XRZuMJzuoDaz7fADb", "id": "16Uiu2HAmJ3TSfPnrJNedHy2DMsjTqwBiVAQQqPo579DuMgGxmG99",
"addrs": [ "addrs": [
"/ip4/127.0.0.1/tcp/8070" "/ip4/127.0.0.1/tcp/8070"
], ],
"repo": "Data1", "repo": "/Users/user/projects/nim-codex/Data1",
"spr": "spr:CiUIAhIhA0BhMXo12O4h8DSdfnvU6MWUQx3kd-xw_2sCZrWOWChOEgIDARo8CicAJQgCEiEDQGExejXY7iHwNJ1-e9ToxZRDHeR37HD_awJmtY5YKE4Q7aqInwYaCwoJBH8AAAGRAh-aKkYwRAIgSHGvrb4mxQbOTU5wdcJJYz3fErkVx4v09nqHE4n9d4ECIGWyfF58pmfUKeC7MWCtIhBDCgNJkjHz2JkKfJoYgqHW" "spr": "spr:CiUIAhIhA1AL2J7EWfg7x77iOrR9YYBisY6CDtU2nEhuwDaQyjpkEgIDARo8CicAJQgCEiEDUAvYnsRZ-DvHvuI6tH1hgGKxjoIO1TacSG7ANpDKOmQQ2MWasAYaCwoJBH8AAAGRAh-aKkYwRAIgB2ooPfAyzWEJDe8hD2OXKOBnyTOPakc4GzqKqjM2OGoCICraQLPWf0oSEuvmSroFebVQx-3SDtMqDoIyWhjq1XFF",
"announceAddresses": [
"/ip4/127.0.0.1/tcp/8070"
],
"table": {
"localNode": {
"nodeId": "f6e6d48fa7cd171688249a57de0c1aba15e88308c07538c91e1310c9f48c860a",
"peerId": "16Uiu2HAmJ3TSfPnrJNedHy2DMsjTqwBiVAQQqPo579DuMgGxmG99",
"record": "...",
"address": "0.0.0.0:8090",
"seen": false
},
"nodes": []
},
"codex": {
"version": "untagged build",
"revision": "b3e626a5"
}
} }
``` ```
@ -72,33 +90,53 @@ This GET request will return the node's debug information. The response JSON sho
| `addrs` | Multiaddresses currently open to accept connections from other nodes. | | `addrs` | Multiaddresses currently open to accept connections from other nodes. |
| `repo` | Path of this node's data folder. | | `repo` | Path of this node's data folder. |
| `spr` | Signed Peer Record, encoded information about this node and its location in the network. | | `spr` | Signed Peer Record, encoded information about this node and its location in the network. |
| `announceAddresses` | Multiaddresses used for annoucning this node
| `table` | Table of nodes present in the node's DHT
| `codex` | Codex version information
### 3. Launch Node #2 ### 3. Launch Node #2
Replace `<SPR HERE>` in the next command with the string value for `spr`, returned by the first node's `debug/info` response. Retreive the SPR by running:
```bash
curl -H "Accept: text/plain" http://127.0.0.1:8080/api/codex/v1/spr
```
Next replace `<SPR HERE>` in the following command with the SPR returned from the previous command. (Note that it should include the `spr:` at the beginning.)
Open a new terminal and run: Open a new terminal and run:
- Mac/Unx: `"build/codex" --data-dir="$(pwd)/Data2" --listen-addrs=/ip4/127.0.0.1/tcp/8071 --api-port=8081 --disc-port=8091 --bootstrap-node=<SPR HERE>` - Mac/Linux: `"build/codex" --data-dir="$(pwd)/Data2" --listen-addrs=/ip4/127.0.0.1/tcp/8071 --api-port=8081 --disc-port=8091 --bootstrap-node=<SPR HERE>`
- Windows: `"build/codex.exe" --data-dir="Data2" --listen-addrs=/ip4/127.0.0.1/tcp/8071 --api-port=8081 --disc-port=8091 --bootstrap-node=<SPR HERE>` - Windows: `"build/codex.exe" --data-dir="Data2" --listen-addrs=/ip4/127.0.0.1/tcp/8071 --api-port=8081 --disc-port=8091 --bootstrap-node=<SPR HERE>`
Alternatively on Mac, Linux, or MSYS2 you can run it in one command like:
```sh
"build/codex" --data-dir="$(pwd)/Data2" --listen-addrs=/ip4/127.0.0.1/tcp/8071 --api-port=8081 --disc-port=8091 --bootstrap-node=$(curl -H "Accept: text/plain" http://127.0.0.1:8080/api/codex/v1/spr)
```
Notice we're using a new data-dir, and we've increased each port number by one. This is needed so that the new node won't try to open ports already in use by the first node. Notice we're using a new data-dir, and we've increased each port number by one. This is needed so that the new node won't try to open ports already in use by the first node.
We're now also including the `bootstrap-node` argument. This allows us to link the new node to another one, bootstrapping our own little peer-to-peer network. (SPR strings always start with "spr:".) We're now also including the `bootstrap-node` argument. This allows us to link the new node to another one, bootstrapping our own little peer-to-peer network. (SPR strings always start with "spr:".)
### 4. Connect The Two ### 4. Connect The Two
Use the command we've used in step 2 to retrieve the debug information from node 2: Normally the two nodes will automatically connect. If they do not automatically connect or you want to manually connect nodes you can use the peerId to connect nodes.
You can get the first node's peer id by running:
```bash ```bash
curl --request GET \ curl -X GET -H "Accept: text/plain" http://127.0.0.1:8081/api/codex/v1/peerid
--url http://127.0.0.1:8081/api/codex/v1/debug/info
``` ```
In the JSON response copy the value for `id`, and use it to replace `<PEER ID HERE>` in the following command: Next replace `<PEER ID HERE>` in the following command with the peerId returned from the previous command:
```bash ```bash
curl --request GET \ curl -X GET http://127.0.0.1:8080/api/codex/v1/connect/<PEER ID HERE>?addrs=/ip4/127.0.0.1/tcp/8071
--url http://127.0.0.1:8080/api/codex/v1/connect/<PEER ID HERE>?addrs=/ip4/127.0.0.1/tcp/8071 ```
Alternatively on Mac, Linux, or MSYS2 you can run it in one command like:
```bash
curl -X GET http://127.0.0.1:8080/api/codex/v1/connect/$(curl -X GET -H "Accept: text/plain" http://127.0.0.1:8081/api/codex/v1/peerid)\?addrs=/ip4/127.0.0.1/tcp/8071
``` ```
Notice that we are sending the peerId and the multiaddress of node 2 to the `/connect` endpoint of node 1. This provides node 1 all the information it needs to communicate with node 2. The response to this request should be `Successfully connected to peer`. Notice that we are sending the peerId and the multiaddress of node 2 to the `/connect` endpoint of node 1. This provides node 1 all the information it needs to communicate with node 2. The response to this request should be `Successfully connected to peer`.
@ -107,10 +145,10 @@ Notice that we are sending the peerId and the multiaddress of node 2 to the `/co
We're now ready to upload a file to the network. In this example we'll use node 1 for uploading and node 2 for downloading. But the reverse also works. We're now ready to upload a file to the network. In this example we'll use node 1 for uploading and node 2 for downloading. But the reverse also works.
Replace `<FILE PATH>` with the path to the file you want to upload in the following command: Next replace `<FILE PATH>` with the path to the file you want to upload in the following command:
```bash ```bash
curl -H "content-type: application/octet-stream" -H "Expect: 100-continue" -T "<FILE PATH>" 127.0.0.1:8080/api/codex/v1/data -X POST curl -H "Content-Type: application/octet-stream" -H "Expect: 100-continue" -T "<FILE PATH>" 127.0.0.1:8080/api/codex/v1/data -X POST
``` ```
(Hint: if curl is reluctant to show you the response, add `-o <FILENAME>` to write the result to a file.) (Hint: if curl is reluctant to show you the response, add `-o <FILENAME>` to write the result to a file.)
@ -130,50 +168,3 @@ Notice we are connecting to the second node in order to download the file. The C
### 7. Verify The Results ### 7. Verify The Results
If your file is downloaded and identical to the file you uploaded, then this manual test has passed. Rejoice! If on the other hand that didn't happen or you were unable to complete any of these steps, please leave us a message detailing your troubles. If your file is downloaded and identical to the file you uploaded, then this manual test has passed. Rejoice! If on the other hand that didn't happen or you were unable to complete any of these steps, please leave us a message detailing your troubles.
### 8. Offer your storage for sale (optional)
```bash
curl --location 'http://localhost:8081/api/codex/v1/sales/availability' \
--header 'Content-Type: application/json' \
--data '{
"size": "1000000",
"duration": "3600",
"minPrice": "1000",
"maxCollateral": "1"
}'
```
This informs your node that you are available to store 1MB of data for a duration of one hour (3600 seconds) at a minimum price of 1,000 tokens, automatically matching any storage requests announced on the network.
### 9. Create storage Request (optional)
```bash
curl --location 'http://localhost:8080/api/codex/v1/storage/request/<CID>' \
--header 'Content-Type: application/json' \
--data '{
"reward": "1024",
"duration": "120",
"proofProbability": "8"
"collateral": "1"
}'
```
This creates a storage Request for `<CID>` (that you have to fill in) for
duration of 2 minutes and with reward of 1024 tokens. It expects hosts to
provide a storage proof once every 8 periods on average.
It returns Request ID which you can then use to query for the Request's state as follows:
```bash
curl --location 'http://localhost:8080/api/codex/v1/storage/purchases/<RequestID>'
```
## Notes
When using the Ganache blockchain, there are some deviations from the expected behavior, mainly linked to how blocks are mined, which affects certain functionalities in the Sales module.
Therefore, if you are manually testing processes such as payout collection after a request is finished or proof submissions, you need to mine some blocks manually for it to work correctly. You can do this by using the following curl command:
```bash
$ curl -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"evm_mine","params":[],"id":67}' 127.0.0.1:8545
```

53
openapi.yaml
View File

@ -65,6 +65,22 @@ components:
description: A timestamp as seconds since unix epoch at which this request expires if the Request does not find requested amount of nodes to host the data. description: A timestamp as seconds since unix epoch at which this request expires if the Request does not find requested amount of nodes to host the data.
default: 10 minutes default: 10 minutes
SPR:
type: string
description: Signed Peer Record (libp2p)
SPRRead:
type: object
properties:
spr:
$ref: "#/components/schemas/SPR"
PeerIdRead:
type: object
properties:
id:
$ref: "#/components/schemas/PeerId"
ErasureParameters: ErasureParameters:
type: object type: object
properties: properties:
@ -106,8 +122,7 @@ components:
type: string type: string
description: Path of the data repository where all nodes data are stored description: Path of the data repository where all nodes data are stored
spr: spr:
type: string $ref: "#/components/schemas/SPR"
description: Signed Peer Record to advertise DHT connection information
SalesAvailability: SalesAvailability:
type: object type: object
@ -685,6 +700,40 @@ paths:
"503": "503":
description: Purchasing is unavailable description: Purchasing is unavailable
"/node/spr":
get:
summary: "Get Node's SPR"
operationId: getSPR
tags: [ Node ]
responses:
"200":
description: Node's SPR
content:
plain/text:
schema:
$ref: "#/components/schemas/SPR"
application/json:
schema:
$ref: "#/components/schemas/SPRRead"
"503":
description: Node SPR not ready, try again later
"/node/peerid":
get:
summary: "Get Node's PeerID"
operationId: getPeerId
tags: [ Node ]
responses:
"200":
description: Node's Peer ID
content:
plain/text:
schema:
$ref: "#/components/schemas/PeerId"
application/json:
schema:
$ref: "#/components/schemas/PeerIdRead"
"/debug/chronicles/loglevel": "/debug/chronicles/loglevel":
post: post:
summary: "Set log level at run time" summary: "Set log level at run time"