From 76e9a98841d07f2a5c461cca9189924e9c060d5a Mon Sep 17 00:00:00 2001 From: Lorenzo Delgado Date: Thu, 2 Jun 2022 11:45:00 +0200 Subject: [PATCH] feat(rest): Add HTTP REST API (#727). Debug API POC --- .gitmodules | 5 ++ tests/all_tests_v2.nim | 3 + tests/v2/test_rest_debug_api.nim | 52 ++++++++++++++ tests/v2/test_rest_debug_api_serdes.nim | 39 ++++++++++ tests/v2/test_rest_serdes.nim | 68 ++++++++++++++++++ vendor/nim-presto | 1 + waku.nimble | 3 +- waku/v2/node/rest/client.nim | 6 ++ waku/v2/node/rest/debug_api.nim | 96 +++++++++++++++++++++++++ waku/v2/node/rest/debug_api.yaml | 35 +++++++++ waku/v2/node/rest/serdes.nim | 72 +++++++++++++++++++ waku/v2/node/rest/server.nim | 86 ++++++++++++++++++++++ waku/v2/node/rest/utils.nim | 18 +++++ 13 files changed, 483 insertions(+), 1 deletion(-) create mode 100644 tests/v2/test_rest_debug_api.nim create mode 100644 tests/v2/test_rest_debug_api_serdes.nim create mode 100644 tests/v2/test_rest_serdes.nim create mode 160000 vendor/nim-presto create mode 100644 waku/v2/node/rest/client.nim create mode 100644 waku/v2/node/rest/debug_api.nim create mode 100644 waku/v2/node/rest/debug_api.yaml create mode 100644 waku/v2/node/rest/serdes.nim create mode 100644 waku/v2/node/rest/server.nim create mode 100644 waku/v2/node/rest/utils.nim diff --git a/.gitmodules b/.gitmodules index acc36aff9..65e22f1eb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -143,3 +143,8 @@ [submodule "vendor/nim-toml-serialization"] path = vendor/nim-toml-serialization url = https://github.com/status-im/nim-toml-serialization.git +[submodule "vendor/nim-presto"] + path = vendor/nim-presto + url = https://github.com/status-im/nim-presto.git + ignore = untracked + branch = master diff --git a/tests/all_tests_v2.nim b/tests/all_tests_v2.nim index 022bd52ae..85356bfb3 100644 --- a/tests/all_tests_v2.nim +++ b/tests/all_tests_v2.nim @@ -10,6 +10,9 @@ import ./v2/test_waku_swap, ./v2/test_message_store, ./v2/test_jsonrpc_waku, + ./v2/test_rest_serdes, + ./v2/test_rest_debug_api_serdes, + ./v2/test_rest_debug_api, ./v2/test_peer_manager, ./v2/test_web3, # TODO remove it when rln-relay tests get finalized ./v2/test_waku_bridge, diff --git a/tests/v2/test_rest_debug_api.nim b/tests/v2/test_rest_debug_api.nim new file mode 100644 index 000000000..c1a0cf629 --- /dev/null +++ b/tests/v2/test_rest_debug_api.nim @@ -0,0 +1,52 @@ +import + stew/shims/net, + chronicles, + testutils/unittests, + presto, + libp2p/crypto/crypto +import + ../../waku/v2/node/wakunode2, + ../../waku/v2/node/rest/[server, client, debug_api] + + +proc testWakuNode(): WakuNode = + let + rng = crypto.newRng() + privkey = crypto.PrivateKey.random(Secp256k1, rng[]).tryGet() + bindIp = ValidIpAddress.init("0.0.0.0") + extIp = ValidIpAddress.init("127.0.0.1") + port = Port(9000) + + WakuNode.new(privkey, bindIp, port, some(extIp), some(port)) + + +suite "REST API - Debug": + asyncTest "Get node info - GET /debug/v1/info": + # Given + let node = testWakuNode() + await node.start() + node.mountRelay() + + let restPort = Port(8546) + let restAddress = ValidIpAddress.init("0.0.0.0") + let restServer = RestServerRef.init( + restAddress, + restPort, + none(string), + none(RestServerConf) + ) + + installDebugApiHandlers(restServer.router, node) + restServer.start() + + # When + let client = newRestHttpClient(initTAddress(restAddress, restPort)) + let response = await client.debugInfoV1() + + # Then + check: + response.listenAddresses == @[$node.switch.peerInfo.addrs[^1] & "/p2p/" & $node.switch.peerInfo.peerId] + + await restServer.stop() + await restServer.closeWait() + await node.stop() \ No newline at end of file diff --git a/tests/v2/test_rest_debug_api_serdes.nim b/tests/v2/test_rest_debug_api_serdes.nim new file mode 100644 index 000000000..35397b6cf --- /dev/null +++ b/tests/v2/test_rest_debug_api_serdes.nim @@ -0,0 +1,39 @@ +import std/typetraits +import chronicles, + unittest2, + stew/[results, byteutils], + json_serialization +import + ../../waku/v2/node/rest/[serdes, debug_api] + + +suite "Debug API - serialization": + + suite "DebugWakuInfo - decode": + test "optional field is not provided": + # Given + let jsonBytes = toBytes("""{ "listenAddresses":["123"] }""") + + # When + let res = decodeFromJsonBytes(DebugWakuInfo, jsonBytes, requireAllFields = true) + + # Then + require(res.isOk) + let value = res.get() + check: + value.listenAddresses == @["123"] + value.enrUri.isNone + + suite "DebugWakuInfo - encode": + test "optional field is none": + # Given + let data = DebugWakuInfo(listenAddresses: @["GO"], enrUri: none(string)) + + # When + let res = encodeIntoJsonBytes(data) + + # Then + require(res.isOk) + let value = res.get() + check: + value == toBytes("""{"listenAddresses":["GO"]}""") diff --git a/tests/v2/test_rest_serdes.nim b/tests/v2/test_rest_serdes.nim new file mode 100644 index 000000000..3dfc16024 --- /dev/null +++ b/tests/v2/test_rest_serdes.nim @@ -0,0 +1,68 @@ +import std/typetraits +import chronicles, + unittest2, + stew/[results, byteutils], + json_serialization +import + ../../waku/v2/node/rest/[serdes, debug_api] + + +# TODO: Decouple this test suite from the `debug_api` module by defining +# private custom types for this test suite module +suite "Serdes": + + suite "decode": + test "decodeFromJsonString - use the corresponding readValue template": + # Given + let jsonString = JsonString("""{ "listenAddresses":["123"] }""") + + # When + let res = decodeFromJsonString(DebugWakuInfo, jsonString, requireAllFields = true) + + # Then + require(res.isOk) + let value = res.get() + check: + value.listenAddresses == @["123"] + value.enrUri.isNone + + test "decodeFromJsonBytes - use the corresponding readValue template": + # Given + let jsonBytes = toBytes("""{ "listenAddresses":["123"] }""") + + # When + let res = decodeFromJsonBytes(DebugWakuInfo, jsonBytes, requireAllFields = true) + + # Then + require(res.isOk) + let value = res.get() + check: + value.listenAddresses == @["123"] + value.enrUri.isNone + + suite "encode": + test "encodeIntoJsonString - use the corresponding writeValue template": + # Given + let data = DebugWakuInfo(listenAddresses: @["GO"]) + + # When + let res = encodeIntoJsonString(data) + + # Then + require(res.isOk) + let value = res.get() + check: + value == """{"listenAddresses":["GO"]}""" + + test "encodeIntoJsonBytes - use the corresponding writeValue template": + # Given + let data = DebugWakuInfo(listenAddresses: @["ABC"]) + + # When + let res = encodeIntoJsonBytes(data) + + # Then + require(res.isOk) + let value = res.get() + check: + value == toBytes("""{"listenAddresses":["ABC"]}""" ) diff --git a/vendor/nim-presto b/vendor/nim-presto new file mode 160000 index 000000000..1dba6dd6f --- /dev/null +++ b/vendor/nim-presto @@ -0,0 +1 @@ +Subproject commit 1dba6dd6f466cd4e7b793b0e473c237ce453d82a diff --git a/waku.nimble b/waku.nimble index 5599d32c4..ff6a71380 100644 --- a/waku.nimble +++ b/waku.nimble @@ -20,7 +20,8 @@ requires "nim >= 1.2.0", "stint", "metrics", "libp2p", # Only for Waku v2 - "web3" + "web3", + "presto" ### Helper functions proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") = diff --git a/waku/v2/node/rest/client.nim b/waku/v2/node/rest/client.nim new file mode 100644 index 000000000..230e7fc39 --- /dev/null +++ b/waku/v2/node/rest/client.nim @@ -0,0 +1,6 @@ +{.push raises: [Defect].} + +import presto/client + +proc newRestHttpClient*(address: TransportAddress): RestClientRef = + RestClientRef.new(address, HttpClientScheme.NonSecure) diff --git a/waku/v2/node/rest/debug_api.nim b/waku/v2/node/rest/debug_api.nim new file mode 100644 index 000000000..41a113aa9 --- /dev/null +++ b/waku/v2/node/rest/debug_api.nim @@ -0,0 +1,96 @@ +{.push raises: [Defect].} + +import + stew/byteutils, + chronicles, + json_serialization, + json_serialization/std/options, + presto/[route, client] +import "."/[serdes, utils], + ../wakunode2 + +logScope: topics = "rest_api_debug" + + +#### Types + +type + DebugWakuInfo* = object + listenAddresses*: seq[string] + enrUri*: Option[string] + + +#### Serialization and deserialization + +proc writeValue*(writer: var JsonWriter[RestJson], value: DebugWakuInfo) + {.raises: [IOError, Defect].} = + writer.beginRecord() + writer.writeField("listenAddresses", value.listenAddresses) + if value.enrUri.isSome: + writer.writeField("enrUri", value.enrUri) + writer.endRecord() + +proc readValue*(reader: var JsonReader[RestJson], value: var DebugWakuInfo) + {.raises: [SerializationError, IOError, Defect].} = + var + listenAddresses: Option[seq[string]] + enrUri: Option[string] + + for fieldName in readObjectFields(reader): + case fieldName + of "listenAddresses": + if listenAddresses.isSome(): + reader.raiseUnexpectedField("Multiple `listenAddresses` fields found", "DebugWakuInfo") + listenAddresses = some(reader.readValue(seq[string])) + of "enrUri": + if enrUri.isSome(): + reader.raiseUnexpectedField("Multiple `enrUri` fields found", "DebugWakuInfo") + enrUri = some(reader.readValue(string)) + else: + unrecognizedFieldWarning() + + if listenAddresses.isNone(): + reader.raiseUnexpectedValue("Field `listenAddresses` is missing") + + value = DebugWakuInfo( + listenAddresses: listenAddresses.get, + enrUri: enrUri + ) + + +#### Server request handlers + +proc toDebugWakuInfo(nodeInfo: WakuInfo): DebugWakuInfo = + DebugWakuInfo( + listenAddresses: nodeInfo.listenAddresses, + enrUri: some(nodeInfo.enrUri) + ) + +const ROUTE_DEBUG_INFOV1* = "/debug/v1/info" + +proc installDebugInfoV1Handler(router: var RestRouter, node: WakuNode) = + router.api(MethodGet, ROUTE_DEBUG_INFOV1) do () -> RestApiResponse: + let info = node.info().toDebugWakuInfo() + let resp = RestApiResponse.jsonResponse(info, status=Http200) + if resp.isErr(): + debug "An error ocurred while building the json respose", error=resp.error() + return RestApiResponse.internalServerError() + + return resp.get() + +proc installDebugApiHandlers*(router: var RestRouter, node: WakuNode) = + installDebugInfoV1Handler(router, node) + + +#### Client + +proc decodeBytes*(t: typedesc[DebugWakuInfo], data: openArray[byte], contentType: string): RestResult[DebugWakuInfo] = + if MediaType.init(contentType) != MIMETYPE_JSON: + error "Unsupported respose contentType value", contentType = contentType + return err("Unsupported response contentType") + + let decoded = ?decodeFromJsonBytes(DebugWakuInfo, data) + return ok(decoded) + +# TODO: Check how we can use a constant to set the method endpoint (improve "rest" pragma under nim-presto) +proc debugInfoV1*(): DebugWakuInfo {.rest, endpoint: "/debug/v1/info", meth: HttpMethod.MethodGet.} \ No newline at end of file diff --git a/waku/v2/node/rest/debug_api.yaml b/waku/v2/node/rest/debug_api.yaml new file mode 100644 index 000000000..cc1fdf1f3 --- /dev/null +++ b/waku/v2/node/rest/debug_api.yaml @@ -0,0 +1,35 @@ +openapi: 3.1.0 +info: + title: Waku REST API - Debug + version: 1.0.0 + +paths: + /debug/v1/info: + get: + summary: Get node info + description: Retrieve information about a Waku v2 node. + tags: + - Debug + responses: + '200': + description: Information about a Waku v2 node. + content: + application/json: + schema: + $ref: '#/components/schemas/WakuInfo' + '5XX': + description: Unexpected error. + +components: + schemas: + WakuInfo: + type: object + properties: + listenAddresses: + type: array + items: + type: string + enrUri: + type: string + required: + - listenAddresses diff --git a/waku/v2/node/rest/serdes.nim b/waku/v2/node/rest/serdes.nim new file mode 100644 index 000000000..fa90b58dc --- /dev/null +++ b/waku/v2/node/rest/serdes.nim @@ -0,0 +1,72 @@ +{.push raises: [Defect].} + +import std/typetraits +import + stew/results, + chronicles, + serialization, + json_serialization, + json_serialization/std/[options, net, sets], + presto/common + +logScope: topics = "rest_api_serdes" + +Json.createFlavor RestJson + +template unrecognizedFieldWarning* = + # TODO: There should be a different notification mechanism for informing the + # caller of a deserialization routine for unexpected fields. + # The chonicles import in this module should be removed. + debug "JSON field not recognized by the current version of nwaku. Consider upgrading", + fieldName, typeName = typetraits.name(typeof value) + + +type SerdesResult*[T] = Result[T, cstring] + +proc decodeFromJsonString*[T](t: typedesc[T], + data: JsonString, + requireAllFields = true): SerdesResult[T] = + try: + ok(RestJson.decode(string(data), T, + requireAllFields = requireAllFields, + allowUnknownFields = true)) + except SerializationError: + # TODO: Do better error reporting here + err("Unable to deserialize data") + +proc decodeFromJsonBytes*[T](t: typedesc[T], + data: openArray[byte], + requireAllFields = true): SerdesResult[T] = + try: + ok(RestJson.decode(string.fromBytes(data), T, + requireAllFields = requireAllFields, + allowUnknownFields = true)) + except SerializationError: + # TODO: Do better error reporting here + err("Unable to deserialize data") + +proc encodeIntoJsonString*(value: auto): SerdesResult[string] = + var encoded: string + try: + var stream = memoryOutput() + var writer = JsonWriter[RestJson].init(stream) + writer.writeValue(value) + encoded = stream.getOutput(string) + except SerializationError, IOError: + # TODO: Do better error reporting here + return err("unable to serialize data") + + ok(encoded) + +proc encodeIntoJsonBytes*(value: auto): SerdesResult[seq[byte]] = + var encoded: seq[byte] + try: + var stream = memoryOutput() + var writer = JsonWriter[RestJson].init(stream) + writer.writeValue(value) + encoded = stream.getOutput(seq[byte]) + except SerializationError, IOError: + # TODO: Do better error reporting here + return err("unable to serialize data") + + ok(encoded) \ No newline at end of file diff --git a/waku/v2/node/rest/server.nim b/waku/v2/node/rest/server.nim new file mode 100644 index 000000000..8d633c9f4 --- /dev/null +++ b/waku/v2/node/rest/server.nim @@ -0,0 +1,86 @@ +{.push raises: [Defect].} + +import + std/[os, times], + std/typetraits, + stew/[byteutils, io2], + stew/shims/net, + chronicles, chronos, + metrics, metrics/chronos_httpserver, + bearssl, + presto + + +proc getRouter(allowedOrigin: Option[string]): RestRouter = + # TODO: Review this `validate` method. Check in nim-presto what is this used for. + proc validate(key: string, value: string): int = + ## This is rough validation procedure which should be simple and fast, + ## because it will be used for query routing. + 1 + + RestRouter.init(validate, allowedOrigin = allowedOrigin) + + +type RestServerConf* = object + cacheSize*: Natural ## \ + ## The maximum number of recently accessed states that are kept in \ + ## memory. Speeds up requests obtaining information for consecutive + ## slots or epochs. + + cacheTtl*: Natural ## \ + ## The number of seconds to keep recently accessed states in memory + + requestTimeout*: Natural ## \ + ## The number of seconds to wait until complete REST request will be received + + maxRequestBodySize*: Natural ## \ + ## Maximum size of REST request body (kilobytes) + + maxRequestHeadersSize*: Natural ## \ + ## Maximum size of REST request headers (kilobytes) + +proc default(T: type RestServerConf): RestServerConf = + RestServerConf( + cacheSize: 3, + cacheTtl: 60, + requestTimeout: 0, + maxRequestBodySize: 16_384, + maxRequestHeadersSize: 64 + ) + + +template init*(T: type RestServerRef, + ip: ValidIpAddress, port: Port, + allowedOrigin: Option[string], + config: Option[RestServerConf]): T = + let address = initTAddress(ip, port) + let serverFlags = { + HttpServerFlags.QueryCommaSeparatedArray, + HttpServerFlags.NotifyDisconnect + } + + let conf = if config.isSome: config.get() + else: RestServerConf.default() + + let + headersTimeout = if conf.requestTimeout == 0: chronos.InfiniteDuration + else: seconds(int64(conf.requestTimeout)) + maxHeadersSize = conf.maxRequestHeadersSize * 1024 + maxRequestBodySize = conf.maxRequestBodySize * 1024 + + let router = getRouter(allowedOrigin) + let res = RestServerRef.new( + router, + address, + serverFlags = serverFlags, + httpHeadersTimeout = headersTimeout, + maxHeadersSize = maxHeadersSize, + maxRequestBodySize = maxRequestBodySize + ) + + if res.isErr(): + notice "Rest server could not be started", address = $address, reason = res.error() + nil + else: + notice "Starting REST HTTP server", url = "http://" & $ip & ":" & $port & "/" + res.get() diff --git a/waku/v2/node/rest/utils.nim b/waku/v2/node/rest/utils.nim new file mode 100644 index 000000000..2e787d718 --- /dev/null +++ b/waku/v2/node/rest/utils.nim @@ -0,0 +1,18 @@ +{.push raises: [Defect].} + +import std/typetraits +import + chronicles, + stew/results, + presto/common +import "."/serdes + + +const MIMETYPE_JSON* = MediaType.init("application/json") + +proc jsonResponse*(t: typedesc[RestApiResponse], data: auto, status: HttpCode = Http200): SerdesResult[RestApiResponse] = + let encoded = ?encodeIntoJsonBytes(data) + ok(RestApiResponse.response(encoded, status, $MIMETYPE_JSON)) + +proc internalServerError*(t: typedesc[RestApiResponse]): RestApiResponse = + RestApiResponse.error(Http500) \ No newline at end of file