feat(rest): Add HTTP REST API (#727). Debug API POC

.gitmodules

@@ -143,3 +143,8 @@
 [submodule "vendor/nim-toml-serialization"]
 	path = vendor/nim-toml-serialization
 	url = https://github.com/status-im/nim-toml-serialization.git
+[submodule "vendor/nim-presto"]
+	path = vendor/nim-presto
+	url = https://github.com/status-im/nim-presto.git
+	ignore = untracked
+	branch = master

tests/all_tests_v2.nim

@@ -10,6 +10,9 @@ import
   ./v2/test_waku_swap,
   ./v2/test_message_store,
   ./v2/test_jsonrpc_waku,
+  ./v2/test_rest_serdes,
+  ./v2/test_rest_debug_api_serdes,
+  ./v2/test_rest_debug_api,
   ./v2/test_peer_manager,
   ./v2/test_web3, # TODO  remove it when rln-relay tests get finalized
   ./v2/test_waku_bridge,

tests/v2/test_rest_debug_api.nim

@@ -0,0 +1,52 @@
+import
+  stew/shims/net,
+  chronicles,
+  testutils/unittests,
+  presto,
+  libp2p/crypto/crypto
+import
+  ../../waku/v2/node/wakunode2,
+  ../../waku/v2/node/rest/[server, client, debug_api]
+
+
+proc testWakuNode(): WakuNode = 
+  let 
+    rng = crypto.newRng()
+    privkey = crypto.PrivateKey.random(Secp256k1, rng[]).tryGet()
+    bindIp = ValidIpAddress.init("0.0.0.0")
+    extIp = ValidIpAddress.init("127.0.0.1")
+    port = Port(9000)
+
+  WakuNode.new(privkey, bindIp, port, some(extIp), some(port))
+
+
+suite "REST API - Debug":
+  asyncTest "Get node info - GET /debug/v1/info": 
+    # Given
+    let node = testWakuNode()
+    await node.start()
+    node.mountRelay()
+
+    let restPort = Port(8546)
+    let restAddress = ValidIpAddress.init("0.0.0.0")
+    let restServer = RestServerRef.init(
+      restAddress,
+      restPort,
+      none(string),
+      none(RestServerConf)
+    )
+
+    installDebugApiHandlers(restServer.router, node)
+    restServer.start()
+
+    # When
+    let client = newRestHttpClient(initTAddress(restAddress, restPort))
+    let response = await client.debugInfoV1()
+
+    # Then
+    check:
+      response.listenAddresses == @[$node.switch.peerInfo.addrs[^1] & "/p2p/" & $node.switch.peerInfo.peerId]
+
+    await restServer.stop()
+    await restServer.closeWait()
+    await node.stop()

tests/v2/test_rest_debug_api_serdes.nim

@@ -0,0 +1,39 @@
+import std/typetraits
+import chronicles,
+  unittest2,
+  stew/[results, byteutils],
+  json_serialization
+import 
+  ../../waku/v2/node/rest/[serdes, debug_api]
+
+
+suite "Debug API - serialization":
+
+  suite "DebugWakuInfo - decode":
+    test "optional field is not provided":
+      # Given
+      let jsonBytes = toBytes("""{ "listenAddresses":["123"] }""")
+
+      # When
+      let res = decodeFromJsonBytes(DebugWakuInfo, jsonBytes, requireAllFields = true)
+
+      # Then
+      require(res.isOk)
+      let value = res.get()
+      check:
+        value.listenAddresses == @["123"]
+        value.enrUri.isNone
+
+  suite "DebugWakuInfo - encode":
+    test "optional field is none":
+      # Given
+      let data = DebugWakuInfo(listenAddresses: @["GO"], enrUri: none(string))
+
+      # When
+      let res = encodeIntoJsonBytes(data)
+
+      # Then
+      require(res.isOk)
+      let value = res.get()
+      check:
+        value == toBytes("""{"listenAddresses":["GO"]}""")

tests/v2/test_rest_serdes.nim

@@ -0,0 +1,68 @@
+import std/typetraits
+import chronicles,
+  unittest2,
+  stew/[results, byteutils],
+  json_serialization
+import 
+  ../../waku/v2/node/rest/[serdes, debug_api]
+
+
+# TODO: Decouple this test suite from the `debug_api` module by defining
+#  private custom types for this test suite module
+suite "Serdes":
+
+  suite "decode":
+    test "decodeFromJsonString - use the corresponding readValue template":
+      # Given
+      let jsonString = JsonString("""{ "listenAddresses":["123"] }""")
+
+      # When
+      let res = decodeFromJsonString(DebugWakuInfo, jsonString, requireAllFields = true)
+
+      # Then
+      require(res.isOk)
+      let value = res.get()
+      check:
+        value.listenAddresses == @["123"]
+        value.enrUri.isNone
+    
+    test "decodeFromJsonBytes - use the corresponding readValue template":
+      # Given
+      let jsonBytes = toBytes("""{ "listenAddresses":["123"] }""")
+
+      # When
+      let res = decodeFromJsonBytes(DebugWakuInfo, jsonBytes, requireAllFields = true)
+
+      # Then
+      require(res.isOk)
+      let value = res.get()
+      check:
+        value.listenAddresses == @["123"]
+        value.enrUri.isNone
+
+  suite "encode":
+    test "encodeIntoJsonString - use the corresponding writeValue template":
+      # Given
+      let data = DebugWakuInfo(listenAddresses: @["GO"])
+
+      # When
+      let res = encodeIntoJsonString(data)
+
+      # Then
+      require(res.isOk)
+      let value = res.get()
+      check:
+        value == """{"listenAddresses":["GO"]}"""
+
+    test "encodeIntoJsonBytes - use the corresponding writeValue template":
+      # Given
+      let data = DebugWakuInfo(listenAddresses: @["ABC"])
+
+      # When
+      let res = encodeIntoJsonBytes(data)
+
+      # Then
+      require(res.isOk)
+      let value = res.get()
+      check:
+        value == toBytes("""{"listenAddresses":["ABC"]}""" )

vendor/nim-presto

@@ -0,0 +1 @@
+Subproject commit 1dba6dd6f466cd4e7b793b0e473c237ce453d82a

waku.nimble

@@ -20,7 +20,8 @@ requires "nim >= 1.2.0",
   "stint",
   "metrics",
   "libp2p", # Only for Waku v2
-  "web3"
+  "web3",
+  "presto"
 
 ### Helper functions
 proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") =

waku/v2/node/rest/client.nim

@@ -0,0 +1,6 @@
+{.push raises: [Defect].}
+
+import presto/client
+
+proc newRestHttpClient*(address: TransportAddress): RestClientRef =
+  RestClientRef.new(address, HttpClientScheme.NonSecure)

waku/v2/node/rest/debug_api.nim

@@ -0,0 +1,96 @@
+{.push raises: [Defect].}
+
+import
+  stew/byteutils,
+  chronicles,
+  json_serialization,
+  json_serialization/std/options,
+  presto/[route, client]
+import "."/[serdes, utils],
+  ../wakunode2
+
+logScope: topics = "rest_api_debug"
+
+
+#### Types
+
+type
+  DebugWakuInfo* = object
+    listenAddresses*: seq[string]
+    enrUri*: Option[string]
+
+
+#### Serialization and deserialization
+
+proc writeValue*(writer: var JsonWriter[RestJson], value: DebugWakuInfo)
+  {.raises: [IOError, Defect].} =
+  writer.beginRecord()
+  writer.writeField("listenAddresses", value.listenAddresses)
+  if value.enrUri.isSome:
+    writer.writeField("enrUri", value.enrUri)
+  writer.endRecord()
+
+proc readValue*(reader: var JsonReader[RestJson], value: var DebugWakuInfo)
+  {.raises: [SerializationError, IOError, Defect].} =
+  var
+    listenAddresses: Option[seq[string]]
+    enrUri: Option[string]
+
+  for fieldName in readObjectFields(reader):
+    case fieldName
+    of "listenAddresses":
+      if listenAddresses.isSome():
+        reader.raiseUnexpectedField("Multiple `listenAddresses` fields found", "DebugWakuInfo")
+      listenAddresses = some(reader.readValue(seq[string]))
+    of "enrUri":
+      if enrUri.isSome():
+        reader.raiseUnexpectedField("Multiple `enrUri` fields found", "DebugWakuInfo")
+      enrUri = some(reader.readValue(string))
+    else:
+      unrecognizedFieldWarning()
+
+  if listenAddresses.isNone():
+    reader.raiseUnexpectedValue("Field `listenAddresses` is missing")
+
+  value = DebugWakuInfo(
+    listenAddresses: listenAddresses.get,
+    enrUri: enrUri
+  )
+
+
+####  Server request handlers
+
+proc toDebugWakuInfo(nodeInfo: WakuInfo): DebugWakuInfo = 
+    DebugWakuInfo(
+      listenAddresses: nodeInfo.listenAddresses,
+      enrUri: some(nodeInfo.enrUri)
+    )
+
+const ROUTE_DEBUG_INFOV1* = "/debug/v1/info"
+
+proc installDebugInfoV1Handler(router: var RestRouter, node: WakuNode) =
+  router.api(MethodGet, ROUTE_DEBUG_INFOV1) do () -> RestApiResponse:
+    let info = node.info().toDebugWakuInfo()
+    let resp = RestApiResponse.jsonResponse(info, status=Http200)
+    if resp.isErr():
+      debug "An error ocurred while building the json respose", error=resp.error()
+      return RestApiResponse.internalServerError()
+
+    return resp.get()
+
+proc installDebugApiHandlers*(router: var RestRouter, node: WakuNode) =
+  installDebugInfoV1Handler(router, node)
+
+
+#### Client
+
+proc decodeBytes*(t: typedesc[DebugWakuInfo], data: openArray[byte], contentType: string): RestResult[DebugWakuInfo] =
+  if MediaType.init(contentType) != MIMETYPE_JSON:
+    error "Unsupported respose contentType value", contentType = contentType
+    return err("Unsupported response contentType")
+  
+  let decoded = ?decodeFromJsonBytes(DebugWakuInfo, data)
+  return ok(decoded)
+
+# TODO: Check how we can use a constant to set the method endpoint (improve "rest" pragma under nim-presto)
+proc debugInfoV1*(): DebugWakuInfo {.rest, endpoint: "/debug/v1/info", meth: HttpMethod.MethodGet.}

waku/v2/node/rest/debug_api.yaml

@@ -0,0 +1,35 @@
+openapi: 3.1.0
+info:
+  title: Waku REST API - Debug
+  version: 1.0.0
+
+paths:
+  /debug/v1/info:
+    get:
+      summary: Get node info
+      description: Retrieve information about a Waku v2 node.
+      tags:
+        - Debug
+      responses:
+        '200':
+          description: Information about a Waku v2 node.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/WakuInfo'
+        '5XX':
+          description: Unexpected error.
+
+components:
+  schemas:
+    WakuInfo:
+      type: object
+      properties:
+        listenAddresses:
+          type: array
+          items:
+            type: string
+        enrUri:
+          type: string
+      required:
+        - listenAddresses

waku/v2/node/rest/serdes.nim

@@ -0,0 +1,72 @@
+{.push raises: [Defect].}
+
+import std/typetraits
+import 
+  stew/results,
+  chronicles,
+  serialization,
+  json_serialization,
+  json_serialization/std/[options, net, sets],
+  presto/common
+
+logScope: topics = "rest_api_serdes"
+
+Json.createFlavor RestJson
+
+template unrecognizedFieldWarning* =
+  # TODO: There should be a different notification mechanism for informing the
+  #       caller of a deserialization routine for unexpected fields.
+  #       The chonicles import in this module should be removed.
+  debug "JSON field not recognized by the current version of nwaku. Consider upgrading",
+        fieldName, typeName = typetraits.name(typeof value)
+
+
+type SerdesResult*[T] = Result[T, cstring]
+
+proc decodeFromJsonString*[T](t: typedesc[T],
+                          data: JsonString,
+                          requireAllFields = true): SerdesResult[T] =
+  try:
+    ok(RestJson.decode(string(data), T,
+                       requireAllFields = requireAllFields,
+                       allowUnknownFields = true))
+  except SerializationError:
+    # TODO: Do better error reporting here
+    err("Unable to deserialize data")
+
+proc decodeFromJsonBytes*[T](t: typedesc[T],
+                         data: openArray[byte],
+                         requireAllFields = true): SerdesResult[T] =
+  try:
+    ok(RestJson.decode(string.fromBytes(data), T,
+                       requireAllFields = requireAllFields,
+                       allowUnknownFields = true))
+  except SerializationError:
+    # TODO: Do better error reporting here
+    err("Unable to deserialize data")
+
+proc encodeIntoJsonString*(value: auto): SerdesResult[string] =
+  var encoded: string
+  try:
+    var stream = memoryOutput()
+    var writer = JsonWriter[RestJson].init(stream)
+    writer.writeValue(value)
+    encoded = stream.getOutput(string)
+  except SerializationError, IOError:
+    # TODO: Do better error reporting here
+    return err("unable to serialize data")
+
+  ok(encoded)
+
+proc encodeIntoJsonBytes*(value: auto): SerdesResult[seq[byte]] =
+  var encoded: seq[byte]
+  try:
+    var stream = memoryOutput()
+    var writer = JsonWriter[RestJson].init(stream)
+    writer.writeValue(value)
+    encoded = stream.getOutput(seq[byte])
+  except SerializationError, IOError:
+    # TODO: Do better error reporting here
+    return err("unable to serialize data")
+
+  ok(encoded)

waku/v2/node/rest/server.nim

@@ -0,0 +1,86 @@
+{.push raises: [Defect].}
+
+import
+  std/[os, times],
+  std/typetraits,
+  stew/[byteutils, io2],
+  stew/shims/net,
+  chronicles, chronos,
+  metrics, metrics/chronos_httpserver,
+  bearssl,
+  presto
+
+
+proc getRouter(allowedOrigin: Option[string]): RestRouter =
+  # TODO: Review this `validate` method. Check in nim-presto what is this used for.
+  proc validate(key: string, value: string): int =
+    ## This is rough validation procedure which should be simple and fast,
+    ## because it will be used for query routing.
+    1
+
+  RestRouter.init(validate, allowedOrigin = allowedOrigin)
+
+
+type RestServerConf* = object 
+      cacheSize*: Natural ## \
+        ## The maximum number of recently accessed states that are kept in \
+        ## memory. Speeds up requests obtaining information for consecutive  
+        ## slots or epochs.
+
+      cacheTtl*: Natural ## \
+        ## The number of seconds to keep recently accessed states in memory
+
+      requestTimeout*: Natural ## \
+        ## The number of seconds to wait until complete REST request will be received
+
+      maxRequestBodySize*: Natural ## \
+        ## Maximum size of REST request body (kilobytes)
+
+      maxRequestHeadersSize*: Natural ## \
+        ## Maximum size of REST request headers (kilobytes)
+
+proc default(T: type RestServerConf): RestServerConf =
+  RestServerConf(
+      cacheSize: 3,
+      cacheTtl: 60,
+      requestTimeout: 0,
+      maxRequestBodySize: 16_384,
+      maxRequestHeadersSize: 64
+  )
+
+
+template init*(T: type RestServerRef,
+              ip: ValidIpAddress, port: Port,
+              allowedOrigin: Option[string],
+              config: Option[RestServerConf]): T =
+  let address = initTAddress(ip, port)
+  let serverFlags = {
+    HttpServerFlags.QueryCommaSeparatedArray,
+    HttpServerFlags.NotifyDisconnect
+  }
+  
+  let conf = if config.isSome: config.get() 
+             else: RestServerConf.default()
+
+  let 
+    headersTimeout = if conf.requestTimeout == 0: chronos.InfiniteDuration
+                     else: seconds(int64(conf.requestTimeout))
+    maxHeadersSize = conf.maxRequestHeadersSize * 1024
+    maxRequestBodySize = conf.maxRequestBodySize * 1024
+
+  let router = getRouter(allowedOrigin)
+  let res = RestServerRef.new(
+    router, 
+    address, 
+    serverFlags = serverFlags,
+    httpHeadersTimeout = headersTimeout,
+    maxHeadersSize = maxHeadersSize,
+    maxRequestBodySize = maxRequestBodySize
+  )
+
+  if res.isErr():
+    notice "Rest server could not be started", address = $address, reason = res.error()
+    nil
+  else:
+    notice "Starting REST HTTP server", url = "http://" & $ip & ":" & $port & "/"
+    res.get()

waku/v2/node/rest/utils.nim

@@ -0,0 +1,18 @@
+{.push raises: [Defect].}
+
+import std/typetraits
+import 
+  chronicles,
+  stew/results,
+  presto/common
+import "."/serdes
+
+
+const MIMETYPE_JSON* = MediaType.init("application/json")
+
+proc jsonResponse*(t: typedesc[RestApiResponse], data: auto, status: HttpCode = Http200): SerdesResult[RestApiResponse] =
+  let encoded = ?encodeIntoJsonBytes(data)
+  ok(RestApiResponse.response(encoded, status, $MIMETYPE_JSON))
+
+proc internalServerError*(t: typedesc[RestApiResponse]): RestApiResponse =
+  RestApiResponse.error(Http500)