From 9f05f6305423810feea5ef0e61aef1f736538235 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 14 Jun 2018 16:52:03 +0100 Subject: [PATCH 01/60] Rename `eth-rpc` to `rpc` --- eth-rpc/client.nim | 212 ------------------ eth-rpc/server.nim | 354 ------------------------------- {eth-rpc => rpc}/jsonmarshal.nim | 0 rpcclient.nim | 2 +- rpcserver.nim | 2 +- 5 files changed, 2 insertions(+), 568 deletions(-) delete mode 100644 eth-rpc/client.nim delete mode 100644 eth-rpc/server.nim rename {eth-rpc => rpc}/jsonmarshal.nim (100%) diff --git a/eth-rpc/client.nim b/eth-rpc/client.nim deleted file mode 100644 index e4bb85b..0000000 --- a/eth-rpc/client.nim +++ /dev/null @@ -1,212 +0,0 @@ -import tables, json, macros -import asyncdispatch2 -import jsonmarshal - -type - RpcClient* = ref object - transp: StreamTransport - awaiting: Table[string, Future[Response]] - address: TransportAddress - nextId: int64 - Response* = tuple[error: bool, result: JsonNode] - -const maxRequestLength = 1024 * 128 - -proc newRpcClient*(): RpcClient = - ## Creates a new ``RpcClient`` instance. - result = RpcClient(awaiting: initTable[string, Future[Response]](), nextId: 1) - -proc call*(self: RpcClient, name: string, - params: JsonNode): Future[Response] {.async.} = - ## Remotely calls the specified RPC method. - let id = $self.nextId - self.nextId.inc - var msg = $ %{"jsonrpc": %"2.0", "method": %name, "params": params, - "id": %id} & "\c\l" - let res = await self.transp.write(msg) - # TODO: Add actions when not full packet was send, e.g. disconnect peer. - assert(res == len(msg)) - - # completed by processMessage. - var newFut = newFuture[Response]() - # add to awaiting responses - self.awaiting[id] = newFut - result = await newFut - -template handleRaise[T](fut: Future[T], errType: typedesc, msg: string) = - # complete future before raising - fut.complete((true, %msg)) - raise newException(errType, msg) - -macro checkGet(node: JsonNode, fieldName: string, - jKind: static[JsonNodeKind]): untyped = - let n = genSym(ident = "n") #`node`{`fieldName`} - result = quote: - let `n` = `node`{`fieldname`} - if `n`.isNil or `n`.kind == JNull: - raise newException(ValueError, - "Message is missing required field \"" & `fieldName` & "\"") - if `n`.kind != `jKind`.JsonNodeKind: - raise newException(ValueError, - "Expected " & $(`jKind`.JsonNodeKind) & ", got " & $`n`.kind) - case jKind - of JBool: result.add(quote do: `n`.getBool) - of JInt: result.add(quote do: `n`.getInt) - of JString: result.add(quote do: `n`.getStr) - of JFloat: result.add(quote do: `n`.getFloat) - of JObject: result.add(quote do: `n`.getObject) - else: discard - -proc processMessage(self: RpcClient, line: string) = - let node = parseJson(line) - - # TODO: Use more appropriate exception objects - let id = checkGet(node, "id", JString) - if not self.awaiting.hasKey(id): - raise newException(ValueError, - "Cannot find message id \"" & node["id"].str & "\"") - - let version = checkGet(node, "jsonrpc", JString) - if version != "2.0": - self.awaiting[id].handleRaise(ValueError, - "Unsupported version of JSON, expected 2.0, received \"" & version & "\"") - - let errorNode = node{"error"} - if errorNode.isNil or errorNode.kind == JNull: - var res = node{"result"} - if not res.isNil: - self.awaiting[id].complete((false, res)) - self.awaiting.del(id) - # TODO: actions on unable find result node - else: - self.awaiting[id].complete((true, errorNode)) - self.awaiting.del(id) - -proc connect*(self: RpcClient, address: string, port: Port): Future[void] - -proc processData(self: RpcClient) {.async.} = - while true: - let line = await self.transp.readLine(maxRequestLength) - if line == "": - # transmission ends - self.transp.close() - break - - processMessage(self, line) - # async loop reconnection and waiting - self.transp = await connect(self.address) - -proc connect*(self: RpcClient, address: string, port: Port) {.async.} = - # TODO: `address` hostname can be resolved to many IP addresses, we are using - # first one, but maybe it would be better to iterate over all IP addresses - # and try to establish connection until it will not be established. - let addresses = resolveTAddress(address, port) - self.transp = await connect(addresses[0]) - self.address = addresses[0] - asyncCheck processData(self) - -proc createRpcProc(procName, parameters, callBody: NimNode): NimNode = - # parameters come as a tree - var paramList = newSeq[NimNode]() - for p in parameters: paramList.add(p) - - # build proc - result = newProc(procName, paramList, callBody) - # make proc async - result.addPragma ident"async" - # export this proc - result[0] = nnkPostFix.newTree(ident"*", newIdentNode($procName)) - -proc toJsonArray(parameters: NimNode): NimNode = - # outputs an array of jsonified parameters - # ie; %[%a, %b, %c] - parameters.expectKind nnkFormalParams - var items = newNimNode(nnkBracket) - for i in 2 ..< parameters.len: - let curParam = parameters[i][0] - if curParam.kind != nnkEmpty: - items.add(nnkPrefix.newTree(ident"%", curParam)) - result = nnkPrefix.newTree(bindSym("%", brForceOpen), items) - -proc createRpcFromSig*(rpcDecl: NimNode): NimNode = - # Each input parameter in the rpc signature is converted - # to json with `%`. - # Return types are then converted back to native Nim types. - let iJsonNode = newIdentNode("JsonNode") - - var parameters = rpcDecl.findChild(it.kind == nnkFormalParams).copy - # ensure we have at least space for a return parameter - if parameters.isNil or parameters.kind == nnkEmpty or parameters.len == 0: - parameters = nnkFormalParams.newTree(iJsonNode) - - let - procName = rpcDecl.name - pathStr = $procName - returnType = - # if no return type specified, defaults to JsonNode - if parameters[0].kind == nnkEmpty: iJsonNode - else: parameters[0] - customReturnType = returnType != iJsonNode - - # insert rpc client as first parameter - parameters.insert(1, nnkIdentDefs.newTree(ident"client", ident"RpcClient", - newEmptyNode())) - - let - # variable used to send json to the server - jsonParamIdent = genSym(nskVar, "jsonParam") - # json array of marshalled parameters - jsonParamArray = parameters.toJsonArray() - var - # populate json params - even rpcs with no parameters have an empty json - # array node sent - callBody = newStmtList().add(quote do: - var `jsonParamIdent` = `jsonParamArray` - ) - - # convert return type to Future - parameters[0] = nnkBracketExpr.newTree(ident"Future", returnType) - # create rpc proc - result = createRpcProc(procName, parameters, callBody) - - let - # temporary variable to hold `Response` from rpc call - rpcResult = genSym(nskLet, "res") - clientIdent = newIdentNode("client") - # proc return variable - procRes = ident"result" - # actual return value, `rpcResult`.result - jsonRpcResult = nnkDotExpr.newTree(rpcResult, newIdentNode("result")) - - # perform rpc call - callBody.add(quote do: - # `rpcResult` is of type `Response` - let `rpcResult` = await `clientIdent`.call(`pathStr`, `jsonParamIdent`) - if `rpcResult`.error: raise newException(ValueError, $`rpcResult`.result) - ) - - if customReturnType: - # marshal json to native Nim type - callBody.add(jsonToNim(procRes, returnType, jsonRpcResult, "result")) - else: - # native json expected so no work - callBody.add(quote do: - `procRes` = `rpcResult`.result - ) - when defined(nimDumpRpcs): - echo pathStr, ":\n", result.repr - -proc processRpcSigs(parsedCode: NimNode): NimNode = - result = newStmtList() - - for line in parsedCode: - if line.kind == nnkProcDef: - var procDef = createRpcFromSig(line) - result.add(procDef) - -macro createRpcSigs*(filePath: static[string]): untyped = - ## Takes a file of forward declarations in Nim and builds them into RPC - ## calls, based on their parameters. - ## Inputs are marshalled to json, and results are put into the signature's - ## Nim type. - result = processRpcSigs(staticRead($filePath).parseStmt()) diff --git a/eth-rpc/server.nim b/eth-rpc/server.nim deleted file mode 100644 index 294f317..0000000 --- a/eth-rpc/server.nim +++ /dev/null @@ -1,354 +0,0 @@ -import json, tables, strutils, options, macros #, chronicles -import asyncdispatch2 -import jsonmarshal - -export asyncdispatch2, json, jsonmarshal - -# Temporarily disable logging -macro debug(body: varargs[untyped]): untyped = newStmtList() -macro info(body: varargs[untyped]): untyped = newStmtList() -macro error(body: varargs[untyped]): untyped = newStmtList() - -#logScope: -# topics = "RpcServer" - -type - RpcJsonError* = enum rjeInvalidJson, rjeVersionError, rjeNoMethod, rjeNoId - - RpcJsonErrorContainer* = tuple[err: RpcJsonError, msg: string] - - # Procedure signature accepted as an RPC call by server - RpcProc* = proc (params: JsonNode): Future[JsonNode] - - RpcServer* = ref object - servers*: seq[StreamServer] - procs*: TableRef[string, RpcProc] - - RpcProcError* = ref object of Exception - code*: int - data*: JsonNode - - RpcBindError* = object of Exception - RpcAddressUnresolvableError* = object of Exception - -const - JSON_PARSE_ERROR* = -32700 - INVALID_REQUEST* = -32600 - METHOD_NOT_FOUND* = -32601 - INVALID_PARAMS* = -32602 - INTERNAL_ERROR* = -32603 - SERVER_ERROR* = -32000 - - maxRequestLength = 1024 * 128 - - jsonErrorMessages*: array[RpcJsonError, (int, string)] = - [ - (JSON_PARSE_ERROR, "Invalid JSON"), - (INVALID_REQUEST, "JSON 2.0 required"), - (INVALID_REQUEST, "No method requested"), - (INVALID_REQUEST, "No id specified") - ] - -# Utility functions -# TODO: Move outside server -func `%`*(p: Port): JsonNode = %(p.int) - -# Json state checking - -template jsonValid*(jsonString: string, node: var JsonNode): (bool, string) = - var - valid = true - msg = "" - try: node = parseJson(line) - except: - valid = false - msg = getCurrentExceptionMsg() - debug "Cannot process json", json = jsonString, msg = msg - (valid, msg) - -proc checkJsonErrors*(line: string, - node: var JsonNode): Option[RpcJsonErrorContainer] = - ## Tries parsing line into node, if successful checks required fields - ## Returns: error state or none - let res = jsonValid(line, node) - if not res[0]: - return some((rjeInvalidJson, res[1])) - if not node.hasKey("id"): - return some((rjeNoId, "")) - if node{"jsonrpc"} != %"2.0": - return some((rjeVersionError, "")) - if not node.hasKey("method"): - return some((rjeNoMethod, "")) - return none(RpcJsonErrorContainer) - -# Json reply wrappers - -proc wrapReply*(id: JsonNode, value: JsonNode, error: JsonNode): string = - let node = %{"jsonrpc": %"2.0", "result": value, "error": error, "id": id} - return $node & "\c\l" - -proc sendError*(client: StreamTransport, code: int, msg: string, id: JsonNode, - data: JsonNode = newJNull()) {.async.} = - ## Send error message to client - let error = %{"code": %(code), "id": id, "message": %msg, "data": data} - debug "Error generated", error = error, id = id - var res = wrapReply(id, newJNull(), error) - result = client.write(res) - -proc sendJsonError*(state: RpcJsonError, client: StreamTransport, id: JsonNode, - data = newJNull()) {.async.} = - ## Send client response for invalid json state - let errMsgs = jsonErrorMessages[state] - await client.sendError(errMsgs[0], errMsgs[1], id, data) - -# Server message processing -proc processMessage(server: RpcServer, client: StreamTransport, - line: string) {.async.} = - var - node: JsonNode - # set up node and/or flag errors - jsonErrorState = checkJsonErrors(line, node) - - if jsonErrorState.isSome: - let errState = jsonErrorState.get - var id = - if errState.err == rjeInvalidJson or errState.err == rjeNoId: - newJNull() - else: - node["id"] - await errState.err.sendJsonError(client, id, %errState.msg) - else: - let - methodName = node["method"].str - id = node["id"] - - if not server.procs.hasKey(methodName): - await client.sendError(METHOD_NOT_FOUND, "Method not found", %id, - %(methodName & " is not a registered method.")) - else: - let callRes = await server.procs[methodName](node["params"]) - var res = wrapReply(id, callRes, newJNull()) - discard await client.write(res) - -proc processClient(server: StreamServer, client: StreamTransport) {.async, gcsafe.} = - var rpc = getUserData[RpcServer](server) - while true: - let line = await client.readLine(maxRequestLength) - if line == "": - client.close() - break - - debug "Processing client", addresss = client.remoteAddress(), line - - let future = processMessage(rpc, client, line) - yield future - if future.failed: - if future.readError of RpcProcError: - let err = future.readError.RpcProcError - await client.sendError(err.code, err.msg, err.data) - elif future.readError of ValueError: - let err = future.readError[].ValueError - await client.sendError(INVALID_PARAMS, err.msg, %"") - else: - await client.sendError(SERVER_ERROR, - "Error: Unknown error occurred", %"") - -proc newRpcServer*(addresses: openarray[TransportAddress]): RpcServer = - ## Create new server and assign it to addresses ``addresses``. - result = RpcServer() - result.procs = newTable[string, RpcProc]() - result.servers = newSeq[StreamServer]() - - for item in addresses: - try: - info "Creating server on ", address = $item - var server = createStreamServer(item, processClient, {ReuseAddr}, - udata = result) - result.servers.add(server) - except: - error "Failed to create server", address = $item, message = getCurrentExceptionMsg() - - if len(result.servers) == 0: - # Server was not bound, critical error. - # TODO: Custom RpcException error - raise newException(RpcBindError, "Unable to create server!") - -proc newRpcServer*(addresses: openarray[string]): RpcServer = - ## Create new server and assign it to addresses ``addresses``. - var - tas4: seq[TransportAddress] - tas6: seq[TransportAddress] - baddrs: seq[TransportAddress] - - for a in addresses: - # Attempt to resolve `address` for IPv4 address space. - try: - tas4 = resolveTAddress(a, IpAddressFamily.IPv4) - except: - discard - - # Attempt to resolve `address` for IPv6 address space. - try: - tas6 = resolveTAddress(a, IpAddressFamily.IPv6) - except: - discard - - for r in tas4: - baddrs.add(r) - for r in tas6: - baddrs.add(r) - - if len(baddrs) == 0: - # Addresses could not be resolved, critical error. - raise newException(RpcAddressUnresolvableError, "Unable to get address!") - - result = newRpcServer(baddrs) - -proc newRpcServer*(address = "localhost", port: Port = Port(8545)): RpcServer = - var - tas4: seq[TransportAddress] - tas6: seq[TransportAddress] - - # Attempt to resolve `address` for IPv4 address space. - try: - tas4 = resolveTAddress(address, port, IpAddressFamily.IPv4) - except: - discard - - # Attempt to resolve `address` for IPv6 address space. - try: - tas6 = resolveTAddress(address, port, IpAddressFamily.IPv6) - except: - discard - - if len(tas4) == 0 and len(tas6) == 0: - # Address was not resolved, critical error. - raise newException(RpcAddressUnresolvableError, - "Address " & address & " could not be resolved!") - - result = RpcServer() - result.procs = newTable[string, RpcProc]() - result.servers = newSeq[StreamServer]() - for item in tas4: - try: - info "Creating server for address", ip4address = $item - var server = createStreamServer(item, processClient, {ReuseAddr}, - udata = result) - result.servers.add(server) - except: - error "Failed to create server for address", address = $item - - for item in tas6: - try: - info "Server created", ip6address = $item - var server = createStreamServer(item, processClient, {ReuseAddr}, - udata = result) - result.servers.add(server) - except: - error "Failed to create server", address = $item - - if len(result.servers) == 0: - # Server was not bound, critical error. - raise newException(RpcBindError, - "Could not setup server on " & address & ":" & $int(port)) - -proc start*(server: RpcServer) = - ## Start the RPC server. - for item in server.servers: - item.start() - -proc stop*(server: RpcServer) = - ## Stop the RPC server. - for item in server.servers: - item.stop() - -proc close*(server: RpcServer) = - ## Cleanup resources of RPC server. - for item in server.servers: - item.close() - -# Server registration and RPC generation - -proc register*(server: RpcServer, name: string, rpc: RpcProc) = - ## Add a name/code pair to the RPC server. - server.procs[name] = rpc - -proc unRegisterAll*(server: RpcServer) = - # Remove all remote procedure calls from this server. - server.procs.clear - -proc makeProcName(s: string): string = - result = "" - for c in s: - if c.isAlphaNumeric: result.add c - -proc hasReturnType(params: NimNode): bool = - if params != nil and params.len > 0 and params[0] != nil and - params[0].kind != nnkEmpty: - result = true - -macro rpc*(server: RpcServer, path: string, body: untyped): untyped = - ## Define a remote procedure call. - ## Input and return parameters are defined using the ``do`` notation. - ## For example: - ## .. code-block:: nim - ## myServer.rpc("path") do(param1: int, param2: float) -> string: - ## result = $param1 & " " & $param2 - ## ``` - ## Input parameters are automatically marshalled from json to Nim types, - ## and output parameters are automatically marshalled to json for transport. - result = newStmtList() - let - parameters = body.findChild(it.kind == nnkFormalParams) - # all remote calls have a single parameter: `params: JsonNode` - paramsIdent = newIdentNode"params" - # procs are generated from the stripped path - pathStr = $path - # strip non alphanumeric - procNameStr = pathStr.makeProcName - # public rpc proc - procName = newIdentNode(procNameStr) - # when parameters present: proc that contains our rpc body - doMain = newIdentNode(procNameStr & "DoMain") - # async result - res = newIdentNode("result") - var - setup = jsonToNim(parameters, paramsIdent) - procBody = if body.kind == nnkStmtList: body else: body.body - - if parameters.hasReturnType: - let returnType = parameters[0] - - # delegate async proc allows return and setting of result as native type - result.add(quote do: - proc `doMain`(`paramsIdent`: JsonNode): Future[`returnType`] {.async.} = - `setup` - `procBody` - ) - - if returnType == ident"JsonNode": - # `JsonNode` results don't need conversion - result.add( quote do: - proc `procName`(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = - `res` = await `doMain`(`paramsIdent`) - ) - else: - result.add(quote do: - proc `procName`(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = - `res` = %await `doMain`(`paramsIdent`) - ) - else: - # no return types, inline contents - result.add(quote do: - proc `procName`(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = - `setup` - `procBody` - ) - result.add( quote do: - `server`.register(`path`, `procName`) - ) - - when defined(nimDumpRpcs): - echo "\n", pathStr, ": ", result.repr - -# TODO: Allow cross checking between client signatures and server calls diff --git a/eth-rpc/jsonmarshal.nim b/rpc/jsonmarshal.nim similarity index 100% rename from eth-rpc/jsonmarshal.nim rename to rpc/jsonmarshal.nim diff --git a/rpcclient.nim b/rpcclient.nim index c1a3431..14712ef 100644 --- a/rpcclient.nim +++ b/rpcclient.nim @@ -1,3 +1,3 @@ -import eth-rpc / client +import rpc / client export client diff --git a/rpcserver.nim b/rpcserver.nim index 905d53e..af3bbfb 100644 --- a/rpcserver.nim +++ b/rpcserver.nim @@ -1,2 +1,2 @@ -import eth-rpc / server +import rpc / server export server From 33b1c61952c8996c2a7bbe37ec9c379d7b5881d6 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 14 Jun 2018 16:52:41 +0100 Subject: [PATCH 02/60] Move client and server to `rpc` folder --- rpc/client.nim | 212 +++++++++++++++++++++++++++++++++++++++ rpc/server.nim | 265 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 477 insertions(+) create mode 100644 rpc/client.nim create mode 100644 rpc/server.nim diff --git a/rpc/client.nim b/rpc/client.nim new file mode 100644 index 0000000..e4bb85b --- /dev/null +++ b/rpc/client.nim @@ -0,0 +1,212 @@ +import tables, json, macros +import asyncdispatch2 +import jsonmarshal + +type + RpcClient* = ref object + transp: StreamTransport + awaiting: Table[string, Future[Response]] + address: TransportAddress + nextId: int64 + Response* = tuple[error: bool, result: JsonNode] + +const maxRequestLength = 1024 * 128 + +proc newRpcClient*(): RpcClient = + ## Creates a new ``RpcClient`` instance. + result = RpcClient(awaiting: initTable[string, Future[Response]](), nextId: 1) + +proc call*(self: RpcClient, name: string, + params: JsonNode): Future[Response] {.async.} = + ## Remotely calls the specified RPC method. + let id = $self.nextId + self.nextId.inc + var msg = $ %{"jsonrpc": %"2.0", "method": %name, "params": params, + "id": %id} & "\c\l" + let res = await self.transp.write(msg) + # TODO: Add actions when not full packet was send, e.g. disconnect peer. + assert(res == len(msg)) + + # completed by processMessage. + var newFut = newFuture[Response]() + # add to awaiting responses + self.awaiting[id] = newFut + result = await newFut + +template handleRaise[T](fut: Future[T], errType: typedesc, msg: string) = + # complete future before raising + fut.complete((true, %msg)) + raise newException(errType, msg) + +macro checkGet(node: JsonNode, fieldName: string, + jKind: static[JsonNodeKind]): untyped = + let n = genSym(ident = "n") #`node`{`fieldName`} + result = quote: + let `n` = `node`{`fieldname`} + if `n`.isNil or `n`.kind == JNull: + raise newException(ValueError, + "Message is missing required field \"" & `fieldName` & "\"") + if `n`.kind != `jKind`.JsonNodeKind: + raise newException(ValueError, + "Expected " & $(`jKind`.JsonNodeKind) & ", got " & $`n`.kind) + case jKind + of JBool: result.add(quote do: `n`.getBool) + of JInt: result.add(quote do: `n`.getInt) + of JString: result.add(quote do: `n`.getStr) + of JFloat: result.add(quote do: `n`.getFloat) + of JObject: result.add(quote do: `n`.getObject) + else: discard + +proc processMessage(self: RpcClient, line: string) = + let node = parseJson(line) + + # TODO: Use more appropriate exception objects + let id = checkGet(node, "id", JString) + if not self.awaiting.hasKey(id): + raise newException(ValueError, + "Cannot find message id \"" & node["id"].str & "\"") + + let version = checkGet(node, "jsonrpc", JString) + if version != "2.0": + self.awaiting[id].handleRaise(ValueError, + "Unsupported version of JSON, expected 2.0, received \"" & version & "\"") + + let errorNode = node{"error"} + if errorNode.isNil or errorNode.kind == JNull: + var res = node{"result"} + if not res.isNil: + self.awaiting[id].complete((false, res)) + self.awaiting.del(id) + # TODO: actions on unable find result node + else: + self.awaiting[id].complete((true, errorNode)) + self.awaiting.del(id) + +proc connect*(self: RpcClient, address: string, port: Port): Future[void] + +proc processData(self: RpcClient) {.async.} = + while true: + let line = await self.transp.readLine(maxRequestLength) + if line == "": + # transmission ends + self.transp.close() + break + + processMessage(self, line) + # async loop reconnection and waiting + self.transp = await connect(self.address) + +proc connect*(self: RpcClient, address: string, port: Port) {.async.} = + # TODO: `address` hostname can be resolved to many IP addresses, we are using + # first one, but maybe it would be better to iterate over all IP addresses + # and try to establish connection until it will not be established. + let addresses = resolveTAddress(address, port) + self.transp = await connect(addresses[0]) + self.address = addresses[0] + asyncCheck processData(self) + +proc createRpcProc(procName, parameters, callBody: NimNode): NimNode = + # parameters come as a tree + var paramList = newSeq[NimNode]() + for p in parameters: paramList.add(p) + + # build proc + result = newProc(procName, paramList, callBody) + # make proc async + result.addPragma ident"async" + # export this proc + result[0] = nnkPostFix.newTree(ident"*", newIdentNode($procName)) + +proc toJsonArray(parameters: NimNode): NimNode = + # outputs an array of jsonified parameters + # ie; %[%a, %b, %c] + parameters.expectKind nnkFormalParams + var items = newNimNode(nnkBracket) + for i in 2 ..< parameters.len: + let curParam = parameters[i][0] + if curParam.kind != nnkEmpty: + items.add(nnkPrefix.newTree(ident"%", curParam)) + result = nnkPrefix.newTree(bindSym("%", brForceOpen), items) + +proc createRpcFromSig*(rpcDecl: NimNode): NimNode = + # Each input parameter in the rpc signature is converted + # to json with `%`. + # Return types are then converted back to native Nim types. + let iJsonNode = newIdentNode("JsonNode") + + var parameters = rpcDecl.findChild(it.kind == nnkFormalParams).copy + # ensure we have at least space for a return parameter + if parameters.isNil or parameters.kind == nnkEmpty or parameters.len == 0: + parameters = nnkFormalParams.newTree(iJsonNode) + + let + procName = rpcDecl.name + pathStr = $procName + returnType = + # if no return type specified, defaults to JsonNode + if parameters[0].kind == nnkEmpty: iJsonNode + else: parameters[0] + customReturnType = returnType != iJsonNode + + # insert rpc client as first parameter + parameters.insert(1, nnkIdentDefs.newTree(ident"client", ident"RpcClient", + newEmptyNode())) + + let + # variable used to send json to the server + jsonParamIdent = genSym(nskVar, "jsonParam") + # json array of marshalled parameters + jsonParamArray = parameters.toJsonArray() + var + # populate json params - even rpcs with no parameters have an empty json + # array node sent + callBody = newStmtList().add(quote do: + var `jsonParamIdent` = `jsonParamArray` + ) + + # convert return type to Future + parameters[0] = nnkBracketExpr.newTree(ident"Future", returnType) + # create rpc proc + result = createRpcProc(procName, parameters, callBody) + + let + # temporary variable to hold `Response` from rpc call + rpcResult = genSym(nskLet, "res") + clientIdent = newIdentNode("client") + # proc return variable + procRes = ident"result" + # actual return value, `rpcResult`.result + jsonRpcResult = nnkDotExpr.newTree(rpcResult, newIdentNode("result")) + + # perform rpc call + callBody.add(quote do: + # `rpcResult` is of type `Response` + let `rpcResult` = await `clientIdent`.call(`pathStr`, `jsonParamIdent`) + if `rpcResult`.error: raise newException(ValueError, $`rpcResult`.result) + ) + + if customReturnType: + # marshal json to native Nim type + callBody.add(jsonToNim(procRes, returnType, jsonRpcResult, "result")) + else: + # native json expected so no work + callBody.add(quote do: + `procRes` = `rpcResult`.result + ) + when defined(nimDumpRpcs): + echo pathStr, ":\n", result.repr + +proc processRpcSigs(parsedCode: NimNode): NimNode = + result = newStmtList() + + for line in parsedCode: + if line.kind == nnkProcDef: + var procDef = createRpcFromSig(line) + result.add(procDef) + +macro createRpcSigs*(filePath: static[string]): untyped = + ## Takes a file of forward declarations in Nim and builds them into RPC + ## calls, based on their parameters. + ## Inputs are marshalled to json, and results are put into the signature's + ## Nim type. + result = processRpcSigs(staticRead($filePath).parseStmt()) diff --git a/rpc/server.nim b/rpc/server.nim new file mode 100644 index 0000000..541d713 --- /dev/null +++ b/rpc/server.nim @@ -0,0 +1,265 @@ +import json, tables, strutils, options, macros #, chronicles +import asyncdispatch2 +import jsonmarshal + +export asyncdispatch2, json, jsonmarshal + +# Temporarily disable logging +macro debug(body: varargs[untyped]): untyped = newStmtList() +macro info(body: varargs[untyped]): untyped = newStmtList() +macro error(body: varargs[untyped]): untyped = newStmtList() + +#logScope: +# topics = "RpcServer" + +type + RpcJsonError* = enum rjeInvalidJson, rjeVersionError, rjeNoMethod, rjeNoId + + RpcJsonErrorContainer* = tuple[err: RpcJsonError, msg: string] + + # Procedure signature accepted as an RPC call by server + RpcProc* = proc (params: JsonNode): Future[JsonNode] + + RpcClientTransport* = concept trans, type t + trans.write(var string) is Future[int] + trans.readLine(int) is Future[string] + #trans.getUserData[t] is t + + RpcServerTransport* = concept t + t.start + t.stop + t.close + + RpcServer*[S: RpcServerTransport] = ref object + servers*: seq[S] + procs*: TableRef[string, RpcProc] + + RpcProcError* = ref object of Exception + code*: int + data*: JsonNode + + RpcBindError* = object of Exception + RpcAddressUnresolvableError* = object of Exception + +const + JSON_PARSE_ERROR* = -32700 + INVALID_REQUEST* = -32600 + METHOD_NOT_FOUND* = -32601 + INVALID_PARAMS* = -32602 + INTERNAL_ERROR* = -32603 + SERVER_ERROR* = -32000 + + maxRequestLength = 1024 * 128 + + jsonErrorMessages*: array[RpcJsonError, (int, string)] = + [ + (JSON_PARSE_ERROR, "Invalid JSON"), + (INVALID_REQUEST, "JSON 2.0 required"), + (INVALID_REQUEST, "No method requested"), + (INVALID_REQUEST, "No id specified") + ] + +# Utility functions +# TODO: Move outside server +func `%`*(p: Port): JsonNode = %(p.int) + +# Json state checking + +template jsonValid*(jsonString: string, node: var JsonNode): (bool, string) = + var + valid = true + msg = "" + try: node = parseJson(line) + except: + valid = false + msg = getCurrentExceptionMsg() + debug "Cannot process json", json = jsonString, msg = msg + (valid, msg) + +proc checkJsonErrors*(line: string, + node: var JsonNode): Option[RpcJsonErrorContainer] = + ## Tries parsing line into node, if successful checks required fields + ## Returns: error state or none + let res = jsonValid(line, node) + if not res[0]: + return some((rjeInvalidJson, res[1])) + if not node.hasKey("id"): + return some((rjeNoId, "")) + if node{"jsonrpc"} != %"2.0": + return some((rjeVersionError, "")) + if not node.hasKey("method"): + return some((rjeNoMethod, "")) + return none(RpcJsonErrorContainer) + +# Json reply wrappers + +proc wrapReply*(id: JsonNode, value: JsonNode, error: JsonNode): string = + let node = %{"jsonrpc": %"2.0", "result": value, "error": error, "id": id} + return $node & "\c\l" + +proc sendError*(client: RpcClientTransport, code: int, msg: string, id: JsonNode, + data: JsonNode = newJNull()) {.async.} = + ## Send error message to client + let error = %{"code": %(code), "id": id, "message": %msg, "data": data} + debug "Error generated", error = error, id = id + var res = wrapReply(id, newJNull(), error) + result = client.write(res) + +proc sendJsonError*(state: RpcJsonError, client: RpcClientTransport, id: JsonNode, + data = newJNull()) {.async.} = + ## Send client response for invalid json state + let errMsgs = jsonErrorMessages[state] + await client.sendError(errMsgs[0], errMsgs[1], id, data) + +# Server message processing +proc processMessage[T](server: RpcServer[T], client: RpcClientTransport, + line: string) {.async.} = + var + node: JsonNode + # set up node and/or flag errors + jsonErrorState = checkJsonErrors(line, node) + + if jsonErrorState.isSome: + let errState = jsonErrorState.get + var id = + if errState.err == rjeInvalidJson or errState.err == rjeNoId: + newJNull() + else: + node["id"] + await errState.err.sendJsonError(client, id, %errState.msg) + else: + let + methodName = node["method"].str + id = node["id"] + + if not server.procs.hasKey(methodName): + await client.sendError(METHOD_NOT_FOUND, "Method not found", %id, + %(methodName & " is not a registered method.")) + else: + let callRes = await server.procs[methodName](node["params"]) + var res = wrapReply(id, callRes, newJNull()) + discard await client.write(res) + +proc processClient*[S: RpcServerTransport, C: RpcClientTransport](server: S, client: C) {.async, gcsafe.} = + var rpc = getUserData[RpcServer[S]](server) + while true: + let line = await client.readLine(maxRequestLength) + if line == "": + client.close() + break + + debug "Processing client", addresss = client.remoteAddress(), line + + let future = processMessage(rpc, client, line) + yield future + if future.failed: + if future.readError of RpcProcError: + let err = future.readError.RpcProcError + await client.sendError(err.code, err.msg, err.data) + elif future.readError of ValueError: + let err = future.readError[].ValueError + await client.sendError(INVALID_PARAMS, err.msg, %"") + else: + await client.sendError(SERVER_ERROR, + "Error: Unknown error occurred", %"") + +proc start*(server: RpcServer) = + ## Start the RPC server. + for item in server.servers: + item.start() + +proc stop*(server: RpcServer) = + ## Stop the RPC server. + for item in server.servers: + item.stop() + +proc close*(server: RpcServer) = + ## Cleanup resources of RPC server. + for item in server.servers: + item.close() + +# Server registration and RPC generation + +proc register*(server: RpcServer, name: string, rpc: RpcProc) = + ## Add a name/code pair to the RPC server. + server.procs[name] = rpc + +proc unRegisterAll*(server: RpcServer) = + # Remove all remote procedure calls from this server. + server.procs.clear + +proc makeProcName(s: string): string = + result = "" + for c in s: + if c.isAlphaNumeric: result.add c + +proc hasReturnType(params: NimNode): bool = + if params != nil and params.len > 0 and params[0] != nil and + params[0].kind != nnkEmpty: + result = true + +macro rpc*(server: RpcServer, path: string, body: untyped): untyped = + ## Define a remote procedure call. + ## Input and return parameters are defined using the ``do`` notation. + ## For example: + ## .. code-block:: nim + ## myServer.rpc("path") do(param1: int, param2: float) -> string: + ## result = $param1 & " " & $param2 + ## ``` + ## Input parameters are automatically marshalled from json to Nim types, + ## and output parameters are automatically marshalled to json for transport. + result = newStmtList() + let + parameters = body.findChild(it.kind == nnkFormalParams) + # all remote calls have a single parameter: `params: JsonNode` + paramsIdent = newIdentNode"params" + # procs are generated from the stripped path + pathStr = $path + # strip non alphanumeric + procNameStr = pathStr.makeProcName + # public rpc proc + procName = newIdentNode(procNameStr) + # when parameters present: proc that contains our rpc body + doMain = newIdentNode(procNameStr & "DoMain") + # async result + res = newIdentNode("result") + var + setup = jsonToNim(parameters, paramsIdent) + procBody = if body.kind == nnkStmtList: body else: body.body + + if parameters.hasReturnType: + let returnType = parameters[0] + + # delegate async proc allows return and setting of result as native type + result.add(quote do: + proc `doMain`(`paramsIdent`: JsonNode): Future[`returnType`] {.async.} = + `setup` + `procBody` + ) + + if returnType == ident"JsonNode": + # `JsonNode` results don't need conversion + result.add( quote do: + proc `procName`(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = + `res` = await `doMain`(`paramsIdent`) + ) + else: + result.add(quote do: + proc `procName`(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = + `res` = %await `doMain`(`paramsIdent`) + ) + else: + # no return types, inline contents + result.add(quote do: + proc `procName`(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = + `setup` + `procBody` + ) + result.add( quote do: + `server`.register(`path`, `procName`) + ) + + when defined(nimDumpRpcs): + echo "\n", pathStr, ": ", result.repr + +# TODO: Allow cross checking between client signatures and server calls From 2f7580f0828310cb41f21fa52144ba495827d432 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 14 Jun 2018 16:53:24 +0100 Subject: [PATCH 03/60] Update to use rpc folder --- tests/debugclient.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/debugclient.nim b/tests/debugclient.nim index cd48d5e..574302f 100644 --- a/tests/debugclient.nim +++ b/tests/debugclient.nim @@ -1,4 +1,4 @@ -include ../ eth-rpc / client +include ../ rpc / client proc nextId*(self: RpcClient): int64 = self.nextId From 7486a542acf15e1e89eb0ee9305f532c0ed55430 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 14 Jun 2018 16:53:57 +0100 Subject: [PATCH 04/60] Stream servers are now separated into new module --- rpcstreamservers.nim | 114 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 rpcstreamservers.nim diff --git a/rpcstreamservers.nim b/rpcstreamservers.nim new file mode 100644 index 0000000..8432288 --- /dev/null +++ b/rpcstreamservers.nim @@ -0,0 +1,114 @@ +import rpcserver, tables, asyncdispatch2 +export rpcserver + +# Temporarily disable logging +import macros +macro debug(body: varargs[untyped]): untyped = newStmtList() +macro info(body: varargs[untyped]): untyped = newStmtList() +macro error(body: varargs[untyped]): untyped = newStmtList() + +type + RpcStreamServer* = RpcServer[StreamServer] + +proc streamServerProcess(server: StreamServer, client: StreamTransport) {.async.} = + result = processClient(server, client) + +proc newRpcStreamServer*(addresses: openarray[TransportAddress]): RpcStreamServer = + ## Create new server and assign it to addresses ``addresses``. + result = RpcServer[StreamServer]() + result.procs = newTable[string, RpcProc]() + result.servers = newSeq[StreamServer]() + + for item in addresses: + try: + info "Creating server on ", address = $item + var server = createStreamServer(item, streamServerProcess, {ReuseAddr}, + udata = result) + result.servers.add(server) + except: + error "Failed to create server", address = $item, message = getCurrentExceptionMsg() + + if len(result.servers) == 0: + # Server was not bound, critical error. + # TODO: Custom RpcException error + raise newException(RpcBindError, "Unable to create server!") + +proc newRpcStreamServer*(addresses: openarray[string]): RpcServer[StreamServer] = + ## Create new server and assign it to addresses ``addresses``. + var + tas4: seq[TransportAddress] + tas6: seq[TransportAddress] + baddrs: seq[TransportAddress] + + for a in addresses: + # Attempt to resolve `address` for IPv4 address space. + try: + tas4 = resolveTAddress(a, IpAddressFamily.IPv4) + except: + discard + + # Attempt to resolve `address` for IPv6 address space. + try: + tas6 = resolveTAddress(a, IpAddressFamily.IPv6) + except: + discard + + for r in tas4: + baddrs.add(r) + for r in tas6: + baddrs.add(r) + + if len(baddrs) == 0: + # Addresses could not be resolved, critical error. + raise newException(RpcAddressUnresolvableError, "Unable to get address!") + + result = newRpcStreamServer(baddrs) + +proc newRpcStreamServer*(address = "localhost", port: Port = Port(8545)): RpcServer[StreamServer] = + var + tas4: seq[TransportAddress] + tas6: seq[TransportAddress] + + # Attempt to resolve `address` for IPv4 address space. + try: + tas4 = resolveTAddress(address, port, IpAddressFamily.IPv4) + except: + discard + + # Attempt to resolve `address` for IPv6 address space. + try: + tas6 = resolveTAddress(address, port, IpAddressFamily.IPv6) + except: + discard + + if len(tas4) == 0 and len(tas6) == 0: + # Address was not resolved, critical error. + raise newException(RpcAddressUnresolvableError, + "Address " & address & " could not be resolved!") + + result = RpcServer[StreamServer]() + result.procs = newTable[string, RpcProc]() + result.servers = newSeq[StreamServer]() + for item in tas4: + try: + info "Creating server for address", ip4address = $item + var server = createStreamServer(item, processClient, {ReuseAddr}, + udata = result) + result.servers.add(server) + except: + error "Failed to create server for address", address = $item + + for item in tas6: + try: + info "Server created", ip6address = $item + var server = createStreamServer(item, processClient, {ReuseAddr}, + udata = result) + result.servers.add(server) + except: + error "Failed to create server", address = $item + + if len(result.servers) == 0: + # Server was not bound, critical error. + raise newException(RpcBindError, + "Could not setup server on " & address & ":" & $int(port)) + From bb07650f3381e5649e810c7dc843f48d3656fcf2 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 14 Jun 2018 16:55:10 +0100 Subject: [PATCH 05/60] Update tests to use streamserver --- tests/testerrors.nim | 4 ++-- tests/testethcalls.nim | 4 ++-- tests/testrpcmacro.nim | 4 ++-- tests/testserverclient.nim | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/testerrors.nim b/tests/testerrors.nim index 9f03777..1887872 100644 --- a/tests/testerrors.nim +++ b/tests/testerrors.nim @@ -3,10 +3,10 @@ allow unchecked and unformatted calls. ]# -import unittest, debugclient, ../rpcserver +import unittest, debugclient, ../rpcstreamservers import strformat, chronicles -var server = newRpcServer("localhost", 8547.Port) +var server = newRpcStreamServer("localhost", 8547.Port) var client = newRpcClient() server.start() diff --git a/tests/testethcalls.nim b/tests/testethcalls.nim index 13d54a1..c55213b 100644 --- a/tests/testethcalls.nim +++ b/tests/testethcalls.nim @@ -1,5 +1,5 @@ import unittest, json, tables -import ../rpcclient, ../rpcserver +import ../rpcclient, ../rpcstreamservers import stint, ethtypes, ethprocs, stintjson from os import getCurrentDir, DirSep @@ -7,7 +7,7 @@ from strutils import rsplit template sourceDir: string = currentSourcePath.rsplit(DirSep, 1)[0] var - server = newRpcServer("localhost", Port(8546)) + server = newRpcStreamServer("localhost", Port(8546)) client = newRpcClient() ## Generate Ethereum server RPCs diff --git a/tests/testrpcmacro.nim b/tests/testrpcmacro.nim index 6d39c78..b8bcb70 100644 --- a/tests/testrpcmacro.nim +++ b/tests/testrpcmacro.nim @@ -1,5 +1,5 @@ import unittest, json, tables -import ../rpcserver +import ../rpcstreamservers type # some nested types to check object parsing @@ -27,7 +27,7 @@ let }, "c": %1.23} -var s = newRpcServer(["localhost:8545"]) +var s = newRpcStreamServer(["localhost:8545"]) # RPC definitions diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index 4706732..4c8f8be 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -1,7 +1,7 @@ import unittest, json -import ../rpcclient, ../rpcserver +import ../rpcclient, ../rpcstreamservers -var srv = newRpcServer(["localhost:8545"]) +var srv = newRpcStreamServer(["localhost:8545"]) var client = newRpcClient() # Create RPC on server From e5c92dd207a89176533d9341caaece5a6f0a67a7 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 14 Jun 2018 17:02:17 +0100 Subject: [PATCH 06/60] Fix "cannot find init" as nimcrypto wasn't visible in ethprocs --- tests/testethcalls.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testethcalls.nim b/tests/testethcalls.nim index c55213b..702d3d6 100644 --- a/tests/testethcalls.nim +++ b/tests/testethcalls.nim @@ -1,6 +1,6 @@ import unittest, json, tables import ../rpcclient, ../rpcstreamservers -import stint, ethtypes, ethprocs, stintjson +import stint, ethtypes, ethprocs, stintjson, nimcrypto from os import getCurrentDir, DirSep from strutils import rsplit From 20ddb0267f3448664de7ea5dc16d2fe5f1270a23 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 14 Jun 2018 17:11:24 +0100 Subject: [PATCH 07/60] Remove redundant comment --- rpc/server.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/rpc/server.nim b/rpc/server.nim index 541d713..18a73bb 100644 --- a/rpc/server.nim +++ b/rpc/server.nim @@ -23,7 +23,6 @@ type RpcClientTransport* = concept trans, type t trans.write(var string) is Future[int] trans.readLine(int) is Future[string] - #trans.getUserData[t] is t RpcServerTransport* = concept t t.start From 30b97e713156697a92177b77b454cf835d2df8e7 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 14 Jun 2018 17:11:58 +0100 Subject: [PATCH 08/60] Remove delegate proc for processClient --- rpcstreamservers.nim | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/rpcstreamservers.nim b/rpcstreamservers.nim index 8432288..13f6cc6 100644 --- a/rpcstreamservers.nim +++ b/rpcstreamservers.nim @@ -7,11 +7,7 @@ macro debug(body: varargs[untyped]): untyped = newStmtList() macro info(body: varargs[untyped]): untyped = newStmtList() macro error(body: varargs[untyped]): untyped = newStmtList() -type - RpcStreamServer* = RpcServer[StreamServer] - -proc streamServerProcess(server: StreamServer, client: StreamTransport) {.async.} = - result = processClient(server, client) +type RpcStreamServer* = RpcServer[StreamServer] proc newRpcStreamServer*(addresses: openarray[TransportAddress]): RpcStreamServer = ## Create new server and assign it to addresses ``addresses``. @@ -22,7 +18,7 @@ proc newRpcStreamServer*(addresses: openarray[TransportAddress]): RpcStreamServe for item in addresses: try: info "Creating server on ", address = $item - var server = createStreamServer(item, streamServerProcess, {ReuseAddr}, + var server = createStreamServer(item, processClient, {ReuseAddr}, udata = result) result.servers.add(server) except: @@ -30,7 +26,6 @@ proc newRpcStreamServer*(addresses: openarray[TransportAddress]): RpcStreamServe if len(result.servers) == 0: # Server was not bound, critical error. - # TODO: Custom RpcException error raise newException(RpcBindError, "Unable to create server!") proc newRpcStreamServer*(addresses: openarray[string]): RpcServer[StreamServer] = From 0dfb4be55c4cbf852370b50971ba89f6539bf921 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Fri, 15 Jun 2018 10:34:24 +0100 Subject: [PATCH 09/60] Rename rpcstreamserver and newStreamServer to rpcsocketserver and newSocketServer --- rpcstreamservers.nim => rpcsocketservers.nim | 14 +++++++------- tests/testerrors.nim | 4 ++-- tests/testethcalls.nim | 4 ++-- tests/testrpcmacro.nim | 4 ++-- tests/testserverclient.nim | 4 ++-- 5 files changed, 15 insertions(+), 15 deletions(-) rename rpcstreamservers.nim => rpcsocketservers.nim (87%) diff --git a/rpcstreamservers.nim b/rpcsocketservers.nim similarity index 87% rename from rpcstreamservers.nim rename to rpcsocketservers.nim index 13f6cc6..c1c7aae 100644 --- a/rpcstreamservers.nim +++ b/rpcsocketservers.nim @@ -9,7 +9,7 @@ macro error(body: varargs[untyped]): untyped = newStmtList() type RpcStreamServer* = RpcServer[StreamServer] -proc newRpcStreamServer*(addresses: openarray[TransportAddress]): RpcStreamServer = +proc newRpcSocketServer*(addresses: openarray[TransportAddress]): RpcStreamServer = ## Create new server and assign it to addresses ``addresses``. result = RpcServer[StreamServer]() result.procs = newTable[string, RpcProc]() @@ -28,7 +28,7 @@ proc newRpcStreamServer*(addresses: openarray[TransportAddress]): RpcStreamServe # Server was not bound, critical error. raise newException(RpcBindError, "Unable to create server!") -proc newRpcStreamServer*(addresses: openarray[string]): RpcServer[StreamServer] = +proc newRpcSocketServer*(addresses: openarray[string]): RpcServer[StreamServer] = ## Create new server and assign it to addresses ``addresses``. var tas4: seq[TransportAddress] @@ -57,9 +57,9 @@ proc newRpcStreamServer*(addresses: openarray[string]): RpcServer[StreamServer] # Addresses could not be resolved, critical error. raise newException(RpcAddressUnresolvableError, "Unable to get address!") - result = newRpcStreamServer(baddrs) + result = newRpcSocketServer(baddrs) -proc newRpcStreamServer*(address = "localhost", port: Port = Port(8545)): RpcServer[StreamServer] = +proc newRpcSocketServer*(address = "localhost", port: Port = Port(8545)): RpcServer[StreamServer] = var tas4: seq[TransportAddress] tas6: seq[TransportAddress] @@ -74,7 +74,7 @@ proc newRpcStreamServer*(address = "localhost", port: Port = Port(8545)): RpcSer try: tas6 = resolveTAddress(address, port, IpAddressFamily.IPv6) except: - discard + error "Failed to create server for address", address = $item, errror = getCurrentException() if len(tas4) == 0 and len(tas6) == 0: # Address was not resolved, critical error. @@ -91,7 +91,7 @@ proc newRpcStreamServer*(address = "localhost", port: Port = Port(8545)): RpcSer udata = result) result.servers.add(server) except: - error "Failed to create server for address", address = $item + error "Failed to create server for address", address = $item, errror = getCurrentException() for item in tas6: try: @@ -100,7 +100,7 @@ proc newRpcStreamServer*(address = "localhost", port: Port = Port(8545)): RpcSer udata = result) result.servers.add(server) except: - error "Failed to create server", address = $item + error "Failed to create server for address", address = $item, errror = getCurrentException() if len(result.servers) == 0: # Server was not bound, critical error. diff --git a/tests/testerrors.nim b/tests/testerrors.nim index 1887872..95acfbb 100644 --- a/tests/testerrors.nim +++ b/tests/testerrors.nim @@ -3,10 +3,10 @@ allow unchecked and unformatted calls. ]# -import unittest, debugclient, ../rpcstreamservers +import unittest, debugclient, ../rpcsocketservers import strformat, chronicles -var server = newRpcStreamServer("localhost", 8547.Port) +var server = newRpcSocketServer("localhost", 8547.Port) var client = newRpcClient() server.start() diff --git a/tests/testethcalls.nim b/tests/testethcalls.nim index 702d3d6..6f2c7e2 100644 --- a/tests/testethcalls.nim +++ b/tests/testethcalls.nim @@ -1,5 +1,5 @@ import unittest, json, tables -import ../rpcclient, ../rpcstreamservers +import ../rpcclient, ../rpcsocketservers import stint, ethtypes, ethprocs, stintjson, nimcrypto from os import getCurrentDir, DirSep @@ -7,7 +7,7 @@ from strutils import rsplit template sourceDir: string = currentSourcePath.rsplit(DirSep, 1)[0] var - server = newRpcStreamServer("localhost", Port(8546)) + server = newRpcSocketServer("localhost", Port(8546)) client = newRpcClient() ## Generate Ethereum server RPCs diff --git a/tests/testrpcmacro.nim b/tests/testrpcmacro.nim index b8bcb70..eab8553 100644 --- a/tests/testrpcmacro.nim +++ b/tests/testrpcmacro.nim @@ -1,5 +1,5 @@ import unittest, json, tables -import ../rpcstreamservers +import ../rpcsocketservers type # some nested types to check object parsing @@ -27,7 +27,7 @@ let }, "c": %1.23} -var s = newRpcStreamServer(["localhost:8545"]) +var s = newRpcSocketServer(["localhost:8545"]) # RPC definitions diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index 4c8f8be..10e8665 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -1,7 +1,7 @@ import unittest, json -import ../rpcclient, ../rpcstreamservers +import ../rpcclient, ../rpcsocketservers -var srv = newRpcStreamServer(["localhost:8545"]) +var srv = newRpcSocketServer(["localhost:8545"]) var client = newRpcClient() # Create RPC on server From 746232a928878341d52df2bd5a737c2e2258e17c Mon Sep 17 00:00:00 2001 From: coffeepots Date: Fri, 15 Jun 2018 11:11:10 +0100 Subject: [PATCH 10/60] Revert "Rename rpcstreamserver and newStreamServer to rpcsocketserver and newSocketServer" This reverts commit 0dfb4be55c4cbf852370b50971ba89f6539bf921. --- rpcsocketservers.nim => rpcstreamservers.nim | 14 +++++++------- tests/testerrors.nim | 4 ++-- tests/testethcalls.nim | 4 ++-- tests/testrpcmacro.nim | 4 ++-- tests/testserverclient.nim | 4 ++-- 5 files changed, 15 insertions(+), 15 deletions(-) rename rpcsocketservers.nim => rpcstreamservers.nim (87%) diff --git a/rpcsocketservers.nim b/rpcstreamservers.nim similarity index 87% rename from rpcsocketservers.nim rename to rpcstreamservers.nim index c1c7aae..13f6cc6 100644 --- a/rpcsocketservers.nim +++ b/rpcstreamservers.nim @@ -9,7 +9,7 @@ macro error(body: varargs[untyped]): untyped = newStmtList() type RpcStreamServer* = RpcServer[StreamServer] -proc newRpcSocketServer*(addresses: openarray[TransportAddress]): RpcStreamServer = +proc newRpcStreamServer*(addresses: openarray[TransportAddress]): RpcStreamServer = ## Create new server and assign it to addresses ``addresses``. result = RpcServer[StreamServer]() result.procs = newTable[string, RpcProc]() @@ -28,7 +28,7 @@ proc newRpcSocketServer*(addresses: openarray[TransportAddress]): RpcStreamServe # Server was not bound, critical error. raise newException(RpcBindError, "Unable to create server!") -proc newRpcSocketServer*(addresses: openarray[string]): RpcServer[StreamServer] = +proc newRpcStreamServer*(addresses: openarray[string]): RpcServer[StreamServer] = ## Create new server and assign it to addresses ``addresses``. var tas4: seq[TransportAddress] @@ -57,9 +57,9 @@ proc newRpcSocketServer*(addresses: openarray[string]): RpcServer[StreamServer] # Addresses could not be resolved, critical error. raise newException(RpcAddressUnresolvableError, "Unable to get address!") - result = newRpcSocketServer(baddrs) + result = newRpcStreamServer(baddrs) -proc newRpcSocketServer*(address = "localhost", port: Port = Port(8545)): RpcServer[StreamServer] = +proc newRpcStreamServer*(address = "localhost", port: Port = Port(8545)): RpcServer[StreamServer] = var tas4: seq[TransportAddress] tas6: seq[TransportAddress] @@ -74,7 +74,7 @@ proc newRpcSocketServer*(address = "localhost", port: Port = Port(8545)): RpcSer try: tas6 = resolveTAddress(address, port, IpAddressFamily.IPv6) except: - error "Failed to create server for address", address = $item, errror = getCurrentException() + discard if len(tas4) == 0 and len(tas6) == 0: # Address was not resolved, critical error. @@ -91,7 +91,7 @@ proc newRpcSocketServer*(address = "localhost", port: Port = Port(8545)): RpcSer udata = result) result.servers.add(server) except: - error "Failed to create server for address", address = $item, errror = getCurrentException() + error "Failed to create server for address", address = $item for item in tas6: try: @@ -100,7 +100,7 @@ proc newRpcSocketServer*(address = "localhost", port: Port = Port(8545)): RpcSer udata = result) result.servers.add(server) except: - error "Failed to create server for address", address = $item, errror = getCurrentException() + error "Failed to create server", address = $item if len(result.servers) == 0: # Server was not bound, critical error. diff --git a/tests/testerrors.nim b/tests/testerrors.nim index 95acfbb..1887872 100644 --- a/tests/testerrors.nim +++ b/tests/testerrors.nim @@ -3,10 +3,10 @@ allow unchecked and unformatted calls. ]# -import unittest, debugclient, ../rpcsocketservers +import unittest, debugclient, ../rpcstreamservers import strformat, chronicles -var server = newRpcSocketServer("localhost", 8547.Port) +var server = newRpcStreamServer("localhost", 8547.Port) var client = newRpcClient() server.start() diff --git a/tests/testethcalls.nim b/tests/testethcalls.nim index 6f2c7e2..702d3d6 100644 --- a/tests/testethcalls.nim +++ b/tests/testethcalls.nim @@ -1,5 +1,5 @@ import unittest, json, tables -import ../rpcclient, ../rpcsocketservers +import ../rpcclient, ../rpcstreamservers import stint, ethtypes, ethprocs, stintjson, nimcrypto from os import getCurrentDir, DirSep @@ -7,7 +7,7 @@ from strutils import rsplit template sourceDir: string = currentSourcePath.rsplit(DirSep, 1)[0] var - server = newRpcSocketServer("localhost", Port(8546)) + server = newRpcStreamServer("localhost", Port(8546)) client = newRpcClient() ## Generate Ethereum server RPCs diff --git a/tests/testrpcmacro.nim b/tests/testrpcmacro.nim index eab8553..b8bcb70 100644 --- a/tests/testrpcmacro.nim +++ b/tests/testrpcmacro.nim @@ -1,5 +1,5 @@ import unittest, json, tables -import ../rpcsocketservers +import ../rpcstreamservers type # some nested types to check object parsing @@ -27,7 +27,7 @@ let }, "c": %1.23} -var s = newRpcSocketServer(["localhost:8545"]) +var s = newRpcStreamServer(["localhost:8545"]) # RPC definitions diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index 10e8665..4c8f8be 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -1,7 +1,7 @@ import unittest, json -import ../rpcclient, ../rpcsocketservers +import ../rpcclient, ../rpcstreamservers -var srv = newRpcSocketServer(["localhost:8545"]) +var srv = newRpcStreamServer(["localhost:8545"]) var client = newRpcClient() # Create RPC on server From bdf47fd2edbd54e77955247e8fcf85efba770dae Mon Sep 17 00:00:00 2001 From: coffeepots Date: Fri, 15 Jun 2018 11:12:34 +0100 Subject: [PATCH 11/60] Add rpc server init proc --- rpc/server.nim | 5 +++++ rpcstreamservers.nim | 4 +--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/rpc/server.nim b/rpc/server.nim index 18a73bb..7a9e4dd 100644 --- a/rpc/server.nim +++ b/rpc/server.nim @@ -58,6 +58,11 @@ const (INVALID_REQUEST, "No id specified") ] +proc newRpcServer*[T]: RpcServer[T] = + result = RpcServer[T]() + result.procs = newTable[string, RpcProc]() + result.servers = @[] + # Utility functions # TODO: Move outside server func `%`*(p: Port): JsonNode = %(p.int) diff --git a/rpcstreamservers.nim b/rpcstreamservers.nim index 13f6cc6..7155a21 100644 --- a/rpcstreamservers.nim +++ b/rpcstreamservers.nim @@ -11,9 +11,7 @@ type RpcStreamServer* = RpcServer[StreamServer] proc newRpcStreamServer*(addresses: openarray[TransportAddress]): RpcStreamServer = ## Create new server and assign it to addresses ``addresses``. - result = RpcServer[StreamServer]() - result.procs = newTable[string, RpcProc]() - result.servers = newSeq[StreamServer]() + result = newRpcServer[StreamServer]() for item in addresses: try: From 6e81e6c5f517dd3fee122fa32667b9bc899fa567 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Fri, 15 Jun 2018 13:02:37 +0100 Subject: [PATCH 12/60] Updated concept to remove type --- rpc/server.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rpc/server.nim b/rpc/server.nim index 7a9e4dd..0bb7504 100644 --- a/rpc/server.nim +++ b/rpc/server.nim @@ -20,9 +20,9 @@ type # Procedure signature accepted as an RPC call by server RpcProc* = proc (params: JsonNode): Future[JsonNode] - RpcClientTransport* = concept trans, type t - trans.write(var string) is Future[int] - trans.readLine(int) is Future[string] + RpcClientTransport* = concept t + t.write(var string) is Future[int] + t.readLine(int) is Future[string] RpcServerTransport* = concept t t.start From d789ca6fee668345bcf76e57bf9d36cf98dbd664 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Fri, 15 Jun 2018 20:33:29 +0100 Subject: [PATCH 13/60] Utilities for validating ethereum values --- tests/ethutils.nim | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 tests/ethutils.nim diff --git a/tests/ethutils.nim b/tests/ethutils.nim new file mode 100644 index 0000000..22e8a61 --- /dev/null +++ b/tests/ethutils.nim @@ -0,0 +1,43 @@ +template stripLeadingZeros(value: string): string = + var cidx = 0 + # ignore the last character so we retain '0' on zero value + while cidx < value.len - 1 and value[cidx] == '0': + cidx.inc + value[cidx .. ^1] + +proc encodeQuantity*(value: SomeUnsignedInt): string = + var hValue = value.toHex.stripLeadingZeros + result = "0x" & hValue + +template hasHexHeader*(value: string): bool = + if value[0] == '0' and value[1] in {'x', 'X'} and value.len > 2: true + else: false + +template isHexChar*(c: char): bool = + if c notin {'0'..'9'} and + c notin {'a'..'f'} and + c notin {'A'..'F'}: false + else: true + +proc validateHexQuantity*(value: string): bool = + if not value.hasHexHeader: + return false + # No leading zeros + if value[2] == '0': return false + for i in 2.. Date: Fri, 15 Jun 2018 20:35:49 +0100 Subject: [PATCH 14/60] Add simple validation for hex strings to web3_sha3 --- tests/ethprocs.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ethprocs.nim b/tests/ethprocs.nim index 215d097..d54ea9c 100644 --- a/tests/ethprocs.nim +++ b/tests/ethprocs.nim @@ -1,4 +1,4 @@ -import ../rpcserver, nimcrypto, json, stint, strutils, ethtypes, stintjson +import ../rpcserver, nimcrypto, json, stint, strutils, ethtypes, stintjson, ethutils #[ For details on available RPC calls, see: https://github.com/ethereum/wiki/wiki/JSON-RPC @@ -38,10 +38,10 @@ proc addEthRpcs*(server: RpcServer) = ## Returns the SHA3 result of the given string. # TODO: Capture error on malformed input var rawData: seq[byte] - if data.len > 2 and data[0] == '0' and data[1] in ['x', 'X']: + if data.validateHexData: rawData = data[2..data.high].fromHex else: - rawData = data.fromHex + raise newException(ValueError, "Invalid hex format") # data will have 0x prefix result = "0x" & $keccak_256.digest(rawData) From 81d52cb3cb56bf662e67ffefb714268516510fc9 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Mon, 18 Jun 2018 18:31:11 +0100 Subject: [PATCH 15/60] Add validated hex string type --- tests/ethhexstrings.nim | 151 ++++++++++++++++++++++++++++++++++++++++ tests/ethprocs.nim | 11 ++- tests/ethutils.nim | 43 ------------ tests/testethcalls.nim | 2 +- 4 files changed, 156 insertions(+), 51 deletions(-) create mode 100644 tests/ethhexstrings.nim delete mode 100644 tests/ethutils.nim diff --git a/tests/ethhexstrings.nim b/tests/ethhexstrings.nim new file mode 100644 index 0000000..50529bf --- /dev/null +++ b/tests/ethhexstrings.nim @@ -0,0 +1,151 @@ +type + HexQuantityStr* = distinct string + HexDataStr* = distinct string + +# Hex validation + +template stripLeadingZeros(value: string): string = + var cidx = 0 + # ignore the last character so we retain '0' on zero value + while cidx < value.len - 1 and value[cidx] == '0': + cidx.inc + value[cidx .. ^1] + +proc encodeQuantity*(value: SomeUnsignedInt): string = + var hValue = value.toHex.stripLeadingZeros + result = "0x" & hValue + +template hasHexHeader*(value: string | HexDataStr | HexQuantityStr): bool = + template strVal: untyped = value.string + if strVal[0] == '0' and strVal[1] in {'x', 'X'} and strVal.len > 2: true + else: false + +template isHexChar*(c: char): bool = + if c notin {'0'..'9'} and + c notin {'a'..'f'} and + c notin {'A'..'F'}: false + else: true + +proc validate*(value: HexQuantityStr): bool = + template strVal: untyped = value.string + if not value.hasHexHeader: + return false + # No leading zeros + if strVal[2] == '0': return false + for i in 2.. string: + server.rpc("web3_sha3") do(data: HexDataStr) -> HexDataStr: ## Returns Keccak-256 (not the standardized SHA3-256) of the given data. ## ## data: the data to convert into a SHA3 hash. ## Returns the SHA3 result of the given string. # TODO: Capture error on malformed input var rawData: seq[byte] - if data.validateHexData: - rawData = data[2..data.high].fromHex - else: - raise newException(ValueError, "Invalid hex format") + rawData = data.string.fromHex # data will have 0x prefix - result = "0x" & $keccak_256.digest(rawData) + result = hexDataStr "0x" & $keccak_256.digest(rawData) server.rpc("net_version") do() -> string: ## Returns string of the current network id: diff --git a/tests/ethutils.nim b/tests/ethutils.nim deleted file mode 100644 index 22e8a61..0000000 --- a/tests/ethutils.nim +++ /dev/null @@ -1,43 +0,0 @@ -template stripLeadingZeros(value: string): string = - var cidx = 0 - # ignore the last character so we retain '0' on zero value - while cidx < value.len - 1 and value[cidx] == '0': - cidx.inc - value[cidx .. ^1] - -proc encodeQuantity*(value: SomeUnsignedInt): string = - var hValue = value.toHex.stripLeadingZeros - result = "0x" & hValue - -template hasHexHeader*(value: string): bool = - if value[0] == '0' and value[1] in {'x', 'X'} and value.len > 2: true - else: false - -template isHexChar*(c: char): bool = - if c notin {'0'..'9'} and - c notin {'a'..'f'} and - c notin {'A'..'F'}: false - else: true - -proc validateHexQuantity*(value: string): bool = - if not value.hasHexHeader: - return false - # No leading zeros - if value[2] == '0': return false - for i in 2.. Date: Mon, 18 Jun 2018 18:39:48 +0100 Subject: [PATCH 16/60] Unified return type --- rpcstreamservers.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rpcstreamservers.nim b/rpcstreamservers.nim index 7155a21..cd8f7a0 100644 --- a/rpcstreamservers.nim +++ b/rpcstreamservers.nim @@ -26,7 +26,7 @@ proc newRpcStreamServer*(addresses: openarray[TransportAddress]): RpcStreamServe # Server was not bound, critical error. raise newException(RpcBindError, "Unable to create server!") -proc newRpcStreamServer*(addresses: openarray[string]): RpcServer[StreamServer] = +proc newRpcStreamServer*(addresses: openarray[string]): RpcStreamServer = ## Create new server and assign it to addresses ``addresses``. var tas4: seq[TransportAddress] @@ -57,7 +57,7 @@ proc newRpcStreamServer*(addresses: openarray[string]): RpcServer[StreamServer] result = newRpcStreamServer(baddrs) -proc newRpcStreamServer*(address = "localhost", port: Port = Port(8545)): RpcServer[StreamServer] = +proc newRpcStreamServer*(address = "localhost", port: Port = Port(8545)): RpcStreamServer = var tas4: seq[TransportAddress] tas6: seq[TransportAddress] @@ -79,7 +79,7 @@ proc newRpcStreamServer*(address = "localhost", port: Port = Port(8545)): RpcSer raise newException(RpcAddressUnresolvableError, "Address " & address & " could not be resolved!") - result = RpcServer[StreamServer]() + result = RpcStreamServer() result.procs = newTable[string, RpcProc]() result.servers = newSeq[StreamServer]() for item in tas4: From ce94ba8b41a837a655b883c5b2570ffdad7ce02a Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 19 Jun 2018 18:15:43 +0100 Subject: [PATCH 17/60] Removed rpcstreamservers and added to rpcserver --- rpcstreamservers.nim | 107 ------------------------------------------- 1 file changed, 107 deletions(-) delete mode 100644 rpcstreamservers.nim diff --git a/rpcstreamservers.nim b/rpcstreamservers.nim deleted file mode 100644 index cd8f7a0..0000000 --- a/rpcstreamservers.nim +++ /dev/null @@ -1,107 +0,0 @@ -import rpcserver, tables, asyncdispatch2 -export rpcserver - -# Temporarily disable logging -import macros -macro debug(body: varargs[untyped]): untyped = newStmtList() -macro info(body: varargs[untyped]): untyped = newStmtList() -macro error(body: varargs[untyped]): untyped = newStmtList() - -type RpcStreamServer* = RpcServer[StreamServer] - -proc newRpcStreamServer*(addresses: openarray[TransportAddress]): RpcStreamServer = - ## Create new server and assign it to addresses ``addresses``. - result = newRpcServer[StreamServer]() - - for item in addresses: - try: - info "Creating server on ", address = $item - var server = createStreamServer(item, processClient, {ReuseAddr}, - udata = result) - result.servers.add(server) - except: - error "Failed to create server", address = $item, message = getCurrentExceptionMsg() - - if len(result.servers) == 0: - # Server was not bound, critical error. - raise newException(RpcBindError, "Unable to create server!") - -proc newRpcStreamServer*(addresses: openarray[string]): RpcStreamServer = - ## Create new server and assign it to addresses ``addresses``. - var - tas4: seq[TransportAddress] - tas6: seq[TransportAddress] - baddrs: seq[TransportAddress] - - for a in addresses: - # Attempt to resolve `address` for IPv4 address space. - try: - tas4 = resolveTAddress(a, IpAddressFamily.IPv4) - except: - discard - - # Attempt to resolve `address` for IPv6 address space. - try: - tas6 = resolveTAddress(a, IpAddressFamily.IPv6) - except: - discard - - for r in tas4: - baddrs.add(r) - for r in tas6: - baddrs.add(r) - - if len(baddrs) == 0: - # Addresses could not be resolved, critical error. - raise newException(RpcAddressUnresolvableError, "Unable to get address!") - - result = newRpcStreamServer(baddrs) - -proc newRpcStreamServer*(address = "localhost", port: Port = Port(8545)): RpcStreamServer = - var - tas4: seq[TransportAddress] - tas6: seq[TransportAddress] - - # Attempt to resolve `address` for IPv4 address space. - try: - tas4 = resolveTAddress(address, port, IpAddressFamily.IPv4) - except: - discard - - # Attempt to resolve `address` for IPv6 address space. - try: - tas6 = resolveTAddress(address, port, IpAddressFamily.IPv6) - except: - discard - - if len(tas4) == 0 and len(tas6) == 0: - # Address was not resolved, critical error. - raise newException(RpcAddressUnresolvableError, - "Address " & address & " could not be resolved!") - - result = RpcStreamServer() - result.procs = newTable[string, RpcProc]() - result.servers = newSeq[StreamServer]() - for item in tas4: - try: - info "Creating server for address", ip4address = $item - var server = createStreamServer(item, processClient, {ReuseAddr}, - udata = result) - result.servers.add(server) - except: - error "Failed to create server for address", address = $item - - for item in tas6: - try: - info "Server created", ip6address = $item - var server = createStreamServer(item, processClient, {ReuseAddr}, - udata = result) - result.servers.add(server) - except: - error "Failed to create server", address = $item - - if len(result.servers) == 0: - # Server was not bound, critical error. - raise newException(RpcBindError, - "Could not setup server on " & address & ":" & $int(port)) - From c162f242531b8666e166fcb5dbdb2fef6548a469 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 19 Jun 2018 18:16:39 +0100 Subject: [PATCH 18/60] Re-enabled chronicles, extended RpcClientTransport, added stream servers --- rpc/server.nim | 133 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 121 insertions(+), 12 deletions(-) diff --git a/rpc/server.nim b/rpc/server.nim index 0bb7504..2f79de0 100644 --- a/rpc/server.nim +++ b/rpc/server.nim @@ -1,16 +1,11 @@ -import json, tables, strutils, options, macros #, chronicles +import json, tables, strutils, options, macros, chronicles import asyncdispatch2 import jsonmarshal export asyncdispatch2, json, jsonmarshal -# Temporarily disable logging -macro debug(body: varargs[untyped]): untyped = newStmtList() -macro info(body: varargs[untyped]): untyped = newStmtList() -macro error(body: varargs[untyped]): untyped = newStmtList() - -#logScope: -# topics = "RpcServer" +logScope: + topics = "RpcServer" type RpcJsonError* = enum rjeInvalidJson, rjeVersionError, rjeNoMethod, rjeNoId @@ -23,12 +18,17 @@ type RpcClientTransport* = concept t t.write(var string) is Future[int] t.readLine(int) is Future[string] + t.close + t.remoteAddress() # Required for logging + t.localAddress() RpcServerTransport* = concept t t.start t.stop t.close + RpcProcessClient* = proc (server: RpcServerTransport, client: RpcClientTransport): Future[void] {.gcsafe.} + RpcServer*[S: RpcServerTransport] = ref object servers*: seq[S] procs*: TableRef[string, RpcProc] @@ -152,7 +152,7 @@ proc processClient*[S: RpcServerTransport, C: RpcClientTransport](server: S, cli client.close() break - debug "Processing client", addresss = client.remoteAddress(), line + debug "Processing client", address = client.remoteAddress(), line let future = processMessage(rpc, client, line) yield future @@ -230,7 +230,11 @@ macro rpc*(server: RpcServer, path: string, body: untyped): untyped = var setup = jsonToNim(parameters, paramsIdent) procBody = if body.kind == nnkStmtList: body else: body.body - + errTrappedBody = quote do: + try: + `procBody` + except: + debug "Error occurred within RPC " & `path` & ": ", getCurrentExceptionMsg() if parameters.hasReturnType: let returnType = parameters[0] @@ -238,7 +242,7 @@ macro rpc*(server: RpcServer, path: string, body: untyped): untyped = result.add(quote do: proc `doMain`(`paramsIdent`: JsonNode): Future[`returnType`] {.async.} = `setup` - `procBody` + `errTrappedBody` ) if returnType == ident"JsonNode": @@ -257,7 +261,7 @@ macro rpc*(server: RpcServer, path: string, body: untyped): untyped = result.add(quote do: proc `procName`(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = `setup` - `procBody` + `errTrappedBody` ) result.add( quote do: `server`.register(`path`, `procName`) @@ -266,4 +270,109 @@ macro rpc*(server: RpcServer, path: string, body: untyped): untyped = when defined(nimDumpRpcs): echo "\n", pathStr, ": ", result.repr +# Utility functions for setting up servers using transport addresses + +proc addStreamServer*[T: RpcServer](server: T, address: TransportAddress, streamCallback: StreamCallback) = + try: + info "Creating server on ", address = $address + var transportServer = createStreamServer(address, streamCallback, {ReuseAddr}, udata = server) + server.servers.add(transportServer) + except: + error "Failed to create server", address = $address, message = getCurrentExceptionMsg() + + if len(server.servers) == 0: + # Server was not bound, critical error. + raise newException(RpcBindError, "Unable to create server!") + +proc addStreamServers*[T: RpcServer](server: T, addresses: openarray[TransportAddress], streamCallback: StreamCallback) = + for item in addresses: + server.addStreamServer(item, streamCallback) + +proc addStreamServer*[T: RpcServer](server: T, address: string, streamCallback: StreamCallback) = + ## Create new server and assign it to addresses ``addresses``. + var + tas4: seq[TransportAddress] + tas6: seq[TransportAddress] + added = 0 + + # Attempt to resolve `address` for IPv4 address space. + try: + tas4 = resolveTAddress(address, IpAddressFamily.IPv4) + except: + discard + + # Attempt to resolve `address` for IPv6 address space. + try: + tas6 = resolveTAddress(address, IpAddressFamily.IPv6) + except: + discard + + for r in tas4: + server.addStreamServer(r, streamCallback) + added.inc + for r in tas6: + server.addStreamServer(r, streamCallback) + added.inc + + if added == 0: + # Addresses could not be resolved, critical error. + raise newException(RpcAddressUnresolvableError, "Unable to get address!") + +proc addStreamServers*[T: RpcServer](server: T, addresses: openarray[string], streamCallback: StreamCallback) = + for address in addresses: + server.addStreamServer(address, streamCallback) + +proc addStreamServer*[T: RpcServer](server: T, address: string, port: Port, streamCallback: StreamCallback) = + var + tas4: seq[TransportAddress] + tas6: seq[TransportAddress] + added = 0 + + # Attempt to resolve `address` for IPv4 address space. + try: + tas4 = resolveTAddress(address, port, IpAddressFamily.IPv4) + except: + discard + + # Attempt to resolve `address` for IPv6 address space. + try: + tas6 = resolveTAddress(address, port, IpAddressFamily.IPv6) + except: + discard + + if len(tas4) == 0 and len(tas6) == 0: + # Address was not resolved, critical error. + raise newException(RpcAddressUnresolvableError, + "Address " & address & " could not be resolved!") + + for r in tas4: + server.addStreamServer(r, streamCallback) + added.inc + for r in tas6: + server.addStreamServer(r, streamCallback) + added.inc + + if len(server.servers) == 0: + # Server was not bound, critical error. + raise newException(RpcBindError, + "Could not setup server on " & address & ":" & $int(port)) + +type RpcStreamServer* = RpcServer[StreamServer] + +proc newRpcStreamServer*(addresses: openarray[TransportAddress]): RpcStreamServer = + ## Create new server and assign it to addresses ``addresses``. + result = newRpcServer[StreamServer]() + result.addStreamServers(addresses, processClient) + +proc newRpcStreamServer*(addresses: openarray[string]): RpcStreamServer = + ## Create new server and assign it to addresses ``addresses``. + result = newRpcServer[StreamServer]() + result.addStreamServers(addresses, processClient) + +proc newRpcStreamServer*(address = "localhost", port: Port = Port(8545)): RpcStreamServer = + # Create server on specified port + result = newRpcServer[StreamServer]() + result.addStreamServer(address, port, processClient) + + # TODO: Allow cross checking between client signatures and server calls From 28ce222e235f237696af27429e72c0a6b47a00bf Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 19 Jun 2018 18:17:12 +0100 Subject: [PATCH 19/60] Client now should fail the future on errors --- rpc/client.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpc/client.nim b/rpc/client.nim index e4bb85b..c8ef0b2 100644 --- a/rpc/client.nim +++ b/rpc/client.nim @@ -79,7 +79,7 @@ proc processMessage(self: RpcClient, line: string) = self.awaiting.del(id) # TODO: actions on unable find result node else: - self.awaiting[id].complete((true, errorNode)) + self.awaiting[id].fail(newException(ValueError, $errorNode)) self.awaiting.del(id) proc connect*(self: RpcClient, address: string, port: Port): Future[void] From 8d1174b1365cd9ae96ccbad145aa644d2ca1d780 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 19 Jun 2018 18:17:56 +0100 Subject: [PATCH 20/60] Updated paths to use rpcserver now that streamservers are in there --- tests/testethcalls.nim | 4 ++-- tests/testrpcmacro.nim | 4 ++-- tests/testserverclient.nim | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/testethcalls.nim b/tests/testethcalls.nim index f3b0c5b..736cbba 100644 --- a/tests/testethcalls.nim +++ b/tests/testethcalls.nim @@ -1,6 +1,6 @@ import unittest, json, tables -import ../rpcclient, ../rpcstreamservers -import stint, ethtypes, ethprocs, stintjson, nimcrypto, ethhexstrings +import ../rpcclient, ../rpcserver +import stint, ethtypes, ethprocs, stintjson, nimcrypto, ethhexstrings, chronicles from os import getCurrentDir, DirSep from strutils import rsplit diff --git a/tests/testrpcmacro.nim b/tests/testrpcmacro.nim index b8bcb70..89573e9 100644 --- a/tests/testrpcmacro.nim +++ b/tests/testrpcmacro.nim @@ -1,5 +1,5 @@ -import unittest, json, tables -import ../rpcstreamservers +import unittest, json, tables, chronicles +import ../rpcserver type # some nested types to check object parsing diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index 4c8f8be..7495d77 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -1,5 +1,5 @@ -import unittest, json -import ../rpcclient, ../rpcstreamservers +import unittest, json, chronicles +import ../rpcclient, ../rpcserver var srv = newRpcStreamServer(["localhost:8545"]) var client = newRpcClient() From e42864ef7ab123e8580f9ebb2409ff700998726d Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 19 Jun 2018 18:18:48 +0100 Subject: [PATCH 21/60] Remove testerrors from tests whilst fixing exceptions in rpcs --- tests/all.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/all.nim b/tests/all.nim index d3923d1..ae9eb62 100644 --- a/tests/all.nim +++ b/tests/all.nim @@ -1,3 +1,3 @@ import - testrpcmacro, testserverclient, testethcalls, testerrors + testrpcmacro, testserverclient, testethcalls #, testerrors From 23911ad80ed9f90f9e595579b5d07dabf08c1b65 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 19 Jun 2018 18:20:46 +0100 Subject: [PATCH 22/60] Fix bad debug log --- rpc/server.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpc/server.nim b/rpc/server.nim index 2f79de0..9591f73 100644 --- a/rpc/server.nim +++ b/rpc/server.nim @@ -234,7 +234,7 @@ macro rpc*(server: RpcServer, path: string, body: untyped): untyped = try: `procBody` except: - debug "Error occurred within RPC " & `path` & ": ", getCurrentExceptionMsg() + debug "Error occurred within RPC ", path = `path`, errorMessage = getCurrentExceptionMsg() if parameters.hasReturnType: let returnType = parameters[0] From 14009846efdc38b3c295db0e0b8d65b2c3972800 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 19 Jun 2018 18:21:37 +0100 Subject: [PATCH 23/60] Add beginnings of HTTP-RPC --- rpchttpservers.nim | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 rpchttpservers.nim diff --git a/rpchttpservers.nim b/rpchttpservers.nim new file mode 100644 index 0000000..9c4489f --- /dev/null +++ b/rpchttpservers.nim @@ -0,0 +1,36 @@ +import rpcserver, tables, chronicles +export rpcserver + +type + ClientHttpWrapper* = StreamTransport + RpcHttpServer* = RpcServer[StreamServer] + +proc httpHeader(host: string, length: int): string = + "Host: " & host & "Content-Type: application/json-rpc Content-Length: " & $length + +proc write(client: ClientHttpWrapper, data: var string): Future[int] = + # TODO: WIP + let d = httpHeader($client.localAddress, data.len) & data + result = client.write(d) + +proc readLine(client: ClientHttpWrapper, bytesToRead: int): Future[string] {.async.} = + result = await client.readLine + # TODO: Strip http + +proc processHtmlClient*(server: StreamServer, client: ClientHttpWrapper) {.async, gcsafe.} = + await server.processClient(client) + +proc newRpcHttpServer*(addresses: openarray[TransportAddress]): RpcHttpServer = + ## Create new server and assign it to addresses ``addresses``. + result = newRpcServer[StreamServer]().RpcHttpServer + result.addStreamServers(addresses, processHtmlClient) + +proc newRpcHttpServer*(addresses: openarray[string]): RpcHttpServer = + ## Create new server and assign it to addresses ``addresses``. + result = newRpcServer[StreamServer]().RpcHttpServer + result.addStreamServers(addresses, processHtmlClient) + +proc newRpcHttpServer*(address = "localhost", port: Port = Port(8545)): RpcHttpServer = + result = newRpcServer[StreamServer]().RpcHttpServer + result.addStreamServer(address, port, processHtmlClient) + From 49afd6ee768c3a92ed9265dee7540363e128ea17 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 19 Jun 2018 18:22:01 +0100 Subject: [PATCH 24/60] Add simple test for HTTP (WIP) --- tests/testhttp.nim | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tests/testhttp.nim diff --git a/tests/testhttp.nim b/tests/testhttp.nim new file mode 100644 index 0000000..9d9f01d --- /dev/null +++ b/tests/testhttp.nim @@ -0,0 +1,19 @@ +import unittest, json, chronicles +import ../rpcclient, ../rpchttpservers + +var srv = newRpcHttpServer(["localhost:8545"]) +var client = newRpcClient() + +# Create RPC on server +srv.rpc("myProc") do(input: string, data: array[0..3, int]): + result = %("Hello " & input & " data: " & $data) + +srv.start() +waitFor client.connect("localhost", Port(8545)) + +var r = waitFor client.call("myProc", %[%"abc", %[1, 2, 3, 4]]) +echo r + +srv.stop() +srv.close() +echo "done" \ No newline at end of file From 6b619472f37d43aa5ebdcee4362b3c14a4687f1a Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 19 Jun 2018 18:22:13 +0100 Subject: [PATCH 25/60] Test errors WIP --- tests/testerrors.nim | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/tests/testerrors.nim b/tests/testerrors.nim index 1887872..11501b8 100644 --- a/tests/testerrors.nim +++ b/tests/testerrors.nim @@ -3,7 +3,7 @@ allow unchecked and unformatted calls. ]# -import unittest, debugclient, ../rpcstreamservers +import unittest, debugclient, ../rpcserver import strformat, chronicles var server = newRpcStreamServer("localhost", 8547.Port) @@ -15,6 +15,10 @@ waitFor client.connect("localhost", Port(8547)) server.rpc("rpc") do(a: int, b: int): result = %(&"a: {a}, b: {b}") +server.rpc("makeError"): + if true: + raise newException(ValueError, "Test") + proc testMissingRpc: Future[Response] {.async.} = var fut = client.call("phantomRpc", %[]) result = await fut @@ -33,18 +37,28 @@ proc testMalformed: Future[Response] {.async.} = if fut.finished: result = fut.read() else: result = (true, %"Timeout") +proc testRaise: Future[Response] {.async.} = + var fut = client.call("rpcMakeError", %[]) + result = await fut + suite "RPC Errors": # Note: We don't expect a exceptions for most of the tests, # because the server should respond with the error in json test "Missing RPC": - let res = waitFor testMissingRpc() - check res.error == true and - res.result["message"] == %"Method not found" and - res.result["data"] == %"phantomRpc is not a registered method." + expect ValueError: + let res = waitFor testMissingRpc() + check res.error == true and + res.result["message"] == %"Method not found" and + res.result["data"] == %"phantomRpc is not a registered method." test "Incorrect json version": - let res = waitFor testInvalidJsonVer() - check res.error == true and res.result["message"] == %"JSON 2.0 required" + expect ValueError: + let res = waitFor testInvalidJsonVer() + check res.error == true and res.result["message"] == %"JSON 2.0 required" + + test "Raising exceptions": + expect ValueError: + let res = waitFor testRaise() test "Malformed json": # TODO: We time out here because the server won't be able to From b245a237450e9cdd6a8f66a07a0994cc1cf42c25 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 21 Jun 2018 18:15:21 +0100 Subject: [PATCH 26/60] Server now allows defining read, write and close code directly --- rpc/server.nim | 255 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 177 insertions(+), 78 deletions(-) diff --git a/rpc/server.nim b/rpc/server.nim index 9591f73..f830ff9 100644 --- a/rpc/server.nim +++ b/rpc/server.nim @@ -2,7 +2,7 @@ import json, tables, strutils, options, macros, chronicles import asyncdispatch2 import jsonmarshal -export asyncdispatch2, json, jsonmarshal +export asyncdispatch2, json, jsonmarshal, options logScope: topics = "RpcServer" @@ -48,7 +48,7 @@ const INTERNAL_ERROR* = -32603 SERVER_ERROR* = -32000 - maxRequestLength = 1024 * 128 + defaultMaxRequestLength = 1024 * 128 jsonErrorMessages*: array[RpcJsonError, (int, string)] = [ @@ -58,8 +58,8 @@ const (INVALID_REQUEST, "No id specified") ] -proc newRpcServer*[T]: RpcServer[T] = - result = RpcServer[T]() +proc newRpcServer*[S](): RpcServer[S] = + new result result.procs = newTable[string, RpcProc]() result.servers = @[] @@ -101,71 +101,166 @@ proc wrapReply*(id: JsonNode, value: JsonNode, error: JsonNode): string = let node = %{"jsonrpc": %"2.0", "result": value, "error": error, "id": id} return $node & "\c\l" -proc sendError*(client: RpcClientTransport, code: int, msg: string, id: JsonNode, - data: JsonNode = newJNull()) {.async.} = - ## Send error message to client - let error = %{"code": %(code), "id": id, "message": %msg, "data": data} - debug "Error generated", error = error, id = id - var res = wrapReply(id, newJNull(), error) - result = client.write(res) +proc addErrorSending(name, writeCode: NimNode): NimNode = + let + res = newIdentNode("result") + sendJsonErr = newIdentNode($name & "Json") + result = quote do: + proc `name`*[T: RpcClientTransport](clientTrans: T, code: int, msg: string, id: JsonNode, + data: JsonNode = newJNull()) {.async.} = + ## Send error message to client + let error = %{"code": %(code), "id": id, "message": %msg, "data": data} + debug "Error generated", error = error, id = id + var + value {.inject.} = wrapReply(id, newJNull(), error) + client {.inject.}: T + shallowCopy(client, clientTrans) + `res` = `writeCode` -proc sendJsonError*(state: RpcJsonError, client: RpcClientTransport, id: JsonNode, - data = newJNull()) {.async.} = - ## Send client response for invalid json state - let errMsgs = jsonErrorMessages[state] - await client.sendError(errMsgs[0], errMsgs[1], id, data) + proc `sendJsonErr`*(state: RpcJsonError, clientTrans: RpcClientTransport, id: JsonNode, + data = newJNull()) {.async.} = + ## Send client response for invalid json state + let errMsgs = jsonErrorMessages[state] + await clientTrans.`name`(errMsgs[0], errMsgs[1], id, data) # Server message processing -proc processMessage[T](server: RpcServer[T], client: RpcClientTransport, - line: string) {.async.} = + +proc genProcessMessages(name, sendErrorName, writeCode: NimNode): NimNode = + let idSendErrJson = newIdentNode($sendErrorName & "Json") + result = quote do: + proc `name`[T: RpcClientTransport](server: RpcServer, clientTrans: T, + line: string) {.async.} = + var + node: JsonNode + # set up node and/or flag errors + jsonErrorState = checkJsonErrors(line, node) + + if jsonErrorState.isSome: + let errState = jsonErrorState.get + var id = + if errState.err == rjeInvalidJson or errState.err == rjeNoId: + newJNull() + else: + node["id"] + await errState.err.`idSendErrJson`(clientTrans, id, %errState.msg) + else: + let + methodName = node["method"].str + id = node["id"] + + if not server.procs.hasKey(methodName): + await clientTrans.`sendErrorName`(METHOD_NOT_FOUND, "Method not found", %id, + %(methodName & " is not a registered method.")) + else: + let callRes = await server.procs[methodName](node["params"]) + var + value {.inject.} = wrapReply(id, callRes, newJNull()) + client {.inject.}: T + shallowCopy(client, clientTrans) + asyncCheck `writeCode` + +proc genProcessClient(nameIdent, procMessagesIdent, sendErrIdent, readCode, closeCode: NimNode): NimNode = + # This generates the processClient proc to match transport. + # processClient is compatible with createStreamServer and thus StreamCallback. + # However the constraints are conceptualised so you only need to match it's interface + # Note: https://github.com/nim-lang/Nim/issues/644 + result = quote do: + proc `nameIdent`[S: RpcServerTransport, C: RpcClientTransport](server: S, clientTrans: C) {.async, gcsafe.} = + var rpc = getUserData[RpcServer[S]](server) + while true: + var + client {.inject}: C + maxRequestLength {.inject.} = defaultMaxRequestLength + shallowCopy(client, clientTrans) + let line = await `readCode` + if line == "": + `closeCode` + break + + debug "Processing message", address = clientTrans.remoteAddress(), line = line + + let future = `procMessagesIdent`(rpc, clientTrans, line) + yield future + if future.failed: + if future.readError of RpcProcError: + let err = future.readError.RpcProcError + await clientTrans.`sendErrIdent`(err.code, err.msg, err.data) + elif future.readError of ValueError: + let err = future.readError[].ValueError + await clientTrans.`sendErrIdent`(INVALID_PARAMS, err.msg, %"") + else: + await clientTrans.`sendErrIdent`(SERVER_ERROR, + "Error: Unknown error occurred", %"") + echo "$$", result.repr + + +#[ + New API: + For custom RpcServers that do their own work upon getting/sending data. + + newServer = defineRpcServer[StreamTransport, StreamServer]: + write: + mySpecialWriter(server, client, line) + # Note: Anything not defined here will use the default code to operate + + Code is directly inserted into processMessages. You can still define your + own transports, but this lets you define operations for existing transports + without needing to rework them. +]# + +import random + +macro defineRpcTransport*(procClientName: untyped, body: untyped = nil): untyped = + ## Build an rpcServer type that inlines data access operations + #[ + Injects: + line: to be populated by the transport + + Example: + defineRpcTransport(myServer): + write: + write("http://" & value) + read: + readLine + ]# + procClientName.expectKind nnkIdent var - node: JsonNode - # set up node and/or flag errors - jsonErrorState = checkJsonErrors(line, node) + writeCode = quote do: + client.write(value) + readCode = quote do: + client.readLine(defaultMaxRequestLength) + closeCode = quote do: + client.close - if jsonErrorState.isSome: - let errState = jsonErrorState.get - var id = - if errState.err == rjeInvalidJson or errState.err == rjeNoId: - newJNull() - else: - node["id"] - await errState.err.sendJsonError(client, id, %errState.msg) - else: - let - methodName = node["method"].str - id = node["id"] + if body != nil: + body.expectKind nnkStmtList + for item in body: + item.expectKind nnkCall + item[0].expectKind nnkIdent + item[1].expectKind nnkStmtList + let + verb = $item[0] + code = item[1] - if not server.procs.hasKey(methodName): - await client.sendError(METHOD_NOT_FOUND, "Method not found", %id, - %(methodName & " is not a registered method.")) - else: - let callRes = await server.procs[methodName](node["params"]) - var res = wrapReply(id, callRes, newJNull()) - discard await client.write(res) + case verb.toLowerAscii + of "write": + writeCode = item[1] + of "read": + readCode = item[1] + of "close": + closeCode = item[1] + else: error("Unknown verb \"" & verb & "\"") + + result = newStmtList() -proc processClient*[S: RpcServerTransport, C: RpcClientTransport](server: S, client: C) {.async, gcsafe.} = - var rpc = getUserData[RpcServer[S]](server) - while true: - let line = await client.readLine(maxRequestLength) - if line == "": - client.close() - break - - debug "Processing client", address = client.remoteAddress(), line - - let future = processMessage(rpc, client, line) - yield future - if future.failed: - if future.readError of RpcProcError: - let err = future.readError.RpcProcError - await client.sendError(err.code, err.msg, err.data) - elif future.readError of ValueError: - let err = future.readError[].ValueError - await client.sendError(INVALID_PARAMS, err.msg, %"") - else: - await client.sendError(SERVER_ERROR, - "Error: Unknown error occurred", %"") + let + sendErr = newIdentNode($procClientName & "sendError") + procMsgs = newIdentNode($procClientName & "processMessages") + result.add(addErrorSending(sendErr, writeCode)) + result.add(genProcessMessages(procMsgs, sendErr, writeCode)) + result.add(genProcessClient(procClientName, procMsgs, sendErr, readCode, closeCode)) + + echo "defineRpc:\n", result.repr proc start*(server: RpcServer) = ## Start the RPC server. @@ -270,12 +365,16 @@ macro rpc*(server: RpcServer, path: string, body: untyped): untyped = when defined(nimDumpRpcs): echo "\n", pathStr, ": ", result.repr -# Utility functions for setting up servers using transport addresses +# Utility functions for setting up servers using stream transport addresses -proc addStreamServer*[T: RpcServer](server: T, address: TransportAddress, streamCallback: StreamCallback) = +# Create a default transport that's suitable for createStreamServer +defineRpcTransport(processStreamClient) + +proc addStreamServer*[S](server: RpcServer[S], address: TransportAddress, callBack: StreamCallback = processStreamClient) = + #makeProcessClient(processClient, StreamTransport) try: info "Creating server on ", address = $address - var transportServer = createStreamServer(address, streamCallback, {ReuseAddr}, udata = server) + var transportServer = createStreamServer(address, callBack, {ReuseAddr}, udata = server) server.servers.add(transportServer) except: error "Failed to create server", address = $address, message = getCurrentExceptionMsg() @@ -284,11 +383,11 @@ proc addStreamServer*[T: RpcServer](server: T, address: TransportAddress, stream # Server was not bound, critical error. raise newException(RpcBindError, "Unable to create server!") -proc addStreamServers*[T: RpcServer](server: T, addresses: openarray[TransportAddress], streamCallback: StreamCallback) = +proc addStreamServers*[T: RpcServer](server: T, addresses: openarray[TransportAddress], callBack: StreamCallback = processStreamClient) = for item in addresses: - server.addStreamServer(item, streamCallback) + server.addStreamServer(item, callBack) -proc addStreamServer*[T: RpcServer](server: T, address: string, streamCallback: StreamCallback) = +proc addStreamServer*[T: RpcServer](server: T, address: string, callBack: StreamCallback = processStreamClient) = ## Create new server and assign it to addresses ``addresses``. var tas4: seq[TransportAddress] @@ -308,21 +407,21 @@ proc addStreamServer*[T: RpcServer](server: T, address: string, streamCallback: discard for r in tas4: - server.addStreamServer(r, streamCallback) + server.addStreamServer(r, callBack) added.inc for r in tas6: - server.addStreamServer(r, streamCallback) + server.addStreamServer(r, callBack) added.inc if added == 0: # Addresses could not be resolved, critical error. raise newException(RpcAddressUnresolvableError, "Unable to get address!") -proc addStreamServers*[T: RpcServer](server: T, addresses: openarray[string], streamCallback: StreamCallback) = +proc addStreamServers*[T: RpcServer](server: T, addresses: openarray[string], callBack: StreamCallback = processStreamClient) = for address in addresses: - server.addStreamServer(address, streamCallback) + server.addStreamServer(address, callBack) -proc addStreamServer*[T: RpcServer](server: T, address: string, port: Port, streamCallback: StreamCallback) = +proc addStreamServer*[T: RpcServer](server: T, address: string, port: Port, callBack: StreamCallback = processStreamClient) = var tas4: seq[TransportAddress] tas6: seq[TransportAddress] @@ -346,10 +445,10 @@ proc addStreamServer*[T: RpcServer](server: T, address: string, port: Port, stre "Address " & address & " could not be resolved!") for r in tas4: - server.addStreamServer(r, streamCallback) + server.addStreamServer(r, callBack) added.inc for r in tas6: - server.addStreamServer(r, streamCallback) + server.addStreamServer(r, callBack) added.inc if len(server.servers) == 0: @@ -362,17 +461,17 @@ type RpcStreamServer* = RpcServer[StreamServer] proc newRpcStreamServer*(addresses: openarray[TransportAddress]): RpcStreamServer = ## Create new server and assign it to addresses ``addresses``. result = newRpcServer[StreamServer]() - result.addStreamServers(addresses, processClient) + result.addStreamServers(addresses) proc newRpcStreamServer*(addresses: openarray[string]): RpcStreamServer = ## Create new server and assign it to addresses ``addresses``. result = newRpcServer[StreamServer]() - result.addStreamServers(addresses, processClient) + result.addStreamServers(addresses) proc newRpcStreamServer*(address = "localhost", port: Port = Port(8545)): RpcStreamServer = # Create server on specified port result = newRpcServer[StreamServer]() - result.addStreamServer(address, port, processClient) + result.addStreamServer(address, port) # TODO: Allow cross checking between client signatures and server calls From 501d5a398cd173782ddaf25054407224a1b112b0 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 21 Jun 2018 18:16:06 +0100 Subject: [PATCH 27/60] Updated to test defining HTTP-RPC (WIP, client needs more work) --- rpchttpservers.nim | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/rpchttpservers.nim b/rpchttpservers.nim index 9c4489f..77af750 100644 --- a/rpchttpservers.nim +++ b/rpchttpservers.nim @@ -1,36 +1,27 @@ -import rpcserver, tables, chronicles +import rpcserver, tables, chronicles, strformat export rpcserver type - ClientHttpWrapper* = StreamTransport RpcHttpServer* = RpcServer[StreamServer] -proc httpHeader(host: string, length: int): string = - "Host: " & host & "Content-Type: application/json-rpc Content-Length: " & $length - -proc write(client: ClientHttpWrapper, data: var string): Future[int] = - # TODO: WIP - let d = httpHeader($client.localAddress, data.len) & data - result = client.write(d) - -proc readLine(client: ClientHttpWrapper, bytesToRead: int): Future[string] {.async.} = - result = await client.readLine - # TODO: Strip http - -proc processHtmlClient*(server: StreamServer, client: ClientHttpWrapper) {.async, gcsafe.} = - await server.processClient(client) +defineRpcTransport(httpProcessClient): + write: + let + msg = &"Host: {$client.localAddress} Content-Type: application/json-rpc Content-Length: {$value.len} {value}" + debug "Http stream", msg = msg + client.write(msg) proc newRpcHttpServer*(addresses: openarray[TransportAddress]): RpcHttpServer = ## Create new server and assign it to addresses ``addresses``. - result = newRpcServer[StreamServer]().RpcHttpServer - result.addStreamServers(addresses, processHtmlClient) + result = newRpcServer[StreamServer]() + result.addStreamServers(addresses, httpProcessClient) proc newRpcHttpServer*(addresses: openarray[string]): RpcHttpServer = ## Create new server and assign it to addresses ``addresses``. - result = newRpcServer[StreamServer]().RpcHttpServer - result.addStreamServers(addresses, processHtmlClient) + result = newRpcServer[StreamServer]() + result.addStreamServers(addresses, httpProcessClient) proc newRpcHttpServer*(address = "localhost", port: Port = Port(8545)): RpcHttpServer = - result = newRpcServer[StreamServer]().RpcHttpServer - result.addStreamServer(address, port, processHtmlClient) + result = newRpcServer[StreamServer]() + result.addStreamServer(address, port, httpProcessClient) From 5208755f63b28ec3db255be1aa6f33034cda9dc9 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 21 Jun 2018 18:17:46 +0100 Subject: [PATCH 28/60] Remove debug echo --- rpc/server.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/rpc/server.nim b/rpc/server.nim index f830ff9..383876f 100644 --- a/rpc/server.nim +++ b/rpc/server.nim @@ -191,8 +191,6 @@ proc genProcessClient(nameIdent, procMessagesIdent, sendErrIdent, readCode, clos else: await clientTrans.`sendErrIdent`(SERVER_ERROR, "Error: Unknown error occurred", %"") - echo "$$", result.repr - #[ New API: @@ -260,7 +258,8 @@ macro defineRpcTransport*(procClientName: untyped, body: untyped = nil): untyped result.add(genProcessMessages(procMsgs, sendErr, writeCode)) result.add(genProcessClient(procClientName, procMsgs, sendErr, readCode, closeCode)) - echo "defineRpc:\n", result.repr + when defined(nimDumpRpcs): + echo "defineRpc:\n", result.repr proc start*(server: RpcServer) = ## Start the RPC server. From d5116b07bb3e7ee2f49d178fb69c39ed4029d3c2 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 21 Jun 2018 18:22:04 +0100 Subject: [PATCH 29/60] Updated comments --- rpc/server.nim | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/rpc/server.nim b/rpc/server.nim index 383876f..6dff5af 100644 --- a/rpc/server.nim +++ b/rpc/server.nim @@ -192,34 +192,22 @@ proc genProcessClient(nameIdent, procMessagesIdent, sendErrIdent, readCode, clos await clientTrans.`sendErrIdent`(SERVER_ERROR, "Error: Unknown error occurred", %"") -#[ - New API: - For custom RpcServers that do their own work upon getting/sending data. - - newServer = defineRpcServer[StreamTransport, StreamServer]: - write: - mySpecialWriter(server, client, line) - # Note: Anything not defined here will use the default code to operate - - Code is directly inserted into processMessages. You can still define your - own transports, but this lets you define operations for existing transports - without needing to rework them. -]# - -import random - macro defineRpcTransport*(procClientName: untyped, body: untyped = nil): untyped = ## Build an rpcServer type that inlines data access operations #[ Injects: - line: to be populated by the transport + client: RpcClientTransport type + maxRequestLength: optional bytes to read + value: Json string to be written to transport Example: defineRpcTransport(myServer): write: - write("http://" & value) + client.write(value) read: - readLine + client.readLine(maxRequestLength) + close: + client.close ]# procClientName.expectKind nnkIdent var From 7839a553a5a6ca4d371ebf9830c5d4a66add986d Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 21 Jun 2018 18:33:39 +0100 Subject: [PATCH 30/60] Added 'afterRead' verb to define rpc, exported defaultMaxRequestLength --- rpc/server.nim | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/rpc/server.nim b/rpc/server.nim index 6dff5af..313263e 100644 --- a/rpc/server.nim +++ b/rpc/server.nim @@ -48,7 +48,7 @@ const INTERNAL_ERROR* = -32603 SERVER_ERROR* = -32000 - defaultMaxRequestLength = 1024 * 128 + defaultMaxRequestLength* = 1024 * 128 jsonErrorMessages*: array[RpcJsonError, (int, string)] = [ @@ -159,7 +159,7 @@ proc genProcessMessages(name, sendErrorName, writeCode: NimNode): NimNode = shallowCopy(client, clientTrans) asyncCheck `writeCode` -proc genProcessClient(nameIdent, procMessagesIdent, sendErrIdent, readCode, closeCode: NimNode): NimNode = +proc genProcessClient(nameIdent, procMessagesIdent, sendErrIdent, readCode, afterReadCode, closeCode: NimNode): NimNode = # This generates the processClient proc to match transport. # processClient is compatible with createStreamServer and thus StreamCallback. # However the constraints are conceptualised so you only need to match it's interface @@ -172,14 +172,15 @@ proc genProcessClient(nameIdent, procMessagesIdent, sendErrIdent, readCode, clos client {.inject}: C maxRequestLength {.inject.} = defaultMaxRequestLength shallowCopy(client, clientTrans) - let line = await `readCode` - if line == "": + let value {.inject.} = await `readCode` + `afterReadCode` + if value == "": `closeCode` break - debug "Processing message", address = clientTrans.remoteAddress(), line = line + debug "Processing message", address = clientTrans.remoteAddress(), line = value - let future = `procMessagesIdent`(rpc, clientTrans, line) + let future = `procMessagesIdent`(rpc, clientTrans, value) yield future if future.failed: if future.readError of RpcProcError: @@ -217,6 +218,7 @@ macro defineRpcTransport*(procClientName: untyped, body: untyped = nil): untyped client.readLine(defaultMaxRequestLength) closeCode = quote do: client.close + afterReadCode = newStmtList() if body != nil: body.expectKind nnkStmtList @@ -235,7 +237,9 @@ macro defineRpcTransport*(procClientName: untyped, body: untyped = nil): untyped readCode = item[1] of "close": closeCode = item[1] - else: error("Unknown verb \"" & verb & "\"") + of "afterread": + afterReadCode = item[1] + else: error("Unknown RPC verb \"" & verb & "\"") result = newStmtList() @@ -244,7 +248,7 @@ macro defineRpcTransport*(procClientName: untyped, body: untyped = nil): untyped procMsgs = newIdentNode($procClientName & "processMessages") result.add(addErrorSending(sendErr, writeCode)) result.add(genProcessMessages(procMsgs, sendErr, writeCode)) - result.add(genProcessClient(procClientName, procMsgs, sendErr, readCode, closeCode)) + result.add(genProcessClient(procClientName, procMsgs, sendErr, readCode, afterReadCode, closeCode)) when defined(nimDumpRpcs): echo "defineRpc:\n", result.repr From b632cbfb25b5a9a0accb68a54cee604f072c35a0 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 21 Jun 2018 18:39:47 +0100 Subject: [PATCH 31/60] Added HTTP afterRead code (WIP) --- rpchttpservers.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rpchttpservers.nim b/rpchttpservers.nim index 77af750..30e1e44 100644 --- a/rpchttpservers.nim +++ b/rpchttpservers.nim @@ -8,8 +8,11 @@ defineRpcTransport(httpProcessClient): write: let msg = &"Host: {$client.localAddress} Content-Type: application/json-rpc Content-Length: {$value.len} {value}" - debug "Http stream", msg = msg + debug "Http write", msg = msg client.write(msg) + afterRead: + # TODO: read: remove http to allow json validation + debug "Http read", msg = value proc newRpcHttpServer*(addresses: openarray[TransportAddress]): RpcHttpServer = ## Create new server and assign it to addresses ``addresses``. From 09b55a5b3252e30fa5610186cc605d55fe17b1ad Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 21 Jun 2018 18:40:49 +0100 Subject: [PATCH 32/60] More error checking (WIP) --- tests/testerrors.nim | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/tests/testerrors.nim b/tests/testerrors.nim index 11501b8..cd6fcf8 100644 --- a/tests/testerrors.nim +++ b/tests/testerrors.nim @@ -45,24 +45,36 @@ suite "RPC Errors": # Note: We don't expect a exceptions for most of the tests, # because the server should respond with the error in json test "Missing RPC": - expect ValueError: + #expect ValueError: + try: let res = waitFor testMissingRpc() check res.error == true and res.result["message"] == %"Method not found" and res.result["data"] == %"phantomRpc is not a registered method." + except: + echo "Error ", getCurrentExceptionMsg() test "Incorrect json version": - expect ValueError: + #expect ValueError: + try: let res = waitFor testInvalidJsonVer() check res.error == true and res.result["message"] == %"JSON 2.0 required" + except: + echo "Error ", getCurrentExceptionMsg() test "Raising exceptions": - expect ValueError: + #expect ValueError: + try: let res = waitFor testRaise() + except: + echo "Error ", getCurrentExceptionMsg() test "Malformed json": # TODO: We time out here because the server won't be able to # find an id to return to us, so we cannot complete the future. - let res = waitFor testMalformed() - check res.error == true and res.result == %"Timeout" + try: + let res = waitFor testMalformed() + check res.error == true and res.result == %"Timeout" + except: + echo "Error ", getCurrentExceptionMsg() From 11e2e1897785aca4448805afdb6a7784cd680185 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 21 Jun 2018 18:44:01 +0100 Subject: [PATCH 33/60] Ignore vscode settings folder --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 0e69085..bf65179 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,7 @@ # Ignore the nimcache folders nimcache/ +# Ignore editor settings +.vscode + + From ffb47528110b6642ba023495b25adf2630e64b61 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Fri, 22 Jun 2018 18:21:58 +0100 Subject: [PATCH 34/60] Name updates, defineRpcTransport to defineRpcServerTransport --- rpc/server.nim | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/rpc/server.nim b/rpc/server.nim index 313263e..42b2cf0 100644 --- a/rpc/server.nim +++ b/rpc/server.nim @@ -101,7 +101,7 @@ proc wrapReply*(id: JsonNode, value: JsonNode, error: JsonNode): string = let node = %{"jsonrpc": %"2.0", "result": value, "error": error, "id": id} return $node & "\c\l" -proc addErrorSending(name, writeCode: NimNode): NimNode = +proc genErrorSending(name, writeCode: NimNode): NimNode = let res = newIdentNode("result") sendJsonErr = newIdentNode($name & "Json") @@ -193,7 +193,7 @@ proc genProcessClient(nameIdent, procMessagesIdent, sendErrIdent, readCode, afte await clientTrans.`sendErrIdent`(SERVER_ERROR, "Error: Unknown error occurred", %"") -macro defineRpcTransport*(procClientName: untyped, body: untyped = nil): untyped = +macro defineRpcServerTransport*(procClientName: untyped, body: untyped = nil): untyped = ## Build an rpcServer type that inlines data access operations #[ Injects: @@ -232,26 +232,26 @@ macro defineRpcTransport*(procClientName: untyped, body: untyped = nil): untyped case verb.toLowerAscii of "write": - writeCode = item[1] + writeCode = code of "read": - readCode = item[1] + readCode = code of "close": - closeCode = item[1] + closeCode = code of "afterread": - afterReadCode = item[1] + afterReadCode = code else: error("Unknown RPC verb \"" & verb & "\"") result = newStmtList() let - sendErr = newIdentNode($procClientName & "sendError") - procMsgs = newIdentNode($procClientName & "processMessages") - result.add(addErrorSending(sendErr, writeCode)) + sendErr = newIdentNode($procClientName & "SendError") + procMsgs = newIdentNode($procClientName & "ProcessMessages") + result.add(genErrorSending(sendErr, writeCode)) result.add(genProcessMessages(procMsgs, sendErr, writeCode)) result.add(genProcessClient(procClientName, procMsgs, sendErr, readCode, afterReadCode, closeCode)) when defined(nimDumpRpcs): - echo "defineRpc:\n", result.repr + echo "defineServer:\n", result.repr proc start*(server: RpcServer) = ## Start the RPC server. @@ -359,7 +359,7 @@ macro rpc*(server: RpcServer, path: string, body: untyped): untyped = # Utility functions for setting up servers using stream transport addresses # Create a default transport that's suitable for createStreamServer -defineRpcTransport(processStreamClient) +defineRpcServerTransport(processStreamClient) proc addStreamServer*[S](server: RpcServer[S], address: TransportAddress, callBack: StreamCallback = processStreamClient) = #makeProcessClient(processClient, StreamTransport) From 99018eede4cbf0914dec6b5cd3499d1623138df8 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Fri, 22 Jun 2018 18:22:17 +0100 Subject: [PATCH 35/60] Update for define rename --- rpchttpservers.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rpchttpservers.nim b/rpchttpservers.nim index 30e1e44..e32ac97 100644 --- a/rpchttpservers.nim +++ b/rpchttpservers.nim @@ -4,10 +4,10 @@ export rpcserver type RpcHttpServer* = RpcServer[StreamServer] -defineRpcTransport(httpProcessClient): +defineRpcServerTransport(httpProcessClient): write: - let - msg = &"Host: {$client.localAddress} Content-Type: application/json-rpc Content-Length: {$value.len} {value}" + const contentType = "Content-Type: application/json-rpc" + let msg = &"Host: {$client.localAddress} {contentType} Content-Length: {$value.len} {value}" debug "Http write", msg = msg client.write(msg) afterRead: From ef6041ea6aed3277849bff8c1c47af5e42aca87c Mon Sep 17 00:00:00 2001 From: coffeepots Date: Fri, 22 Jun 2018 19:05:32 +0100 Subject: [PATCH 36/60] WIP custom clients --- rpc/client.nim | 175 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 133 insertions(+), 42 deletions(-) diff --git a/rpc/client.nim b/rpc/client.nim index c8ef0b2..95a48a9 100644 --- a/rpc/client.nim +++ b/rpc/client.nim @@ -1,37 +1,50 @@ import tables, json, macros import asyncdispatch2 +from strutils import toLowerAscii import jsonmarshal +export asyncdispatch2 type - RpcClient* = ref object - transp: StreamTransport + RpcClient*[T, A] = ref object + transp: T # StreamTransport awaiting: Table[string, Future[Response]] - address: TransportAddress + address: A # TransportAddress nextId: int64 + Response* = tuple[error: bool, result: JsonNode] -const maxRequestLength = 1024 * 128 +const defaultMaxRequestLength* = 1024 * 128 -proc newRpcClient*(): RpcClient = +proc newRpcClient*[T, A](): RpcClient[T, A] = ## Creates a new ``RpcClient`` instance. - result = RpcClient(awaiting: initTable[string, Future[Response]](), nextId: 1) + result = RpcClient[T, A](awaiting: initTable[string, Future[Response]](), nextId: 1) -proc call*(self: RpcClient, name: string, - params: JsonNode): Future[Response] {.async.} = - ## Remotely calls the specified RPC method. - let id = $self.nextId - self.nextId.inc - var msg = $ %{"jsonrpc": %"2.0", "method": %name, "params": params, - "id": %id} & "\c\l" - let res = await self.transp.write(msg) - # TODO: Add actions when not full packet was send, e.g. disconnect peer. - assert(res == len(msg)) +proc genCall(writeCode: NimNode): NimNode = + result = quote do: + proc call*[T, A](self: RpcClient[T, A], name: string, + params: JsonNode): Future[Response] {.async.} = + ## Remotely calls the specified RPC method. + let id = $self.nextId + self.nextId.inc + var + msg = $ %{"jsonrpc": %"2.0", "method": %name, "params": params, + "id": %id} & "\c\l" + value {.inject.} = + $ %{"jsonrpc": %"2.0", + "method": %name, + "params": params, + "id": %id} & "\c\l" + client {.inject.}: RpcClient[T, A] + shallowCopy(client, self) + let res = await `writeCode` #let res = await self.transp.write(msg) + # TODO: Add actions when not full packet was send, e.g. disconnect peer. + assert(res == len(msg)) - # completed by processMessage. - var newFut = newFuture[Response]() - # add to awaiting responses - self.awaiting[id] = newFut - result = await newFut + # completed by processMessage. + var newFut = newFuture[Response]() + # add to awaiting responses + self.awaiting[id] = newFut + result = await newFut template handleRaise[T](fut: Future[T], errType: typedesc, msg: string) = # complete future before raising @@ -57,8 +70,8 @@ macro checkGet(node: JsonNode, fieldName: string, of JObject: result.add(quote do: `n`.getObject) else: discard -proc processMessage(self: RpcClient, line: string) = - let node = parseJson(line) +proc processMessage[T, A](self: RpcClient[T, A], line: string) = + let node = parseJson(line) # TODO: Check errors # TODO: Use more appropriate exception objects let id = checkGet(node, "id", JString) @@ -82,28 +95,106 @@ proc processMessage(self: RpcClient, line: string) = self.awaiting[id].fail(newException(ValueError, $errorNode)) self.awaiting.del(id) -proc connect*(self: RpcClient, address: string, port: Port): Future[void] +#proc connect*(self: RpcClient, address: string, port: Port): Future[void] -proc processData(self: RpcClient) {.async.} = - while true: - let line = await self.transp.readLine(maxRequestLength) - if line == "": - # transmission ends - self.transp.close() - break +proc genProcessData(name, readCode, closeCode: NimNode): NimNode = + result = quote do: + proc `name`[T, A](self: RpcClient[T, A]) {.async.} = + while true: + #let line = await self.transp.readLine(maxRequestLength) + var + maxRequestLength {.inject.} = defaultMaxRequestLength + client {.inject.}: RpcClient[T, A] + shallowCopy(client, self) + let line = await `readCode` # TODO: Make it easier for callers to know this is expecting a future + if line == "": + # transmission ends + `closeCode` #self.transp.close() + break - processMessage(self, line) - # async loop reconnection and waiting - self.transp = await connect(self.address) + processMessage(self, line) + # async loop reconnection and waiting + self.transp = await connect(self.address) -proc connect*(self: RpcClient, address: string, port: Port) {.async.} = - # TODO: `address` hostname can be resolved to many IP addresses, we are using - # first one, but maybe it would be better to iterate over all IP addresses - # and try to establish connection until it will not be established. - let addresses = resolveTAddress(address, port) - self.transp = await connect(addresses[0]) - self.address = addresses[0] - asyncCheck processData(self) +proc genConnect(procDataName, connectCode: NimNode): NimNode = + result = quote do: + proc `procDataName`[T, A](self: RpcClient[T, A]) {.async.} + + proc connect*[T, A](self: RpcClient[T, A], address: string, port: Port) {.async.} = + # TODO: `address` hostname can be resolved to many IP addresses, we are using + # first one, but maybe it would be better to iterate over all IP addresses + # and try to establish connection until it will not be established. + var + client {.inject.}: RpcClient[T, A] + address {.inject.} = address + port {.inject.} = port + shallowCopy(client, self) + `connectCode` + #let addresses = resolveTAddress(address, port) + #self.transp = await connect(addresses[0]) + #self.address = addresses[0] + asyncCheck `procDataName`[T, A](self) + +macro defineRpcClientTransport*(procDataName: untyped, body: untyped = nil): untyped = + procDataName.expectKind nnkIdent + var + writeCode = quote do: + client.write(value) + readCode = quote do: + client.readLine(defaultMaxRequestLength) + closeCode = quote do: + client.close + connectCode = quote do: + # TODO: Even as a default this is too tied to StreamServer + let addresses = resolveTAddress(address, port) + client.transp = await connect(addresses[0]) + client.address = addresses[0] + + if body != nil: + body.expectKind nnkStmtList + for item in body: + item.expectKind nnkCall + item[0].expectKind nnkIdent + item[1].expectKind nnkStmtList + let + verb = $item[0] + code = item[1] + + case verb.toLowerAscii + of "write": + writeCode = code + of "read": + readCode = code + of "close": + closeCode = code + of "connect": + connectCode = code + else: error("Unknown RPC verb \"" & verb & "\"") + + result = newStmtList() + + let + procData = newIdentNode($procDataName) + result.add(genConnect(procData, connectCode)) + result.add(genCall(writeCode)) + result.add(genProcessData(procData, readCode, closeCode)) + + when defined(nimDumpRpcs): + echo "defineClient:\n", result.repr + +## + +# Default stream server + +defineRpcClientTransport(processStreamData) + +type RpcStreamClient* = RpcClient[StreamTransport, TransportAddress] + +proc newRpcStreamClient*(): RpcStreamClient = + ## Create new server and assign it to addresses ``addresses``. + result = newRpcClient[StreamTransport, TransportAddress]() + +## proc createRpcProc(procName, parameters, callBody: NimNode): NimNode = # parameters come as a tree From 79e756bd3c813ef37acc3aca5801ad94f035d5ab Mon Sep 17 00:00:00 2001 From: coffeepots Date: Fri, 22 Jun 2018 19:06:03 +0100 Subject: [PATCH 37/60] Update tests to use rpcStreamClient --- tests/testerrors.nim | 2 +- tests/testethcalls.nim | 2 +- tests/testhttp.nim | 2 +- tests/testserverclient.nim | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/testerrors.nim b/tests/testerrors.nim index cd6fcf8..9f44a80 100644 --- a/tests/testerrors.nim +++ b/tests/testerrors.nim @@ -7,7 +7,7 @@ import unittest, debugclient, ../rpcserver import strformat, chronicles var server = newRpcStreamServer("localhost", 8547.Port) -var client = newRpcClient() +var client = newRpcStreamClient() server.start() waitFor client.connect("localhost", Port(8547)) diff --git a/tests/testethcalls.nim b/tests/testethcalls.nim index 736cbba..538a32f 100644 --- a/tests/testethcalls.nim +++ b/tests/testethcalls.nim @@ -8,7 +8,7 @@ template sourceDir: string = currentSourcePath.rsplit(DirSep, 1)[0] var server = newRpcStreamServer("localhost", Port(8546)) - client = newRpcClient() + client = newRpcStreamClient() ## Generate Ethereum server RPCs server.addEthRpcs() diff --git a/tests/testhttp.nim b/tests/testhttp.nim index 9d9f01d..86e1f90 100644 --- a/tests/testhttp.nim +++ b/tests/testhttp.nim @@ -2,7 +2,7 @@ import unittest, json, chronicles import ../rpcclient, ../rpchttpservers var srv = newRpcHttpServer(["localhost:8545"]) -var client = newRpcClient() +var client = newRpcStreamClient() # Create RPC on server srv.rpc("myProc") do(input: string, data: array[0..3, int]): diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index 7495d77..d48f9cb 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -2,7 +2,7 @@ import unittest, json, chronicles import ../rpcclient, ../rpcserver var srv = newRpcStreamServer(["localhost:8545"]) -var client = newRpcClient() +var client = newRpcStreamClient() # Create RPC on server srv.rpc("myProc") do(input: string, data: array[0..3, int]): From 13d560738aee017f0ffe92356f67f1a7f48b00e2 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Mon, 25 Jun 2018 17:21:33 +0100 Subject: [PATCH 38/60] Remove shallow copy in place of a template --- rpc/server.nim | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/rpc/server.nim b/rpc/server.nim index 42b2cf0..0bbd1fe 100644 --- a/rpc/server.nim +++ b/rpc/server.nim @@ -113,8 +113,7 @@ proc genErrorSending(name, writeCode: NimNode): NimNode = debug "Error generated", error = error, id = id var value {.inject.} = wrapReply(id, newJNull(), error) - client {.inject.}: T - shallowCopy(client, clientTrans) + template client: untyped = clientTrans `res` = `writeCode` proc `sendJsonErr`*(state: RpcJsonError, clientTrans: RpcClientTransport, id: JsonNode, @@ -153,10 +152,8 @@ proc genProcessMessages(name, sendErrorName, writeCode: NimNode): NimNode = %(methodName & " is not a registered method.")) else: let callRes = await server.procs[methodName](node["params"]) - var - value {.inject.} = wrapReply(id, callRes, newJNull()) - client {.inject.}: T - shallowCopy(client, clientTrans) + var value {.inject.} = wrapReply(id, callRes, newJNull()) + template client: untyped = clientTrans asyncCheck `writeCode` proc genProcessClient(nameIdent, procMessagesIdent, sendErrIdent, readCode, afterReadCode, closeCode: NimNode): NimNode = @@ -168,10 +165,9 @@ proc genProcessClient(nameIdent, procMessagesIdent, sendErrIdent, readCode, afte proc `nameIdent`[S: RpcServerTransport, C: RpcClientTransport](server: S, clientTrans: C) {.async, gcsafe.} = var rpc = getUserData[RpcServer[S]](server) while true: - var - client {.inject}: C - maxRequestLength {.inject.} = defaultMaxRequestLength - shallowCopy(client, clientTrans) + var maxRequestLength {.inject.} = defaultMaxRequestLength + template client: untyped = clientTrans + let value {.inject.} = await `readCode` `afterReadCode` if value == "": From ac9964370e176a8997b6d7b81cf9e60e13883873 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Mon, 25 Jun 2018 17:54:28 +0100 Subject: [PATCH 39/60] Allow custom client transports (WIP) --- rpc/client.nim | 110 +++++++++++++++++++++---------------------------- 1 file changed, 47 insertions(+), 63 deletions(-) diff --git a/rpc/client.nim b/rpc/client.nim index 95a48a9..1f7c345 100644 --- a/rpc/client.nim +++ b/rpc/client.nim @@ -6,9 +6,9 @@ export asyncdispatch2 type RpcClient*[T, A] = ref object - transp: T # StreamTransport + transp*: T awaiting: Table[string, Future[Response]] - address: A # TransportAddress + address: A nextId: int64 Response* = tuple[error: bool, result: JsonNode] @@ -19,37 +19,33 @@ proc newRpcClient*[T, A](): RpcClient[T, A] = ## Creates a new ``RpcClient`` instance. result = RpcClient[T, A](awaiting: initTable[string, Future[Response]](), nextId: 1) -proc genCall(writeCode: NimNode): NimNode = +proc genCall(rpcType, writeCode: NimNode): NimNode = + let res = newIdentNode("result") result = quote do: - proc call*[T, A](self: RpcClient[T, A], name: string, + proc call*(self: `rpcType`, name: string, params: JsonNode): Future[Response] {.async.} = ## Remotely calls the specified RPC method. let id = $self.nextId self.nextId.inc var - msg = $ %{"jsonrpc": %"2.0", "method": %name, "params": params, - "id": %id} & "\c\l" value {.inject.} = $ %{"jsonrpc": %"2.0", "method": %name, "params": params, "id": %id} & "\c\l" - client {.inject.}: RpcClient[T, A] - shallowCopy(client, self) - let res = await `writeCode` #let res = await self.transp.write(msg) + template client: untyped = self + let res = await `writeCode` # TODO: Add actions when not full packet was send, e.g. disconnect peer. - assert(res == len(msg)) + assert(res == len(value)) # completed by processMessage. var newFut = newFuture[Response]() # add to awaiting responses self.awaiting[id] = newFut - result = await newFut + `res` = await newFut -template handleRaise[T](fut: Future[T], errType: typedesc, msg: string) = - # complete future before raising - fut.complete((true, %msg)) - raise newException(errType, msg) +template asyncRaise[T](fut: Future[T], errType: typedesc, msg: string) = + fut.fail(newException(errType, msg)) macro checkGet(node: JsonNode, fieldName: string, jKind: static[JsonNodeKind]): untyped = @@ -81,7 +77,7 @@ proc processMessage[T, A](self: RpcClient[T, A], line: string) = let version = checkGet(node, "jsonrpc", JString) if version != "2.0": - self.awaiting[id].handleRaise(ValueError, + self.awaiting[id].asyncRaise(ValueError, "Unsupported version of JSON, expected 2.0, received \"" & version & "\"") let errorNode = node{"error"} @@ -95,56 +91,46 @@ proc processMessage[T, A](self: RpcClient[T, A], line: string) = self.awaiting[id].fail(newException(ValueError, $errorNode)) self.awaiting.del(id) -#proc connect*(self: RpcClient, address: string, port: Port): Future[void] - -proc genProcessData(name, readCode, closeCode: NimNode): NimNode = +proc genProcessData(rpcType, processDataName, readCode, closeCode: NimNode): NimNode = result = quote do: - proc `name`[T, A](self: RpcClient[T, A]) {.async.} = + proc `processDataName`(clientTransport: `rpcType`) {.async.} = while true: - #let line = await self.transp.readLine(maxRequestLength) - var - maxRequestLength {.inject.} = defaultMaxRequestLength - client {.inject.}: RpcClient[T, A] - shallowCopy(client, self) - let line = await `readCode` # TODO: Make it easier for callers to know this is expecting a future + var maxRequestLength {.inject.} = defaultMaxRequestLength + template client: untyped = clientTransport + let line = await `readCode` if line == "": # transmission ends - `closeCode` #self.transp.close() + `closeCode` break - processMessage(self, line) + processMessage(clientTransport, line) # async loop reconnection and waiting - self.transp = await connect(self.address) + clientTransport.transp = await connect(clientTransport.address) -proc genConnect(procDataName, connectCode: NimNode): NimNode = +proc genConnectAndProcess(rpcType, processDataName, connectCode, readCode, closeCode: NimNode): NimNode = result = quote do: - proc `procDataName`[T, A](self: RpcClient[T, A]) {.async.} + proc connect*(clientTransport: `rpcType`, address: string, port: Port) {.async.} = + var + address {.inject.} = address + port {.inject.} = port + template client: untyped = clientTransport + `connectCode` + asyncCheck `processDataName`(clientTransport) - proc connect*[T, A](self: RpcClient[T, A], address: string, port: Port) {.async.} = +macro defineRpcClientTransport*(transType, addrType: untyped, body: untyped = nil): untyped = + let processDataName = newIdentNode("processData") + var + writeCode = quote do: + client.transp.write(value) + readCode = quote do: + # looks like this is being checked before insertion + client.transp.readLine(defaultMaxRequestLength) + closeCode = quote do: + client.transp.close + connectCode = quote do: # TODO: `address` hostname can be resolved to many IP addresses, we are using # first one, but maybe it would be better to iterate over all IP addresses # and try to establish connection until it will not be established. - var - client {.inject.}: RpcClient[T, A] - address {.inject.} = address - port {.inject.} = port - shallowCopy(client, self) - `connectCode` - #let addresses = resolveTAddress(address, port) - #self.transp = await connect(addresses[0]) - #self.address = addresses[0] - asyncCheck `procDataName`[T, A](self) - -macro defineRpcClientTransport*(procDataName: untyped, body: untyped = nil): untyped = - procDataName.expectKind nnkIdent - var - writeCode = quote do: - client.write(value) - readCode = quote do: - client.readLine(defaultMaxRequestLength) - closeCode = quote do: - client.close - connectCode = quote do: # TODO: Even as a default this is too tied to StreamServer let addresses = resolveTAddress(address, port) client.transp = await connect(addresses[0]) @@ -172,21 +158,19 @@ macro defineRpcClientTransport*(procDataName: untyped, body: untyped = nil): unt else: error("Unknown RPC verb \"" & verb & "\"") result = newStmtList() + let rpcType = quote: RpcClient[`transType`, `addrType`] - let - procData = newIdentNode($procDataName) - result.add(genConnect(procData, connectCode)) - result.add(genCall(writeCode)) - result.add(genProcessData(procData, readCode, closeCode)) + let procData = newIdentNode($processDataName) + result.add(genProcessData(rpcType, procData, readCode, closeCode)) + result.add(genConnectAndProcess(rpcType, procData, connectCode, readCode, closeCode)) + result.add(genCall(rpcType, writeCode)) when defined(nimDumpRpcs): echo "defineClient:\n", result.repr -## +# Define default stream server -# Default stream server - -defineRpcClientTransport(processStreamData) +defineRpcClientTransport(StreamTransport, TransportAddress) type RpcStreamClient* = RpcClient[StreamTransport, TransportAddress] @@ -194,7 +178,7 @@ proc newRpcStreamClient*(): RpcStreamClient = ## Create new server and assign it to addresses ``addresses``. result = newRpcClient[StreamTransport, TransportAddress]() -## +# Signature processing proc createRpcProc(procName, parameters, callBody: NimNode): NimNode = # parameters come as a tree From e508cc077d6ada1c5d760e6a4f7a9d4458e9e5de Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 26 Jun 2018 15:38:49 +0100 Subject: [PATCH 40/60] Custom client transports operational --- rpc/client.nim | 53 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 17 deletions(-) diff --git a/rpc/client.nim b/rpc/client.nim index 1f7c345..58e4904 100644 --- a/rpc/client.nim +++ b/rpc/client.nim @@ -19,10 +19,10 @@ proc newRpcClient*[T, A](): RpcClient[T, A] = ## Creates a new ``RpcClient`` instance. result = RpcClient[T, A](awaiting: initTable[string, Future[Response]](), nextId: 1) -proc genCall(rpcType, writeCode: NimNode): NimNode = +proc genCall(rpcType, callName, writeCode: NimNode): NimNode = let res = newIdentNode("result") result = quote do: - proc call*(self: `rpcType`, name: string, + proc `callName`*(self: `rpcType`, name: string, params: JsonNode): Future[Response] {.async.} = ## Remotely calls the specified RPC method. let id = $self.nextId @@ -67,6 +67,7 @@ macro checkGet(node: JsonNode, fieldName: string, else: discard proc processMessage[T, A](self: RpcClient[T, A], line: string) = + # Note: this doesn't use any transport code so doesn't need to be differentiated. let node = parseJson(line) # TODO: Check errors # TODO: Use more appropriate exception objects @@ -91,25 +92,26 @@ proc processMessage[T, A](self: RpcClient[T, A], line: string) = self.awaiting[id].fail(newException(ValueError, $errorNode)) self.awaiting.del(id) -proc genProcessData(rpcType, processDataName, readCode, closeCode: NimNode): NimNode = +proc genProcessData(rpcType, processDataName, readCode, afterReadCode, closeCode: NimNode): NimNode = result = quote do: proc `processDataName`(clientTransport: `rpcType`) {.async.} = while true: var maxRequestLength {.inject.} = defaultMaxRequestLength template client: untyped = clientTransport - let line = await `readCode` - if line == "": + var value {.inject.} = await `readCode` + `afterReadCode` + if value == "": # transmission ends `closeCode` break - processMessage(clientTransport, line) + processMessage(clientTransport, value) # async loop reconnection and waiting clientTransport.transp = await connect(clientTransport.address) -proc genConnectAndProcess(rpcType, processDataName, connectCode, readCode, closeCode: NimNode): NimNode = +proc genConnect(rpcType, connectName, processDataName, connectCode: NimNode): NimNode = result = quote do: - proc connect*(clientTransport: `rpcType`, address: string, port: Port) {.async.} = + proc `connectName`*(clientTransport: `rpcType`, address: string, port: Port) {.async.} = var address {.inject.} = address port {.inject.} = port @@ -117,13 +119,11 @@ proc genConnectAndProcess(rpcType, processDataName, connectCode, readCode, close `connectCode` asyncCheck `processDataName`(clientTransport) -macro defineRpcClientTransport*(transType, addrType: untyped, body: untyped = nil): untyped = - let processDataName = newIdentNode("processData") +macro defineRpcClientTransport*(transType, addrType: untyped, prefix: string = "", body: untyped = nil): untyped = var writeCode = quote do: client.transp.write(value) readCode = quote do: - # looks like this is being checked before insertion client.transp.readLine(defaultMaxRequestLength) closeCode = quote do: client.transp.close @@ -131,10 +131,10 @@ macro defineRpcClientTransport*(transType, addrType: untyped, body: untyped = ni # TODO: `address` hostname can be resolved to many IP addresses, we are using # first one, but maybe it would be better to iterate over all IP addresses # and try to establish connection until it will not be established. - # TODO: Even as a default this is too tied to StreamServer let addresses = resolveTAddress(address, port) client.transp = await connect(addresses[0]) client.address = addresses[0] + afterReadCode = newStmtList() if body != nil: body.expectKind nnkStmtList @@ -148,27 +148,46 @@ macro defineRpcClientTransport*(transType, addrType: untyped, body: untyped = ni case verb.toLowerAscii of "write": + # `client`, the RpcClient + # `value`, the data to be sent to the server writeCode = code of "read": + # `client`, the RpcClient + # `maxRequestLength`, initially set to defaultMaxRequestLength readCode = code of "close": + # `client`, the RpcClient + # `value`, the data returned from the server + # `maxRequestLength`, initially set to defaultMaxRequestLength closeCode = code of "connect": + # `client`, the RpcClient + # `address`, server destination address string + # `port`, server destination port connectCode = code + of "afterread": + # `client`, the RpcClient + # `value`, the data returned from the server + # `maxRequestLength`, initially set to defaultMaxRequestLength + afterReadCode = code else: error("Unknown RPC verb \"" & verb & "\"") result = newStmtList() - let rpcType = quote: RpcClient[`transType`, `addrType`] + let + rpcType = quote: RpcClient[`transType`, `addrType`] + processDataName = newIdentNode(prefix.strVal & "processData") + connectName = newIdentNode(prefix.strVal & "connect") + callName = newIdentNode(prefix.strVal & "call") - let procData = newIdentNode($processDataName) - result.add(genProcessData(rpcType, procData, readCode, closeCode)) - result.add(genConnectAndProcess(rpcType, procData, connectCode, readCode, closeCode)) - result.add(genCall(rpcType, writeCode)) + result.add(genProcessData(rpcType, processDataName, readCode, afterReadCode, closeCode)) + result.add(genConnect(rpcType, connectName, processDataName, connectCode)) + result.add(genCall(rpcType, callName, writeCode)) when defined(nimDumpRpcs): echo "defineClient:\n", result.repr # Define default stream server +# TODO: Move this into a separate unit so users can define 'connect', 'call' etc without requiring a prefix? defineRpcClientTransport(StreamTransport, TransportAddress) From 8685135fe90de4bebc497377fb215196fc1537e0 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 26 Jun 2018 15:40:09 +0100 Subject: [PATCH 41/60] Update comments --- rpc/server.nim | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/rpc/server.nim b/rpc/server.nim index 0bbd1fe..db7d309 100644 --- a/rpc/server.nim +++ b/rpc/server.nim @@ -111,9 +111,9 @@ proc genErrorSending(name, writeCode: NimNode): NimNode = ## Send error message to client let error = %{"code": %(code), "id": id, "message": %msg, "data": data} debug "Error generated", error = error, id = id - var - value {.inject.} = wrapReply(id, newJNull(), error) + template client: untyped = clientTrans + var value {.inject.} = wrapReply(id, newJNull(), error) `res` = `writeCode` proc `sendJsonErr`*(state: RpcJsonError, clientTrans: RpcClientTransport, id: JsonNode, @@ -152,8 +152,8 @@ proc genProcessMessages(name, sendErrorName, writeCode: NimNode): NimNode = %(methodName & " is not a registered method.")) else: let callRes = await server.procs[methodName](node["params"]) - var value {.inject.} = wrapReply(id, callRes, newJNull()) template client: untyped = clientTrans + var value {.inject.} = wrapReply(id, callRes, newJNull()) asyncCheck `writeCode` proc genProcessClient(nameIdent, procMessagesIdent, sendErrIdent, readCode, afterReadCode, closeCode: NimNode): NimNode = @@ -228,12 +228,21 @@ macro defineRpcServerTransport*(procClientName: untyped, body: untyped = nil): u case verb.toLowerAscii of "write": + # `client`, the RpcClient + # `value`, the data returned from the invoked RPC writeCode = code of "read": + # `client`, the RpcClient + # `maxRequestLength`, set to defaultMaxRequestLength + # Result of expression is awaited readCode = code of "close": + # `client`, the RpcClient + # Access to `value`, which contains the data read by `readCode` closeCode = code of "afterread": + # `client`, the RpcClient + # Access to `value`, which contains the data read by `readCode` afterReadCode = code else: error("Unknown RPC verb \"" & verb & "\"") From 9e8121b40505ba674b7296486b36c875d7116bbd Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 26 Jun 2018 15:44:58 +0100 Subject: [PATCH 42/60] Added http rpc client definition --- rpchttpservers.nim | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/rpchttpservers.nim b/rpchttpservers.nim index e32ac97..f80b158 100644 --- a/rpchttpservers.nim +++ b/rpchttpservers.nim @@ -1,5 +1,5 @@ -import rpcserver, tables, chronicles, strformat -export rpcserver +import rpcserver, rpcclient, tables, chronicles, strformat, strutils +export rpcserver, rpcclient type RpcHttpServer* = RpcServer[StreamServer] @@ -28,3 +28,25 @@ proc newRpcHttpServer*(address = "localhost", port: Port = Port(8545)): RpcHttpS result = newRpcServer[StreamServer]() result.addStreamServer(address, port, httpProcessClient) +type RpcHttpClient* = RpcClient[StreamTransport, TransportAddress] + +defineRpcClientTransport(StreamTransport, TransportAddress, "http"): + read: + client.transp.readLine() + afterRead: + # Strip out http header + # TODO: Performance + let p1 = find(value, '{') + if p1 > -1: + let p2 = rFind(value, '}') + if p2 == -1: + info "Cannot find json end brace", msg = value + else: + value = value[p1 .. p2] + debug "Extracted json", json = value + else: + info "Cannot find json start brace", msg = value + +proc newRpcHttpClient*(): RpcHttpClient = + result = newRpcClient[StreamTransport, TransportAddress]() + From d36da9d14c27ec49c4e13d9dcd8142812c413f6f Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 26 Jun 2018 15:45:40 +0100 Subject: [PATCH 43/60] Added testing, uses http client and server --- tests/testhttp.nim | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/testhttp.nim b/tests/testhttp.nim index 86e1f90..e504ac5 100644 --- a/tests/testhttp.nim +++ b/tests/testhttp.nim @@ -1,19 +1,21 @@ -import unittest, json, chronicles -import ../rpcclient, ../rpchttpservers +import unittest, json, chronicles, unittest +import ../rpchttpservers var srv = newRpcHttpServer(["localhost:8545"]) -var client = newRpcStreamClient() +var client = newRpcHttpClient() # Create RPC on server srv.rpc("myProc") do(input: string, data: array[0..3, int]): result = %("Hello " & input & " data: " & $data) srv.start() -waitFor client.connect("localhost", Port(8545)) +waitFor client.httpConnect("localhost", Port(8545)) -var r = waitFor client.call("myProc", %[%"abc", %[1, 2, 3, 4]]) -echo r +suite "HTTP RPC transport": + test "Call": + var r = waitFor client.httpcall("myProc", %[%"abc", %[1, 2, 3, 4]]) + check r.error == false and r.result == %"Hello abc data: [1, 2, 3, 4]" -srv.stop() +srv.stop() srv.close() echo "done" \ No newline at end of file From f8e178e5b3df27ad809ba294984c790e729b07fe Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 26 Jun 2018 16:23:34 +0100 Subject: [PATCH 44/60] Json extraction for http server read --- rpchttpservers.nim | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/rpchttpservers.nim b/rpchttpservers.nim index f80b158..00183ee 100644 --- a/rpchttpservers.nim +++ b/rpchttpservers.nim @@ -1,18 +1,31 @@ import rpcserver, rpcclient, tables, chronicles, strformat, strutils export rpcserver, rpcclient +proc extractJsonStr(msgSource: string, value: string): string = + result = "" + let p1 = find(value, '{') + if p1 > -1: + let p2 = rFind(value, '}') + if p2 == -1: + info "Cannot find json end brace", source = msgSource, msg = value + else: + result = value[p1 .. p2] + debug "Extracted json", source = msgSource, json = result + else: + info "Cannot find json start brace", source = msgSource, msg = value + type RpcHttpServer* = RpcServer[StreamServer] defineRpcServerTransport(httpProcessClient): write: const contentType = "Content-Type: application/json-rpc" - let msg = &"Host: {$client.localAddress} {contentType} Content-Length: {$value.len} {value}" - debug "Http write", msg = msg - client.write(msg) + let msg = &"Host: {$transport.localAddress} {contentType} Content-Length: {$value.len} {value}" + debug "HTTP server: write", msg = msg + transport.write(msg) afterRead: - # TODO: read: remove http to allow json validation - debug "Http read", msg = value + debug "HTTP server: read", msg = value + value = "HTTP Server".extractJsonStr(value) proc newRpcHttpServer*(addresses: openarray[TransportAddress]): RpcHttpServer = ## Create new server and assign it to addresses ``addresses``. @@ -31,21 +44,16 @@ proc newRpcHttpServer*(address = "localhost", port: Port = Port(8545)): RpcHttpS type RpcHttpClient* = RpcClient[StreamTransport, TransportAddress] defineRpcClientTransport(StreamTransport, TransportAddress, "http"): - read: - client.transp.readLine() + write: + const contentType = "Content-Type: application/json-rpc" + value = &"Host: {$client.transp.localAddress} {contentType} Content-Length: {$value.len} {value}" + debug "HTTP client: write", msg = value + client.transp.write(value) afterRead: # Strip out http header # TODO: Performance - let p1 = find(value, '{') - if p1 > -1: - let p2 = rFind(value, '}') - if p2 == -1: - info "Cannot find json end brace", msg = value - else: - value = value[p1 .. p2] - debug "Extracted json", json = value - else: - info "Cannot find json start brace", msg = value + debug "HTTP client: read", msg = value + value = "HTTP Client".extractJsonStr(value) proc newRpcHttpClient*(): RpcHttpClient = result = newRpcClient[StreamTransport, TransportAddress]() From d5f397104efbec84347653ac05ea1fc3f56c51ed Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 26 Jun 2018 16:24:04 +0100 Subject: [PATCH 45/60] Added note comment --- rpc/client.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/rpc/client.nim b/rpc/client.nim index 58e4904..70842f8 100644 --- a/rpc/client.nim +++ b/rpc/client.nim @@ -150,6 +150,7 @@ macro defineRpcClientTransport*(transType, addrType: untyped, prefix: string = " of "write": # `client`, the RpcClient # `value`, the data to be sent to the server + # Note: Update `value` so it's length can be sent afterwards writeCode = code of "read": # `client`, the RpcClient From 1feca64ca88f55ba434094e730572171ebd7c2aa Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 26 Jun 2018 16:24:35 +0100 Subject: [PATCH 46/60] Renamed `client` access variable to `transport` --- rpc/server.nim | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/rpc/server.nim b/rpc/server.nim index db7d309..131e147 100644 --- a/rpc/server.nim +++ b/rpc/server.nim @@ -112,7 +112,7 @@ proc genErrorSending(name, writeCode: NimNode): NimNode = let error = %{"code": %(code), "id": id, "message": %msg, "data": data} debug "Error generated", error = error, id = id - template client: untyped = clientTrans + template transport: untyped = clientTrans var value {.inject.} = wrapReply(id, newJNull(), error) `res` = `writeCode` @@ -152,7 +152,7 @@ proc genProcessMessages(name, sendErrorName, writeCode: NimNode): NimNode = %(methodName & " is not a registered method.")) else: let callRes = await server.procs[methodName](node["params"]) - template client: untyped = clientTrans + template transport: untyped = clientTrans var value {.inject.} = wrapReply(id, callRes, newJNull()) asyncCheck `writeCode` @@ -166,9 +166,9 @@ proc genProcessClient(nameIdent, procMessagesIdent, sendErrIdent, readCode, afte var rpc = getUserData[RpcServer[S]](server) while true: var maxRequestLength {.inject.} = defaultMaxRequestLength - template client: untyped = clientTrans + template transport: untyped = clientTrans - let value {.inject.} = await `readCode` + var value {.inject.} = await `readCode` `afterReadCode` if value == "": `closeCode` @@ -209,11 +209,11 @@ macro defineRpcServerTransport*(procClientName: untyped, body: untyped = nil): u procClientName.expectKind nnkIdent var writeCode = quote do: - client.write(value) + transport.write(value) readCode = quote do: - client.readLine(defaultMaxRequestLength) + transport.readLine(defaultMaxRequestLength) closeCode = quote do: - client.close + transport.close afterReadCode = newStmtList() if body != nil: @@ -228,21 +228,22 @@ macro defineRpcServerTransport*(procClientName: untyped, body: untyped = nil): u case verb.toLowerAscii of "write": - # `client`, the RpcClient + # `transport`, the client transport # `value`, the data returned from the invoked RPC + # Note: Update `value` so it's length can be sent afterwards writeCode = code of "read": - # `client`, the RpcClient + # `transport`, the client transport # `maxRequestLength`, set to defaultMaxRequestLength - # Result of expression is awaited + # Note: Result of expression is awaited readCode = code of "close": - # `client`, the RpcClient - # Access to `value`, which contains the data read by `readCode` + # `transport`, the client transport + # `value`, which contains the data read by `readCode` closeCode = code of "afterread": - # `client`, the RpcClient - # Access to `value`, which contains the data read by `readCode` + # `transport`, the client transport + # `value`, which contains the data read by `readCode` afterReadCode = code else: error("Unknown RPC verb \"" & verb & "\"") From 80ef987852999c270491803a545dca657d2626a3 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 26 Jun 2018 16:28:34 +0100 Subject: [PATCH 47/60] Remove let, use `value` --- rpchttpservers.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rpchttpservers.nim b/rpchttpservers.nim index 00183ee..2568bb2 100644 --- a/rpchttpservers.nim +++ b/rpchttpservers.nim @@ -20,9 +20,9 @@ type defineRpcServerTransport(httpProcessClient): write: const contentType = "Content-Type: application/json-rpc" - let msg = &"Host: {$transport.localAddress} {contentType} Content-Length: {$value.len} {value}" + value = &"Host: {$transport.localAddress} {contentType} Content-Length: {$value.len} {value}" debug "HTTP server: write", msg = msg - transport.write(msg) + transport.write(value) afterRead: debug "HTTP server: read", msg = value value = "HTTP Server".extractJsonStr(value) From c4b27e40bf9afc03b6ff746d8ef671d129231e23 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 26 Jun 2018 16:28:49 +0100 Subject: [PATCH 48/60] Add http test --- tests/all.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/all.nim b/tests/all.nim index ae9eb62..69a109c 100644 --- a/tests/all.nim +++ b/tests/all.nim @@ -1,3 +1,3 @@ import - testrpcmacro, testserverclient, testethcalls #, testerrors + testrpcmacro, testserverclient, testethcalls, testhttp #, testerrors From 2c92487e1a188a8d1682c3de1ed16f70379ed99e Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 26 Jun 2018 16:29:09 +0100 Subject: [PATCH 49/60] Remove redundant "done" echo --- tests/testhttp.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/testhttp.nim b/tests/testhttp.nim index e504ac5..0f59837 100644 --- a/tests/testhttp.nim +++ b/tests/testhttp.nim @@ -18,4 +18,3 @@ suite "HTTP RPC transport": srv.stop() srv.close() -echo "done" \ No newline at end of file From bed76ed00fbdae8fa08505610740cd0ed53392f9 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 26 Jun 2018 17:15:14 +0100 Subject: [PATCH 50/60] Fix for errant `msg` --- rpchttpservers.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpchttpservers.nim b/rpchttpservers.nim index 2568bb2..4b308a1 100644 --- a/rpchttpservers.nim +++ b/rpchttpservers.nim @@ -21,7 +21,7 @@ defineRpcServerTransport(httpProcessClient): write: const contentType = "Content-Type: application/json-rpc" value = &"Host: {$transport.localAddress} {contentType} Content-Length: {$value.len} {value}" - debug "HTTP server: write", msg = msg + debug "HTTP server: write", msg = value transport.write(value) afterRead: debug "HTTP server: read", msg = value From 68d428a0a9ec778dc3858d4a82b82339dc141a6a Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 26 Jun 2018 17:15:28 +0100 Subject: [PATCH 51/60] Add support for `error` verb --- rpc/server.nim | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/rpc/server.nim b/rpc/server.nim index 131e147..d675903 100644 --- a/rpc/server.nim +++ b/rpc/server.nim @@ -101,7 +101,7 @@ proc wrapReply*(id: JsonNode, value: JsonNode, error: JsonNode): string = let node = %{"jsonrpc": %"2.0", "result": value, "error": error, "id": id} return $node & "\c\l" -proc genErrorSending(name, writeCode: NimNode): NimNode = +proc genErrorSending(name, writeCode, errorCode: NimNode): NimNode = let res = newIdentNode("result") sendJsonErr = newIdentNode($name & "Json") @@ -114,6 +114,7 @@ proc genErrorSending(name, writeCode: NimNode): NimNode = template transport: untyped = clientTrans var value {.inject.} = wrapReply(id, newJNull(), error) + `errorCode` `res` = `writeCode` proc `sendJsonErr`*(state: RpcJsonError, clientTrans: RpcClientTransport, id: JsonNode, @@ -215,6 +216,7 @@ macro defineRpcServerTransport*(procClientName: untyped, body: untyped = nil): u closeCode = quote do: transport.close afterReadCode = newStmtList() + errorCode = newStmtList() if body != nil: body.expectKind nnkStmtList @@ -245,6 +247,11 @@ macro defineRpcServerTransport*(procClientName: untyped, body: untyped = nil): u # `transport`, the client transport # `value`, which contains the data read by `readCode` afterReadCode = code + of "error": + # `transport`, the client transport + # `value`, the data returned from the invoked RPC + # Note: Update `value` so it's length can be sent afterwards + errorCode = code else: error("Unknown RPC verb \"" & verb & "\"") result = newStmtList() @@ -252,7 +259,7 @@ macro defineRpcServerTransport*(procClientName: untyped, body: untyped = nil): u let sendErr = newIdentNode($procClientName & "SendError") procMsgs = newIdentNode($procClientName & "ProcessMessages") - result.add(genErrorSending(sendErr, writeCode)) + result.add(genErrorSending(sendErr, writeCode, errorCode)) result.add(genProcessMessages(procMsgs, sendErr, writeCode)) result.add(genProcessClient(procClientName, procMsgs, sendErr, readCode, afterReadCode, closeCode)) From 0c98f79d9cff304143ffc22c134db5ccec95b645 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 26 Jun 2018 19:08:11 +0100 Subject: [PATCH 52/60] Rename to json_rpc --- eth_rpc.nimble => json_rpc.nimble | 4 ++-- {rpc => json_rpc}/client.nim | 0 {rpc => json_rpc}/jsonmarshal.nim | 0 {rpc => json_rpc}/server.nim | 0 rpcclient.nim | 2 +- rpcserver.nim | 2 +- 6 files changed, 4 insertions(+), 4 deletions(-) rename eth_rpc.nimble => json_rpc.nimble (90%) rename {rpc => json_rpc}/client.nim (100%) rename {rpc => json_rpc}/jsonmarshal.nim (100%) rename {rpc => json_rpc}/server.nim (100%) diff --git a/eth_rpc.nimble b/json_rpc.nimble similarity index 90% rename from eth_rpc.nimble rename to json_rpc.nimble index f6b2a08..c840ac7 100644 --- a/eth_rpc.nimble +++ b/json_rpc.nimble @@ -1,5 +1,5 @@ -packageName = "eth_rpc" -version = "0.0.1" +packageName = "json_rpc" +version = "0.0.2" author = "Status Research & Development GmbH" description = "Ethereum remote procedure calls" license = "Apache License 2.0" diff --git a/rpc/client.nim b/json_rpc/client.nim similarity index 100% rename from rpc/client.nim rename to json_rpc/client.nim diff --git a/rpc/jsonmarshal.nim b/json_rpc/jsonmarshal.nim similarity index 100% rename from rpc/jsonmarshal.nim rename to json_rpc/jsonmarshal.nim diff --git a/rpc/server.nim b/json_rpc/server.nim similarity index 100% rename from rpc/server.nim rename to json_rpc/server.nim diff --git a/rpcclient.nim b/rpcclient.nim index 14712ef..ebecba9 100644 --- a/rpcclient.nim +++ b/rpcclient.nim @@ -1,3 +1,3 @@ -import rpc / client +import json_rpc / client export client diff --git a/rpcserver.nim b/rpcserver.nim index af3bbfb..ef197d1 100644 --- a/rpcserver.nim +++ b/rpcserver.nim @@ -1,2 +1,2 @@ -import rpc / server +import json_rpc / server export server From b968d96923e39592eeb780c496fb92f434ca388d Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 5 Jul 2018 12:40:11 +0100 Subject: [PATCH 53/60] Fix to allow compilation of testerrors --- tests/debugclient.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/debugclient.nim b/tests/debugclient.nim index 574302f..36b2b21 100644 --- a/tests/debugclient.nim +++ b/tests/debugclient.nim @@ -1,4 +1,4 @@ -include ../ rpc / client +include ../ json_rpc / client proc nextId*(self: RpcClient): int64 = self.nextId From cf44cc552d8a02c6a0139a79a8f73bca815205f1 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Fri, 6 Jul 2018 17:47:43 +0100 Subject: [PATCH 54/60] Remove DSL, add router and simplify server --- json_rpc/client.nim | 168 ++++---------- json_rpc/router.nim | 138 +++++++++++ json_rpc/server.nim | 437 +++++------------------------------ json_rpc/sockettransport.nim | 142 ++++++++++++ rpchttpservers.nim | 60 ----- rpcsockets.nim | 2 + tests/all.nim | 2 +- tests/debugclient.nim | 2 +- tests/testerrors.nim | 2 +- tests/testethcalls.nim | 2 +- tests/testhttp.nim | 20 -- tests/testrpcmacro.nim | 18 +- tests/testserverclient.nim | 2 +- 13 files changed, 396 insertions(+), 599 deletions(-) create mode 100644 json_rpc/router.nim create mode 100644 json_rpc/sockettransport.nim delete mode 100644 rpchttpservers.nim create mode 100644 rpcsockets.nim delete mode 100644 tests/testhttp.nim diff --git a/json_rpc/client.nim b/json_rpc/client.nim index 70842f8..524ca30 100644 --- a/json_rpc/client.nim +++ b/json_rpc/client.nim @@ -6,8 +6,8 @@ export asyncdispatch2 type RpcClient*[T, A] = ref object - transp*: T awaiting: Table[string, Future[Response]] + transport: T address: A nextId: int64 @@ -15,34 +15,30 @@ type const defaultMaxRequestLength* = 1024 * 128 -proc newRpcClient*[T, A](): RpcClient[T, A] = +proc newRpcClient*[T, A]: RpcClient[T, A] = ## Creates a new ``RpcClient`` instance. result = RpcClient[T, A](awaiting: initTable[string, Future[Response]](), nextId: 1) -proc genCall(rpcType, callName, writeCode: NimNode): NimNode = - let res = newIdentNode("result") - result = quote do: - proc `callName`*(self: `rpcType`, name: string, - params: JsonNode): Future[Response] {.async.} = - ## Remotely calls the specified RPC method. - let id = $self.nextId - self.nextId.inc - var - value {.inject.} = - $ %{"jsonrpc": %"2.0", - "method": %name, - "params": params, - "id": %id} & "\c\l" - template client: untyped = self - let res = await `writeCode` - # TODO: Add actions when not full packet was send, e.g. disconnect peer. - assert(res == len(value)) +proc call*(self: RpcClient, name: string, + params: JsonNode): Future[Response] {.async.} = + ## Remotely calls the specified RPC method. + let id = $self.nextId + self.nextId.inc + var + value = + $ %{"jsonrpc": %"2.0", + "method": %name, + "params": params, + "id": %id} & "\c\l" + let res = await self.transport.write(value) + # TODO: Add actions when not full packet was send, e.g. disconnect peer. + assert(res == len(value)) - # completed by processMessage. - var newFut = newFuture[Response]() - # add to awaiting responses - self.awaiting[id] = newFut - `res` = await newFut + # completed by processMessage. + var newFut = newFuture[Response]() + # add to awaiting responses + self.awaiting[id] = newFut + result = await newFut template asyncRaise[T](fut: Future[T], errType: typedesc, msg: string) = fut.fail(newException(errType, msg)) @@ -66,12 +62,12 @@ macro checkGet(node: JsonNode, fieldName: string, of JObject: result.add(quote do: `n`.getObject) else: discard -proc processMessage[T, A](self: RpcClient[T, A], line: string) = +proc processMessage(self: RpcClient, line: string) = # Note: this doesn't use any transport code so doesn't need to be differentiated. - let node = parseJson(line) # TODO: Check errors + let + node = parseJson(line) # TODO: Check errors + id = checkGet(node, "id", JString) - # TODO: Use more appropriate exception objects - let id = checkGet(node, "id", JString) if not self.awaiting.hasKey(id): raise newException(ValueError, "Cannot find message id \"" & node["id"].str & "\"") @@ -92,108 +88,26 @@ proc processMessage[T, A](self: RpcClient[T, A], line: string) = self.awaiting[id].fail(newException(ValueError, $errorNode)) self.awaiting.del(id) -proc genProcessData(rpcType, processDataName, readCode, afterReadCode, closeCode: NimNode): NimNode = - result = quote do: - proc `processDataName`(clientTransport: `rpcType`) {.async.} = - while true: - var maxRequestLength {.inject.} = defaultMaxRequestLength - template client: untyped = clientTransport - var value {.inject.} = await `readCode` - `afterReadCode` - if value == "": - # transmission ends - `closeCode` - break +proc processData(client: RpcClient) {.async.} = + while true: + var value = await client.transport.readLine(defaultMaxRequestLength) + if value == "": + # transmission ends + client.transport.close + break - processMessage(clientTransport, value) - # async loop reconnection and waiting - clientTransport.transp = await connect(clientTransport.address) - -proc genConnect(rpcType, connectName, processDataName, connectCode: NimNode): NimNode = - result = quote do: - proc `connectName`*(clientTransport: `rpcType`, address: string, port: Port) {.async.} = - var - address {.inject.} = address - port {.inject.} = port - template client: untyped = clientTransport - `connectCode` - asyncCheck `processDataName`(clientTransport) - -macro defineRpcClientTransport*(transType, addrType: untyped, prefix: string = "", body: untyped = nil): untyped = - var - writeCode = quote do: - client.transp.write(value) - readCode = quote do: - client.transp.readLine(defaultMaxRequestLength) - closeCode = quote do: - client.transp.close - connectCode = quote do: - # TODO: `address` hostname can be resolved to many IP addresses, we are using - # first one, but maybe it would be better to iterate over all IP addresses - # and try to establish connection until it will not be established. - let addresses = resolveTAddress(address, port) - client.transp = await connect(addresses[0]) - client.address = addresses[0] - afterReadCode = newStmtList() - - if body != nil: - body.expectKind nnkStmtList - for item in body: - item.expectKind nnkCall - item[0].expectKind nnkIdent - item[1].expectKind nnkStmtList - let - verb = $item[0] - code = item[1] - - case verb.toLowerAscii - of "write": - # `client`, the RpcClient - # `value`, the data to be sent to the server - # Note: Update `value` so it's length can be sent afterwards - writeCode = code - of "read": - # `client`, the RpcClient - # `maxRequestLength`, initially set to defaultMaxRequestLength - readCode = code - of "close": - # `client`, the RpcClient - # `value`, the data returned from the server - # `maxRequestLength`, initially set to defaultMaxRequestLength - closeCode = code - of "connect": - # `client`, the RpcClient - # `address`, server destination address string - # `port`, server destination port - connectCode = code - of "afterread": - # `client`, the RpcClient - # `value`, the data returned from the server - # `maxRequestLength`, initially set to defaultMaxRequestLength - afterReadCode = code - else: error("Unknown RPC verb \"" & verb & "\"") - - result = newStmtList() - let - rpcType = quote: RpcClient[`transType`, `addrType`] - processDataName = newIdentNode(prefix.strVal & "processData") - connectName = newIdentNode(prefix.strVal & "connect") - callName = newIdentNode(prefix.strVal & "call") - - result.add(genProcessData(rpcType, processDataName, readCode, afterReadCode, closeCode)) - result.add(genConnect(rpcType, connectName, processDataName, connectCode)) - result.add(genCall(rpcType, callName, writeCode)) - - when defined(nimDumpRpcs): - echo "defineClient:\n", result.repr - -# Define default stream server -# TODO: Move this into a separate unit so users can define 'connect', 'call' etc without requiring a prefix? - -defineRpcClientTransport(StreamTransport, TransportAddress) + client.processMessage(value) + # async loop reconnection and waiting + client.transport = await connect(client.address) type RpcStreamClient* = RpcClient[StreamTransport, TransportAddress] +proc connect*(client: RpcStreamClient, address: string, port: Port) {.async.} = + let addresses = resolveTAddress(address, port) + client.transport = await connect(addresses[0]) + client.address = addresses[0] + asyncCheck processData(client) + proc newRpcStreamClient*(): RpcStreamClient = ## Create new server and assign it to addresses ``addresses``. result = newRpcClient[StreamTransport, TransportAddress]() diff --git a/json_rpc/router.nim b/json_rpc/router.nim new file mode 100644 index 0000000..7ac0614 --- /dev/null +++ b/json_rpc/router.nim @@ -0,0 +1,138 @@ +import json, tables, asyncdispatch2, jsonmarshal, strutils, macros +export asyncdispatch2, json, jsonmarshal + +type + # Procedure signature accepted as an RPC call by server + RpcProc* = proc(input: JsonNode): Future[JsonNode] + + RpcRouter* = object + procs*: TableRef[string, RpcProc] + +const + methodField = "method" + paramsField = "params" + +proc newRpcRouter*: RpcRouter = + result.procs = newTable[string, RpcProc]() + +proc register*(router: var RpcRouter, path: string, call: RpcProc) = + router.procs.add(path, call) + +proc clear*(router: var RpcRouter) = router.procs.clear + +proc hasMethod*(router: RpcRouter, methodName: string): bool = router.procs.hasKey(methodName) + +template isEmpty(node: JsonNode): bool = node.isNil or node.kind == JNull + +proc route*(router: RpcRouter, data: JsonNode): Future[JsonNode] {.async, gcsafe.} = + ## Route to RPC, raises exceptions on missing data + let jPath = data{methodField} + if jPath.isEmpty: + raise newException(ValueError, "No " & methodField & " field found") + + let jParams = data{paramsField} + if jParams.isEmpty: + raise newException(ValueError, "No " & paramsField & " field found") + + let + path = jPath.getStr + rpc = router.procs.getOrDefault(path) + # TODO: not GC-safe as it accesses 'rpc' which is a global using GC'ed memory! + if rpc != nil: + result = await rpc(jParams) + else: + raise newException(ValueError, "Method \"" & path & "\" not found") + +proc ifRoute*(router: RpcRouter, data: JsonNode, fut: var Future[JsonNode]): bool = + ## Route to RPC, returns false if the method or params cannot be found + # TODO: This is already checked in processMessages, but allows safer use externally + let + jPath = data{methodField} + jParams = data{paramsField} + if jPath.isEmpty or jParams.isEmpty: + return false + + let + path = jPath.getStr + rpc = router.procs.getOrDefault(path) + if rpc != nil: + fut = rpc(jParams) + return true + +proc makeProcName(s: string): string = + result = "" + for c in s: + if c.isAlphaNumeric: result.add c + +proc hasReturnType(params: NimNode): bool = + if params != nil and params.len > 0 and params[0] != nil and + params[0].kind != nnkEmpty: + result = true + +macro rpc*(server: RpcRouter, path: string, body: untyped): untyped = + ## Define a remote procedure call. + ## Input and return parameters are defined using the ``do`` notation. + ## For example: + ## .. code-block:: nim + ## myServer.rpc("path") do(param1: int, param2: float) -> string: + ## result = $param1 & " " & $param2 + ## ``` + ## Input parameters are automatically marshalled from json to Nim types, + ## and output parameters are automatically marshalled to json for transport. + result = newStmtList() + let + parameters = body.findChild(it.kind == nnkFormalParams) + # all remote calls have a single parameter: `params: JsonNode` + paramsIdent = newIdentNode"params" + # procs are generated from the stripped path + pathStr = $path + # strip non alphanumeric + procNameStr = pathStr.makeProcName + # public rpc proc + procName = newIdentNode(procNameStr) + # when parameters present: proc that contains our rpc body + doMain = newIdentNode(procNameStr & "DoMain") + # async result + res = newIdentNode("result") + var + setup = jsonToNim(parameters, paramsIdent) + procBody = if body.kind == nnkStmtList: body else: body.body + errTrappedBody = quote do: + try: + `procBody` + except: + debug "Error occurred within RPC ", path = `path`, errorMessage = getCurrentExceptionMsg() + if parameters.hasReturnType: + let returnType = parameters[0] + + # delegate async proc allows return and setting of result as native type + result.add(quote do: + proc `doMain`(`paramsIdent`: JsonNode): Future[`returnType`] {.async.} = + `setup` + `errTrappedBody` + ) + + if returnType == ident"JsonNode": + # `JsonNode` results don't need conversion + result.add( quote do: + proc `procName`(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = + `res` = await `doMain`(`paramsIdent`) + ) + else: + result.add(quote do: + proc `procName`(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = + `res` = %await `doMain`(`paramsIdent`) + ) + else: + # no return types, inline contents + result.add(quote do: + proc `procName`(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = + `setup` + `errTrappedBody` + ) + result.add( quote do: + `server`.register(`path`, `procName`) + ) + + when defined(nimDumpRpcs): + echo "\n", pathStr, ": ", result.repr diff --git a/json_rpc/server.nim b/json_rpc/server.nim index d675903..a564957 100644 --- a/json_rpc/server.nim +++ b/json_rpc/server.nim @@ -1,37 +1,20 @@ -import json, tables, strutils, options, macros, chronicles -import asyncdispatch2 +import json, tables, options, macros, chronicles +import asyncdispatch2, router import jsonmarshal -export asyncdispatch2, json, jsonmarshal, options +export asyncdispatch2, json, jsonmarshal, router logScope: topics = "RpcServer" type - RpcJsonError* = enum rjeInvalidJson, rjeVersionError, rjeNoMethod, rjeNoId + RpcJsonError* = enum rjeInvalidJson, rjeVersionError, rjeNoMethod, rjeNoId, rjeNoParams RpcJsonErrorContainer* = tuple[err: RpcJsonError, msg: string] - # Procedure signature accepted as an RPC call by server - RpcProc* = proc (params: JsonNode): Future[JsonNode] - - RpcClientTransport* = concept t - t.write(var string) is Future[int] - t.readLine(int) is Future[string] - t.close - t.remoteAddress() # Required for logging - t.localAddress() - - RpcServerTransport* = concept t - t.start - t.stop - t.close - - RpcProcessClient* = proc (server: RpcServerTransport, client: RpcClientTransport): Future[void] {.gcsafe.} - - RpcServer*[S: RpcServerTransport] = ref object + RpcServer*[S] = ref object servers*: seq[S] - procs*: TableRef[string, RpcProc] + router*: RpcRouter RpcProcError* = ref object of Exception code*: int @@ -49,23 +32,28 @@ const SERVER_ERROR* = -32000 defaultMaxRequestLength* = 1024 * 128 - jsonErrorMessages*: array[RpcJsonError, (int, string)] = [ (JSON_PARSE_ERROR, "Invalid JSON"), (INVALID_REQUEST, "JSON 2.0 required"), (INVALID_REQUEST, "No method requested"), - (INVALID_REQUEST, "No id specified") + (INVALID_REQUEST, "No id specified"), + (INVALID_PARAMS, "No parameters specified") ] proc newRpcServer*[S](): RpcServer[S] = new result - result.procs = newTable[string, RpcProc]() + result.router = newRpcRouter() result.servers = @[] # Utility functions -# TODO: Move outside server -func `%`*(p: Port): JsonNode = %(p.int) +# TODO: Move outside server? +#func `%`*(p: Port): JsonNode = %(p.int) + +template rpc*(server: RpcServer, path: string, body: untyped): untyped = + server.router.rpc(path, body) + +template hasMethod*(server: RpcServer, methodName: string): bool = server.router.hasMethod(methodName) # Json state checking @@ -80,7 +68,7 @@ template jsonValid*(jsonString: string, node: var JsonNode): (bool, string) = debug "Cannot process json", json = jsonString, msg = msg (valid, msg) -proc checkJsonErrors*(line: string, +proc checkJsonState*(line: string, node: var JsonNode): Option[RpcJsonErrorContainer] = ## Tries parsing line into node, if successful checks required fields ## Returns: error state or none @@ -89,10 +77,13 @@ proc checkJsonErrors*(line: string, return some((rjeInvalidJson, res[1])) if not node.hasKey("id"): return some((rjeNoId, "")) - if node{"jsonrpc"} != %"2.0": + let jVer = node{"jsonrpc"} + if jVer != nil and jVer.kind != JNull and jVer != %"2.0": return some((rjeVersionError, "")) if not node.hasKey("method"): return some((rjeNoMethod, "")) + if not node.hasKey("params"): + return some((rjeNoParams, "")) return none(RpcJsonErrorContainer) # Json reply wrappers @@ -101,170 +92,47 @@ proc wrapReply*(id: JsonNode, value: JsonNode, error: JsonNode): string = let node = %{"jsonrpc": %"2.0", "result": value, "error": error, "id": id} return $node & "\c\l" -proc genErrorSending(name, writeCode, errorCode: NimNode): NimNode = - let - res = newIdentNode("result") - sendJsonErr = newIdentNode($name & "Json") - result = quote do: - proc `name`*[T: RpcClientTransport](clientTrans: T, code: int, msg: string, id: JsonNode, - data: JsonNode = newJNull()) {.async.} = - ## Send error message to client - let error = %{"code": %(code), "id": id, "message": %msg, "data": data} - debug "Error generated", error = error, id = id - - template transport: untyped = clientTrans - var value {.inject.} = wrapReply(id, newJNull(), error) - `errorCode` - `res` = `writeCode` - - proc `sendJsonErr`*(state: RpcJsonError, clientTrans: RpcClientTransport, id: JsonNode, - data = newJNull()) {.async.} = - ## Send client response for invalid json state - let errMsgs = jsonErrorMessages[state] - await clientTrans.`name`(errMsgs[0], errMsgs[1], id, data) +proc wrapError*(code: int, msg: string, id: JsonNode, + data: JsonNode = newJNull()): JsonNode = + # Create standardised error json + result = %{"code": %(code), "id": id, "message": %msg, "data": data} + debug "Error generated", error = result, id = id # Server message processing -proc genProcessMessages(name, sendErrorName, writeCode: NimNode): NimNode = - let idSendErrJson = newIdentNode($sendErrorName & "Json") - result = quote do: - proc `name`[T: RpcClientTransport](server: RpcServer, clientTrans: T, - line: string) {.async.} = - var - node: JsonNode - # set up node and/or flag errors - jsonErrorState = checkJsonErrors(line, node) - - if jsonErrorState.isSome: - let errState = jsonErrorState.get - var id = - if errState.err == rjeInvalidJson or errState.err == rjeNoId: - newJNull() - else: - node["id"] - await errState.err.`idSendErrJson`(clientTrans, id, %errState.msg) - else: - let - methodName = node["method"].str - id = node["id"] - - if not server.procs.hasKey(methodName): - await clientTrans.`sendErrorName`(METHOD_NOT_FOUND, "Method not found", %id, - %(methodName & " is not a registered method.")) - else: - let callRes = await server.procs[methodName](node["params"]) - template transport: untyped = clientTrans - var value {.inject.} = wrapReply(id, callRes, newJNull()) - asyncCheck `writeCode` - -proc genProcessClient(nameIdent, procMessagesIdent, sendErrIdent, readCode, afterReadCode, closeCode: NimNode): NimNode = - # This generates the processClient proc to match transport. - # processClient is compatible with createStreamServer and thus StreamCallback. - # However the constraints are conceptualised so you only need to match it's interface - # Note: https://github.com/nim-lang/Nim/issues/644 - result = quote do: - proc `nameIdent`[S: RpcServerTransport, C: RpcClientTransport](server: S, clientTrans: C) {.async, gcsafe.} = - var rpc = getUserData[RpcServer[S]](server) - while true: - var maxRequestLength {.inject.} = defaultMaxRequestLength - template transport: untyped = clientTrans - - var value {.inject.} = await `readCode` - `afterReadCode` - if value == "": - `closeCode` - break - - debug "Processing message", address = clientTrans.remoteAddress(), line = value - - let future = `procMessagesIdent`(rpc, clientTrans, value) - yield future - if future.failed: - if future.readError of RpcProcError: - let err = future.readError.RpcProcError - await clientTrans.`sendErrIdent`(err.code, err.msg, err.data) - elif future.readError of ValueError: - let err = future.readError[].ValueError - await clientTrans.`sendErrIdent`(INVALID_PARAMS, err.msg, %"") - else: - await clientTrans.`sendErrIdent`(SERVER_ERROR, - "Error: Unknown error occurred", %"") - -macro defineRpcServerTransport*(procClientName: untyped, body: untyped = nil): untyped = - ## Build an rpcServer type that inlines data access operations - #[ - Injects: - client: RpcClientTransport type - maxRequestLength: optional bytes to read - value: Json string to be written to transport - - Example: - defineRpcTransport(myServer): - write: - client.write(value) - read: - client.readLine(maxRequestLength) - close: - client.close - ]# - procClientName.expectKind nnkIdent +proc processMessages*[T](server: RpcServer[T], line: string): Future[string] {.async, gcsafe.} = var - writeCode = quote do: - transport.write(value) - readCode = quote do: - transport.readLine(defaultMaxRequestLength) - closeCode = quote do: - transport.close - afterReadCode = newStmtList() - errorCode = newStmtList() + node: JsonNode + # parse json node and/or flag missing fields and errors + jsonErrorState = checkJsonState(line, node) - if body != nil: - body.expectKind nnkStmtList - for item in body: - item.expectKind nnkCall - item[0].expectKind nnkIdent - item[1].expectKind nnkStmtList + if jsonErrorState.isSome: + let errState = jsonErrorState.get + var id = + if errState.err == rjeInvalidJson or errState.err == rjeNoId: + newJNull() + else: + node["id"] + let errMsg = jsonErrorMessages[errState.err] + # return error state as json + result = $wrapError( + code = errMsg[0], + msg = errMsg[1], + id = id) + else: + let + methodName = node["method"].str + id = node["id"] + var callRes: Future[JsonNode] + + if server.router.ifRoute(node, callRes): + let res = await callRes + result = $wrapReply(id, res, newJNull()) + else: let - verb = $item[0] - code = item[1] - - case verb.toLowerAscii - of "write": - # `transport`, the client transport - # `value`, the data returned from the invoked RPC - # Note: Update `value` so it's length can be sent afterwards - writeCode = code - of "read": - # `transport`, the client transport - # `maxRequestLength`, set to defaultMaxRequestLength - # Note: Result of expression is awaited - readCode = code - of "close": - # `transport`, the client transport - # `value`, which contains the data read by `readCode` - closeCode = code - of "afterread": - # `transport`, the client transport - # `value`, which contains the data read by `readCode` - afterReadCode = code - of "error": - # `transport`, the client transport - # `value`, the data returned from the invoked RPC - # Note: Update `value` so it's length can be sent afterwards - errorCode = code - else: error("Unknown RPC verb \"" & verb & "\"") - - result = newStmtList() - - let - sendErr = newIdentNode($procClientName & "SendError") - procMsgs = newIdentNode($procClientName & "ProcessMessages") - result.add(genErrorSending(sendErr, writeCode, errorCode)) - result.add(genProcessMessages(procMsgs, sendErr, writeCode)) - result.add(genProcessClient(procClientName, procMsgs, sendErr, readCode, afterReadCode, closeCode)) - - when defined(nimDumpRpcs): - echo "defineServer:\n", result.repr + methodNotFound = %(methodName & " is not a registered method.") + error = wrapError(METHOD_NOT_FOUND, "Method not found", id, methodNotFound) + result = $wrapReply(id, newJNull(), error) proc start*(server: RpcServer) = ## Start the RPC server. @@ -281,201 +149,14 @@ proc close*(server: RpcServer) = for item in server.servers: item.close() -# Server registration and RPC generation +# Server registration proc register*(server: RpcServer, name: string, rpc: RpcProc) = ## Add a name/code pair to the RPC server. - server.procs[name] = rpc + server.router.addRoute(name, rpc) proc unRegisterAll*(server: RpcServer) = # Remove all remote procedure calls from this server. - server.procs.clear - -proc makeProcName(s: string): string = - result = "" - for c in s: - if c.isAlphaNumeric: result.add c - -proc hasReturnType(params: NimNode): bool = - if params != nil and params.len > 0 and params[0] != nil and - params[0].kind != nnkEmpty: - result = true - -macro rpc*(server: RpcServer, path: string, body: untyped): untyped = - ## Define a remote procedure call. - ## Input and return parameters are defined using the ``do`` notation. - ## For example: - ## .. code-block:: nim - ## myServer.rpc("path") do(param1: int, param2: float) -> string: - ## result = $param1 & " " & $param2 - ## ``` - ## Input parameters are automatically marshalled from json to Nim types, - ## and output parameters are automatically marshalled to json for transport. - result = newStmtList() - let - parameters = body.findChild(it.kind == nnkFormalParams) - # all remote calls have a single parameter: `params: JsonNode` - paramsIdent = newIdentNode"params" - # procs are generated from the stripped path - pathStr = $path - # strip non alphanumeric - procNameStr = pathStr.makeProcName - # public rpc proc - procName = newIdentNode(procNameStr) - # when parameters present: proc that contains our rpc body - doMain = newIdentNode(procNameStr & "DoMain") - # async result - res = newIdentNode("result") - var - setup = jsonToNim(parameters, paramsIdent) - procBody = if body.kind == nnkStmtList: body else: body.body - errTrappedBody = quote do: - try: - `procBody` - except: - debug "Error occurred within RPC ", path = `path`, errorMessage = getCurrentExceptionMsg() - if parameters.hasReturnType: - let returnType = parameters[0] - - # delegate async proc allows return and setting of result as native type - result.add(quote do: - proc `doMain`(`paramsIdent`: JsonNode): Future[`returnType`] {.async.} = - `setup` - `errTrappedBody` - ) - - if returnType == ident"JsonNode": - # `JsonNode` results don't need conversion - result.add( quote do: - proc `procName`(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = - `res` = await `doMain`(`paramsIdent`) - ) - else: - result.add(quote do: - proc `procName`(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = - `res` = %await `doMain`(`paramsIdent`) - ) - else: - # no return types, inline contents - result.add(quote do: - proc `procName`(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = - `setup` - `errTrappedBody` - ) - result.add( quote do: - `server`.register(`path`, `procName`) - ) - - when defined(nimDumpRpcs): - echo "\n", pathStr, ": ", result.repr - -# Utility functions for setting up servers using stream transport addresses - -# Create a default transport that's suitable for createStreamServer -defineRpcServerTransport(processStreamClient) - -proc addStreamServer*[S](server: RpcServer[S], address: TransportAddress, callBack: StreamCallback = processStreamClient) = - #makeProcessClient(processClient, StreamTransport) - try: - info "Creating server on ", address = $address - var transportServer = createStreamServer(address, callBack, {ReuseAddr}, udata = server) - server.servers.add(transportServer) - except: - error "Failed to create server", address = $address, message = getCurrentExceptionMsg() - - if len(server.servers) == 0: - # Server was not bound, critical error. - raise newException(RpcBindError, "Unable to create server!") - -proc addStreamServers*[T: RpcServer](server: T, addresses: openarray[TransportAddress], callBack: StreamCallback = processStreamClient) = - for item in addresses: - server.addStreamServer(item, callBack) - -proc addStreamServer*[T: RpcServer](server: T, address: string, callBack: StreamCallback = processStreamClient) = - ## Create new server and assign it to addresses ``addresses``. - var - tas4: seq[TransportAddress] - tas6: seq[TransportAddress] - added = 0 - - # Attempt to resolve `address` for IPv4 address space. - try: - tas4 = resolveTAddress(address, IpAddressFamily.IPv4) - except: - discard - - # Attempt to resolve `address` for IPv6 address space. - try: - tas6 = resolveTAddress(address, IpAddressFamily.IPv6) - except: - discard - - for r in tas4: - server.addStreamServer(r, callBack) - added.inc - for r in tas6: - server.addStreamServer(r, callBack) - added.inc - - if added == 0: - # Addresses could not be resolved, critical error. - raise newException(RpcAddressUnresolvableError, "Unable to get address!") - -proc addStreamServers*[T: RpcServer](server: T, addresses: openarray[string], callBack: StreamCallback = processStreamClient) = - for address in addresses: - server.addStreamServer(address, callBack) - -proc addStreamServer*[T: RpcServer](server: T, address: string, port: Port, callBack: StreamCallback = processStreamClient) = - var - tas4: seq[TransportAddress] - tas6: seq[TransportAddress] - added = 0 - - # Attempt to resolve `address` for IPv4 address space. - try: - tas4 = resolveTAddress(address, port, IpAddressFamily.IPv4) - except: - discard - - # Attempt to resolve `address` for IPv6 address space. - try: - tas6 = resolveTAddress(address, port, IpAddressFamily.IPv6) - except: - discard - - if len(tas4) == 0 and len(tas6) == 0: - # Address was not resolved, critical error. - raise newException(RpcAddressUnresolvableError, - "Address " & address & " could not be resolved!") - - for r in tas4: - server.addStreamServer(r, callBack) - added.inc - for r in tas6: - server.addStreamServer(r, callBack) - added.inc - - if len(server.servers) == 0: - # Server was not bound, critical error. - raise newException(RpcBindError, - "Could not setup server on " & address & ":" & $int(port)) - -type RpcStreamServer* = RpcServer[StreamServer] - -proc newRpcStreamServer*(addresses: openarray[TransportAddress]): RpcStreamServer = - ## Create new server and assign it to addresses ``addresses``. - result = newRpcServer[StreamServer]() - result.addStreamServers(addresses) - -proc newRpcStreamServer*(addresses: openarray[string]): RpcStreamServer = - ## Create new server and assign it to addresses ``addresses``. - result = newRpcServer[StreamServer]() - result.addStreamServers(addresses) - -proc newRpcStreamServer*(address = "localhost", port: Port = Port(8545)): RpcStreamServer = - # Create server on specified port - result = newRpcServer[StreamServer]() - result.addStreamServer(address, port) + server.router.clear -# TODO: Allow cross checking between client signatures and server calls diff --git a/json_rpc/sockettransport.nim b/json_rpc/sockettransport.nim new file mode 100644 index 0000000..948e554 --- /dev/null +++ b/json_rpc/sockettransport.nim @@ -0,0 +1,142 @@ +import server, json, chronicles + +proc sendError*[T](transport: T, code: int, msg: string, id: JsonNode, + data: JsonNode = newJNull()) {.async.} = + ## Send error message to client + let error = wrapError(code, msg, id, data) + var value = wrapReply(id, newJNull(), error) + result = transport.write(value) + +proc processClient(server: StreamServer, transport: StreamTransport) {.async, gcsafe.} = + ## Process transport data to the RPC server + var rpc = getUserData[RpcServer[StreamTransport]](server) + while true: + var + maxRequestLength = defaultMaxRequestLength + value = await transport.readLine(defaultMaxRequestLength) + if value == "": + transport.close + break + + debug "Processing message", address = transport.remoteAddress(), line = value + + let future = processMessages(rpc, value) + yield future + if future.failed: + if future.readError of RpcProcError: + let err = future.readError.RpcProcError + await transport.sendError(err.code, err.msg, err.data) + elif future.readError of ValueError: + let err = future.readError[].ValueError + await transport.sendError(INVALID_PARAMS, err.msg, %"") + else: + await transport.sendError(SERVER_ERROR, + "Error: Unknown error occurred", %"") + else: + let res = await future + result = transport.write(res) + +# Utility functions for setting up servers using stream transport addresses + +proc addStreamServer*(server: RpcServer[StreamServer], address: TransportAddress, callBack: StreamCallback) = + try: + info "Creating server on ", address = $address + var transportServer = createStreamServer(address, callBack, {ReuseAddr}, udata = server) + server.servers.add(transportServer) + except: + error "Failed to create server", address = $address, message = getCurrentExceptionMsg() + + if len(server.servers) == 0: + # Server was not bound, critical error. + raise newException(RpcBindError, "Unable to create server!") + +proc addStreamServers*(server: RpcServer[StreamServer], addresses: openarray[TransportAddress], callBack: StreamCallback) = + for item in addresses: + server.addStreamServer(item, callBack) + +proc addStreamServer*(server: RpcServer[StreamServer], address: string, callBack: StreamCallback) = + ## Create new server and assign it to addresses ``addresses``. + var + tas4: seq[TransportAddress] + tas6: seq[TransportAddress] + added = 0 + + # Attempt to resolve `address` for IPv4 address space. + try: + tas4 = resolveTAddress(address, IpAddressFamily.IPv4) + except: + discard + + # Attempt to resolve `address` for IPv6 address space. + try: + tas6 = resolveTAddress(address, IpAddressFamily.IPv6) + except: + discard + + for r in tas4: + server.addStreamServer(r, callBack) + added.inc + for r in tas6: + server.addStreamServer(r, callBack) + added.inc + + if added == 0: + # Addresses could not be resolved, critical error. + raise newException(RpcAddressUnresolvableError, "Unable to get address!") + +proc addStreamServers*(server: RpcServer[StreamServer], addresses: openarray[string], callBack: StreamCallback) = + for address in addresses: + server.addStreamServer(address, callBack) + +proc addStreamServer*(server: RpcServer[StreamServer], address: string, port: Port, callBack: StreamCallback) = + var + tas4: seq[TransportAddress] + tas6: seq[TransportAddress] + added = 0 + + # Attempt to resolve `address` for IPv4 address space. + try: + tas4 = resolveTAddress(address, port, IpAddressFamily.IPv4) + except: + discard + + # Attempt to resolve `address` for IPv6 address space. + try: + tas6 = resolveTAddress(address, port, IpAddressFamily.IPv6) + except: + discard + + if len(tas4) == 0 and len(tas6) == 0: + # Address was not resolved, critical error. + raise newException(RpcAddressUnresolvableError, + "Address " & address & " could not be resolved!") + + for r in tas4: + server.addStreamServer(r, callBack) + added.inc + for r in tas6: + server.addStreamServer(r, callBack) + added.inc + + if len(server.servers) == 0: + # Server was not bound, critical error. + raise newException(RpcBindError, + "Could not setup server on " & address & ":" & $int(port)) + +type RpcStreamServer* = RpcServer[StreamServer] + +proc newRpcStreamServer*(addresses: openarray[TransportAddress]): RpcStreamServer = + ## Create new server and assign it to addresses ``addresses``. + result = newRpcServer[StreamServer]() + result.addStreamServers(addresses, processClient) + +proc newRpcStreamServer*(addresses: openarray[string]): RpcStreamServer = + ## Create new server and assign it to addresses ``addresses``. + result = newRpcServer[StreamServer]() + result.addStreamServers(addresses, processClient) + +proc newRpcStreamServer*(address = "localhost", port: Port = Port(8545)): RpcStreamServer = + # Create server on specified port + result = newRpcServer[StreamServer]() + result.addStreamServer(address, port, processClient) + diff --git a/rpchttpservers.nim b/rpchttpservers.nim deleted file mode 100644 index 4b308a1..0000000 --- a/rpchttpservers.nim +++ /dev/null @@ -1,60 +0,0 @@ -import rpcserver, rpcclient, tables, chronicles, strformat, strutils -export rpcserver, rpcclient - -proc extractJsonStr(msgSource: string, value: string): string = - result = "" - let p1 = find(value, '{') - if p1 > -1: - let p2 = rFind(value, '}') - if p2 == -1: - info "Cannot find json end brace", source = msgSource, msg = value - else: - result = value[p1 .. p2] - debug "Extracted json", source = msgSource, json = result - else: - info "Cannot find json start brace", source = msgSource, msg = value - -type - RpcHttpServer* = RpcServer[StreamServer] - -defineRpcServerTransport(httpProcessClient): - write: - const contentType = "Content-Type: application/json-rpc" - value = &"Host: {$transport.localAddress} {contentType} Content-Length: {$value.len} {value}" - debug "HTTP server: write", msg = value - transport.write(value) - afterRead: - debug "HTTP server: read", msg = value - value = "HTTP Server".extractJsonStr(value) - -proc newRpcHttpServer*(addresses: openarray[TransportAddress]): RpcHttpServer = - ## Create new server and assign it to addresses ``addresses``. - result = newRpcServer[StreamServer]() - result.addStreamServers(addresses, httpProcessClient) - -proc newRpcHttpServer*(addresses: openarray[string]): RpcHttpServer = - ## Create new server and assign it to addresses ``addresses``. - result = newRpcServer[StreamServer]() - result.addStreamServers(addresses, httpProcessClient) - -proc newRpcHttpServer*(address = "localhost", port: Port = Port(8545)): RpcHttpServer = - result = newRpcServer[StreamServer]() - result.addStreamServer(address, port, httpProcessClient) - -type RpcHttpClient* = RpcClient[StreamTransport, TransportAddress] - -defineRpcClientTransport(StreamTransport, TransportAddress, "http"): - write: - const contentType = "Content-Type: application/json-rpc" - value = &"Host: {$client.transp.localAddress} {contentType} Content-Length: {$value.len} {value}" - debug "HTTP client: write", msg = value - client.transp.write(value) - afterRead: - # Strip out http header - # TODO: Performance - debug "HTTP client: read", msg = value - value = "HTTP Client".extractJsonStr(value) - -proc newRpcHttpClient*(): RpcHttpClient = - result = newRpcClient[StreamTransport, TransportAddress]() - diff --git a/rpcsockets.nim b/rpcsockets.nim new file mode 100644 index 0000000..06ee707 --- /dev/null +++ b/rpcsockets.nim @@ -0,0 +1,2 @@ +import json_rpc / [server, sockettransport] +export server, sockettransport diff --git a/tests/all.nim b/tests/all.nim index 69a109c..ae9eb62 100644 --- a/tests/all.nim +++ b/tests/all.nim @@ -1,3 +1,3 @@ import - testrpcmacro, testserverclient, testethcalls, testhttp #, testerrors + testrpcmacro, testserverclient, testethcalls #, testerrors diff --git a/tests/debugclient.nim b/tests/debugclient.nim index 36b2b21..15c5f90 100644 --- a/tests/debugclient.nim +++ b/tests/debugclient.nim @@ -9,7 +9,7 @@ proc rawCall*(self: RpcClient, name: string, self.nextId.inc var s = msg & "\c\l" - let res = await self.transp.write(s) + let res = await self.transport.write(s) assert res == len(s) # completed by processMessage. diff --git a/tests/testerrors.nim b/tests/testerrors.nim index 9f44a80..1304cf6 100644 --- a/tests/testerrors.nim +++ b/tests/testerrors.nim @@ -3,7 +3,7 @@ allow unchecked and unformatted calls. ]# -import unittest, debugclient, ../rpcserver +import unittest, debugclient, ../rpcsockets import strformat, chronicles var server = newRpcStreamServer("localhost", 8547.Port) diff --git a/tests/testethcalls.nim b/tests/testethcalls.nim index 538a32f..ec23e46 100644 --- a/tests/testethcalls.nim +++ b/tests/testethcalls.nim @@ -1,5 +1,5 @@ import unittest, json, tables -import ../rpcclient, ../rpcserver +import ../rpcclient, ../rpcsockets import stint, ethtypes, ethprocs, stintjson, nimcrypto, ethhexstrings, chronicles from os import getCurrentDir, DirSep diff --git a/tests/testhttp.nim b/tests/testhttp.nim deleted file mode 100644 index 0f59837..0000000 --- a/tests/testhttp.nim +++ /dev/null @@ -1,20 +0,0 @@ -import unittest, json, chronicles, unittest -import ../rpchttpservers - -var srv = newRpcHttpServer(["localhost:8545"]) -var client = newRpcHttpClient() - -# Create RPC on server -srv.rpc("myProc") do(input: string, data: array[0..3, int]): - result = %("Hello " & input & " data: " & $data) - -srv.start() -waitFor client.httpConnect("localhost", Port(8545)) - -suite "HTTP RPC transport": - test "Call": - var r = waitFor client.httpcall("myProc", %[%"abc", %[1, 2, 3, 4]]) - check r.error == false and r.result == %"Hello abc data: [1, 2, 3, 4]" - -srv.stop() -srv.close() diff --git a/tests/testrpcmacro.nim b/tests/testrpcmacro.nim index 89573e9..d3cb07c 100644 --- a/tests/testrpcmacro.nim +++ b/tests/testrpcmacro.nim @@ -1,5 +1,5 @@ import unittest, json, tables, chronicles -import ../rpcserver +import ../rpcsockets type # some nested types to check object parsing @@ -67,14 +67,14 @@ s.rpc("rpc.testreturns") do() -> int: suite "Server types": test "On macro registration": - check s.procs.hasKey("rpc.simplepath") - check s.procs.hasKey("rpc.differentparams") - check s.procs.hasKey("rpc.arrayparam") - check s.procs.hasKey("rpc.seqparam") - check s.procs.hasKey("rpc.objparam") - check s.procs.hasKey("rpc.returntypesimple") - check s.procs.hasKey("rpc.returntypecomplex") - check s.procs.hasKey("rpc.testreturns") + check s.hasMethod("rpc.simplepath") + check s.hasMethod("rpc.differentparams") + check s.hasMethod("rpc.arrayparam") + check s.hasMethod("rpc.seqparam") + check s.hasMethod("rpc.objparam") + check s.hasMethod("rpc.returntypesimple") + check s.hasMethod("rpc.returntypecomplex") + check s.hasMethod("rpc.testreturns") test "Simple paths": let r = waitFor rpcSimplePath(%[]) diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index d48f9cb..bb2a0f7 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -1,5 +1,5 @@ import unittest, json, chronicles -import ../rpcclient, ../rpcserver +import ../rpcclient, ../rpcsockets var srv = newRpcStreamServer(["localhost:8545"]) var client = newRpcStreamClient() From 4e300f15396807ca02fb1629ab0aec48223b0a7d Mon Sep 17 00:00:00 2001 From: coffeepots Date: Mon, 9 Jul 2018 09:38:26 +0100 Subject: [PATCH 55/60] Remove callback arguement, addStreamServer should be using processClient --- json_rpc/sockettransport.nim | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/json_rpc/sockettransport.nim b/json_rpc/sockettransport.nim index 948e554..9409eb4 100644 --- a/json_rpc/sockettransport.nim +++ b/json_rpc/sockettransport.nim @@ -38,10 +38,10 @@ proc processClient(server: StreamServer, transport: StreamTransport) {.async, gc # Utility functions for setting up servers using stream transport addresses -proc addStreamServer*(server: RpcServer[StreamServer], address: TransportAddress, callBack: StreamCallback) = +proc addStreamServer*(server: RpcServer[StreamServer], address: TransportAddress) = try: info "Creating server on ", address = $address - var transportServer = createStreamServer(address, callBack, {ReuseAddr}, udata = server) + var transportServer = createStreamServer(address, processClient, {ReuseAddr}, udata = server) server.servers.add(transportServer) except: error "Failed to create server", address = $address, message = getCurrentExceptionMsg() @@ -50,11 +50,11 @@ proc addStreamServer*(server: RpcServer[StreamServer], address: TransportAddress # Server was not bound, critical error. raise newException(RpcBindError, "Unable to create server!") -proc addStreamServers*(server: RpcServer[StreamServer], addresses: openarray[TransportAddress], callBack: StreamCallback) = +proc addStreamServers*(server: RpcServer[StreamServer], addresses: openarray[TransportAddress]) = for item in addresses: - server.addStreamServer(item, callBack) + server.addStreamServer(item) -proc addStreamServer*(server: RpcServer[StreamServer], address: string, callBack: StreamCallback) = +proc addStreamServer*(server: RpcServer[StreamServer], address: string) = ## Create new server and assign it to addresses ``addresses``. var tas4: seq[TransportAddress] @@ -74,21 +74,21 @@ proc addStreamServer*(server: RpcServer[StreamServer], address: string, callBack discard for r in tas4: - server.addStreamServer(r, callBack) + server.addStreamServer(r) added.inc for r in tas6: - server.addStreamServer(r, callBack) + server.addStreamServer(r) added.inc if added == 0: # Addresses could not be resolved, critical error. raise newException(RpcAddressUnresolvableError, "Unable to get address!") -proc addStreamServers*(server: RpcServer[StreamServer], addresses: openarray[string], callBack: StreamCallback) = +proc addStreamServers*(server: RpcServer[StreamServer], addresses: openarray[string]) = for address in addresses: - server.addStreamServer(address, callBack) + server.addStreamServer(address) -proc addStreamServer*(server: RpcServer[StreamServer], address: string, port: Port, callBack: StreamCallback) = +proc addStreamServer*(server: RpcServer[StreamServer], address: string, port: Port) = var tas4: seq[TransportAddress] tas6: seq[TransportAddress] @@ -112,10 +112,10 @@ proc addStreamServer*(server: RpcServer[StreamServer], address: string, port: Po "Address " & address & " could not be resolved!") for r in tas4: - server.addStreamServer(r, callBack) + server.addStreamServer(r) added.inc for r in tas6: - server.addStreamServer(r, callBack) + server.addStreamServer(r) added.inc if len(server.servers) == 0: @@ -128,15 +128,15 @@ type RpcStreamServer* = RpcServer[StreamServer] proc newRpcStreamServer*(addresses: openarray[TransportAddress]): RpcStreamServer = ## Create new server and assign it to addresses ``addresses``. result = newRpcServer[StreamServer]() - result.addStreamServers(addresses, processClient) + result.addStreamServers(addresses) proc newRpcStreamServer*(addresses: openarray[string]): RpcStreamServer = ## Create new server and assign it to addresses ``addresses``. result = newRpcServer[StreamServer]() - result.addStreamServers(addresses, processClient) + result.addStreamServers(addresses) proc newRpcStreamServer*(address = "localhost", port: Port = Port(8545)): RpcStreamServer = # Create server on specified port result = newRpcServer[StreamServer]() - result.addStreamServer(address, port, processClient) + result.addStreamServer(address, port) From f943c584c7bbcacf607f98b9064cc50c4643eed3 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Mon, 9 Jul 2018 09:43:23 +0100 Subject: [PATCH 56/60] Add check for empty string --- tests/ethhexstrings.nim | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/ethhexstrings.nim b/tests/ethhexstrings.nim index 50529bf..f711689 100644 --- a/tests/ethhexstrings.nim +++ b/tests/ethhexstrings.nim @@ -17,7 +17,7 @@ proc encodeQuantity*(value: SomeUnsignedInt): string = template hasHexHeader*(value: string | HexDataStr | HexQuantityStr): bool = template strVal: untyped = value.string - if strVal[0] == '0' and strVal[1] in {'x', 'X'} and strVal.len > 2: true + if strVal != "" and strVal[0] == '0' and strVal[1] in {'x', 'X'} and strVal.len > 2: true else: false template isHexChar*(c: char): bool = @@ -94,6 +94,12 @@ proc fromJson*(n: JsonNode, argName: string, result: var HexQuantityStr) = when isMainModule: import unittest suite "Hex quantity": + test "Empty string": + expect ValueError: + let + source = "" + x = hexQuantityStr source + check %x == %source test "Even length": let source = "0x123" From 9075b967d12e79e4b276f96d227fd55f1817a5c7 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Mon, 9 Jul 2018 09:58:39 +0100 Subject: [PATCH 57/60] Move start, stop and close to transport specific sockettransport --- json_rpc/server.nim | 15 --------------- json_rpc/sockettransport.nim | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/json_rpc/server.nim b/json_rpc/server.nim index a564957..98d8b12 100644 --- a/json_rpc/server.nim +++ b/json_rpc/server.nim @@ -134,21 +134,6 @@ proc processMessages*[T](server: RpcServer[T], line: string): Future[string] {.a error = wrapError(METHOD_NOT_FOUND, "Method not found", id, methodNotFound) result = $wrapReply(id, newJNull(), error) -proc start*(server: RpcServer) = - ## Start the RPC server. - for item in server.servers: - item.start() - -proc stop*(server: RpcServer) = - ## Stop the RPC server. - for item in server.servers: - item.stop() - -proc close*(server: RpcServer) = - ## Cleanup resources of RPC server. - for item in server.servers: - item.close() - # Server registration proc register*(server: RpcServer, name: string, rpc: RpcProc) = diff --git a/json_rpc/sockettransport.nim b/json_rpc/sockettransport.nim index 9409eb4..54c6be2 100644 --- a/json_rpc/sockettransport.nim +++ b/json_rpc/sockettransport.nim @@ -140,3 +140,17 @@ proc newRpcStreamServer*(address = "localhost", port: Port = Port(8545)): RpcStr result = newRpcServer[StreamServer]() result.addStreamServer(address, port) +proc start*(server: RpcStreamServer) = + ## Start the RPC server. + for item in server.servers: + item.start() + +proc stop*(server: RpcStreamServer) = + ## Stop the RPC server. + for item in server.servers: + item.stop() + +proc close*(server: RpcStreamServer) = + ## Cleanup resources of RPC server. + for item in server.servers: + item.close() \ No newline at end of file From f60a648968d51d4d2f173516bcdb64952d90bee3 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 10 Jul 2018 10:39:09 +0100 Subject: [PATCH 58/60] Move remaining rpc routing to router.nim --- json_rpc/router.nim | 135 ++++++++++++++++++++++++++++++----- json_rpc/server.nim | 118 ++---------------------------- json_rpc/sockettransport.nim | 4 +- 3 files changed, 122 insertions(+), 135 deletions(-) diff --git a/json_rpc/router.nim b/json_rpc/router.nim index 7ac0614..0902489 100644 --- a/json_rpc/router.nim +++ b/json_rpc/router.nim @@ -1,16 +1,53 @@ -import json, tables, asyncdispatch2, jsonmarshal, strutils, macros +import + json, tables, asyncdispatch2, jsonmarshal, strutils, macros, + chronicles, options export asyncdispatch2, json, jsonmarshal type + RpcJsonError* = enum rjeInvalidJson, rjeVersionError, rjeNoMethod, rjeNoId, rjeNoParams + RpcJsonErrorContainer* = tuple[err: RpcJsonError, msg: string] + # Procedure signature accepted as an RPC call by server RpcProc* = proc(input: JsonNode): Future[JsonNode] + + RpcProcError* = ref object of Exception + code*: int + data*: JsonNode + + RpcBindError* = object of Exception + RpcAddressUnresolvableError* = object of Exception RpcRouter* = object procs*: TableRef[string, RpcProc] - + const methodField = "method" paramsField = "params" + jsonRpcField = "jsonrpc" + idField = "id" + resultField = "result" + errorField = "error" + codeField = "code" + messageField = "message" + dataField = "data" + messageTerminator = "\c\l" + + JSON_PARSE_ERROR* = -32700 + INVALID_REQUEST* = -32600 + METHOD_NOT_FOUND* = -32601 + INVALID_PARAMS* = -32602 + INTERNAL_ERROR* = -32603 + SERVER_ERROR* = -32000 + + defaultMaxRequestLength* = 1024 * 128 + jsonErrorMessages*: array[RpcJsonError, (int, string)] = + [ + (JSON_PARSE_ERROR, "Invalid JSON"), + (INVALID_REQUEST, "JSON 2.0 required"), + (INVALID_REQUEST, "No method requested"), + (INVALID_REQUEST, "No id specified"), + (INVALID_PARAMS, "No parameters specified") + ] proc newRpcRouter*: RpcRouter = result.procs = newTable[string, RpcProc]() @@ -24,28 +61,88 @@ proc hasMethod*(router: RpcRouter, methodName: string): bool = router.procs.hasK template isEmpty(node: JsonNode): bool = node.isNil or node.kind == JNull -proc route*(router: RpcRouter, data: JsonNode): Future[JsonNode] {.async, gcsafe.} = - ## Route to RPC, raises exceptions on missing data - let jPath = data{methodField} - if jPath.isEmpty: - raise newException(ValueError, "No " & methodField & " field found") +# Json state checking - let jParams = data{paramsField} - if jParams.isEmpty: - raise newException(ValueError, "No " & paramsField & " field found") +template jsonValid*(jsonString: string, node: var JsonNode): (bool, string) = + var + valid = true + msg = "" + try: node = parseJson(line) + except: + valid = false + msg = getCurrentExceptionMsg() + debug "Cannot process json", json = jsonString, msg = msg + (valid, msg) - let - path = jPath.getStr - rpc = router.procs.getOrDefault(path) - # TODO: not GC-safe as it accesses 'rpc' which is a global using GC'ed memory! - if rpc != nil: - result = await rpc(jParams) +proc checkJsonState*(line: string, + node: var JsonNode): Option[RpcJsonErrorContainer] = + ## Tries parsing line into node, if successful checks required fields + ## Returns: error state or none + let res = jsonValid(line, node) + if not res[0]: + return some((rjeInvalidJson, res[1])) + if not node.hasKey(idField): + return some((rjeNoId, "")) + let jVer = node{jsonRpcField} + if jVer != nil and jVer.kind != JNull and jVer != %"2.0": + return some((rjeVersionError, "")) + if not node.hasKey(methodField): + return some((rjeNoMethod, "")) + if not node.hasKey(paramsField): + return some((rjeNoParams, "")) + return none(RpcJsonErrorContainer) + +# Json reply wrappers + +proc wrapReply*(id: JsonNode, value: JsonNode, error: JsonNode): JsonNode = + let node = %{jsonRpcField: %"2.0", resultField: value, errorField: error, idField: id} + return node + +proc wrapError*(code: int, msg: string, id: JsonNode, + data: JsonNode = newJNull()): JsonNode = + # Create standardised error json + result = %{codeField: %(code), idField: id, messageField: %msg, dataField: data} + debug "Error generated", error = result, id = id + +proc route*(router: RpcRouter, data: string): Future[string] {.async, gcsafe.} = + ## Route to RPC, returns Json string of RPC result or error node + var + node: JsonNode + # parse json node and/or flag missing fields and errors + jsonErrorState = checkJsonState(data, node) + + if jsonErrorState.isSome: + let errState = jsonErrorState.get + var id = + if errState.err == rjeInvalidJson or errState.err == rjeNoId: + newJNull() + else: + node["id"] + let + errMsg = jsonErrorMessages[errState.err] + res = $wrapError(code = errMsg[0], msg = errMsg[1], id = id) & messageTerminator + # return error state as json + result = res else: - raise newException(ValueError, "Method \"" & path & "\" not found") + let + methodName = node[methodField].str + id = node[idField] + rpcProc = router.procs.getOrDefault(methodName) + + if rpcProc.isNil: + let + methodNotFound = %(methodName & " is not a registered RPC method.") + error = wrapError(METHOD_NOT_FOUND, "Method not found", id, methodNotFound) + result = $wrapReply(id, newJNull(), error) & messageTerminator + else: + let + jParams = node[paramsField] + res = await rpcProc(jParams) + result = $wrapReply(id, res, newJNull()) & messageTerminator proc ifRoute*(router: RpcRouter, data: JsonNode, fut: var Future[JsonNode]): bool = - ## Route to RPC, returns false if the method or params cannot be found - # TODO: This is already checked in processMessages, but allows safer use externally + ## Route to RPC, returns false if the method or params cannot be found. + ## Expects json input and returns json output. let jPath = data{methodField} jParams = data{paramsField} diff --git a/json_rpc/server.nim b/json_rpc/server.nim index 98d8b12..6ef2720 100644 --- a/json_rpc/server.nim +++ b/json_rpc/server.nim @@ -1,138 +1,28 @@ -import json, tables, options, macros, chronicles +import json, tables, options, macros import asyncdispatch2, router import jsonmarshal export asyncdispatch2, json, jsonmarshal, router -logScope: - topics = "RpcServer" - type - RpcJsonError* = enum rjeInvalidJson, rjeVersionError, rjeNoMethod, rjeNoId, rjeNoParams - - RpcJsonErrorContainer* = tuple[err: RpcJsonError, msg: string] - RpcServer*[S] = ref object servers*: seq[S] router*: RpcRouter - RpcProcError* = ref object of Exception - code*: int - data*: JsonNode - - RpcBindError* = object of Exception - RpcAddressUnresolvableError* = object of Exception - -const - JSON_PARSE_ERROR* = -32700 - INVALID_REQUEST* = -32600 - METHOD_NOT_FOUND* = -32601 - INVALID_PARAMS* = -32602 - INTERNAL_ERROR* = -32603 - SERVER_ERROR* = -32000 - - defaultMaxRequestLength* = 1024 * 128 - jsonErrorMessages*: array[RpcJsonError, (int, string)] = - [ - (JSON_PARSE_ERROR, "Invalid JSON"), - (INVALID_REQUEST, "JSON 2.0 required"), - (INVALID_REQUEST, "No method requested"), - (INVALID_REQUEST, "No id specified"), - (INVALID_PARAMS, "No parameters specified") - ] - proc newRpcServer*[S](): RpcServer[S] = new result result.router = newRpcRouter() result.servers = @[] -# Utility functions -# TODO: Move outside server? -#func `%`*(p: Port): JsonNode = %(p.int) - template rpc*(server: RpcServer, path: string, body: untyped): untyped = server.router.rpc(path, body) template hasMethod*(server: RpcServer, methodName: string): bool = server.router.hasMethod(methodName) -# Json state checking +# Wrapper for message processing -template jsonValid*(jsonString: string, node: var JsonNode): (bool, string) = - var - valid = true - msg = "" - try: node = parseJson(line) - except: - valid = false - msg = getCurrentExceptionMsg() - debug "Cannot process json", json = jsonString, msg = msg - (valid, msg) - -proc checkJsonState*(line: string, - node: var JsonNode): Option[RpcJsonErrorContainer] = - ## Tries parsing line into node, if successful checks required fields - ## Returns: error state or none - let res = jsonValid(line, node) - if not res[0]: - return some((rjeInvalidJson, res[1])) - if not node.hasKey("id"): - return some((rjeNoId, "")) - let jVer = node{"jsonrpc"} - if jVer != nil and jVer.kind != JNull and jVer != %"2.0": - return some((rjeVersionError, "")) - if not node.hasKey("method"): - return some((rjeNoMethod, "")) - if not node.hasKey("params"): - return some((rjeNoParams, "")) - return none(RpcJsonErrorContainer) - -# Json reply wrappers - -proc wrapReply*(id: JsonNode, value: JsonNode, error: JsonNode): string = - let node = %{"jsonrpc": %"2.0", "result": value, "error": error, "id": id} - return $node & "\c\l" - -proc wrapError*(code: int, msg: string, id: JsonNode, - data: JsonNode = newJNull()): JsonNode = - # Create standardised error json - result = %{"code": %(code), "id": id, "message": %msg, "data": data} - debug "Error generated", error = result, id = id - -# Server message processing - -proc processMessages*[T](server: RpcServer[T], line: string): Future[string] {.async, gcsafe.} = - var - node: JsonNode - # parse json node and/or flag missing fields and errors - jsonErrorState = checkJsonState(line, node) - - if jsonErrorState.isSome: - let errState = jsonErrorState.get - var id = - if errState.err == rjeInvalidJson or errState.err == rjeNoId: - newJNull() - else: - node["id"] - let errMsg = jsonErrorMessages[errState.err] - # return error state as json - result = $wrapError( - code = errMsg[0], - msg = errMsg[1], - id = id) - else: - let - methodName = node["method"].str - id = node["id"] - var callRes: Future[JsonNode] - - if server.router.ifRoute(node, callRes): - let res = await callRes - result = $wrapReply(id, res, newJNull()) - else: - let - methodNotFound = %(methodName & " is not a registered method.") - error = wrapError(METHOD_NOT_FOUND, "Method not found", id, methodNotFound) - result = $wrapReply(id, newJNull(), error) +proc route*[T](server: RpcServer[T], line: string): Future[string] {.async, gcsafe.} = + result = await server.router.route(line) # Server registration diff --git a/json_rpc/sockettransport.nim b/json_rpc/sockettransport.nim index 54c6be2..26608fa 100644 --- a/json_rpc/sockettransport.nim +++ b/json_rpc/sockettransport.nim @@ -4,7 +4,7 @@ proc sendError*[T](transport: T, code: int, msg: string, id: JsonNode, data: JsonNode = newJNull()) {.async.} = ## Send error message to client let error = wrapError(code, msg, id, data) - var value = wrapReply(id, newJNull(), error) + var value = $wrapReply(id, newJNull(), error) result = transport.write(value) proc processClient(server: StreamServer, transport: StreamTransport) {.async, gcsafe.} = @@ -20,7 +20,7 @@ proc processClient(server: StreamServer, transport: StreamTransport) {.async, gc debug "Processing message", address = transport.remoteAddress(), line = value - let future = processMessages(rpc, value) + let future = rpc.route(value) yield future if future.failed: if future.readError of RpcProcError: From eb23c469490643d9a880bac92def5e1d8a9ba5cc Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 10 Jul 2018 16:07:47 +0100 Subject: [PATCH 59/60] Split route into json only and string version, fix lack of terminator --- json_rpc/router.nim | 47 +++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/json_rpc/router.nim b/json_rpc/router.nim index 0902489..a1f7cd2 100644 --- a/json_rpc/router.nim +++ b/json_rpc/router.nim @@ -8,7 +8,7 @@ type RpcJsonErrorContainer* = tuple[err: RpcJsonError, msg: string] # Procedure signature accepted as an RPC call by server - RpcProc* = proc(input: JsonNode): Future[JsonNode] + RpcProc* = proc(input: JsonNode): Future[JsonNode] {.gcsafe.} RpcProcError* = ref object of Exception code*: int @@ -95,8 +95,7 @@ proc checkJsonState*(line: string, # Json reply wrappers proc wrapReply*(id: JsonNode, value: JsonNode, error: JsonNode): JsonNode = - let node = %{jsonRpcField: %"2.0", resultField: value, errorField: error, idField: id} - return node + return %{jsonRpcField: %"2.0", resultField: value, errorField: error, idField: id} proc wrapError*(code: int, msg: string, id: JsonNode, data: JsonNode = newJNull()): JsonNode = @@ -104,8 +103,27 @@ proc wrapError*(code: int, msg: string, id: JsonNode, result = %{codeField: %(code), idField: id, messageField: %msg, dataField: data} debug "Error generated", error = result, id = id +proc route*(router: RpcRouter, node: JsonNode): Future[JsonNode] {.async, gcsafe.} = + ## Assumes correct setup of node + let + methodName = node[methodField].str + id = node[idField] + rpcProc = router.procs.getOrDefault(methodName) + + if rpcProc.isNil: + let + methodNotFound = %(methodName & " is not a registered RPC method.") + error = wrapError(METHOD_NOT_FOUND, "Method not found", id, methodNotFound) + result = wrapReply(id, newJNull(), error) + else: + let + jParams = node[paramsField] + res = await rpcProc(jParams) + result = wrapReply(id, res, newJNull()) + proc route*(router: RpcRouter, data: string): Future[string] {.async, gcsafe.} = - ## Route to RPC, returns Json string of RPC result or error node + ## Route to RPC from string data. Data is expected to be able to be converted to Json. + ## Returns string of Json from RPC result/error node var node: JsonNode # parse json node and/or flag missing fields and errors @@ -120,25 +138,12 @@ proc route*(router: RpcRouter, data: string): Future[string] {.async, gcsafe.} = node["id"] let errMsg = jsonErrorMessages[errState.err] - res = $wrapError(code = errMsg[0], msg = errMsg[1], id = id) & messageTerminator + res = wrapError(code = errMsg[0], msg = errMsg[1], id = id) # return error state as json - result = res + result = $res & messageTerminator else: - let - methodName = node[methodField].str - id = node[idField] - rpcProc = router.procs.getOrDefault(methodName) - - if rpcProc.isNil: - let - methodNotFound = %(methodName & " is not a registered RPC method.") - error = wrapError(METHOD_NOT_FOUND, "Method not found", id, methodNotFound) - result = $wrapReply(id, newJNull(), error) & messageTerminator - else: - let - jParams = node[paramsField] - res = await rpcProc(jParams) - result = $wrapReply(id, res, newJNull()) & messageTerminator + let res = await router.route(node) + result = $res & messageTerminator proc ifRoute*(router: RpcRouter, data: JsonNode, fut: var Future[JsonNode]): bool = ## Route to RPC, returns false if the method or params cannot be found. From ceec0e76906341fccf71613bae1ba79d9fb1ce69 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 10 Jul 2018 16:51:26 +0100 Subject: [PATCH 60/60] Renamed ifRoute to tryRoute --- json_rpc/router.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/json_rpc/router.nim b/json_rpc/router.nim index a1f7cd2..bc811cb 100644 --- a/json_rpc/router.nim +++ b/json_rpc/router.nim @@ -145,7 +145,7 @@ proc route*(router: RpcRouter, data: string): Future[string] {.async, gcsafe.} = let res = await router.route(node) result = $res & messageTerminator -proc ifRoute*(router: RpcRouter, data: JsonNode, fut: var Future[JsonNode]): bool = +proc tryRoute*(router: RpcRouter, data: JsonNode, fut: var Future[JsonNode]): bool = ## Route to RPC, returns false if the method or params cannot be found. ## Expects json input and returns json output. let