From 8d8bd1172ff2bc3555589c4926cb1e3b9e73f3cb Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 11 Apr 2018 20:07:24 +0100 Subject: [PATCH 001/116] Raise more errors, id is counter, simplify generateCalls --- eth-rpc/client/clientdispatch.nim | 70 ++++++++++--------------------- 1 file changed, 23 insertions(+), 47 deletions(-) diff --git a/eth-rpc/client/clientdispatch.nim b/eth-rpc/client/clientdispatch.nim index a96765e..b266d06 100644 --- a/eth-rpc/client/clientdispatch.nim +++ b/eth-rpc/client/clientdispatch.nim @@ -6,29 +6,29 @@ type awaiting: Table[string, Future[Response]] address: string port: Port + nextId: int64 Response* = tuple[error: bool, result: JsonNode] + proc newRpcClient*(): RpcClient = ## Creates a new ``RpcClient`` instance. RpcClient( socket: newAsyncSocket(), - awaiting: initTable[string, Future[Response]]() + awaiting: initTable[string, Future[Response]](), + nextId: 1 ) proc call*(self: RpcClient, name: string, params: JsonNode): Future[Response] {.async.} = ## Remotely calls the specified RPC method. - # REVIEW: is there a reason why a simple counter is not used here? - # genOid takes CPU cycles and the output is larger - let id = $genOid() - let msg = %{"jsonrpc": %"2.0", "method": %name, "params": params, "id": %id} - # REVIEW: it would be more efficient if you append the terminating new line to - # the `msg` variable in-place. This way, a copy won't be performed most of the - # time because the string is likely to have 2 bytes of unused capacity. - await self.socket.send($msg & "\c\l") + let id = $self.nextId + self.nextId.inc + let msg = $ %{"jsonrpc": %"2.0", "method": %name, "params": params, "id": %id} & "\c\l" + await self.socket.send(msg) - # Completed by processMessage. + # completed by processMessage. var newFut = newFuture[Response]() - self.awaiting[id] = newFut # add to awaiting responses + # add to awaiting responses + self.awaiting[id] = newFut result = await newFut proc isNull(node: JsonNode): bool = node.kind == JNull @@ -36,23 +36,18 @@ proc isNull(node: JsonNode): bool = node.kind == JNull proc processMessage(self: RpcClient, line: string) = let node = parseJson(line) - # REVIEW: These shouldn't be just asserts. You cannot count - # that the other side implements the protocol correctly, so - # you must perform validation even in release builds. - assert node.hasKey("jsonrpc") - assert node["jsonrpc"].str == "2.0" - assert node.hasKey("id") - assert self.awaiting.hasKey(node["id"].str) + # TODO: Use more appropriate exception objects + if not node.hasKey("jsonrpc"): raise newException(ValueError, "Message is missing rpc version field") + elif node["jsonrpc"].str != "2.0": raise newException(ValueError, "Unsupported version of JSON, expected 2.0, received \"" & node["jsonrpc"].str & "\"") + elif not node.hasKey("id"): raise newException(ValueError, "Message is missing id field") + elif not self.awaiting.hasKey(node["id"].str): raise newException(ValueError, "Cannot find message id \"" & node["id"].str & "\"") if node["error"].kind == JNull: self.awaiting[node["id"].str].complete((false, node["result"])) self.awaiting.del(node["id"].str) else: - # If the node id is null, we cannot complete the future. - if not node["id"].isNull: - self.awaiting[node["id"].str].complete((true, node["error"])) - # TODO: Safe to delete here? - self.awaiting.del(node["id"].str) + self.awaiting[node["id"].str].complete((true, node["error"])) + self.awaiting.del(node["id"].str) proc connect*(self: RpcClient, address: string, port: Port): Future[void] @@ -77,20 +72,6 @@ proc connect*(self: RpcClient, address: string, port: Port) {.async.} = self.port = port asyncCheck processData(self) -proc makeTemplate(name: string, params: NimNode, body: NimNode, starred: bool): NimNode = - # set up template AST - result = newNimNode(nnkTemplateDef) - if starred: result.add postFix(ident(name), "*") - else: result.add ident(name) - result.add newEmptyNode(), newEmptyNode(), params, newEmptyNode(), newEmptyNode(), body - -proc appendFormalParam(formalParams: NimNode, identName, typeName: string) = - # set up formal params AST - formalParams.expectKind(nnkFormalParams) - if formalParams.len == 0: formalParams.add newEmptyNode() - var identDef = newIdentDefs(ident(identName), ident(typeName)) - formalParams.add identDef - macro generateCalls: untyped = ## Generate templates for client calls so that: ## client.call("web3_clientVersion", params) @@ -98,16 +79,11 @@ macro generateCalls: untyped = ## client.web3_clientVersion(params) result = newStmtList() for callName in ETHEREUM_RPC_CALLS: - # REVIEW: `macros.quote` would have worked well here to make the code easier to understand/maintain - var - params = newNimNode(nnkFormalParams) - call = newCall(newDotExpr(ident("client"), ident("call")), newStrLitNode(callName), ident("params")) - body = newStmtList().add call - templ = makeTemplate(callName, params, body, true) - params.add newNimNode(nnkBracketExpr).add(ident("Future"), ident("Response")) - params.appendFormalParam("client", "RpcClient") - params.appendFormalParam("params", "JsonNode") - result.add templ + let nameLit = ident(callName) + result.add(quote do: + template `nameLit`*(client: RpcClient, params: JsonNode): Future[Response] = client.call(`callName`, params) + ) + echo result.repr # generate all client ethereum rpc calls generateCalls() From af45820594d38f76c7a1c1d150283645b88d829b Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 11 Apr 2018 20:08:12 +0100 Subject: [PATCH 002/116] Make rpc transform use async --- eth-rpc/server/rpcregistration.nim | 18 ++++++++++++++---- eth-rpc/server/serverdispatch.nim | 13 ++++++------- eth-rpc/server/servertypes.nim | 2 +- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/eth-rpc/server/rpcregistration.nim b/eth-rpc/server/rpcregistration.nim index 28285f9..1f9fb6e 100644 --- a/eth-rpc/server/rpcregistration.nim +++ b/eth-rpc/server/rpcregistration.nim @@ -1,10 +1,14 @@ -import macros, servertypes +import macros, servertypes, strutils var rpcCallRefs {.compiletime.} = newSeq[(string)]() macro rpc*(prc: untyped): untyped = - # REVIEW: (IMPORTANT) I think the rpc procs should be async. - # they may need to call into other async procs of the VM + ## Converts a procedure into the following format: + ## *(params: JsonNode): Future[JsonNode] {.async.} + ## This procedure is then added into a compile-time list + ## so that it is automatically registered for every server that + ## calls registerRpcs(server) + prc.expectKind nnkProcDef result = prc let params = prc.findChild(it.kind == nnkFormalParams) @@ -20,8 +24,13 @@ macro rpc*(prc: untyped): untyped = identDefs.add ident("params"), ident("JsonNode"), newEmptyNode() # check there isn't already a result type assert params.len == 1 and params[0].kind == nnkEmpty - params[0] = ident("JsonNode") + params[0] = newNimNode(nnkBracketExpr) + params[0].add ident("Future"), ident("JsonNode") params.add identDefs + # add async pragma, we can assume there isn't an existing .async. + # as this would fail the result check above. + prc.addPragma(newIdentNode("async")) + # Adds to compiletime list of rpc calls so we can register them in bulk # for multiple servers using `registerRpcs`. rpcCallRefs.add $procName @@ -33,3 +42,4 @@ macro registerRpcs*(server: RpcServer): untyped = for procName in rpcCallRefs: let de = newDotExpr(ident($server), ident("register")) result.add(newCall(de, newStrLitNode(procName), ident(procName))) + echo result.repr diff --git a/eth-rpc/server/serverdispatch.nim b/eth-rpc/server/serverdispatch.nim index 020fd30..54ce450 100644 --- a/eth-rpc/server/serverdispatch.nim +++ b/eth-rpc/server/serverdispatch.nim @@ -20,7 +20,7 @@ proc processMessage(server: RpcServer, client: AsyncSocket, line: string) {.asyn if not server.procs.hasKey(methodName): await client.sendError(METHOD_NOT_FOUND, "Method not found", id, %(methodName & " is not a registered method.")) else: - let callRes = server.procs[methodName](node["params"]) + let callRes = await server.procs[methodName](node["params"]) await client.send($wrapReply(id, callRes, newJNull()) & "\c\l") proc processClient(server: RpcServer, client: AsyncSocket) {.async.} = @@ -33,12 +33,11 @@ proc processClient(server: RpcServer, client: AsyncSocket) {.async.} = ifDebug: echo "Process client: ", server.port, ":" & line - let fut = processMessage(server, client, line) - await fut - if fut.failed: - if fut.readError of RpcProcError: - # TODO: Currently exceptions in rpc calls are not properly handled - let err = fut.readError.RpcProcError + let future = processMessage(server, client, line) + await future + if future.failed: + if future.readError of RpcProcError: + let err = future.readError.RpcProcError await client.sendError(err.code, err.msg, err.data) else: await client.sendError(-32000, "Error", %getCurrentExceptionMsg()) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 1fc1954..61ee711 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -1,7 +1,7 @@ import asyncdispatch, asyncnet, json, tables type - RpcProc* = proc (params: JsonNode): JsonNode + RpcProc* = proc (params: JsonNode): Future[JsonNode] RpcServer* = ref object socket*: AsyncSocket From 7e679f1aab0e13329a5fcbcdc5c03d28b6cf2e2b Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 11 Apr 2018 20:08:48 +0100 Subject: [PATCH 003/116] WIP dummy async RPC calls --- tests/testserverclient.nim | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index e11e577..5b1ad38 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -1,10 +1,13 @@ -import ../eth-rpc / rpcclient, ../eth-rpc / rpcserver, asyncdispatch, json, unittest +import ../eth-rpc / rpcclient, ../eth-rpc / rpcserver, asyncdispatch, json, unittest, ../eth-rpc/server/ethprocs # REVIEW: I'd like to see some dummy implementations of RPC calls handled in async fashion. +proc myProc {.rpc.} = + return %"hello" when isMainModule: # create on localhost, default port var srv = newRpcServer("") + registerEthereumRpcs(srv) asyncCheck srv.serve() suite "RPC": @@ -19,5 +22,8 @@ when isMainModule: test "SHA3": response = waitFor client.web3_sha3(%"abc") check response.result.getStr == "3A985DA74FE225B2045C172D6BD390BD855F086E3E9D525B46BFE24511431532" + response = waitFor client.call("myProc", %"abc") + echo response.result.getStr + waitFor main() From 04e5765a65b1b051242fdecfdbf94722bfaf32cc Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 12 Apr 2018 18:48:46 +0100 Subject: [PATCH 004/116] Migrate registration macros and ethprocs to servertypes --- eth-rpc/rpcserver.nim | 4 +- eth-rpc/server/ethprocs.nim | 117 ++++++++++++++--------------- eth-rpc/server/rpcregistration.nim | 45 ----------- eth-rpc/server/serverdispatch.nim | 3 +- eth-rpc/server/servertypes.nim | 70 +++++++++++++++-- 5 files changed, 123 insertions(+), 116 deletions(-) delete mode 100644 eth-rpc/server/rpcregistration.nim diff --git a/eth-rpc/rpcserver.nim b/eth-rpc/rpcserver.nim index 8724e94..ba40b91 100644 --- a/eth-rpc/rpcserver.nim +++ b/eth-rpc/rpcserver.nim @@ -1,2 +1,2 @@ -import server / [servertypes, rpcconsts, serverdispatch, rpcregistration] -export servertypes, rpcconsts, serverdispatch, rpcregistration +import server / [servertypes, rpcconsts, serverdispatch] +export servertypes, rpcconsts, serverdispatch diff --git a/eth-rpc/server/ethprocs.nim b/eth-rpc/server/ethprocs.nim index 629a42e..19a8e5e 100644 --- a/eth-rpc/server/ethprocs.nim +++ b/eth-rpc/server/ethprocs.nim @@ -1,185 +1,184 @@ -import servertypes, json, asyncdispatch, rpcregistration import cryptoutils -proc web3_clientVersion {.rpc.} = +proc web3_clientVersion* {.rpc.} = return %("Nimbus-RPC-Test") -proc web3_sha3 {.rpc.} = +proc web3_sha3* {.rpc.} = var data = params.getStr let kres = k256(data) return %kres -proc net_version {.rpc.} = +proc net_version* {.rpc.} = #[ See: https://github.com/ethereum/interfaces/issues/6 https://github.com/ethereum/EIPs/issues/611 ]# discard -proc net_listening {.rpc.} = +proc net_listening* {.rpc.} = return %"true" -proc net_peerCount {.rpc.} = +proc net_peerCount* {.rpc.} = # TODO: Discovery integration discard -proc eth_protocolVersion {.rpc.} = +proc eth_protocolVersion* {.rpc.} = discard -proc eth_syncing {.rpc.} = +proc eth_syncing* {.rpc.} = discard -proc eth_coinbase {.rpc.} = +proc eth_coinbase* {.rpc.} = discard -proc eth_mining {.rpc.} = +proc eth_mining* {.rpc.} = discard -proc eth_hashrate {.rpc.} = +proc eth_hashrate* {.rpc.} = discard -proc eth_gasPrice {.rpc.} = +proc eth_gasPrice* {.rpc.} = discard -proc eth_accounts {.rpc.} = +proc eth_accounts* {.rpc.} = discard -proc eth_blockNumber {.rpc.} = +proc eth_blockNumber* {.rpc.} = discard -proc eth_getBalance {.rpc.} = +proc eth_getBalance* {.rpc.} = discard -proc eth_getStorageAt {.rpc.} = +proc eth_getStorageAt* {.rpc.} = discard -proc eth_getTransactionCount {.rpc.} = +proc eth_getTransactionCount* {.rpc.} = discard -proc eth_getBlockTransactionCountByHash {.rpc.} = +proc eth_getBlockTransactionCountByHash* {.rpc.} = discard -proc eth_getBlockTransactionCountByNumber {.rpc.} = +proc eth_getBlockTransactionCountByNumber* {.rpc.} = discard -proc eth_getUncleCountByBlockHash {.rpc.} = +proc eth_getUncleCountByBlockHash* {.rpc.} = discard -proc eth_getUncleCountByBlockNumber {.rpc.} = +proc eth_getUncleCountByBlockNumber* {.rpc.} = discard -proc eth_getCode {.rpc.} = +proc eth_getCode* {.rpc.} = discard -proc eth_sign {.rpc.} = +proc eth_sign* {.rpc.} = discard -proc eth_sendTransaction {.rpc.} = +proc eth_sendTransaction* {.rpc.} = discard -proc eth_sendRawTransaction {.rpc.} = +proc eth_sendRawTransaction* {.rpc.} = discard -proc eth_call {.rpc.} = +proc eth_call* {.rpc.} = discard -proc eth_estimateGas {.rpc.} = +proc eth_estimateGas* {.rpc.} = discard -proc eth_getBlockByHash {.rpc.} = +proc eth_getBlockByHash* {.rpc.} = discard -proc eth_getBlockByNumber {.rpc.} = +proc eth_getBlockByNumber* {.rpc.} = discard -proc eth_getTransactionByHash {.rpc.} = +proc eth_getTransactionByHash* {.rpc.} = discard -proc eth_getTransactionByBlockHashAndIndex {.rpc.} = +proc eth_getTransactionByBlockHashAndIndex* {.rpc.} = discard -proc eth_getTransactionByBlockNumberAndIndex {.rpc.} = +proc eth_getTransactionByBlockNumberAndIndex* {.rpc.} = discard -proc eth_getTransactionReceipt {.rpc.} = +proc eth_getTransactionReceipt* {.rpc.} = discard -proc eth_getUncleByBlockHashAndIndex {.rpc.} = +proc eth_getUncleByBlockHashAndIndex* {.rpc.} = discard -proc eth_getUncleByBlockNumberAndIndex {.rpc.} = +proc eth_getUncleByBlockNumberAndIndex* {.rpc.} = discard -proc eth_getCompilers {.rpc.} = +proc eth_getCompilers* {.rpc.} = discard -proc eth_compileLLL {.rpc.} = +proc eth_compileLLL* {.rpc.} = discard -proc eth_compileSolidity {.rpc.} = +proc eth_compileSolidity* {.rpc.} = discard -proc eth_compileSerpent {.rpc.} = +proc eth_compileSerpent* {.rpc.} = discard -proc eth_newFilter {.rpc.} = +proc eth_newFilter* {.rpc.} = discard -proc eth_newBlockFilter {.rpc.} = +proc eth_newBlockFilter* {.rpc.} = discard -proc eth_newPendingTransactionFilter {.rpc.} = +proc eth_newPendingTransactionFilter* {.rpc.} = discard -proc eth_uninstallFilter {.rpc.} = +proc eth_uninstallFilter* {.rpc.} = discard -proc eth_getFilterChanges {.rpc.} = +proc eth_getFilterChanges* {.rpc.} = discard -proc eth_getFilterLogs {.rpc.} = +proc eth_getFilterLogs* {.rpc.} = discard -proc eth_getLogs {.rpc.} = +proc eth_getLogs* {.rpc.} = discard -proc eth_getWork {.rpc.} = +proc eth_getWork* {.rpc.} = discard -proc eth_submitWork {.rpc.} = +proc eth_submitWork* {.rpc.} = discard -proc eth_submitHashrate {.rpc.} = +proc eth_submitHashrate* {.rpc.} = discard -proc shh_post {.rpc.} = +proc shh_post* {.rpc.} = discard -proc shh_version {.rpc.} = +proc shh_version* {.rpc.} = discard -proc shh_newIdentity {.rpc.} = +proc shh_newIdentity* {.rpc.} = discard -proc shh_hasIdentity {.rpc.} = +proc shh_hasIdentity* {.rpc.} = discard -proc shh_newGroup {.rpc.} = +proc shh_newGroup* {.rpc.} = discard -proc shh_addToGroup {.rpc.} = +proc shh_addToGroup* {.rpc.} = discard -proc shh_newFilter {.rpc.} = +proc shh_newFilter* {.rpc.} = discard -proc shh_uninstallFilter {.rpc.} = +proc shh_uninstallFilter* {.rpc.} = discard -proc shh_getFilterChanges {.rpc.} = +proc shh_getFilterChanges* {.rpc.} = discard -proc shh_getMessages {.rpc.} = +proc shh_getMessages* {.rpc.} = discard proc registerEthereumRpcs*(server: RpcServer) = diff --git a/eth-rpc/server/rpcregistration.nim b/eth-rpc/server/rpcregistration.nim deleted file mode 100644 index 1f9fb6e..0000000 --- a/eth-rpc/server/rpcregistration.nim +++ /dev/null @@ -1,45 +0,0 @@ -import macros, servertypes, strutils - -var rpcCallRefs {.compiletime.} = newSeq[(string)]() - -macro rpc*(prc: untyped): untyped = - ## Converts a procedure into the following format: - ## *(params: JsonNode): Future[JsonNode] {.async.} - ## This procedure is then added into a compile-time list - ## so that it is automatically registered for every server that - ## calls registerRpcs(server) - prc.expectKind nnkProcDef - result = prc - let - params = prc.findChild(it.kind == nnkFormalParams) - procName = prc.findChild(it.kind == nnkIdent) - - assert params != nil - procName.expectKind(nnkIdent) - for param in params.children: - if param.kind == nnkIdentDefs: - if param[1] == ident("JsonNode"): - return - var identDefs = newNimNode(nnkIdentDefs) - identDefs.add ident("params"), ident("JsonNode"), newEmptyNode() - # check there isn't already a result type - assert params.len == 1 and params[0].kind == nnkEmpty - params[0] = newNimNode(nnkBracketExpr) - params[0].add ident("Future"), ident("JsonNode") - params.add identDefs - # add async pragma, we can assume there isn't an existing .async. - # as this would fail the result check above. - prc.addPragma(newIdentNode("async")) - - # Adds to compiletime list of rpc calls so we can register them in bulk - # for multiple servers using `registerRpcs`. - rpcCallRefs.add $procName - -macro registerRpcs*(server: RpcServer): untyped = - ## Step through procs currently registered with {.rpc.} and add a register call for server - result = newNimNode(nnkStmtList) - result.add newCall(newDotExpr(ident($server), ident("unRegisterAll"))) - for procName in rpcCallRefs: - let de = newDotExpr(ident($server), ident("register")) - result.add(newCall(de, newStrLitNode(procName), ident(procName))) - echo result.repr diff --git a/eth-rpc/server/serverdispatch.nim b/eth-rpc/server/serverdispatch.nim index 54ce450..692f1f3 100644 --- a/eth-rpc/server/serverdispatch.nim +++ b/eth-rpc/server/serverdispatch.nim @@ -1,5 +1,5 @@ import asyncdispatch, asyncnet, json, tables, strutils, - servertypes, rpcconsts, private / [transportutils, debugutils], jsonutils, asyncutils, ethprocs, + servertypes, rpcconsts, private / [transportutils, debugutils], jsonutils, asyncutils, options proc processMessage(server: RpcServer, client: AsyncSocket, line: string) {.async.} = @@ -43,7 +43,6 @@ proc processClient(server: RpcServer, client: AsyncSocket) {.async.} = await client.sendError(-32000, "Error", %getCurrentExceptionMsg()) proc serve*(server: RpcServer) {.async.} = - server.registerEthereumRpcs server.socket.bindAddr(server.port, server.address) server.socket.listen() diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 61ee711..75c25b9 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -1,4 +1,4 @@ -import asyncdispatch, asyncnet, json, tables +import asyncdispatch, asyncnet, json, tables, macros, strutils type RpcProc* = proc (params: JsonNode): Future[JsonNode] @@ -13,15 +13,69 @@ type code*: int data*: JsonNode -proc newRpcServer*(address: string, port: Port = Port(8545)): RpcServer = - RpcServer( - socket: newAsyncSocket(), - port: port, - address: address, - procs: newTable[string, RpcProc]() - ) +var rpcCallRefs {.compiletime.} = newSeq[(string)]() proc register*(server: RpcServer, name: string, rpc: RpcProc) = server.procs[name] = rpc proc unRegisterAll*(server: RpcServer) = server.procs.clear + +macro rpc*(prc: untyped): untyped = + ## Converts a procedure into the following format: + ## *(params: JsonNode): Future[JsonNode] {.async.} + ## This procedure is then added into a compile-time list + ## so that it is automatically registered for every server that + ## calls registerRpcs(server) + prc.expectKind nnkProcDef + result = prc + let + params = prc.params + procName = prc.name + + procName.expectKind(nnkIdent) + + # check there isn't already a result type + assert params[0].kind == nnkEmpty + + # add parameter + params.add nnkIdentDefs.newTree( + newIdentNode("params"), + newIdentNode("JsonNode"), + newEmptyNode() + ) + # set result type + params[0] = nnkBracketExpr.newTree( + newIdentNode("Future"), + newIdentNode("JsonNode") + ) + # add async pragma; we can assume there isn't an existing .async. + # as this would mean there's a return type and fail the result check above. + prc.addPragma(newIdentNode("async")) + + # Adds to compiletime list of rpc calls so we can register them in bulk + # for multiple servers using `registerRpcs`. + rpcCallRefs.add $procName + +macro registerRpcs*(server: RpcServer): untyped = + ## Step through procs currently registered with {.rpc.} and add a register call for this server + result = newStmtList() + result.add(quote do: + `server`.unRegisterAll + ) + for procName in rpcCallRefs: + let i = ident(procName) + result.add(quote do: + `server`.register(`procName`, `i`) + ) + +include ethprocs # TODO: This isn't ideal as it means editing code in ethprocs shows errors + +proc newRpcServer*(address: string, port: Port = Port(8545)): RpcServer = + result = RpcServer( + socket: newAsyncSocket(), + port: port, + address: address, + procs: newTable[string, RpcProc]() + ) + result.registerRpcs + From 9983e385743d8a1b7a3ff0efb95a234e2e1393ca Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 12 Apr 2018 18:49:58 +0100 Subject: [PATCH 005/116] 'Fix' template issue reporting argument error, remove unused isNull proc --- eth-rpc/client/clientdispatch.nim | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/eth-rpc/client/clientdispatch.nim b/eth-rpc/client/clientdispatch.nim index b266d06..6ec195f 100644 --- a/eth-rpc/client/clientdispatch.nim +++ b/eth-rpc/client/clientdispatch.nim @@ -31,8 +31,6 @@ proc call*(self: RpcClient, name: string, params: JsonNode): Future[Response] {. self.awaiting[id] = newFut result = await newFut -proc isNull(node: JsonNode): bool = node.kind == JNull - proc processMessage(self: RpcClient, line: string) = let node = parseJson(line) @@ -81,9 +79,8 @@ macro generateCalls: untyped = for callName in ETHEREUM_RPC_CALLS: let nameLit = ident(callName) result.add(quote do: - template `nameLit`*(client: RpcClient, params: JsonNode): Future[Response] = client.call(`callName`, params) + proc `nameLit`*(client: RpcClient, params: JsonNode): Future[Response] {.inline.} = client.call(`callName`, params) # TODO: Back to template ) - echo result.repr # generate all client ethereum rpc calls generateCalls() From f0ad484c6f0ea8d2011690fedb0f139f3c936fbf Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 12 Apr 2018 18:51:06 +0100 Subject: [PATCH 006/116] Add some testing for custom rpc calls --- tests/testserverclient.nim | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index 5b1ad38..d356a27 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -1,15 +1,19 @@ -import ../eth-rpc / rpcclient, ../eth-rpc / rpcserver, asyncdispatch, json, unittest, ../eth-rpc/server/ethprocs +import ../eth-rpc / rpcclient, ../eth-rpc / rpcserver, + asyncdispatch, json, unittest, tables # REVIEW: I'd like to see some dummy implementations of RPC calls handled in async fashion. -proc myProc {.rpc.} = - return %"hello" +proc myProc* {.rpc.} = + # Custom async RPC call + return %"Hello" + +var srv = newRpcServer("") +# This is required to automatically register `myProc` to new servers +registerRpcs(srv) +# TODO: Avoid having to add procs twice, once for the ethprocs in newRpcServer, +# and again with the extra `myProc` rpc when isMainModule: # create on localhost, default port - var srv = newRpcServer("") - registerEthereumRpcs(srv) - asyncCheck srv.serve() - suite "RPC": proc main {.async.} = var client = newRpcClient() From e304abae2d08501d90f4cfa38fb1d4f8af546af1 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 12 Apr 2018 19:29:05 +0100 Subject: [PATCH 007/116] Fix wrong path --- tests/testutils.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testutils.nim b/tests/testutils.nim index d9a25b4..cb27129 100644 --- a/tests/testutils.nim +++ b/tests/testutils.nim @@ -1,4 +1,4 @@ -import strutils, eth-rpc/server/private/transportutils, unittest +import strutils, ../eth-rpc/server/private/transportutils, unittest suite "Encoding": test "Encode quantity": From d280dcf5aae3aec95ae6f45e122bc8dd2aeb4253 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 12 Apr 2018 19:29:48 +0100 Subject: [PATCH 008/116] Re-add serve --- tests/testserverclient.nim | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index d356a27..ff14f60 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -9,9 +9,9 @@ proc myProc* {.rpc.} = var srv = newRpcServer("") # This is required to automatically register `myProc` to new servers registerRpcs(srv) +asyncCheck srv.serve # TODO: Avoid having to add procs twice, once for the ethprocs in newRpcServer, # and again with the extra `myProc` rpc - when isMainModule: # create on localhost, default port suite "RPC": @@ -26,8 +26,10 @@ when isMainModule: test "SHA3": response = waitFor client.web3_sha3(%"abc") check response.result.getStr == "3A985DA74FE225B2045C172D6BD390BD855F086E3E9D525B46BFE24511431532" - response = waitFor client.call("myProc", %"abc") - echo response.result.getStr + test "Custom RPC": + response = waitFor client.call("myProc", %"abc") + check response.result.getStr == "Hello" waitFor main() + \ No newline at end of file From 93f93c72c9cdf576a992488fd49cc7c1ae75885f Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 12 Apr 2018 19:29:48 +0100 Subject: [PATCH 009/116] Re-add serve, add test for custom RPCs --- tests/testserverclient.nim | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index d356a27..ff14f60 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -9,9 +9,9 @@ proc myProc* {.rpc.} = var srv = newRpcServer("") # This is required to automatically register `myProc` to new servers registerRpcs(srv) +asyncCheck srv.serve # TODO: Avoid having to add procs twice, once for the ethprocs in newRpcServer, # and again with the extra `myProc` rpc - when isMainModule: # create on localhost, default port suite "RPC": @@ -26,8 +26,10 @@ when isMainModule: test "SHA3": response = waitFor client.web3_sha3(%"abc") check response.result.getStr == "3A985DA74FE225B2045C172D6BD390BD855F086E3E9D525B46BFE24511431532" - response = waitFor client.call("myProc", %"abc") - echo response.result.getStr + test "Custom RPC": + response = waitFor client.call("myProc", %"abc") + check response.result.getStr == "Hello" waitFor main() + \ No newline at end of file From 961702e8e280573aa79eaaa11457f10ad1ad5610 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Fri, 20 Apr 2018 21:19:08 +0100 Subject: [PATCH 010/116] Prototype 'on' transformation - work in progress --- eth-rpc/server/servertypes.nim | 93 +++++++++++++++++++++++++--------- 1 file changed, 70 insertions(+), 23 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 75c25b9..c51b48a 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -13,8 +13,6 @@ type code*: int data*: JsonNode -var rpcCallRefs {.compiletime.} = newSeq[(string)]() - proc register*(server: RpcServer, name: string, rpc: RpcProc) = server.procs[name] = rpc @@ -52,30 +50,79 @@ macro rpc*(prc: untyped): untyped = # as this would mean there's a return type and fail the result check above. prc.addPragma(newIdentNode("async")) - # Adds to compiletime list of rpc calls so we can register them in bulk - # for multiple servers using `registerRpcs`. - rpcCallRefs.add $procName - -macro registerRpcs*(server: RpcServer): untyped = - ## Step through procs currently registered with {.rpc.} and add a register call for this server - result = newStmtList() - result.add(quote do: - `server`.unRegisterAll - ) - for procName in rpcCallRefs: - let i = ident(procName) - result.add(quote do: - `server`.register(`procName`, `i`) - ) - -include ethprocs # TODO: This isn't ideal as it means editing code in ethprocs shows errors - -proc newRpcServer*(address: string, port: Port = Port(8545)): RpcServer = +proc newRpcServer*(address = "localhost", port: Port = Port(8545)): RpcServer = result = RpcServer( socket: newAsyncSocket(), port: port, address: address, procs: newTable[string, RpcProc]() ) - result.registerRpcs - + +var sharedServer: RpcServer + +proc sharedRpcServer*(): RpcServer = + if sharedServer.isNil: sharedServer = newRpcServer("") + result = sharedServer + +macro multiRemove(s: string, values: varargs[string]): untyped = + ## wrapper for multiReplace + result = newStmtList() + var call = newNimNode(nnkCall) + call.add(ident"multiReplace") + call.add(s) + for item in values: + let sItem = $item + # generate tuples with empty strings + call.add(newPar(newStrLitNode(sItem), newStrLitNode(""))) + result.add call + +macro on*(server: var RpcServer, path: string, body: untyped): untyped = + var paramTemplates = newStmtList() + # process parameters of body into templates + let parameters = body.findChild(it.kind == nnkFormalParams) + if not parameters.isNil: + # marshall result to json + var resType = parameters[0] + if resType.kind != nnkEmpty: + # TODO: transform result type and/or return to json + discard + # convert input parameters to json fetch templates + for i in 1.. int: + echo "hello2" + s.on("the/path3"): + echo "hello3" + assert s.procs.hasKey("the/path") + assert s.procs.hasKey("the/path2") + assert s.procs.hasKey("the/path3") From edf214bc4ff99e8444fd8766a8c5357935c9a90a Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 24 Apr 2018 13:41:59 +0100 Subject: [PATCH 011/116] `on` macro now fetches from params as array as per ethereum rpc standard --- eth-rpc/server/servertypes.nim | 45 ++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index c51b48a..edba4a6 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -1,4 +1,5 @@ import asyncdispatch, asyncnet, json, tables, macros, strutils +export asyncdispatch, asyncnet, json type RpcProc* = proc (params: JsonNode): Future[JsonNode] @@ -78,51 +79,65 @@ macro multiRemove(s: string, values: varargs[string]): untyped = macro on*(server: var RpcServer, path: string, body: untyped): untyped = var paramTemplates = newStmtList() - # process parameters of body into templates let parameters = body.findChild(it.kind == nnkFormalParams) if not parameters.isNil: - # marshall result to json + # process parameters of body into json fetch templates var resType = parameters[0] if resType.kind != nnkEmpty: # TODO: transform result type and/or return to json discard - # convert input parameters to json fetch templates + # for i in 1.. int: echo "hello2" s.on("the/path3"): echo "hello3" - assert s.procs.hasKey("the/path") - assert s.procs.hasKey("the/path2") - assert s.procs.hasKey("the/path3") + result = %1 + suite "Server types": + test "On macro registration": + check s.procs.hasKey("the/path") + check s.procs.hasKey("the/path2") + check s.procs.hasKey("the/path3") From ae2a28ac8ce92e043856e00c14b0b1b67e6c501c Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 24 Apr 2018 13:42:39 +0100 Subject: [PATCH 012/116] Updated test rpc calls to use new `on` macro --- eth-rpc/server/ethprocs.nim | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/eth-rpc/server/ethprocs.nim b/eth-rpc/server/ethprocs.nim index 19a8e5e..eed5af9 100644 --- a/eth-rpc/server/ethprocs.nim +++ b/eth-rpc/server/ethprocs.nim @@ -1,12 +1,13 @@ -import cryptoutils +import servertypes, cryptoutils, json, macros -proc web3_clientVersion* {.rpc.} = - return %("Nimbus-RPC-Test") +var s = sharedRpcServer() -proc web3_sha3* {.rpc.} = - var data = params.getStr - let kres = k256(data) - return %kres +s.on("web3_clientVersion"): + result = %"Nimbus-RPC-Test" + +s.on("web3_sha3") do(input: string): + let kres = k256(input) + result = %kres proc net_version* {.rpc.} = #[ See: @@ -181,7 +182,3 @@ proc shh_getFilterChanges* {.rpc.} = proc shh_getMessages* {.rpc.} = discard -proc registerEthereumRpcs*(server: RpcServer) = - ## Register all ethereum rpc calls to the server - # TODO: Automate this - registerRpcs(server) From d8fa488e24259ac4f4208bc4a69c08524fbbd90c Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 24 Apr 2018 13:43:04 +0100 Subject: [PATCH 013/116] Updated test to use new `on` macro --- tests/testserverclient.nim | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index ff14f60..55ba552 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -1,19 +1,18 @@ import ../eth-rpc / rpcclient, ../eth-rpc / rpcserver, - asyncdispatch, json, unittest, tables + asyncdispatch, json, unittest, tables, ../eth-rpc / server / ethprocs -# REVIEW: I'd like to see some dummy implementations of RPC calls handled in async fashion. -proc myProc* {.rpc.} = +# TODO: dummy implementations of RPC calls handled in async fashion. +var srv = sharedRpcServer() +srv.address = "localhost" +srv.port = Port(8545) + +srv.on("myProc") do(input: string): # Custom async RPC call - return %"Hello" + result = %("Hello " & input) -var srv = newRpcServer("") -# This is required to automatically register `myProc` to new servers -registerRpcs(srv) asyncCheck srv.serve -# TODO: Avoid having to add procs twice, once for the ethprocs in newRpcServer, -# and again with the extra `myProc` rpc + when isMainModule: - # create on localhost, default port suite "RPC": proc main {.async.} = var client = newRpcClient() @@ -24,11 +23,11 @@ when isMainModule: response = waitFor client.web3_clientVersion(newJNull()) check response.result == %"Nimbus-RPC-Test" test "SHA3": - response = waitFor client.web3_sha3(%"abc") + response = waitFor client.web3_sha3(%["abc"]) check response.result.getStr == "3A985DA74FE225B2045C172D6BD390BD855F086E3E9D525B46BFE24511431532" test "Custom RPC": - response = waitFor client.call("myProc", %"abc") - check response.result.getStr == "Hello" + response = waitFor client.call("myProc", %["abc"]) + check response.result.getStr == "Hello abc" waitFor main() From 189c8c70a8ee9bd3152dc3fee185347e57520296 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 24 Apr 2018 13:50:56 +0100 Subject: [PATCH 014/116] Update tests to run with `nimble test` --- tests/testserverclient.nim | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index 55ba552..88b0886 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -12,23 +12,22 @@ srv.on("myProc") do(input: string): asyncCheck srv.serve -when isMainModule: - suite "RPC": - proc main {.async.} = - var client = newRpcClient() - await client.connect("localhost", Port(8545)) - var response: Response +suite "RPC": + proc main {.async.} = + var client = newRpcClient() + await client.connect("localhost", Port(8545)) + var response: Response - test "Version": - response = waitFor client.web3_clientVersion(newJNull()) - check response.result == %"Nimbus-RPC-Test" - test "SHA3": - response = waitFor client.web3_sha3(%["abc"]) - check response.result.getStr == "3A985DA74FE225B2045C172D6BD390BD855F086E3E9D525B46BFE24511431532" - test "Custom RPC": - response = waitFor client.call("myProc", %["abc"]) - check response.result.getStr == "Hello abc" - + test "Version": + response = waitFor client.web3_clientVersion(newJNull()) + check response.result == %"Nimbus-RPC-Test" + test "SHA3": + response = waitFor client.web3_sha3(%["abc"]) + check response.result.getStr == "3A985DA74FE225B2045C172D6BD390BD855F086E3E9D525B46BFE24511431532" + test "Custom RPC": + response = waitFor client.call("myProc", %["abc"]) + check response.result.getStr == "Hello abc" + - waitFor main() - \ No newline at end of file + waitFor main() + \ No newline at end of file From 6b8fb96de04385fb250f978ece886358a238c671 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 24 Apr 2018 13:52:34 +0100 Subject: [PATCH 015/116] Add configuration for appveyor --- .appveyor.yml | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .appveyor.yml diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 0000000..109e779 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,48 @@ +version: '{build}' + +cache: +- x86_64-4.9.2-release-win32-seh-rt_v4-rev4.7z +- i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z + +# We always want 32 and 64-bit compilation +matrix: + fast_finish: false # set this flag to immediately finish build once one of the jobs fails. + +environment: + + matrix: + - MINGW_DIR: mingw32 + MINGW_URL: https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Personal%20Builds/mingw-builds/4.9.2/threads-win32/dwarf/i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z/download + MINGW_ARCHIVE: i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z + platform: x86 + - MINGW_DIR: mingw64 + MINGW_URL: https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win64/Personal%20Builds/mingw-builds/4.9.2/threads-win32/seh/x86_64-4.9.2-release-win32-seh-rt_v4-rev4.7z/download + MINGW_ARCHIVE: x86_64-4.9.2-release-win32-seh-rt_v4-rev4.7z + platform: x64 + +install: + - setlocal EnableExtensions EnableDelayedExpansion + - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%" + - 7z x -y "%MINGW_ARCHIVE%" > nul + - SET PATH=%CD%\%MINGW_DIR%\bin;%CD%\Nim\bin;%PATH% + - git clone https://github.com/nim-lang/Nim.git + - cd %CD%\Nim + - git remote add statusim https://github.com/status-im/Nim.git + - git fetch statusim + - git config --global user.email "you@example.com" + - git config --global user.name "Your Name" + - for /f "tokens=*" %%G IN ('git branch -a --list ^"statusim/status-autopatch-*^"') DO (git merge %%G) + - git clone --depth 1 https://github.com/nim-lang/csources + - cd csources + - IF "%PLATFORM%" == "x64" ( build64.bat ) else ( build.bat ) + - cd .. + - bin\nim c koch + - koch boot -d:release + - koch nimble +build_script: + - cd C:\projects\%APPVEYOR_PROJECT_SLUG% + - nimble install -y +test_script: + - nimble test + +deploy: off \ No newline at end of file From 3999f64d92ffdffc97529859bc9af5d93dc9fe36 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 24 Apr 2018 14:01:28 +0100 Subject: [PATCH 016/116] Initialise `ethprocs` automatically on `import rpcserver` --- eth-rpc/rpcserver.nim | 4 ++-- tests/testserverclient.nim | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eth-rpc/rpcserver.nim b/eth-rpc/rpcserver.nim index ba40b91..14844f1 100644 --- a/eth-rpc/rpcserver.nim +++ b/eth-rpc/rpcserver.nim @@ -1,2 +1,2 @@ -import server / [servertypes, rpcconsts, serverdispatch] -export servertypes, rpcconsts, serverdispatch +import server / [servertypes, rpcconsts, serverdispatch, ethprocs] +export servertypes, rpcconsts, serverdispatch, ethprocs diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index 88b0886..558a90c 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -1,5 +1,5 @@ import ../eth-rpc / rpcclient, ../eth-rpc / rpcserver, - asyncdispatch, json, unittest, tables, ../eth-rpc / server / ethprocs + asyncdispatch, json, unittest, tables # TODO: dummy implementations of RPC calls handled in async fashion. var srv = sharedRpcServer() From 641ee92aa1697472e1bdda979d9246ded64524a8 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 24 Apr 2018 16:28:01 +0100 Subject: [PATCH 017/116] Updated multiRemove macro --- eth-rpc/server/servertypes.nim | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index edba4a6..8829701 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -66,16 +66,20 @@ proc sharedRpcServer*(): RpcServer = result = sharedServer macro multiRemove(s: string, values: varargs[string]): untyped = - ## wrapper for multiReplace - result = newStmtList() - var call = newNimNode(nnkCall) - call.add(ident"multiReplace") - call.add(s) + ## Wrapper for multiReplace + var + body = newStmtList() + multiReplaceCall = newCall(ident"multiReplace", s) + + body.add(newVarStmt(ident"eStr", newStrLitNode(""))) + let emptyStr = ident"eStr" for item in values: + # generate tuples of values with the empty string `eStr` let sItem = $item - # generate tuples with empty strings - call.add(newPar(newStrLitNode(sItem), newStrLitNode(""))) - result.add call + multiReplaceCall.add(newPar(newStrLitNode(sItem), emptyStr)) + + body.add multiReplaceCall + result = newBlockStmt(body) macro on*(server: var RpcServer, path: string, body: untyped): untyped = var paramTemplates = newStmtList() From 9674cac4711cad8e0eed1fc89e56591944c3c134 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 24 Apr 2018 17:37:01 +0100 Subject: [PATCH 018/116] Arrays now marshalled to native types and other params are prefetched --- eth-rpc/server/servertypes.nim | 93 ++++++++++++++++++++++++---------- 1 file changed, 65 insertions(+), 28 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 8829701..05c1f07 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -81,6 +81,15 @@ macro multiRemove(s: string, values: varargs[string]): untyped = body.add multiReplaceCall result = newBlockStmt(body) +proc jsonGetFunc(paramType: string): NimNode = + case paramType + of "string": result = ident"getStr" + of "int": result = ident"getInt" + of "float": result = ident"getFloat" + of "bool": result = ident"getBool" + of "byte": result = ident"getInt" + else: result = nil + macro on*(server: var RpcServer, path: string, body: untyped): untyped = var paramTemplates = newStmtList() let parameters = body.findChild(it.kind == nnkFormalParams) @@ -90,29 +99,49 @@ macro on*(server: var RpcServer, path: string, body: untyped): untyped = if resType.kind != nnkEmpty: # TODO: transform result type and/or return to json discard - # + + var paramsIdent = ident"params" + for i in 1.. `name`.len: + raise newException(ValueError, "Array longer than parameter allows. Expected " & $`arrayLen` & ", data length is " & $`paramsIdent`.len) + else: + for `idx` in 0 ..< `paramsIdent`.len: + `name`[`idx`] = `arrayType`(`paramsIdent`.elems[`idx`].`getFunc`) + ) + else: + # other types + var getFuncName = jsonGetFunc($paramType) + assert getFuncName != nil + # fetch parameter + let getFunc = newIdentNode($getFuncName) + paramTemplates.add(quote do: + var `name`: `paramType` = `paramsIdent`.elems[`pos`].`getFunc` + ) # create RPC proc let @@ -132,16 +161,24 @@ macro on*(server: var RpcServer, path: string, body: untyped): untyped = when isMainModule: import unittest var s = newRpcServer("localhost") - s.on("the/path") do(a: int, b: string): - var node = %"test" - result = node - s.on("the/path2") do() -> int: - echo "hello2" - s.on("the/path3"): + s.on("the/path1"): echo "hello3" result = %1 + s.on("the/path2") do() -> int: + echo "hello2" + s.on("the/path3") do(a: int, b: string): + var node = %"test" + result = node + s.on("the/path4") do(arr: array[6, byte], b: string): + var res = newJArray() + for item in arr: + res.add %int(item) + result = res suite "Server types": test "On macro registration": - check s.procs.hasKey("the/path") + check s.procs.hasKey("the/path1") check s.procs.hasKey("the/path2") check s.procs.hasKey("the/path3") + test "Processing arrays": + let r = waitfor thepath4(%[1, 2, 3]) + check r == %[1, 2, 3, 0, 0, 0] \ No newline at end of file From 7ba38414956e05c3aa1e771450fcde4e1aaa77ea Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 24 Apr 2018 17:37:31 +0100 Subject: [PATCH 019/116] Updated params to array for `eth_getBalance` --- eth-rpc/server/ethprocs.nim | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eth-rpc/server/ethprocs.nim b/eth-rpc/server/ethprocs.nim index eed5af9..e38c1c7 100644 --- a/eth-rpc/server/ethprocs.nim +++ b/eth-rpc/server/ethprocs.nim @@ -1,12 +1,12 @@ import servertypes, cryptoutils, json, macros -var s = sharedRpcServer() +var server = sharedRpcServer() -s.on("web3_clientVersion"): +server.on("web3_clientVersion"): result = %"Nimbus-RPC-Test" -s.on("web3_sha3") do(input: string): - let kres = k256(input) +server.on("web3_sha3") do(data: string): + let kres = k256(data) result = %kres proc net_version* {.rpc.} = @@ -47,7 +47,7 @@ proc eth_accounts* {.rpc.} = proc eth_blockNumber* {.rpc.} = discard -proc eth_getBalance* {.rpc.} = +server.on("eth_getBalance") do(data: array[20, byte], quantity: int): discard proc eth_getStorageAt* {.rpc.} = From e8a7f34289f6ab801db0e4727620da85e69d3ff6 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 24 Apr 2018 19:21:51 +0100 Subject: [PATCH 020/116] Added seq, fixed some wrong index bugs --- eth-rpc/server/servertypes.nim | 95 +++++++++++++++++++++++++--------- 1 file changed, 71 insertions(+), 24 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 05c1f07..d08ea3b 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -91,7 +91,9 @@ proc jsonGetFunc(paramType: string): NimNode = else: result = nil macro on*(server: var RpcServer, path: string, body: untyped): untyped = - var paramTemplates = newStmtList() + var + paramTemplates = newStmtList() + expectedParams = 0 let parameters = body.findChild(it.kind == nnkFormalParams) if not parameters.isNil: # process parameters of body into json fetch templates @@ -101,6 +103,7 @@ macro on*(server: var RpcServer, path: string, body: untyped): untyped = discard var paramsIdent = ident"params" + expectedParams = parameters.len - 1 for i in 1.. `name`.len: - raise newException(ValueError, "Array longer than parameter allows. Expected " & $`arrayLen` & ", data length is " & $`paramsIdent`.len) + raise newException(ValueError, "Array longer than parameter allows. Expected " & $`arrayLen` & ", data length is " & $`paramsIdent`.elems[`pos`].len) else: - for `idx` in 0 ..< `paramsIdent`.len: - `name`[`idx`] = `arrayType`(`paramsIdent`.elems[`idx`].`getFunc`) - ) + for `idx` in 0 ..< `paramsIdent`.elems[`pos`].len: + `name`[`idx`] = `arrayType`(`paramsIdent`.elems[`pos`].elems[`idx`].`getFunc`) + ) + else: + paramType.expectLen 2 + let + seqType = paramType[1] + getFunc = jsonGetFunc($seqType) + paramTemplates.add(quote do: + if `paramsIdent`.elems[`pos`].kind != JArray: + raise newException(ValueError, "Expected array but got " & $`paramsIdent`.elems[`pos`].kind) + var `name` = newSeq[`seqType`](`paramsIdent`.elems[`pos`].len) + for `idx` in 0 ..< `paramsIdent`.elems[`pos`].len: + `name`[`idx`] = `seqType`(`paramsIdent`.elems[`pos`].elems[`idx`].`getFunc`) + ) else: # other types var getFuncName = jsonGetFunc($paramType) @@ -142,7 +166,6 @@ macro on*(server: var RpcServer, path: string, body: untyped): untyped = paramTemplates.add(quote do: var `name`: `paramType` = `paramsIdent`.elems[`pos`].`getFunc` ) - # create RPC proc let pathStr = $path @@ -152,11 +175,20 @@ macro on*(server: var RpcServer, path: string, body: untyped): untyped = if body.kind == nnkStmtList: procBody = body else: procBody = body.body # + var checkTypeError: NimNode + if expectedParams > 0: + checkTypeError = quote do: + if `paramsIdent`.kind != JArray: + raise newException(ValueError, "Expected array but got " & $`paramsIdent`.kind) + else: checkTypeError = newStmtList() + result = quote do: proc `procName`*(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = + `checkTypeError` `paramTemplates` `procBody` `server`.register(`path`, `procName`) + echo result.repr when isMainModule: import unittest @@ -171,6 +203,13 @@ when isMainModule: result = node s.on("the/path4") do(arr: array[6, byte], b: string): var res = newJArray() + for item in arr: + res.add %int(item) + res.add %b + result = %res + s.on("the/path5") do(b: string, arr: seq[int]): + var res = newJArray() + res.add %b for item in arr: res.add %int(item) result = res @@ -179,6 +218,14 @@ when isMainModule: check s.procs.hasKey("the/path1") check s.procs.hasKey("the/path2") check s.procs.hasKey("the/path3") - test "Processing arrays": - let r = waitfor thepath4(%[1, 2, 3]) - check r == %[1, 2, 3, 0, 0, 0] \ No newline at end of file + test "Array/seq parameters": + let r1 = waitfor thepath4(%[%[1, 2, 3], %"hello"]) + var ckR1 = %[1, 2, 3, 0, 0, 0] + ckR1.elems.add %"hello" + check r1 == ckR1 + + let r2 = waitfor thepath5(%[%"abc", %[1, 2, 3, 4, 5]]) + var ckR2 = %["abc"] + for i in 0..4: ckR2.add %(i + 1) + check r2 == ckR2 + \ No newline at end of file From bbe89b5c59adc17a39bfbb2a361c47fff26534f1 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 24 Apr 2018 19:22:16 +0100 Subject: [PATCH 021/116] Updated test to return input data --- tests/testserverclient.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index 558a90c..8f55b33 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -6,9 +6,9 @@ var srv = sharedRpcServer() srv.address = "localhost" srv.port = Port(8545) -srv.on("myProc") do(input: string): +srv.on("myProc") do(input: string, data: array[0..3, int]): # Custom async RPC call - result = %("Hello " & input) + result = %("Hello " & input & " data: " & $data) asyncCheck srv.serve @@ -25,8 +25,8 @@ suite "RPC": response = waitFor client.web3_sha3(%["abc"]) check response.result.getStr == "3A985DA74FE225B2045C172D6BD390BD855F086E3E9D525B46BFE24511431532" test "Custom RPC": - response = waitFor client.call("myProc", %["abc"]) - check response.result.getStr == "Hello abc" + response = waitFor client.call("myProc", %[%"abc", %[1, 2, 3, 4]]) + check response.result.getStr == "Hello abc data: [1, 2, 3, 4]" waitFor main() From 46cb9e416b70325f00111368714313e5fa619c52 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 24 Apr 2018 19:38:25 +0100 Subject: [PATCH 022/116] Added some parameters to the procs, more to do --- eth-rpc/server/ethprocs.nim | 112 ++++++++++++++++++------------------ 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/eth-rpc/server/ethprocs.nim b/eth-rpc/server/ethprocs.nim index e38c1c7..4647c64 100644 --- a/eth-rpc/server/ethprocs.nim +++ b/eth-rpc/server/ethprocs.nim @@ -9,176 +9,176 @@ server.on("web3_sha3") do(data: string): let kres = k256(data) result = %kres -proc net_version* {.rpc.} = +server.on("net_version"): #[ See: https://github.com/ethereum/interfaces/issues/6 https://github.com/ethereum/EIPs/issues/611 ]# discard -proc net_listening* {.rpc.} = +server.on("net_listening"): return %"true" -proc net_peerCount* {.rpc.} = +server.on("net_peerCount"): # TODO: Discovery integration discard -proc eth_protocolVersion* {.rpc.} = +server.on("eth_protocolVersion"): discard -proc eth_syncing* {.rpc.} = +server.on("eth_syncing"): discard -proc eth_coinbase* {.rpc.} = +server.on("eth_coinbase"): discard -proc eth_mining* {.rpc.} = +server.on("eth_mining"): discard -proc eth_hashrate* {.rpc.} = +server.on("eth_hashrate"): discard -proc eth_gasPrice* {.rpc.} = +server.on("eth_gasPrice"): discard -proc eth_accounts* {.rpc.} = +server.on("eth_accounts"): discard -proc eth_blockNumber* {.rpc.} = +server.on("eth_blockNumber"): discard -server.on("eth_getBalance") do(data: array[20, byte], quantity: int): +server.on("eth_getBalance") do(data: array[20, byte], quantityTag: string): discard -proc eth_getStorageAt* {.rpc.} = +server.on("eth_getStorageAt") do(data: array[20, byte], quantity: int, quantityTag: string): discard -proc eth_getTransactionCount* {.rpc.} = +server.on("eth_getTransactionCount") do(data: array[20, byte], quantityTag: string): discard -proc eth_getBlockTransactionCountByHash* {.rpc.} = +server.on("eth_getBlockTransactionCountByHash") do(data: array[32, byte]): discard -proc eth_getBlockTransactionCountByNumber* {.rpc.} = +server.on("eth_getBlockTransactionCountByNumber") do(quantityTag: string): discard -proc eth_getUncleCountByBlockHash* {.rpc.} = +server.on("eth_getUncleCountByBlockHash") do(data: array[32, byte]): discard -proc eth_getUncleCountByBlockNumber* {.rpc.} = +server.on("eth_getUncleCountByBlockNumber") do(quantityTag: string): discard -proc eth_getCode* {.rpc.} = +server.on("eth_getCode") do(data: array[20, byte], quantityTag: string): discard -proc eth_sign* {.rpc.} = +server.on("eth_sign") do(data: array[20, byte], message: seq[byte]): discard -proc eth_sendTransaction* {.rpc.} = +server.on("eth_sendTransaction"): # TODO: Object discard -proc eth_sendRawTransaction* {.rpc.} = +server.on("eth_sendRawTransaction") do(data: string): # TODO: string or array of byte? discard -proc eth_call* {.rpc.} = +server.on("eth_call"): # TODO: Object discard -proc eth_estimateGas* {.rpc.} = +server.on("eth_estimateGas"): # TODO: Object discard -proc eth_getBlockByHash* {.rpc.} = +server.on("eth_getBlockByHash"): discard -proc eth_getBlockByNumber* {.rpc.} = +server.on("eth_getBlockByNumber"): discard -proc eth_getTransactionByHash* {.rpc.} = +server.on("eth_getTransactionByHash"): discard -proc eth_getTransactionByBlockHashAndIndex* {.rpc.} = +server.on("eth_getTransactionByBlockHashAndIndex"): discard -proc eth_getTransactionByBlockNumberAndIndex* {.rpc.} = +server.on("eth_getTransactionByBlockNumberAndIndex"): discard -proc eth_getTransactionReceipt* {.rpc.} = +server.on("eth_getTransactionReceipt"): discard -proc eth_getUncleByBlockHashAndIndex* {.rpc.} = +server.on("eth_getUncleByBlockHashAndIndex"): discard -proc eth_getUncleByBlockNumberAndIndex* {.rpc.} = +server.on("eth_getUncleByBlockNumberAndIndex"): discard -proc eth_getCompilers* {.rpc.} = +server.on("eth_getCompilers"): discard -proc eth_compileLLL* {.rpc.} = +server.on("eth_compileLLL"): discard -proc eth_compileSolidity* {.rpc.} = +server.on("eth_compileSolidity"): discard -proc eth_compileSerpent* {.rpc.} = +server.on("eth_compileSerpent"): discard -proc eth_newFilter* {.rpc.} = +server.on("eth_newFilter"): discard -proc eth_newBlockFilter* {.rpc.} = +server.on("eth_newBlockFilter"): discard -proc eth_newPendingTransactionFilter* {.rpc.} = +server.on("eth_newPendingTransactionFilter"): discard -proc eth_uninstallFilter* {.rpc.} = +server.on("eth_uninstallFilter"): discard -proc eth_getFilterChanges* {.rpc.} = +server.on("eth_getFilterChanges"): discard -proc eth_getFilterLogs* {.rpc.} = +server.on("eth_getFilterLogs"): discard -proc eth_getLogs* {.rpc.} = +server.on("eth_getLogs"): discard -proc eth_getWork* {.rpc.} = +server.on("eth_getWork"): discard -proc eth_submitWork* {.rpc.} = +server.on("eth_submitWork"): discard -proc eth_submitHashrate* {.rpc.} = +server.on("eth_submitHashrate"): discard -proc shh_post* {.rpc.} = +server.on("shh_post"): discard -proc shh_version* {.rpc.} = +server.on("shh_version"): discard -proc shh_newIdentity* {.rpc.} = +server.on("shh_newIdentity"): discard -proc shh_hasIdentity* {.rpc.} = +server.on("shh_hasIdentity"): discard -proc shh_newGroup* {.rpc.} = +server.on("shh_newGroup"): discard -proc shh_addToGroup* {.rpc.} = +server.on("shh_addToGroup"): discard -proc shh_newFilter* {.rpc.} = +server.on("shh_newFilter"): discard -proc shh_uninstallFilter* {.rpc.} = +server.on("shh_uninstallFilter"): discard -proc shh_getFilterChanges* {.rpc.} = +server.on("shh_getFilterChanges"): discard -proc shh_getMessages* {.rpc.} = +server.on("shh_getMessages"): discard From b0cd557d3e1044467d8c12807eef67ca7c810356 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 25 Apr 2018 19:18:42 +0100 Subject: [PATCH 023/116] Reworked seq/array parameter construction --- eth-rpc/server/servertypes.nim | 92 ++++++++++++++++++++-------------- 1 file changed, 55 insertions(+), 37 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index d08ea3b..92d921e 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -92,18 +92,24 @@ proc jsonGetFunc(paramType: string): NimNode = macro on*(server: var RpcServer, path: string, body: untyped): untyped = var - paramTemplates = newStmtList() + paramFetch = newStmtList() expectedParams = 0 let parameters = body.findChild(it.kind == nnkFormalParams) if not parameters.isNil: # process parameters of body into json fetch templates var resType = parameters[0] + if resType.kind != nnkEmpty: # TODO: transform result type and/or return to json discard var paramsIdent = ident"params" expectedParams = parameters.len - 1 + let expectedStr = "Expected " & $`expectedParams` & " Json parameter(s) but got " + paramFetch.add(quote do: + if `paramsIdent`.len != `expectedParams`: + raise newException(ValueError, `expectedStr` & $`paramsIdent`.len) + ) for i in 1.. `name`.len: - raise newException(ValueError, "Array longer than parameter allows. Expected " & $`arrayLen` & ", data length is " & $`paramsIdent`.elems[`pos`].len) - else: - for `idx` in 0 ..< `paramsIdent`.elems[`pos`].len: - `name`[`idx`] = `arrayType`(`paramsIdent`.elems[`pos`].elems[`idx`].`getFunc`) - ) - else: - paramType.expectLen 2 - let - seqType = paramType[1] - getFunc = jsonGetFunc($seqType) - paramTemplates.add(quote do: - if `paramsIdent`.elems[`pos`].kind != JArray: - raise newException(ValueError, "Expected array but got " & $`paramsIdent`.elems[`pos`].kind) - var `name` = newSeq[`seqType`](`paramsIdent`.elems[`pos`].len) - for `idx` in 0 ..< `paramsIdent`.elems[`pos`].len: - `name`[`idx`] = `seqType`(`paramsIdent`.elems[`pos`].elems[`idx`].`getFunc`) + # arrays can only be up to the defined length + # note that passing smaller arrays is still valid and are padded with zeros + checks.add(quote do: + if `paramsIdent`.elems[`pos`].len > `name`.len: + raise newException(ValueError, "Provided array is longer than parameter allows. Expected " & `arrayLenStr` & ", data length is " & $`paramsIdent`.elems[`pos`].len) ) + of ltSeq: + listType = paramType[1] + varDecl = quote do: + var `name` = newSeq[`listType`](`paramsIdent`.elems[`pos`].len) + + let + getFunc = jsonGetFunc($listType) + idx = ident"i" + listParse = quote do: + for `idx` in 0 ..< `paramsIdent`.elems[`pos`].len: + `name`[`idx`] = `listType`(`paramsIdent`.elems[`pos`].elems[`idx`].`getFunc`) + # assemble fetch parameters code + paramFetch.add(quote do: + `varDecl` + `checks` + `listParse` + ) else: # other types var getFuncName = jsonGetFunc($paramType) assert getFuncName != nil # fetch parameter let getFunc = newIdentNode($getFuncName) - paramTemplates.add(quote do: + paramFetch.add(quote do: var `name`: `paramType` = `paramsIdent`.elems[`pos`].`getFunc` ) # create RPC proc @@ -185,7 +200,7 @@ macro on*(server: var RpcServer, path: string, body: untyped): untyped = result = quote do: proc `procName`*(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = `checkTypeError` - `paramTemplates` + `paramFetch` `procBody` `server`.register(`path`, `procName`) echo result.repr @@ -201,7 +216,7 @@ when isMainModule: s.on("the/path3") do(a: int, b: string): var node = %"test" result = node - s.on("the/path4") do(arr: array[6, byte], b: string): + s.on("the/path4") do(arr: array[0..5, byte], b: string): var res = newJArray() for item in arr: res.add %int(item) @@ -228,4 +243,7 @@ when isMainModule: var ckR2 = %["abc"] for i in 0..4: ckR2.add %(i + 1) check r2 == ckR2 + test "Runtime errors": + expect ValueError: + let r1 = waitfor thepath4(%[%[1, 2, 3, 4, 5, 6, 7, 8, 9, 0], %"hello"]) \ No newline at end of file From 12bdedc3bf7c40ee9caf539d264db47cd44890a2 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 26 Apr 2018 19:32:14 +0100 Subject: [PATCH 024/116] Moved rpcclient/rpcserver to more accessible place --- eth-rpc/rpcclient.nim => rpcclient.nim | 0 eth-rpc/rpcserver.nim => rpcserver.nim | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename eth-rpc/rpcclient.nim => rpcclient.nim (100%) rename eth-rpc/rpcserver.nim => rpcserver.nim (100%) diff --git a/eth-rpc/rpcclient.nim b/rpcclient.nim similarity index 100% rename from eth-rpc/rpcclient.nim rename to rpcclient.nim diff --git a/eth-rpc/rpcserver.nim b/rpcserver.nim similarity index 100% rename from eth-rpc/rpcserver.nim rename to rpcserver.nim From 097129cc7531d0ef6420286d0a215fe4fc2cb118 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 26 Apr 2018 19:33:45 +0100 Subject: [PATCH 025/116] Stops errors when nimble test is using different backend --- eth_rpc.nimble | 1 + 1 file changed, 1 insertion(+) diff --git a/eth_rpc.nimble b/eth_rpc.nimble index 7455c6e..2ba4177 100644 --- a/eth_rpc.nimble +++ b/eth_rpc.nimble @@ -14,6 +14,7 @@ proc configForTests() = --debuginfo --path: "." --run + --forceBuild task test, "run tests": configForTests() From 117445858cecaecc2ab5456cd21badb334e9aec3 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 26 Apr 2018 19:35:57 +0100 Subject: [PATCH 026/116] Added object support (no nesting yet), also flag for dumping macro gen --- eth-rpc/server/servertypes.nim | 100 +++++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 23 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 92d921e..8428a77 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -90,6 +90,37 @@ proc jsonGetFunc(paramType: string): NimNode = of "byte": result = ident"getInt" else: result = nil +proc jsonCheckType(paramType: string): NimNode = + case paramType + of "string": result = ident"JString" + of "int": result = ident"JInt" + of "float": result = ident"JFloat" + of "bool": result = ident"JBool" + of "byte": result = ident"JInt" + else: result = nil + +# TODO: Nested complex fields in objects +# Probably going to need to make it recursive + +macro bindObj*(objInst: untyped, objType: typedesc, paramsArg: typed, elemIdx: int): untyped = + result = newNimNode(nnkStmtList) + let typeDesc = getType(getType(objType)[1]) + for field in typeDesc[2].children: + let + fieldStr = $field + fieldTypeStr = $field.getType() + getFunc = jsonGetFunc(fieldTypeStr) + expectedKind = fieldTypeStr.jsonCheckType + expectedStr = "Expected " & $expectedKind & " but got " + result.add(quote do: + let jParam = `paramsArg`.elems[`elemIdx`][`fieldStr`] + if jParam.kind != `expectedKind`: + raise newException(ValueError, `expectedStr` & $jParam.kind) + `objInst`.`field` = jParam.`getFunc` + ) + when defined(nimDumpRpcs): + echo "BindObj expansion: ", result.repr + macro on*(server: var RpcServer, path: string, body: untyped): untyped = var paramFetch = newStmtList() @@ -119,7 +150,6 @@ macro on*(server: var RpcServer, path: string, body: untyped): untyped = let name = parameters[i][0] var paramType = parameters[i][1] - # TODO: marshalling for object types # TODO: Replace exception with async error return values # Requires passing the server in local parameters to access the socket @@ -138,7 +168,8 @@ macro on*(server: var RpcServer, path: string, body: untyped): untyped = listType: NimNode checks = newStmtList() varDecl: NimNode - # always include check for array type + # always include check for array type for parameters + # TODO: If defined as single params, relax array check checks.add quote do: if `paramsIdent`.elems[`pos`].kind != JArray: raise newException(ValueError, "Expected " & `paramTypeStr` & " but got " & $`paramsIdent`.elems[`pos`].kind) @@ -175,12 +206,23 @@ macro on*(server: var RpcServer, path: string, body: untyped): untyped = else: # other types var getFuncName = jsonGetFunc($paramType) - assert getFuncName != nil - # fetch parameter - let getFunc = newIdentNode($getFuncName) - paramFetch.add(quote do: - var `name`: `paramType` = `paramsIdent`.elems[`pos`].`getFunc` - ) + if not getFuncName.isNil: + # fetch parameter + let getFunc = newIdentNode($getFuncName) + paramFetch.add(quote do: + var `name`: `paramType` = `paramsIdent`.elems[`pos`].`getFunc` + ) + else: + # this type is probably a custom type, eg object + # bindObj creates assignments to the object fields + let paramTypeStr = $paramType + paramFetch.add(quote do: + var `name`: `paramType` + if `paramsIdent`.elems[`pos`].kind != JObject: + raise newException(ValueError, "Expected " & `paramTypeStr` & " but got " & $`paramsIdent`.elems[`pos`].kind) + + bindObj(`name`, `paramType`, `paramsIdent`, `pos`) + ) # create RPC proc let pathStr = $path @@ -203,47 +245,59 @@ macro on*(server: var RpcServer, path: string, body: untyped): untyped = `paramFetch` `procBody` `server`.register(`path`, `procName`) - echo result.repr - + when defined(nimDumpRpcs): + echo result.repr +#[ when isMainModule: import unittest var s = newRpcServer("localhost") - s.on("the/path1"): + s.on("rpc.simplepath"): echo "hello3" result = %1 - s.on("the/path2") do() -> int: + s.on("rpc.returnint") do() -> int: echo "hello2" - s.on("the/path3") do(a: int, b: string): + s.on("rpc.differentparams") do(a: int, b: string): var node = %"test" result = node - s.on("the/path4") do(arr: array[0..5, byte], b: string): + s.on("rpc.arrayparam") do(arr: array[0..5, byte], b: string): var res = newJArray() for item in arr: res.add %int(item) res.add %b result = %res - s.on("the/path5") do(b: string, arr: seq[int]): + s.on("rpc.seqparam") do(b: string, s: seq[int]): var res = newJArray() res.add %b - for item in arr: + for item in s: res.add %int(item) result = res + type MyObject* = object + a: int + b: string + c: float + s.on("rpc.objparam") do(b: string, obj: MyObject): + result = %obj suite "Server types": test "On macro registration": - check s.procs.hasKey("the/path1") - check s.procs.hasKey("the/path2") - check s.procs.hasKey("the/path3") + check s.procs.hasKey("rpc.simplepath") + check s.procs.hasKey("rpc.returnint") + check s.procs.hasKey("rpc.returnint") test "Array/seq parameters": - let r1 = waitfor thepath4(%[%[1, 2, 3], %"hello"]) + let r1 = waitfor rpcArrayParam(%[%[1, 2, 3], %"hello"]) var ckR1 = %[1, 2, 3, 0, 0, 0] ckR1.elems.add %"hello" check r1 == ckR1 - let r2 = waitfor thepath5(%[%"abc", %[1, 2, 3, 4, 5]]) + let r2 = waitfor rpcSeqParam(%[%"abc", %[1, 2, 3, 4, 5]]) var ckR2 = %["abc"] for i in 0..4: ckR2.add %(i + 1) check r2 == ckR2 + test "Object parameters": + let + obj = %*{"a": %1, "b": %"hello", "c": %1.23} + r = waitfor rpcObjParam(%[%"abc", obj]) + check r == obj test "Runtime errors": expect ValueError: - let r1 = waitfor thepath4(%[%[1, 2, 3, 4, 5, 6, 7, 8, 9, 0], %"hello"]) - \ No newline at end of file + discard waitfor rpcArrayParam(%[%[0, 1, 2, 3, 4, 5, 6], %"hello"]) +]# \ No newline at end of file From 3629e434b84fe6bd9390170a6f0ee1d7e509dcc1 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 26 Apr 2018 19:36:27 +0100 Subject: [PATCH 027/116] Updating paths --- rpcclient.nim | 2 +- rpcserver.nim | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/rpcclient.nim b/rpcclient.nim index 8cbb260..4c36611 100644 --- a/rpcclient.nim +++ b/rpcclient.nim @@ -1,3 +1,3 @@ -import client / clientdispatch +import eth-rpc / client / clientdispatch export clientdispatch diff --git a/rpcserver.nim b/rpcserver.nim index 14844f1..6764a93 100644 --- a/rpcserver.nim +++ b/rpcserver.nim @@ -1,2 +1,6 @@ -import server / [servertypes, rpcconsts, serverdispatch, ethprocs] +import + eth-rpc / server / servertypes, + eth-rpc / server / rpcconsts, + eth-rpc / server / serverdispatch, + eth-rpc / server / ethprocs export servertypes, rpcconsts, serverdispatch, ethprocs From 14c5d369cf5dc564d2c7b276e000cc2e21bd9e9f Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 26 Apr 2018 19:37:02 +0100 Subject: [PATCH 028/116] Moved rpc macro tests out into separate module --- tests/testrpcmacro.nim | 53 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 tests/testrpcmacro.nim diff --git a/tests/testrpcmacro.nim b/tests/testrpcmacro.nim new file mode 100644 index 0000000..c537eb9 --- /dev/null +++ b/tests/testrpcmacro.nim @@ -0,0 +1,53 @@ +import ../ eth-rpc / server / servertypes, unittest, asyncdispatch, json, tables + +var s = newRpcServer("localhost") +s.on("rpc.simplepath"): + echo "hello3" + result = %1 +s.on("rpc.returnint") do() -> int: + echo "hello2" +s.on("rpc.differentparams") do(a: int, b: string): + var node = %"test" + result = node +s.on("rpc.arrayparam") do(arr: array[0..5, byte], b: string): + var res = newJArray() + for item in arr: + res.add %int(item) + res.add %b + result = %res +s.on("rpc.seqparam") do(b: string, s: seq[int]): + var res = newJArray() + res.add %b + for item in s: + res.add %int(item) + result = res +type MyObject* = object + a: int + b: string + c: float +s.on("rpc.objparam") do(b: string, obj: MyObject): + result = %obj +suite "Server types": + test "On macro registration": + check s.procs.hasKey("rpc.simplepath") + check s.procs.hasKey("rpc.returnint") + check s.procs.hasKey("rpc.returnint") + test "Array/seq parameters": + let r1 = waitfor rpcArrayParam(%[%[1, 2, 3], %"hello"]) + var ckR1 = %[1, 2, 3, 0, 0, 0] + ckR1.elems.add %"hello" + check r1 == ckR1 + + let r2 = waitfor rpcSeqParam(%[%"abc", %[1, 2, 3, 4, 5]]) + var ckR2 = %["abc"] + for i in 0..4: ckR2.add %(i + 1) + check r2 == ckR2 + test "Object parameters": + let + obj = %*{"a": %1, "b": %"hello", "c": %1.23} + r = waitfor rpcObjParam(%[%"abc", obj]) + check r == obj + test "Runtime errors": + expect ValueError: + discard waitfor rpcArrayParam(%[%[0, 1, 2, 3, 4, 5, 6], %"hello"]) + # TODO: Add other errors \ No newline at end of file From 79417b74ab75c9d4ebe8d115e2c405441682f29e Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 26 Apr 2018 19:37:19 +0100 Subject: [PATCH 029/116] Updated paths --- tests/testserverclient.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index 8f55b33..fd32548 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -1,4 +1,4 @@ -import ../eth-rpc / rpcclient, ../eth-rpc / rpcserver, +import ../ rpcclient, ../ rpcserver, asyncdispatch, json, unittest, tables # TODO: dummy implementations of RPC calls handled in async fashion. @@ -27,7 +27,8 @@ suite "RPC": test "Custom RPC": response = waitFor client.call("myProc", %[%"abc", %[1, 2, 3, 4]]) check response.result.getStr == "Hello abc data: [1, 2, 3, 4]" - + + waitFor main() \ No newline at end of file From 327a9b85514e4a2cf80429f28ff4938e995d94b8 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 26 Apr 2018 19:43:36 +0100 Subject: [PATCH 030/116] Added testrpcmacro to test module --- tests/all.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/all.nim b/tests/all.nim index 31cf7bd..50cec98 100644 --- a/tests/all.nim +++ b/tests/all.nim @@ -1,3 +1,3 @@ import - testutils, testserverclient + testutils, testrpcmacro, testserverclient From e446b8eea831d2cf70b3d099f0c94cf2edf35642 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 1 May 2018 20:32:28 +0100 Subject: [PATCH 031/116] WIP traversing objects --- eth-rpc/server/servertypes.nim | 253 ++++++++++++++++----------------- 1 file changed, 121 insertions(+), 132 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 8428a77..414075f 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -99,155 +99,139 @@ proc jsonCheckType(paramType: string): NimNode = of "byte": result = ident"JInt" else: result = nil -# TODO: Nested complex fields in objects -# Probably going to need to make it recursive +proc preParseTypes(typeNode: var NimNode, typeName: NimNode, errorCheck: var NimNode): bool {.compileTime.} = + # handle byte + for i, item in typeNode: + if item.kind == nnkIdent and item.basename == ident"byte": + typeNode[i] = ident"int" + # add some extra checks + result = true + else: + var t = typeNode[i] + if preParseTypes(t, typeName, errorCheck): + typeNode[i] = t -macro bindObj*(objInst: untyped, objType: typedesc, paramsArg: typed, elemIdx: int): untyped = - result = newNimNode(nnkStmtList) - let typeDesc = getType(getType(objType)[1]) - for field in typeDesc[2].children: +proc expect(node, jsonIdent, fieldName: NimNode, tn: JsonNodeKind) = + let + expectedStr = "Expected parameter `" & fieldName.repr & "` to be " & $tn & " but got " + tnIdent = ident($tn) + node.add(quote do: + if `jsonIdent`.kind != `tnIdent`: + raise newException(ValueError, `expectedStr` & $`jsonIdent`.kind) + ) + +macro processFields(jsonIdent, fieldName, fieldType: typed): untyped = + result = newStmtList() + let typeInfo = getType(getType(fieldType)) + typeInfo.expectKind nnkBracketExpr + let typeDesc = getType(typeInfo[1]) + case typeDesc.kind + of nnkObjectTy: let - fieldStr = $field - fieldTypeStr = $field.getType() - getFunc = jsonGetFunc(fieldTypeStr) - expectedKind = fieldTypeStr.jsonCheckType - expectedStr = "Expected " & $expectedKind & " but got " + recs = typeDesc.findChild it.kind == nnkRecList + assert recs != nil, "Detected object type but no RecList found to traverse fields" + result.expect(jsonIdent, fieldName, JObject) + for i in 0 ..< recs.len: + var + objFieldName = recs[i] + fns = objFieldName.toStrLit + objFieldType = getType(recs[i]) + var fieldRealType = nnkBracketExpr.newTree(typeInfo[0], objFieldType) + result.add(quote do: + processFields(`jsonIdent`[`fns`], `fieldName`.`objfieldName`, `fieldRealType`) + ) + of nnkBracketExpr: + let + brType = typeInfo[1][1] + brFormat = $typeInfo[1][0] # seq/array + var typeName: NimNode + + if brType.kind == nnkBracketExpr: typeName = typeInfo[1][2] + else: typeName = brType + result.expect(jsonIdent, fieldName, JArray) + + let jFunc = jsonGetFunc($typeName) + if brFormat == "seq": + result.add(quote do: + `fieldName` = @[] + `fieldName`.setLen(`jsonIdent`.len) + ) + else: + # array + let + expectedParams = typeInfo[1][1][2] + expectedParamsStr = expectedParams.toStrLit + expectedLenStr = "Expected parameter `" & fieldName.repr & "` to have a length of " & $expectedParamsStr & " but got " + # TODO: Note, currently only raising if greater than value, not different size + result.add(quote do: + if `jsonIdent`.len > `expectedParams`: + raise newException(ValueError, `expectedLenStr` & $`jsonIdent`.len) + ) + result.add(quote do: - let jParam = `paramsArg`.elems[`elemIdx`][`fieldStr`] - if jParam.kind != `expectedKind`: - raise newException(ValueError, `expectedStr` & $jParam.kind) - `objInst`.`field` = jParam.`getFunc` + for i in 0 ..< `jsonIdent`.len: + `fieldName`[i] = `jsonIdent`.elems[i].`jFunc` ) - when defined(nimDumpRpcs): - echo "BindObj expansion: ", result.repr + of nnkSym: + let + typeName = $typeDesc + jFetch = jsonGetFunc(typeName) + result.add(quote do: + `fieldName` = `jsonIdent`.`jFetch` + ) + else: echo "Unhandled type: ", typeDesc.kind + echo result.repr -macro on*(server: var RpcServer, path: string, body: untyped): untyped = - var - paramFetch = newStmtList() - expectedParams = 0 - let parameters = body.findChild(it.kind == nnkFormalParams) +proc setupParams(node, parameters, paramsIdent: NimNode) = + # recurse parameter's fields until we only have symbols if not parameters.isNil: - # process parameters of body into json fetch templates - var resType = parameters[0] - - if resType.kind != nnkEmpty: - # TODO: transform result type and/or return to json - discard - - var paramsIdent = ident"params" - expectedParams = parameters.len - 1 + var + errorCheck = newStmtList() + expectedParams = parameters.len - 1 let expectedStr = "Expected " & $`expectedParams` & " Json parameter(s) but got " - paramFetch.add(quote do: + node.add(quote do: if `paramsIdent`.len != `expectedParams`: raise newException(ValueError, `expectedStr` & $`paramsIdent`.len) ) - for i in 1.. `name`.len: - raise newException(ValueError, "Provided array is longer than parameter allows. Expected " & `arrayLenStr` & ", data length is " & $`paramsIdent`.elems[`pos`].len) - ) - of ltSeq: - listType = paramType[1] - varDecl = quote do: - var `name` = newSeq[`listType`](`paramsIdent`.elems[`pos`].len) - - let - getFunc = jsonGetFunc($listType) - idx = ident"i" - listParse = quote do: - for `idx` in 0 ..< `paramsIdent`.elems[`pos`].len: - `name`[`idx`] = `listType`(`paramsIdent`.elems[`pos`].elems[`idx`].`getFunc`) - # assemble fetch parameters code - paramFetch.add(quote do: - `varDecl` - `checks` - `listParse` - ) - else: - # other types - var getFuncName = jsonGetFunc($paramType) - if not getFuncName.isNil: - # fetch parameter - let getFunc = newIdentNode($getFuncName) - paramFetch.add(quote do: - var `name`: `paramType` = `paramsIdent`.elems[`pos`].`getFunc` - ) - else: - # this type is probably a custom type, eg object - # bindObj creates assignments to the object fields - let paramTypeStr = $paramType - paramFetch.add(quote do: - var `name`: `paramType` - if `paramsIdent`.elems[`pos`].kind != JObject: - raise newException(ValueError, "Expected " & `paramTypeStr` & " but got " & $`paramsIdent`.elems[`pos`].kind) - - bindObj(`name`, `paramType`, `paramsIdent`, `pos`) - ) - # create RPC proc + # wrapping proc let pathStr = $path procName = ident(pathStr.multiRemove(".", "/")) # TODO: Make this unique to avoid potential clashes, or allow people to know the name for calling? - paramsIdent = ident("params") var procBody: NimNode if body.kind == nnkStmtList: procBody = body else: procBody = body.body - # - var checkTypeError: NimNode - if expectedParams > 0: - checkTypeError = quote do: - if `paramsIdent`.kind != JArray: - raise newException(ValueError, "Expected array but got " & $`paramsIdent`.kind) - else: checkTypeError = newStmtList() - result = quote do: proc `procName`*(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = - `checkTypeError` - `paramFetch` + #`checkTypeError` + `setup` `procBody` `server`.register(`path`, `procName`) when defined(nimDumpRpcs): - echo result.repr -#[ + echo pathStr, ": ", result.repr + when isMainModule: import unittest var s = newRpcServer("localhost") @@ -265,17 +249,21 @@ when isMainModule: res.add %int(item) res.add %b result = %res - s.on("rpc.seqparam") do(b: string, s: seq[int]): + echo result + s.on("rpc.seqparam") do(a: string, s: seq[int]): var res = newJArray() - res.add %b + res.add %a for item in s: res.add %int(item) result = res + type Test = object + a: int #array[0..10, int] + type MyObject* = object a: int - b: string + b: Test c: float - s.on("rpc.objparam") do(b: string, obj: MyObject): + s.on("rpc.objparam") do(a: string, obj: MyObject): result = %obj suite "Server types": test "On macro registration": @@ -294,10 +282,11 @@ when isMainModule: check r2 == ckR2 test "Object parameters": let - obj = %*{"a": %1, "b": %"hello", "c": %1.23} + obj = %*{"a": %1, "b": %*{"a": %5}, "c": %1.23} r = waitfor rpcObjParam(%[%"abc", obj]) check r == obj test "Runtime errors": expect ValueError: - discard waitfor rpcArrayParam(%[%[0, 1, 2, 3, 4, 5, 6], %"hello"]) -]# \ No newline at end of file + echo waitfor rpcArrayParam(%[%[0, 1, 2, 3, 4, 5, 6], %"hello"]) + + From 7e0abead630c9f03c574f219fb6c60d4c976f503 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 1 May 2018 20:59:10 +0100 Subject: [PATCH 032/116] Fix off by one error in runtime check for array length --- eth-rpc/server/servertypes.nim | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 414075f..5457cf3 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -159,13 +159,17 @@ macro processFields(jsonIdent, fieldName, fieldType: typed): untyped = else: # array let - expectedParams = typeInfo[1][1][2] - expectedParamsStr = expectedParams.toStrLit - expectedLenStr = "Expected parameter `" & fieldName.repr & "` to have a length of " & $expectedParamsStr & " but got " + startLen = typeInfo[1][1][1] + endLen = typeInfo[1][1][2] + expectedLen = genSym(nskConst) + expectedParams = quote do: + const `expectedLen` = `endLen` - `startLen` + 1 + expectedLenStr = "Expected parameter `" & fieldName.repr & "` to have a length of " # TODO: Note, currently only raising if greater than value, not different size result.add(quote do: - if `jsonIdent`.len > `expectedParams`: - raise newException(ValueError, `expectedLenStr` & $`jsonIdent`.len) + `expectedParams` + if `jsonIdent`.len > `expectedLen`: + raise newException(ValueError, `expectedLenStr` & $`expectedLen` & " but got " & $`jsonIdent`.len) ) result.add(quote do: @@ -257,7 +261,7 @@ when isMainModule: res.add %int(item) result = res type Test = object - a: int #array[0..10, int] + a: seq[int] #array[0..10, int] type MyObject* = object a: int @@ -282,7 +286,7 @@ when isMainModule: check r2 == ckR2 test "Object parameters": let - obj = %*{"a": %1, "b": %*{"a": %5}, "c": %1.23} + obj = %*{"a": %1, "b": %*{"a": %[5]}, "c": %1.23} r = waitfor rpcObjParam(%[%"abc", obj]) check r == obj test "Runtime errors": From e2d2e7402c4084b9445d0f1a4ecb584be38a82e3 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 2 May 2018 16:21:05 +0100 Subject: [PATCH 033/116] Nested objects with arrays now parse --- eth-rpc/server/servertypes.nim | 173 +++++++++++++++++---------------- 1 file changed, 88 insertions(+), 85 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 5457cf3..a1be21e 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -87,17 +87,16 @@ proc jsonGetFunc(paramType: string): NimNode = of "int": result = ident"getInt" of "float": result = ident"getFloat" of "bool": result = ident"getBool" - of "byte": result = ident"getInt" + of "uint8": result = ident"getInt()" else: result = nil -proc jsonCheckType(paramType: string): NimNode = +proc jsonCheckType(paramType: string): JsonNodeKind = case paramType - of "string": result = ident"JString" - of "int": result = ident"JInt" - of "float": result = ident"JFloat" - of "bool": result = ident"JBool" - of "byte": result = ident"JInt" - else: result = nil + of "string": result = JString + of "int": result = JInt + of "float": result = JFloat + of "bool": result = JBool + of "uint8": result = JInt proc preParseTypes(typeNode: var NimNode, typeName: NimNode, errorCheck: var NimNode): bool {.compileTime.} = # handle byte @@ -119,73 +118,78 @@ proc expect(node, jsonIdent, fieldName: NimNode, tn: JsonNodeKind) = if `jsonIdent`.kind != `tnIdent`: raise newException(ValueError, `expectedStr` & $`jsonIdent`.kind) ) - + macro processFields(jsonIdent, fieldName, fieldType: typed): untyped = result = newStmtList() - let typeInfo = getType(getType(fieldType)) - typeInfo.expectKind nnkBracketExpr - let typeDesc = getType(typeInfo[1]) - case typeDesc.kind - of nnkObjectTy: - let - recs = typeDesc.findChild it.kind == nnkRecList - assert recs != nil, "Detected object type but no RecList found to traverse fields" - result.expect(jsonIdent, fieldName, JObject) - for i in 0 ..< recs.len: - var - objFieldName = recs[i] - fns = objFieldName.toStrLit - objFieldType = getType(recs[i]) - var fieldRealType = nnkBracketExpr.newTree(typeInfo[0], objFieldType) - result.add(quote do: - processFields(`jsonIdent`[`fns`], `fieldName`.`objfieldName`, `fieldRealType`) - ) - of nnkBracketExpr: - let - brType = typeInfo[1][1] - brFormat = $typeInfo[1][0] # seq/array - var typeName: NimNode - - if brType.kind == nnkBracketExpr: typeName = typeInfo[1][2] - else: typeName = brType - result.expect(jsonIdent, fieldName, JArray) - - let jFunc = jsonGetFunc($typeName) - if brFormat == "seq": - result.add(quote do: - `fieldName` = @[] - `fieldName`.setLen(`jsonIdent`.len) - ) - else: - # array - let - startLen = typeInfo[1][1][1] - endLen = typeInfo[1][1][2] - expectedLen = genSym(nskConst) - expectedParams = quote do: - const `expectedLen` = `endLen` - `startLen` + 1 - expectedLenStr = "Expected parameter `" & fieldName.repr & "` to have a length of " - # TODO: Note, currently only raising if greater than value, not different size - result.add(quote do: - `expectedParams` - if `jsonIdent`.len > `expectedLen`: - raise newException(ValueError, `expectedLenStr` & $`expectedLen` & " but got " & $`jsonIdent`.len) - ) - - result.add(quote do: - for i in 0 ..< `jsonIdent`.len: - `fieldName`[i] = `jsonIdent`.elems[i].`jFunc` - ) - of nnkSym: - let - typeName = $typeDesc - jFetch = jsonGetFunc(typeName) + let + fieldTypeStr = fieldType.repr + jFetch = jsonGetFunc(fieldTypeStr) + if not jFetch.isNil: + result.expect(jsonIdent, fieldName, jsonCheckType(fieldTypeStr)) result.add(quote do: `fieldName` = `jsonIdent`.`jFetch` ) - else: echo "Unhandled type: ", typeDesc.kind - echo result.repr + else: + var fetchedType = getType(fieldType) + var derivedType: NimNode + if fetchedType[0].repr == "typeDesc": + derivedType = getType(fetchedType[1]) + else: + derivedType = fetchedType + if derivedType.kind == nnkObjectTy: + result.expect(jsonIdent, fieldName, JObject) + let recs = derivedType.findChild it.kind == nnkRecList + for i in 0.. `expectedLen`: + raise newException(ValueError, `expectedLenStr` & $`expectedLen` & " but got " & $`jsonIdent`.len) + ) + jFunc = jsonGetFunc($derivedType[2]) + of "seq": + result.add(quote do: + `fieldName` = @[] + `fieldName`.setLen(`jsonIdent`.len) + ) + jFunc = jsonGetFunc($derivedType[1]) + else: + raise newException(ValueError, "Cannot determine bracket expression type of \"" & derivedType.treerepr & "\"") + # add fetch code for array/seq + result.add(quote do: + for i in 0 ..< `jsonIdent`.len: + `fieldName`[i] = `jsonIdent`.elems[i].`jFunc` + ) + else: + raise newException(ValueError, "Unknown type \"" & derivedType.treerepr & "\"") + proc setupParams(node, parameters, paramsIdent: NimNode) = # recurse parameter's fields until we only have symbols if not parameters.isNil: @@ -239,29 +243,25 @@ macro on*(server: var RpcServer, path: string, body: untyped): untyped = when isMainModule: import unittest var s = newRpcServer("localhost") - s.on("rpc.simplepath"): - echo "hello3" - result = %1 - s.on("rpc.returnint") do() -> int: - echo "hello2" - s.on("rpc.differentparams") do(a: int, b: string): - var node = %"test" - result = node s.on("rpc.arrayparam") do(arr: array[0..5, byte], b: string): var res = newJArray() for item in arr: res.add %int(item) res.add %b result = %res - echo result s.on("rpc.seqparam") do(a: string, s: seq[int]): var res = newJArray() res.add %a for item in s: res.add %int(item) result = res - type Test = object - a: seq[int] #array[0..10, int] + + type + Test2 = object + x: array[2, int] + Test = object + d: array[0..1, int] + e: Test2 type MyObject* = object a: int @@ -270,10 +270,6 @@ when isMainModule: s.on("rpc.objparam") do(a: string, obj: MyObject): result = %obj suite "Server types": - test "On macro registration": - check s.procs.hasKey("rpc.simplepath") - check s.procs.hasKey("rpc.returnint") - check s.procs.hasKey("rpc.returnint") test "Array/seq parameters": let r1 = waitfor rpcArrayParam(%[%[1, 2, 3], %"hello"]) var ckR1 = %[1, 2, 3, 0, 0, 0] @@ -286,9 +282,16 @@ when isMainModule: check r2 == ckR2 test "Object parameters": let - obj = %*{"a": %1, "b": %*{"a": %[5]}, "c": %1.23} + obj = %*{"a": %1, "b": %*{"d": %[5, 0], "e": %*{"x": %[1, 1]}}, "c": %1.23} r = waitfor rpcObjParam(%[%"abc", obj]) check r == obj + expect ValueError: + # here we fail to provide one of the nested fields in json to the rpc + # TODO: Should this work? We either allow partial non-ambiguous parsing or not + # Currently, as long as the Nim fields are satisfied, other fields are ignored + let + obj = %*{"a": %1, "b": %*{"a": %[5, 0]}, "c": %1.23} + discard waitFor rpcObjParam(%[%"abc", obj]) # Why doesn't asyncCheck raise? test "Runtime errors": expect ValueError: echo waitfor rpcArrayParam(%[%[0, 1, 2, 3, 4, 5, 6], %"hello"]) From 0198cf680b37df640ce0ce940290645e32eb1382 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 2 May 2018 23:12:07 +0100 Subject: [PATCH 034/116] Added prototype for processing of byte params --- eth-rpc/server/servertypes.nim | 39 ++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index a1be21e..93d44e9 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -87,9 +87,24 @@ proc jsonGetFunc(paramType: string): NimNode = of "int": result = ident"getInt" of "float": result = ident"getFloat" of "bool": result = ident"getBool" - of "uint8": result = ident"getInt()" + of "uint8": result = ident"getInt" else: result = nil +proc jsonTranslate(translation: var NimNode, paramType: string): NimNode = + case paramType + of "uint8": + result = genSym(nskTemplate) + translation = quote do: + template `result`(value: int): uint8 = + if value > 255 or value < 0: + raise newException(ValueError, "Value out of range of byte, expected 0-255, got " & $value) + uint8(value and 0xff) + else: + result = genSym(nskTemplate) + translation = quote do: + template `result`(value: untyped): untyped = value + + proc jsonCheckType(paramType: string): JsonNodeKind = case paramType of "string": result = JString @@ -124,10 +139,13 @@ macro processFields(jsonIdent, fieldName, fieldType: typed): untyped = let fieldTypeStr = fieldType.repr jFetch = jsonGetFunc(fieldTypeStr) + var translation: NimNode if not jFetch.isNil: result.expect(jsonIdent, fieldName, jsonCheckType(fieldTypeStr)) + let transIdent = translation.jsonTranslate(fieldTypeStr) result.add(quote do: - `fieldName` = `jsonIdent`.`jFetch` + `translation` + `fieldName` = `transIdent`(`jsonIdent`.`jFetch`) ) else: var fetchedType = getType(fieldType) @@ -157,7 +175,8 @@ macro processFields(jsonIdent, fieldName, fieldType: typed): untyped = let formatType = derivedType[0].repr expectedLen = genSym(nskConst) - var jFunc: NimNode + var + jFunc, rootType: NimNode case formatType of "array": let @@ -166,30 +185,32 @@ macro processFields(jsonIdent, fieldName, fieldType: typed): untyped = expectedParamLen = quote do: const `expectedLen` = `endLen` - `startLen` + 1 expectedLenStr = "Expected parameter `" & fieldName.repr & "` to have a length of " - # TODO: Note, currently only raising if greater than value, not different size + # TODO: Note, currently only raising if greater than length, not different size result.add(quote do: `expectedParamLen` if `jsonIdent`.len > `expectedLen`: raise newException(ValueError, `expectedLenStr` & $`expectedLen` & " but got " & $`jsonIdent`.len) ) - jFunc = jsonGetFunc($derivedType[2]) + rootType = derivedType[2] of "seq": result.add(quote do: `fieldName` = @[] `fieldName`.setLen(`jsonIdent`.len) ) - jFunc = jsonGetFunc($derivedType[1]) + rootType = derivedType[1] else: raise newException(ValueError, "Cannot determine bracket expression type of \"" & derivedType.treerepr & "\"") # add fetch code for array/seq + jFunc = jsonGetFunc($rootType) + let transIdent = translation.jsonTranslate($rootType) result.add(quote do: + `translation` for i in 0 ..< `jsonIdent`.len: - `fieldName`[i] = `jsonIdent`.elems[i].`jFunc` + `fieldName`[i] = `transIdent`(`jsonIdent`.elems[i].`jFunc`) ) else: raise newException(ValueError, "Unknown type \"" & derivedType.treerepr & "\"") - proc setupParams(node, parameters, paramsIdent: NimNode) = # recurse parameter's fields until we only have symbols if not parameters.isNil: @@ -208,7 +229,7 @@ proc setupParams(node, parameters, paramsIdent: NimNode) = pos = i - 1 var paramType = parameters[i][1] - discard paramType.preParseTypes(paramName, errorCheck) + #discard paramType.preParseTypes(paramName, errorCheck) node.add(quote do: var `paramName`: `paramType` processFields(`paramsIdent`[`pos`], `paramName`, `paramType`) From 75d41585d051d71743960e0350d00943a5fbdc1e Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 3 May 2018 20:20:10 +0100 Subject: [PATCH 035/116] Add conversion for json -> byte --- eth-rpc/server/servertypes.nim | 75 ++++++++++++++++++++++++++-------- 1 file changed, 58 insertions(+), 17 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 93d44e9..109059d 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -81,7 +81,16 @@ macro multiRemove(s: string, values: varargs[string]): untyped = body.add multiReplaceCall result = newBlockStmt(body) -proc jsonGetFunc(paramType: string): NimNode = +proc jsonGetFunc(paramType: string): (NimNode, JsonNodeKind) = + case paramType + of "string": result = (ident"getStr", JString) + of "int": result = (ident"getInt", JInt) + of "float": result = (ident"getFloat", JFloat) + of "bool": result = (ident"getBool", JBool) + of "uint8", "byte": result = (ident"getInt", JInt) + else: result = (nil, JInt) + +#[proc jsonGetFunc(paramType: string): NimNode = case paramType of "string": result = ident"getStr" of "int": result = ident"getInt" @@ -89,7 +98,7 @@ proc jsonGetFunc(paramType: string): NimNode = of "bool": result = ident"getBool" of "uint8": result = ident"getInt" else: result = nil - +]# proc jsonTranslate(translation: var NimNode, paramType: string): NimNode = case paramType of "uint8": @@ -104,14 +113,15 @@ proc jsonTranslate(translation: var NimNode, paramType: string): NimNode = translation = quote do: template `result`(value: untyped): untyped = value - +#[ proc jsonCheckType(paramType: string): JsonNodeKind = case paramType of "string": result = JString of "int": result = JInt of "float": result = JFloat of "bool": result = JBool - of "uint8": result = JInt + of "byte": result = JInt +]# proc preParseTypes(typeNode: var NimNode, typeName: NimNode, errorCheck: var NimNode): bool {.compileTime.} = # handle byte @@ -125,7 +135,7 @@ proc preParseTypes(typeNode: var NimNode, typeName: NimNode, errorCheck: var Nim if preParseTypes(t, typeName, errorCheck): typeNode[i] = t -proc expect(node, jsonIdent, fieldName: NimNode, tn: JsonNodeKind) = +proc expectKind(node, jsonIdent, fieldName: NimNode, tn: JsonNodeKind) = let expectedStr = "Expected parameter `" & fieldName.repr & "` to be " & $tn & " but got " tnIdent = ident($tn) @@ -134,19 +144,38 @@ proc expect(node, jsonIdent, fieldName: NimNode, tn: JsonNodeKind) = raise newException(ValueError, `expectedStr` & $`jsonIdent`.kind) ) +proc translate(paramType: string, getField: NimNode): NimNode = + # Note that specific types add extra run time bounds checking code + case paramType + of "byte": + result = quote do: + let x = `getField` + if x > 255 or x < 0: + raise newException(ValueError, "Value out of range of byte, expected 0-255, got " & $x) + uint8(x and 0xff) + else: + result = quote do: `getField` + macro processFields(jsonIdent, fieldName, fieldType: typed): untyped = result = newStmtList() let fieldTypeStr = fieldType.repr - jFetch = jsonGetFunc(fieldTypeStr) + (jFetch, jKind) = jsonGetFunc(fieldTypeStr) var translation: NimNode if not jFetch.isNil: - result.expect(jsonIdent, fieldName, jsonCheckType(fieldTypeStr)) - let transIdent = translation.jsonTranslate(fieldTypeStr) + result.expectKind(jsonIdent, fieldName, jKind) + #let transIdent = translation.jsonTranslate(fieldTypeStr) + let + getField = quote do: `jsonIdent`.`jFetch` + res = translate(`fieldTypeStr`, `getField`) result.add(quote do: - `translation` - `fieldName` = `transIdent`(`jsonIdent`.`jFetch`) + `fieldName` = `res` ) + echo ">>", result.repr, "<<" + #result.add(quote do: + # `translation` + # `fieldName` = `transIdent`(`jsonIdent`.`jFetch`) + #) else: var fetchedType = getType(fieldType) var derivedType: NimNode @@ -155,7 +184,7 @@ macro processFields(jsonIdent, fieldName, fieldType: typed): untyped = else: derivedType = fetchedType if derivedType.kind == nnkObjectTy: - result.expect(jsonIdent, fieldName, JObject) + result.expectKind(jsonIdent, fieldName, JObject) let recs = derivedType.findChild it.kind == nnkRecList for i in 0.. Date: Thu, 3 May 2018 22:40:28 +0100 Subject: [PATCH 036/116] Add processing and checking for uintX and intX types and related tests --- eth-rpc/server/servertypes.nim | 163 ++++++++++++++++++++------------- 1 file changed, 99 insertions(+), 64 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 109059d..a75c81e 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -82,24 +82,20 @@ macro multiRemove(s: string, values: varargs[string]): untyped = result = newBlockStmt(body) proc jsonGetFunc(paramType: string): (NimNode, JsonNodeKind) = + # Unknown types get attempted as int case paramType of "string": result = (ident"getStr", JString) of "int": result = (ident"getInt", JInt) of "float": result = (ident"getFloat", JFloat) of "bool": result = (ident"getBool", JBool) - of "uint8", "byte": result = (ident"getInt", JInt) - else: result = (nil, JInt) + else: + if paramType == "byte" or paramType[0..3] == "uint" or paramType[0..2] == "int": + result = (ident"getInt", JInt) + else: + result = (nil, JInt) -#[proc jsonGetFunc(paramType: string): NimNode = - case paramType - of "string": result = ident"getStr" - of "int": result = ident"getInt" - of "float": result = ident"getFloat" - of "bool": result = ident"getBool" - of "uint8": result = ident"getInt" - else: result = nil -]# proc jsonTranslate(translation: var NimNode, paramType: string): NimNode = + # TODO: Remove or rework this into `translate` case paramType of "uint8": result = genSym(nskTemplate) @@ -108,33 +104,18 @@ proc jsonTranslate(translation: var NimNode, paramType: string): NimNode = if value > 255 or value < 0: raise newException(ValueError, "Value out of range of byte, expected 0-255, got " & $value) uint8(value and 0xff) + of "int8": + result = genSym(nskTemplate) + translation = quote do: + template `result`(value: int): uint8 = + if value > 255 or value < 0: + raise newException(ValueError, "Value out of range of byte, expected 0-255, got " & $value) + uint8(value and 0xff) else: result = genSym(nskTemplate) translation = quote do: template `result`(value: untyped): untyped = value -#[ -proc jsonCheckType(paramType: string): JsonNodeKind = - case paramType - of "string": result = JString - of "int": result = JInt - of "float": result = JFloat - of "bool": result = JBool - of "byte": result = JInt -]# - -proc preParseTypes(typeNode: var NimNode, typeName: NimNode, errorCheck: var NimNode): bool {.compileTime.} = - # handle byte - for i, item in typeNode: - if item.kind == nnkIdent and item.basename == ident"byte": - typeNode[i] = ident"int" - # add some extra checks - result = true - else: - var t = typeNode[i] - if preParseTypes(t, typeName, errorCheck): - typeNode[i] = t - proc expectKind(node, jsonIdent, fieldName: NimNode, tn: JsonNodeKind) = let expectedStr = "Expected parameter `" & fieldName.repr & "` to be " & $tn & " but got " @@ -144,38 +125,79 @@ proc expectKind(node, jsonIdent, fieldName: NimNode, tn: JsonNodeKind) = raise newException(ValueError, `expectedStr` & $`jsonIdent`.kind) ) -proc translate(paramType: string, getField: NimNode): NimNode = - # Note that specific types add extra run time bounds checking code +proc getDigit(s: string): (bool, int) = + if s.len == 0: return (false, 0) + for c in s: + if not c.isDigit: return (false, 0) + return (true, s.parseInt) + + +from math import pow + +proc translate(paramTypeStr: string, getField: NimNode): NimNode = + # Add checking and type conversion for more constrained types + # Note: + # * specific types add extra run time bounds checking code + # * types that map one-one get passed as is + # * any other types get a simple cast, ie; MyType(value) and + # get are assumed to be integer. + # NOTE: However this will never occur because currently jsonFunc + # is required to return nil to process other types. + # TODO: Allow distinct types + var paramType = paramTypeStr + if paramType == "byte": paramType = "uint8" + case paramType - of "byte": - result = quote do: - let x = `getField` - if x > 255 or x < 0: - raise newException(ValueError, "Value out of range of byte, expected 0-255, got " & $x) - uint8(x and 0xff) - else: + of "string", "int", "bool": result = quote do: `getField` + else: + if paramType[0 .. 3].toLowerAscii == "uint": + let (numeric, bitSize) = paramType[4 .. high(paramType)].getDigit + if numeric: + assert bitSize mod 8 == 0 + let + maxSize = 1 shl bitSize - 1 + sizeRangeStr = "0 to " & $maxSize + uintType = ident("uint" & $bitSize) + result = quote do: + let x = `getField` + if x > `maxSize` or x < 0: + raise newException(ValueError, "Value out of range of byte, expected " & `sizeRangeStr` & ", got " & $x) + `uintType`(x) + elif paramType[0 .. 2].toLowerAscii == "int": + let (numeric, bitSize) = paramType[3 .. paramType.high].getDigit + if numeric: + assert bitSize mod 8 == 0 + let + maxSize = 1 shl (bitSize - 1) + minVal = -maxSize + maxVal = maxSize - 1 + sizeRangeStr = $minVal & " to " & $maxVal + intType = ident("int" & $bitSize) + result = quote do: + let x = `getField` + if x < `minVal` or x > `maxVal`: + raise newException(ValueError, "Value out of range of byte, expected " & `sizeRangeStr` & ", got " & $x) + `intType`(x) + else: + let nativeParamType = ident(paramTypeStr) + result = quote do: `nativeParamType`(`getField`) macro processFields(jsonIdent, fieldName, fieldType: typed): untyped = result = newStmtList() let - fieldTypeStr = fieldType.repr + fieldTypeStr = fieldType.repr.toLowerAscii() (jFetch, jKind) = jsonGetFunc(fieldTypeStr) - var translation: NimNode + if not jFetch.isNil: + # TODO: getType(fieldType) to translate byte -> uint8 and avoid special cases result.expectKind(jsonIdent, fieldName, jKind) - #let transIdent = translation.jsonTranslate(fieldTypeStr) let getField = quote do: `jsonIdent`.`jFetch` res = translate(`fieldTypeStr`, `getField`) result.add(quote do: `fieldName` = `res` ) - echo ">>", result.repr, "<<" - #result.add(quote do: - # `translation` - # `fieldName` = `transIdent`(`jsonIdent`.`jFetch`) - #) else: var fetchedType = getType(fieldType) var derivedType: NimNode @@ -229,20 +251,21 @@ macro processFields(jsonIdent, fieldName, fieldType: typed): untyped = else: raise newException(ValueError, "Cannot determine bracket expression type of \"" & derivedType.treerepr & "\"") # add fetch code for array/seq - let (jFunc, jKind) = jsonGetFunc($rootType) - let transIdent = translation.jsonTranslate($rootType) + var translation: NimNode + let + (jFunc, jKind) = jsonGetFunc(($rootType).toLowerAscii) + transIdent = translation.jsonTranslate($rootType) + # TODO: Add checks PER ITEM (performance hit!) in the array, if required by the type + # TODO: Refactor `jsonTranslate` into `translate` result.add(quote do: `translation` for i in 0 ..< `jsonIdent`.len: `fieldName`[i] = `transIdent`(`jsonIdent`.elems[i].`jFunc`) ) else: - echo "DT ", fieldType.treerepr - echo "DT ", fetchedType.treerepr - echo "DT ", derivedType.treerepr - echo "fts ", fieldTypeStr - echo "JN ", jFetch.treerepr raise newException(ValueError, "Unknown type \"" & derivedType.treerepr & "\"") + when defined(nimDumpRpcs): + echo result.repr proc setupParams(node, parameters, paramsIdent: NimNode) = # recurse parameter's fields until we only have symbols @@ -324,9 +347,10 @@ when isMainModule: s.on("rpc.objparam") do(a: string, obj: MyObject): result = %obj - s.on("rpc.specialtypes") do(a: byte): - result = %int(a) - # TODO: Add path as constant for each rpc + s.on("rpc.uinttypes") do(a: byte, b: uint16, c: uint32): + result = %[int(a), int(b), int(c)] + s.on("rpc.inttypes") do(a: int8, b: int16, c: int32, d: int8, e: int16, f: int32): + result = %[int(a), int(b), int(c), int(d), int(e), int(f)] suite "Server types": test "Array/seq parameters": @@ -351,11 +375,22 @@ when isMainModule: let obj = %*{"a": %1, "b": %*{"a": %[5, 0]}, "c": %1.23} discard waitFor rpcObjParam(%[%"abc", obj]) # Why doesn't asyncCheck raise? - test "Special types": - let r = waitfor rpcSpecialTypes(%[%5]) - check r == %5 + test "Uint types": + let + testCase = %[%255, %65534, %4294967295] + r = waitfor rpcUIntTypes(testCase) + check r == testCase + test "Int types": + let + testCase = %[ + %(127), %(32767), %(2147483647), + %(-128), %(-32768), %(-2147483648) + ] + r = waitfor rpcIntTypes(testCase) + check r == testCase test "Runtime errors": expect ValueError: echo waitfor rpcArrayParam(%[%[0, 1, 2, 3, 4, 5, 6], %"hello"]) - +# TODO: Split runtime strictness checking into defines - is there ever a reason to trust input? +# TODO: Add path as constant for each rpc From 5c7d339c21edddc642534d6a9e3ec363525c15af Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 8 May 2018 11:51:24 +0100 Subject: [PATCH 037/116] Parse objects by overload --- eth-rpc/server/servertypes.nim | 304 ++++++++++----------------------- 1 file changed, 87 insertions(+), 217 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index a75c81e..1fcf9a2 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -81,42 +81,19 @@ macro multiRemove(s: string, values: varargs[string]): untyped = body.add multiReplaceCall result = newBlockStmt(body) -proc jsonGetFunc(paramType: string): (NimNode, JsonNodeKind) = - # Unknown types get attempted as int - case paramType - of "string": result = (ident"getStr", JString) - of "int": result = (ident"getInt", JInt) - of "float": result = (ident"getFloat", JFloat) - of "bool": result = (ident"getBool", JBool) - else: - if paramType == "byte" or paramType[0..3] == "uint" or paramType[0..2] == "int": - result = (ident"getInt", JInt) +proc preParseTypes(typeNode: var NimNode, typeName: NimNode, errorCheck: var NimNode): bool {.compileTime.} = + # handle byte + for i, item in typeNode: + if item.kind == nnkIdent and item.basename == ident"byte": + typeNode[i] = ident"int" + # add some extra checks + result = true else: - result = (nil, JInt) + var t = typeNode[i] + if preParseTypes(t, typeName, errorCheck): + typeNode[i] = t -proc jsonTranslate(translation: var NimNode, paramType: string): NimNode = - # TODO: Remove or rework this into `translate` - case paramType - of "uint8": - result = genSym(nskTemplate) - translation = quote do: - template `result`(value: int): uint8 = - if value > 255 or value < 0: - raise newException(ValueError, "Value out of range of byte, expected 0-255, got " & $value) - uint8(value and 0xff) - of "int8": - result = genSym(nskTemplate) - translation = quote do: - template `result`(value: int): uint8 = - if value > 255 or value < 0: - raise newException(ValueError, "Value out of range of byte, expected 0-255, got " & $value) - uint8(value and 0xff) - else: - result = genSym(nskTemplate) - translation = quote do: - template `result`(value: untyped): untyped = value - -proc expectKind(node, jsonIdent, fieldName: NimNode, tn: JsonNodeKind) = +proc expect(node, jsonIdent, fieldName: NimNode, tn: JsonNodeKind) = let expectedStr = "Expected parameter `" & fieldName.repr & "` to be " & $tn & " but got " tnIdent = ident($tn) @@ -124,148 +101,46 @@ proc expectKind(node, jsonIdent, fieldName: NimNode, tn: JsonNodeKind) = if `jsonIdent`.kind != `tnIdent`: raise newException(ValueError, `expectedStr` & $`jsonIdent`.kind) ) - -proc getDigit(s: string): (bool, int) = - if s.len == 0: return (false, 0) - for c in s: - if not c.isDigit: return (false, 0) - return (true, s.parseInt) - - -from math import pow - -proc translate(paramTypeStr: string, getField: NimNode): NimNode = - # Add checking and type conversion for more constrained types - # Note: - # * specific types add extra run time bounds checking code - # * types that map one-one get passed as is - # * any other types get a simple cast, ie; MyType(value) and - # get are assumed to be integer. - # NOTE: However this will never occur because currently jsonFunc - # is required to return nil to process other types. - # TODO: Allow distinct types - var paramType = paramTypeStr - if paramType == "byte": paramType = "uint8" - - case paramType - of "string", "int", "bool": - result = quote do: `getField` - else: - if paramType[0 .. 3].toLowerAscii == "uint": - let (numeric, bitSize) = paramType[4 .. high(paramType)].getDigit - if numeric: - assert bitSize mod 8 == 0 - let - maxSize = 1 shl bitSize - 1 - sizeRangeStr = "0 to " & $maxSize - uintType = ident("uint" & $bitSize) - result = quote do: - let x = `getField` - if x > `maxSize` or x < 0: - raise newException(ValueError, "Value out of range of byte, expected " & `sizeRangeStr` & ", got " & $x) - `uintType`(x) - elif paramType[0 .. 2].toLowerAscii == "int": - let (numeric, bitSize) = paramType[3 .. paramType.high].getDigit - if numeric: - assert bitSize mod 8 == 0 - let - maxSize = 1 shl (bitSize - 1) - minVal = -maxSize - maxVal = maxSize - 1 - sizeRangeStr = $minVal & " to " & $maxVal - intType = ident("int" & $bitSize) - result = quote do: - let x = `getField` - if x < `minVal` or x > `maxVal`: - raise newException(ValueError, "Value out of range of byte, expected " & `sizeRangeStr` & ", got " & $x) - `intType`(x) - else: - let nativeParamType = ident(paramTypeStr) - result = quote do: `nativeParamType`(`getField`) - -macro processFields(jsonIdent, fieldName, fieldType: typed): untyped = - result = newStmtList() - let - fieldTypeStr = fieldType.repr.toLowerAscii() - (jFetch, jKind) = jsonGetFunc(fieldTypeStr) - if not jFetch.isNil: - # TODO: getType(fieldType) to translate byte -> uint8 and avoid special cases - result.expectKind(jsonIdent, fieldName, jKind) - let - getField = quote do: `jsonIdent`.`jFetch` - res = translate(`fieldTypeStr`, `getField`) - result.add(quote do: - `fieldName` = `res` - ) - else: - var fetchedType = getType(fieldType) - var derivedType: NimNode - if fetchedType[0].repr == "typeDesc": - derivedType = getType(fetchedType[1]) - else: - derivedType = fetchedType - if derivedType.kind == nnkObjectTy: - result.expectKind(jsonIdent, fieldName, JObject) - let recs = derivedType.findChild it.kind == nnkRecList - for i in 0.. `expectedLen`: - raise newException(ValueError, `expectedLenStr` & $`expectedLen` & " but got " & $`jsonIdent`.len) - ) - rootType = derivedType[2] - of "seq": - result.add(quote do: - `fieldName` = @[] - `fieldName`.setLen(`jsonIdent`.len) - ) - rootType = derivedType[1] - else: - raise newException(ValueError, "Cannot determine bracket expression type of \"" & derivedType.treerepr & "\"") - # add fetch code for array/seq - var translation: NimNode - let - (jFunc, jKind) = jsonGetFunc(($rootType).toLowerAscii) - transIdent = translation.jsonTranslate($rootType) - # TODO: Add checks PER ITEM (performance hit!) in the array, if required by the type - # TODO: Refactor `jsonTranslate` into `translate` - result.add(quote do: - `translation` - for i in 0 ..< `jsonIdent`.len: - `fieldName`[i] = `transIdent`(`jsonIdent`.elems[i].`jFunc`) - ) - else: - raise newException(ValueError, "Unknown type \"" & derivedType.treerepr & "\"") - when defined(nimDumpRpcs): - echo result.repr +### + +proc fromJson(n: JsonNode, result: var int) = + # TODO: validate... + result = n.getInt() + +proc fromJson(n: JsonNode, result: var byte) = + let v = n.getInt() + if v > 255: raise newException(ValueError, "Parameter value to large for byte: " & $v) + result = byte(v) + +proc fromJson(n: JsonNode, result: var float) = + # TODO: validate... + result = n.getFloat() + +proc fromJson(n: JsonNode, result: var string) = + # TODO: validate... + result = n.getStr() + +proc fromJson[T](n: JsonNode, result: var seq[T]) = + # TODO: validate... + result = newSeq[T](n.len) + for i in 0 ..< n.len: + fromJson(n[i], result[i]) + +proc fromJson[N, T](n: JsonNode, result: var array[N, T]) = + # TODO: validate... + if n.len > result.len: raise newException(ValueError, "Parameter data too big for array") + for i in 0..< n.len: + fromJson(n[i], result[i]) + +proc fromJson[T: object](n: JsonNode, result: var T) = # This reads a custom object + # TODO: validate... + for k, v in fieldpairs(result): + fromJson(n[k], v) + +proc unpackArg[T](argIdx: int, argName: string, argtype: typedesc[T], args: JsonNode): T = + echo argName, " ", args.pretty + fromJson(args[argIdx], result) proc setupParams(node, parameters, paramsIdent: NimNode) = # recurse parameter's fields until we only have symbols @@ -279,19 +154,18 @@ proc setupParams(node, parameters, paramsIdent: NimNode) = raise newException(ValueError, `expectedStr` & $`paramsIdent`.len) ) - for i in 1..< parameters.len: + for i in 1 ..< parameters.len: let paramName = parameters[i][0] pos = i - 1 + paramNameStr = $paramName var paramType = parameters[i][1] - #discard paramType.preParseTypes(paramName, errorCheck) node.add(quote do: - var `paramName`: `paramType` - processFields(`paramsIdent`[`pos`], `paramName`, `paramType`) + var `paramName` = `unpackArg`(`pos`, `paramNameStr`, `paramType`, `paramsIdent`) + `errorCheck` ) - # TODO: Check for byte ranges macro on*(server: var RpcServer, path: string, body: untyped): untyped = result = newStmtList() @@ -320,6 +194,13 @@ macro on*(server: var RpcServer, path: string, body: untyped): untyped = when isMainModule: import unittest var s = newRpcServer("localhost") + s.on("rpc.simplepath"): + result = %1 + s.on("rpc.returnint") do() -> int: + result = %2 + s.on("rpc.differentparams") do(a: int, b: string): + var node = %"test" + result = node s.on("rpc.arrayparam") do(arr: array[0..5, byte], b: string): var res = newJArray() for item in arr: @@ -335,24 +216,25 @@ when isMainModule: type Test2 = object - x: array[2, int] - Test = object - d: array[0..1, int] - e: Test2 + x: array[0..2, int] + y: string - type MyObject* = object - a: int - b: Test - c: float + Test = object + a: array[0..1, int] + b: Test2 + + MyObject* = object + a: int + b: Test + c: float s.on("rpc.objparam") do(a: string, obj: MyObject): result = %obj - s.on("rpc.uinttypes") do(a: byte, b: uint16, c: uint32): - result = %[int(a), int(b), int(c)] - s.on("rpc.inttypes") do(a: int8, b: int16, c: int32, d: int8, e: int16, f: int32): - result = %[int(a), int(b), int(c), int(d), int(e), int(f)] - suite "Server types": + test "On macro registration": + check s.procs.hasKey("rpc.simplepath") + check s.procs.hasKey("rpc.returnint") + check s.procs.hasKey("rpc.returnint") test "Array/seq parameters": let r1 = waitfor rpcArrayParam(%[%[1, 2, 3], %"hello"]) var ckR1 = %[1, 2, 3, 0, 0, 0] @@ -365,32 +247,20 @@ when isMainModule: check r2 == ckR2 test "Object parameters": let - obj = %*{"a": %1, "b": %*{"d": %[5, 0], "e": %*{"x": %[1, 1]}}, "c": %1.23} - r = waitfor rpcObjParam(%[%"Test", obj]) + obj = %*{ + "a": %1, + "b": %*{ + "a": %[5, 0], + "b": %*{ + "x": %[1, 2, 3], + "y": %"test" + } + }, + "c": %1.23} + r = waitfor rpcObjParam(%[%"abc", obj]) check r == obj - expect ValueError: - # here we fail to provide one of the nested fields in json to the rpc - # TODO: Should this be allowed? We either allow partial non-ambiguous parsing or not - # Currently, as long as the Nim fields are satisfied, other fields are ignored - let - obj = %*{"a": %1, "b": %*{"a": %[5, 0]}, "c": %1.23} - discard waitFor rpcObjParam(%[%"abc", obj]) # Why doesn't asyncCheck raise? - test "Uint types": - let - testCase = %[%255, %65534, %4294967295] - r = waitfor rpcUIntTypes(testCase) - check r == testCase - test "Int types": - let - testCase = %[ - %(127), %(32767), %(2147483647), - %(-128), %(-32768), %(-2147483648) - ] - r = waitfor rpcIntTypes(testCase) - check r == testCase test "Runtime errors": expect ValueError: echo waitfor rpcArrayParam(%[%[0, 1, 2, 3, 4, 5, 6], %"hello"]) -# TODO: Split runtime strictness checking into defines - is there ever a reason to trust input? -# TODO: Add path as constant for each rpc + From f49f011d8877b69deca0ed654ce3b48452cca157 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 8 May 2018 15:37:23 +0100 Subject: [PATCH 038/116] Add error checking, tidy up code, added bool processing --- eth-rpc/server/servertypes.nim | 156 ++++++++++++++------------------- 1 file changed, 68 insertions(+), 88 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 1fcf9a2..99505af 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -64,6 +64,72 @@ var sharedServer: RpcServer proc sharedRpcServer*(): RpcServer = if sharedServer.isNil: sharedServer = newRpcServer("") result = sharedServer + +proc fromJson(n: JsonNode, result: var bool) = + if n.kind != JBool: raise newException(ValueError, "Expected JBool but got " & $n.kind) + result = n.getBool() + +proc fromJson(n: JsonNode, result: var int) = + if n.kind != JInt: raise newException(ValueError, "Expected JInt but got " & $n.kind) + result = n.getInt() + +proc fromJson(n: JsonNode, result: var byte) = + if n.kind != JInt: raise newException(ValueError, "Expected JInt but got " & $n.kind) + let v = n.getInt() + if v > 255 or v < 0: raise newException(ValueError, "Parameter value out of range for byte: " & $v) + result = byte(v) + +proc fromJson(n: JsonNode, result: var float) = + if n.kind != JFloat: raise newException(ValueError, "Expected JFloat but got " & $n.kind) + result = n.getFloat() + +proc fromJson(n: JsonNode, result: var string) = + if n.kind != JString: raise newException(ValueError, "Expected JString but got " & $n.kind) + result = n.getStr() + +proc fromJson[T](n: JsonNode, result: var seq[T]) = + if n.kind != JArray: raise newException(ValueError, "Expected JArray but got " & $n.kind) + result = newSeq[T](n.len) + for i in 0 ..< n.len: + fromJson(n[i], result[i]) + +proc fromJson[N, T](n: JsonNode, result: var array[N, T]) = + if n.kind != JArray: raise newException(ValueError, "Expected JArray but got " & $n.kind) + if n.len > result.len: raise newException(ValueError, "Parameter item count is too big for array") + for i in 0 ..< n.len: + fromJson(n[i], result[i]) + +proc fromJson[T: object](n: JsonNode, result: var T) = + if n.kind != JObject: raise newException(ValueError, "Expected JObject but got " & $n.kind) + for k, v in fieldpairs(result): + fromJson(n[k], v) + +proc unpackArg[T](argIdx: int, argName: string, argtype: typedesc[T], args: JsonNode): T = + fromJson(args[argIdx], result) + +proc setupParams(parameters, paramsIdent: NimNode): NimNode = + # Add code to verify input and load parameters into Nim types + result = newStmtList() + if not parameters.isNil: + # initial parameter array length check + var expectedLen = parameters.len - 1 + let expectedStr = "Expected " & $expectedLen & " Json parameter(s) but got " + result.add(quote do: + if `paramsIdent`.len != `expectedLen`: + raise newException(ValueError, `expectedStr` & $`paramsIdent`.len) + ) + # unpack each parameter and provide assignments + for i in 1 ..< parameters.len: + let + paramName = parameters[i][0] + pos = i - 1 + paramNameStr = $paramName + var + paramType = parameters[i][1] + result.add(quote do: + var `paramName` = `unpackArg`(`pos`, `paramNameStr`, `paramType`, `paramsIdent`) + ) + echo result.repr macro multiRemove(s: string, values: varargs[string]): untyped = ## Wrapper for multiReplace @@ -81,99 +147,13 @@ macro multiRemove(s: string, values: varargs[string]): untyped = body.add multiReplaceCall result = newBlockStmt(body) -proc preParseTypes(typeNode: var NimNode, typeName: NimNode, errorCheck: var NimNode): bool {.compileTime.} = - # handle byte - for i, item in typeNode: - if item.kind == nnkIdent and item.basename == ident"byte": - typeNode[i] = ident"int" - # add some extra checks - result = true - else: - var t = typeNode[i] - if preParseTypes(t, typeName, errorCheck): - typeNode[i] = t - -proc expect(node, jsonIdent, fieldName: NimNode, tn: JsonNodeKind) = - let - expectedStr = "Expected parameter `" & fieldName.repr & "` to be " & $tn & " but got " - tnIdent = ident($tn) - node.add(quote do: - if `jsonIdent`.kind != `tnIdent`: - raise newException(ValueError, `expectedStr` & $`jsonIdent`.kind) - ) - -### - -proc fromJson(n: JsonNode, result: var int) = - # TODO: validate... - result = n.getInt() - -proc fromJson(n: JsonNode, result: var byte) = - let v = n.getInt() - if v > 255: raise newException(ValueError, "Parameter value to large for byte: " & $v) - result = byte(v) - -proc fromJson(n: JsonNode, result: var float) = - # TODO: validate... - result = n.getFloat() - -proc fromJson(n: JsonNode, result: var string) = - # TODO: validate... - result = n.getStr() - -proc fromJson[T](n: JsonNode, result: var seq[T]) = - # TODO: validate... - result = newSeq[T](n.len) - for i in 0 ..< n.len: - fromJson(n[i], result[i]) - -proc fromJson[N, T](n: JsonNode, result: var array[N, T]) = - # TODO: validate... - if n.len > result.len: raise newException(ValueError, "Parameter data too big for array") - for i in 0..< n.len: - fromJson(n[i], result[i]) - -proc fromJson[T: object](n: JsonNode, result: var T) = # This reads a custom object - # TODO: validate... - for k, v in fieldpairs(result): - fromJson(n[k], v) - -proc unpackArg[T](argIdx: int, argName: string, argtype: typedesc[T], args: JsonNode): T = - echo argName, " ", args.pretty - fromJson(args[argIdx], result) - -proc setupParams(node, parameters, paramsIdent: NimNode) = - # recurse parameter's fields until we only have symbols - if not parameters.isNil: - var - errorCheck = newStmtList() - expectedParams = parameters.len - 1 - let expectedStr = "Expected " & $`expectedParams` & " Json parameter(s) but got " - node.add(quote do: - if `paramsIdent`.len != `expectedParams`: - raise newException(ValueError, `expectedStr` & $`paramsIdent`.len) - ) - - for i in 1 ..< parameters.len: - let - paramName = parameters[i][0] - pos = i - 1 - paramNameStr = $paramName - var - paramType = parameters[i][1] - node.add(quote do: - var `paramName` = `unpackArg`(`pos`, `paramNameStr`, `paramType`, `paramsIdent`) - - `errorCheck` - ) - macro on*(server: var RpcServer, path: string, body: untyped): untyped = result = newStmtList() - var setup = newStmtList() let parameters = body.findChild(it.kind == nnkFormalParams) paramsIdent = ident"params" - setup.setupParams(parameters, paramsIdent) + var setup = setupParams(parameters, paramsIdent) + #setup.setupParams(parameters, paramsIdent) # wrapping proc let From 9b4e373633a7db52e0985a213756487665252536 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 8 May 2018 15:55:31 +0100 Subject: [PATCH 039/116] Add input error checking and propagate parameter names for errors --- eth-rpc/server/servertypes.nim | 83 ++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 39 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 99505af..44112f2 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -65,47 +65,47 @@ proc sharedRpcServer*(): RpcServer = if sharedServer.isNil: sharedServer = newRpcServer("") result = sharedServer -proc fromJson(n: JsonNode, result: var bool) = - if n.kind != JBool: raise newException(ValueError, "Expected JBool but got " & $n.kind) +proc fromJson(n: JsonNode, argName: string, result: var bool) = + if n.kind != JBool: raise newException(ValueError, "Parameter \"" & argName & "\" expected JBool but got " & $n.kind) result = n.getBool() -proc fromJson(n: JsonNode, result: var int) = - if n.kind != JInt: raise newException(ValueError, "Expected JInt but got " & $n.kind) +proc fromJson(n: JsonNode, argName: string, result: var int) = + if n.kind != JInt: raise newException(ValueError, "Parameter \"" & argName & "\" expected JInt but got " & $n.kind) result = n.getInt() -proc fromJson(n: JsonNode, result: var byte) = - if n.kind != JInt: raise newException(ValueError, "Expected JInt but got " & $n.kind) +proc fromJson(n: JsonNode, argName: string, result: var byte) = + if n.kind != JInt: raise newException(ValueError, "Parameter \"" & argName & "\" expected JInt but got " & $n.kind) let v = n.getInt() - if v > 255 or v < 0: raise newException(ValueError, "Parameter value out of range for byte: " & $v) + if v > 255 or v < 0: raise newException(ValueError, "Parameter \"" & argName & "\" value out of range for byte: " & $v) result = byte(v) -proc fromJson(n: JsonNode, result: var float) = - if n.kind != JFloat: raise newException(ValueError, "Expected JFloat but got " & $n.kind) +proc fromJson(n: JsonNode, argName: string, result: var float) = + if n.kind != JFloat: raise newException(ValueError, "Parameter \"" & argName & "\" expected JFloat but got " & $n.kind) result = n.getFloat() -proc fromJson(n: JsonNode, result: var string) = - if n.kind != JString: raise newException(ValueError, "Expected JString but got " & $n.kind) +proc fromJson(n: JsonNode, argName: string, result: var string) = + if n.kind != JString: raise newException(ValueError, "Parameter \"" & argName & "\" expected JString but got " & $n.kind) result = n.getStr() -proc fromJson[T](n: JsonNode, result: var seq[T]) = - if n.kind != JArray: raise newException(ValueError, "Expected JArray but got " & $n.kind) +proc fromJson[T](n: JsonNode, argName: string, result: var seq[T]) = + if n.kind != JArray: raise newException(ValueError, "Parameter \"" & argName & "\" expected JArray but got " & $n.kind) result = newSeq[T](n.len) for i in 0 ..< n.len: - fromJson(n[i], result[i]) + fromJson(n[i], argName, result[i]) -proc fromJson[N, T](n: JsonNode, result: var array[N, T]) = - if n.kind != JArray: raise newException(ValueError, "Expected JArray but got " & $n.kind) - if n.len > result.len: raise newException(ValueError, "Parameter item count is too big for array") +proc fromJson[N, T](n: JsonNode, argName: string, result: var array[N, T]) = + if n.kind != JArray: raise newException(ValueError, "Parameter \"" & argName & "\" expected JArray but got " & $n.kind) + if n.len > result.len: raise newException(ValueError, "Parameter \"" & argName & "\" item count is too big for array") for i in 0 ..< n.len: - fromJson(n[i], result[i]) + fromJson(n[i], argName, result[i]) -proc fromJson[T: object](n: JsonNode, result: var T) = - if n.kind != JObject: raise newException(ValueError, "Expected JObject but got " & $n.kind) +proc fromJson[T: object](n: JsonNode, argName: string, result: var T) = + if n.kind != JObject: raise newException(ValueError, "Parameter \"" & argName & "\" expected JObject but got " & $n.kind) for k, v in fieldpairs(result): - fromJson(n[k], v) + fromJson(n[k], k, v) proc unpackArg[T](argIdx: int, argName: string, argtype: typedesc[T], args: JsonNode): T = - fromJson(args[argIdx], result) + fromJson(args[argIdx], argName, result) proc setupParams(parameters, paramsIdent: NimNode): NimNode = # Add code to verify input and load parameters into Nim types @@ -115,6 +115,8 @@ proc setupParams(parameters, paramsIdent: NimNode): NimNode = var expectedLen = parameters.len - 1 let expectedStr = "Expected " & $expectedLen & " Json parameter(s) but got " result.add(quote do: + if `paramsIdent`.kind != JArray: + raise newException(ValueError, "Parameter params expected JArray but got " & $`paramsIdent`.kind) if `paramsIdent`.len != `expectedLen`: raise newException(ValueError, `expectedStr` & $`paramsIdent`.len) ) @@ -124,12 +126,17 @@ proc setupParams(parameters, paramsIdent: NimNode): NimNode = paramName = parameters[i][0] pos = i - 1 paramNameStr = $paramName - var paramType = parameters[i][1] result.add(quote do: var `paramName` = `unpackArg`(`pos`, `paramNameStr`, `paramType`, `paramsIdent`) ) - echo result.repr + else: + # no parameters expected + result.add(quote do: + if `paramsIdent`.len != 0: + raise newException(ValueError, "Expected no parameters but got " & $`paramsIdent`.len) + ) + macro multiRemove(s: string, values: varargs[string]): untyped = ## Wrapper for multiReplace @@ -153,7 +160,6 @@ macro on*(server: var RpcServer, path: string, body: untyped): untyped = parameters = body.findChild(it.kind == nnkFormalParams) paramsIdent = ident"params" var setup = setupParams(parameters, paramsIdent) - #setup.setupParams(parameters, paramsIdent) # wrapping proc let @@ -164,7 +170,6 @@ macro on*(server: var RpcServer, path: string, body: untyped): untyped = else: procBody = body.body result = quote do: proc `procName`*(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = - #`checkTypeError` `setup` `procBody` `server`.register(`path`, `procName`) @@ -207,6 +212,17 @@ when isMainModule: a: int b: Test c: float + let + testObj = %*{ + "a": %1, + "b": %*{ + "a": %[5, 0], + "b": %*{ + "x": %[1, 2, 3], + "y": %"test" + } + }, + "c": %1.23} s.on("rpc.objparam") do(a: string, obj: MyObject): result = %obj @@ -226,19 +242,8 @@ when isMainModule: for i in 0..4: ckR2.add %(i + 1) check r2 == ckR2 test "Object parameters": - let - obj = %*{ - "a": %1, - "b": %*{ - "a": %[5, 0], - "b": %*{ - "x": %[1, 2, 3], - "y": %"test" - } - }, - "c": %1.23} - r = waitfor rpcObjParam(%[%"abc", obj]) - check r == obj + let r = waitfor rpcObjParam(%[%"abc", testObj]) + check r == testObj test "Runtime errors": expect ValueError: echo waitfor rpcArrayParam(%[%[0, 1, 2, 3, 4, 5, 6], %"hello"]) From d0c34ba8348349b0cbccd5f14bb4c4e38ef03868 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 8 May 2018 16:03:28 +0100 Subject: [PATCH 040/116] Minor comment tidy up --- eth-rpc/server/servertypes.nim | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 44112f2..d68f244 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -130,13 +130,6 @@ proc setupParams(parameters, paramsIdent: NimNode): NimNode = result.add(quote do: var `paramName` = `unpackArg`(`pos`, `paramNameStr`, `paramType`, `paramsIdent`) ) - else: - # no parameters expected - result.add(quote do: - if `paramsIdent`.len != 0: - raise newException(ValueError, "Expected no parameters but got " & $`paramsIdent`.len) - ) - macro multiRemove(s: string, values: varargs[string]): untyped = ## Wrapper for multiReplace @@ -159,22 +152,22 @@ macro on*(server: var RpcServer, path: string, body: untyped): untyped = let parameters = body.findChild(it.kind == nnkFormalParams) paramsIdent = ident"params" - var setup = setupParams(parameters, paramsIdent) - - # wrapping proc - let pathStr = $path - procName = ident(pathStr.multiRemove(".", "/")) # TODO: Make this unique to avoid potential clashes, or allow people to know the name for calling? - var procBody: NimNode + procName = ident(pathStr.multiRemove(".", "/")) + var + setup = setupParams(parameters, paramsIdent) + procBody: NimNode if body.kind == nnkStmtList: procBody = body else: procBody = body.body + + # wrapping async proc result = quote do: proc `procName`*(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = `setup` `procBody` `server`.register(`path`, `procName`) when defined(nimDumpRpcs): - echo pathStr, ": ", result.repr + echo "\n", pathStr, ": ", result.repr when isMainModule: import unittest From d96f984d5c525f25fd5f3be2cdd2b86565b15ab3 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 8 May 2018 16:26:13 +0100 Subject: [PATCH 041/116] Remove {.rpc.} macro and merged testing into testrpcmacro --- eth-rpc/server/servertypes.nim | 106 --------------------------------- tests/testrpcmacro.nim | 59 ++++++++++++++---- tests/testserverclient.nim | 2 +- 3 files changed, 47 insertions(+), 120 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index d68f244..063304a 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -19,38 +19,6 @@ proc register*(server: RpcServer, name: string, rpc: RpcProc) = proc unRegisterAll*(server: RpcServer) = server.procs.clear -macro rpc*(prc: untyped): untyped = - ## Converts a procedure into the following format: - ## *(params: JsonNode): Future[JsonNode] {.async.} - ## This procedure is then added into a compile-time list - ## so that it is automatically registered for every server that - ## calls registerRpcs(server) - prc.expectKind nnkProcDef - result = prc - let - params = prc.params - procName = prc.name - - procName.expectKind(nnkIdent) - - # check there isn't already a result type - assert params[0].kind == nnkEmpty - - # add parameter - params.add nnkIdentDefs.newTree( - newIdentNode("params"), - newIdentNode("JsonNode"), - newEmptyNode() - ) - # set result type - params[0] = nnkBracketExpr.newTree( - newIdentNode("Future"), - newIdentNode("JsonNode") - ) - # add async pragma; we can assume there isn't an existing .async. - # as this would mean there's a return type and fail the result check above. - prc.addPragma(newIdentNode("async")) - proc newRpcServer*(address = "localhost", port: Port = Port(8545)): RpcServer = result = RpcServer( socket: newAsyncSocket(), @@ -168,77 +136,3 @@ macro on*(server: var RpcServer, path: string, body: untyped): untyped = `server`.register(`path`, `procName`) when defined(nimDumpRpcs): echo "\n", pathStr, ": ", result.repr - -when isMainModule: - import unittest - var s = newRpcServer("localhost") - s.on("rpc.simplepath"): - result = %1 - s.on("rpc.returnint") do() -> int: - result = %2 - s.on("rpc.differentparams") do(a: int, b: string): - var node = %"test" - result = node - s.on("rpc.arrayparam") do(arr: array[0..5, byte], b: string): - var res = newJArray() - for item in arr: - res.add %int(item) - res.add %b - result = %res - s.on("rpc.seqparam") do(a: string, s: seq[int]): - var res = newJArray() - res.add %a - for item in s: - res.add %int(item) - result = res - - type - Test2 = object - x: array[0..2, int] - y: string - - Test = object - a: array[0..1, int] - b: Test2 - - MyObject* = object - a: int - b: Test - c: float - let - testObj = %*{ - "a": %1, - "b": %*{ - "a": %[5, 0], - "b": %*{ - "x": %[1, 2, 3], - "y": %"test" - } - }, - "c": %1.23} - - s.on("rpc.objparam") do(a: string, obj: MyObject): - result = %obj - suite "Server types": - test "On macro registration": - check s.procs.hasKey("rpc.simplepath") - check s.procs.hasKey("rpc.returnint") - check s.procs.hasKey("rpc.returnint") - test "Array/seq parameters": - let r1 = waitfor rpcArrayParam(%[%[1, 2, 3], %"hello"]) - var ckR1 = %[1, 2, 3, 0, 0, 0] - ckR1.elems.add %"hello" - check r1 == ckR1 - - let r2 = waitfor rpcSeqParam(%[%"abc", %[1, 2, 3, 4, 5]]) - var ckR2 = %["abc"] - for i in 0..4: ckR2.add %(i + 1) - check r2 == ckR2 - test "Object parameters": - let r = waitfor rpcObjParam(%[%"abc", testObj]) - check r == testObj - test "Runtime errors": - expect ValueError: - echo waitfor rpcArrayParam(%[%[0, 1, 2, 3, 4, 5, 6], %"hello"]) - - diff --git a/tests/testrpcmacro.nim b/tests/testrpcmacro.nim index c537eb9..5d8d1f5 100644 --- a/tests/testrpcmacro.nim +++ b/tests/testrpcmacro.nim @@ -1,53 +1,86 @@ import ../ eth-rpc / server / servertypes, unittest, asyncdispatch, json, tables +type + # some nested types to check object parsing + Test2 = object + x: array[0..2, int] + y: string + + Test = object + a: array[0..1, int] + b: Test2 + + MyObject* = object + a: int + b: Test + c: float +let + testObj = %*{ + "a": %1, + "b": %*{ + "a": %[5, 0], + "b": %*{ + "x": %[1, 2, 3], + "y": %"test" + } + }, + "c": %1.23} + var s = newRpcServer("localhost") + +# RPC definitions s.on("rpc.simplepath"): echo "hello3" result = %1 + s.on("rpc.returnint") do() -> int: echo "hello2" + s.on("rpc.differentparams") do(a: int, b: string): var node = %"test" result = node + s.on("rpc.arrayparam") do(arr: array[0..5, byte], b: string): var res = newJArray() for item in arr: res.add %int(item) res.add %b result = %res -s.on("rpc.seqparam") do(b: string, s: seq[int]): + +s.on("rpc.seqparam") do(a: string, s: seq[int]): var res = newJArray() - res.add %b + res.add %a for item in s: res.add %int(item) result = res -type MyObject* = object - a: int - b: string - c: float -s.on("rpc.objparam") do(b: string, obj: MyObject): + +s.on("rpc.objparam") do(a: string, obj: MyObject): result = %obj + +# Tests suite "Server types": + test "On macro registration": check s.procs.hasKey("rpc.simplepath") check s.procs.hasKey("rpc.returnint") check s.procs.hasKey("rpc.returnint") - test "Array/seq parameters": + + test "Array parameters": let r1 = waitfor rpcArrayParam(%[%[1, 2, 3], %"hello"]) var ckR1 = %[1, 2, 3, 0, 0, 0] ckR1.elems.add %"hello" check r1 == ckR1 + test "Seq parameters": let r2 = waitfor rpcSeqParam(%[%"abc", %[1, 2, 3, 4, 5]]) var ckR2 = %["abc"] for i in 0..4: ckR2.add %(i + 1) check r2 == ckR2 + test "Object parameters": - let - obj = %*{"a": %1, "b": %"hello", "c": %1.23} - r = waitfor rpcObjParam(%[%"abc", obj]) - check r == obj + let r = waitfor rpcObjParam(%[%"abc", testObj]) + check r == testObj + test "Runtime errors": expect ValueError: discard waitfor rpcArrayParam(%[%[0, 1, 2, 3, 4, 5, 6], %"hello"]) - # TODO: Add other errors \ No newline at end of file diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index fd32548..1a8da47 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -7,7 +7,6 @@ srv.address = "localhost" srv.port = Port(8545) srv.on("myProc") do(input: string, data: array[0..3, int]): - # Custom async RPC call result = %("Hello " & input & " data: " & $data) asyncCheck srv.serve @@ -25,6 +24,7 @@ suite "RPC": response = waitFor client.web3_sha3(%["abc"]) check response.result.getStr == "3A985DA74FE225B2045C172D6BD390BD855F086E3E9D525B46BFE24511431532" test "Custom RPC": + # Custom async RPC call response = waitFor client.call("myProc", %[%"abc", %[1, 2, 3, 4]]) check response.result.getStr == "Hello abc data: [1, 2, 3, 4]" From 735554314723eb61e18eed9ba1e037c82abdb9a2 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 8 May 2018 17:29:23 +0100 Subject: [PATCH 042/116] Now allows return types in on macro --- eth-rpc/server/servertypes.nim | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 063304a..74dde0a 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -125,14 +125,34 @@ macro on*(server: var RpcServer, path: string, body: untyped): untyped = var setup = setupParams(parameters, paramsIdent) procBody: NimNode + bodyWrapper = newStmtList() + if body.kind == nnkStmtList: procBody = body else: procBody = body.body - - # wrapping async proc + + if parameters.len > 0 and parameters[0] != nil: + # when a return type is specified, shadow async's result + # and pass it back jsonified + let + returnType = parameters[0] + res = ident"result" + template doMain(body: untyped): untyped = + # create a new scope to allow shadowing result + block: + body + bodyWrapper = quote do: + `res` = `doMain`: + var `res`: `returnType` + `procBody` + %`res` + else: + bodyWrapper = quote do: `procBody` + + # async proc wrapper around body result = quote do: - proc `procName`*(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = - `setup` - `procBody` - `server`.register(`path`, `procName`) + proc `procName`*(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = + `setup` + `bodyWrapper` + `server`.register(`path`, `procName`) when defined(nimDumpRpcs): echo "\n", pathStr, ": ", result.repr From 2c8faae0b5a685047d2708625014fa4cd02a0678 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 8 May 2018 17:30:05 +0100 Subject: [PATCH 043/116] Updated tests to check return type processing --- tests/testrpcmacro.nim | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/testrpcmacro.nim b/tests/testrpcmacro.nim index 5d8d1f5..9e1803b 100644 --- a/tests/testrpcmacro.nim +++ b/tests/testrpcmacro.nim @@ -57,6 +57,13 @@ s.on("rpc.seqparam") do(a: string, s: seq[int]): s.on("rpc.objparam") do(a: string, obj: MyObject): result = %obj +s.on("rpc.returntypesimple") do(i: int) -> int: + result = i + +s.on("rpc.returntypecomplex") do(i: int) -> Test2: + result.x = [1, i, 3] + result.y = "test" + # Tests suite "Server types": @@ -81,6 +88,18 @@ suite "Server types": let r = waitfor rpcObjParam(%[%"abc", testObj]) check r == testObj + test "Simple return types": + let + inp = %99 + r1 = waitfor rpcReturnTypeSimple(%[%inp]) + check r1 == inp + + test "Complex return types": + let + inp = 99 + r1 = waitfor rpcReturnTypeComplex(%[%inp]) + check r1 == %*{"x": %[1, inp, 3], "y": "test"} + test "Runtime errors": expect ValueError: discard waitfor rpcArrayParam(%[%[0, 1, 2, 3, 4, 5, 6], %"hello"]) From e6e343ab9202fb35378a0aee22ed63a7a911a3a0 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 8 May 2018 20:34:28 +0100 Subject: [PATCH 044/116] Enforce checks in release and refactor to separate macro --- eth-rpc/client/clientdispatch.nim | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/eth-rpc/client/clientdispatch.nim b/eth-rpc/client/clientdispatch.nim index 6ec195f..34c1c69 100644 --- a/eth-rpc/client/clientdispatch.nim +++ b/eth-rpc/client/clientdispatch.nim @@ -31,21 +31,33 @@ proc call*(self: RpcClient, name: string, params: JsonNode): Future[Response] {. self.awaiting[id] = newFut result = await newFut +macro checkGet(node: JsonNode, fieldName: string, jKind: static[JsonNodeKind]): untyped = + result = quote do: + if not node.hasKey(`fieldName`): raise newException(ValueError, "Message is missing required field \"" & `fieldName` & "\"") + if `node`[`fieldName`].kind != `jKind`.JsonNodeKind: raise newException(ValueError, "Expected " & $(`jKind`.JsonNodeKind) & ", got " & $`node`[`fieldName`].kind) + case jKind + of JBool: result.add(quote do: `node`[`fieldName`].getBool) + of JInt: result.add(quote do: `node`[`fieldName`].getInt) + of JString: result.add(quote do: `node`[`fieldName`].getStr) + of JFloat: result.add(quote do: `node`[`fieldName`].getFloat) + of JObject: result.add(quote do: `node`[`fieldName`].getObject) + else: discard + proc processMessage(self: RpcClient, line: string) = let node = parseJson(line) # TODO: Use more appropriate exception objects - if not node.hasKey("jsonrpc"): raise newException(ValueError, "Message is missing rpc version field") - elif node["jsonrpc"].str != "2.0": raise newException(ValueError, "Unsupported version of JSON, expected 2.0, received \"" & node["jsonrpc"].str & "\"") - elif not node.hasKey("id"): raise newException(ValueError, "Message is missing id field") - elif not self.awaiting.hasKey(node["id"].str): raise newException(ValueError, "Cannot find message id \"" & node["id"].str & "\"") + let version = checkGet(node, "jsonrpc", JString) + if version != "2.0": raise newException(ValueError, "Unsupported version of JSON, expected 2.0, received \"" & version & "\"") + let id = checkGet(node, "id", JString) + if not self.awaiting.hasKey(id): raise newException(ValueError, "Cannot find message id \"" & node["id"].str & "\"") if node["error"].kind == JNull: - self.awaiting[node["id"].str].complete((false, node["result"])) - self.awaiting.del(node["id"].str) + self.awaiting[id].complete((false, node["result"])) + self.awaiting.del(id) else: - self.awaiting[node["id"].str].complete((true, node["error"])) - self.awaiting.del(node["id"].str) + self.awaiting[id].complete((true, node["error"])) + self.awaiting.del(id) proc connect*(self: RpcClient, address: string, port: Port): Future[void] From 81909360a65627c792c47890fa23b882020d2093 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 8 May 2018 20:35:30 +0100 Subject: [PATCH 045/116] Move checks out of generic procs, refactor expect len to separate proc --- eth-rpc/server/servertypes.nim | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 74dde0a..1fa750f 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -56,50 +56,54 @@ proc fromJson(n: JsonNode, argName: string, result: var string) = result = n.getStr() proc fromJson[T](n: JsonNode, argName: string, result: var seq[T]) = - if n.kind != JArray: raise newException(ValueError, "Parameter \"" & argName & "\" expected JArray but got " & $n.kind) result = newSeq[T](n.len) for i in 0 ..< n.len: fromJson(n[i], argName, result[i]) proc fromJson[N, T](n: JsonNode, argName: string, result: var array[N, T]) = - if n.kind != JArray: raise newException(ValueError, "Parameter \"" & argName & "\" expected JArray but got " & $n.kind) - if n.len > result.len: raise newException(ValueError, "Parameter \"" & argName & "\" item count is too big for array") for i in 0 ..< n.len: fromJson(n[i], argName, result[i]) proc fromJson[T: object](n: JsonNode, argName: string, result: var T) = - if n.kind != JObject: raise newException(ValueError, "Parameter \"" & argName & "\" expected JObject but got " & $n.kind) for k, v in fieldpairs(result): fromJson(n[k], k, v) proc unpackArg[T](argIdx: int, argName: string, argtype: typedesc[T], args: JsonNode): T = + when argType is array or argType is seq: + if args[argIdx].kind != JArray: raise newException(ValueError, "Parameter \"" & argName & "\" expected JArray but got " & $args[argIdx].kind) + when argType is array: + if args[argIdx].len > result.len: raise newException(ValueError, "Parameter \"" & argName & "\" item count is too big for array") + when argType is object: + if args[argIdx].kind != JObject: raise newException(ValueError, "Parameter \"" & argName & "\" expected JObject but got " & $args[argIdx].kind) fromJson(args[argIdx], argName, result) +proc expectArrayLen(node: NimNode, paramsIdent: untyped, length: int) = + let expectedStr = "Expected " & $length & " Json parameter(s) but got " + node.add(quote do: + if `paramsIdent`.kind != JArray: + raise newException(ValueError, "Parameter params expected JArray but got " & $`paramsIdent`.kind) + if `paramsIdent`.len != `length`: + raise newException(ValueError, `expectedStr` & $`paramsIdent`.len) + ) + proc setupParams(parameters, paramsIdent: NimNode): NimNode = # Add code to verify input and load parameters into Nim types result = newStmtList() if not parameters.isNil: # initial parameter array length check - var expectedLen = parameters.len - 1 - let expectedStr = "Expected " & $expectedLen & " Json parameter(s) but got " - result.add(quote do: - if `paramsIdent`.kind != JArray: - raise newException(ValueError, "Parameter params expected JArray but got " & $`paramsIdent`.kind) - if `paramsIdent`.len != `expectedLen`: - raise newException(ValueError, `expectedStr` & $`paramsIdent`.len) - ) + result.expectArrayLen(paramsIdent, parameters.len - 1) # unpack each parameter and provide assignments for i in 1 ..< parameters.len: let - paramName = parameters[i][0] pos = i - 1 + paramName = parameters[i][0] paramNameStr = $paramName paramType = parameters[i][1] result.add(quote do: var `paramName` = `unpackArg`(`pos`, `paramNameStr`, `paramType`, `paramsIdent`) ) -macro multiRemove(s: string, values: varargs[string]): untyped = +macro multiRemove(s: string, values: varargs[string]): string = ## Wrapper for multiReplace var body = newStmtList() @@ -114,6 +118,7 @@ macro multiRemove(s: string, values: varargs[string]): untyped = body.add multiReplaceCall result = newBlockStmt(body) + echo "!!", result.repr macro on*(server: var RpcServer, path: string, body: untyped): untyped = result = newStmtList() From c9a9eb90ce18d08cfbc0e6556f714d647fed4eda Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 8 May 2018 20:36:13 +0100 Subject: [PATCH 046/116] Remove redundant tests, add more error checks --- tests/testrpcmacro.nim | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/testrpcmacro.nim b/tests/testrpcmacro.nim index 9e1803b..da5f299 100644 --- a/tests/testrpcmacro.nim +++ b/tests/testrpcmacro.nim @@ -30,15 +30,10 @@ var s = newRpcServer("localhost") # RPC definitions s.on("rpc.simplepath"): - echo "hello3" result = %1 -s.on("rpc.returnint") do() -> int: - echo "hello2" - -s.on("rpc.differentparams") do(a: int, b: string): - var node = %"test" - result = node +s.on("rpc.differentparams") do(a: int, b: string): + result = %"test" s.on("rpc.arrayparam") do(arr: array[0..5, byte], b: string): var res = newJArray() @@ -69,8 +64,12 @@ suite "Server types": test "On macro registration": check s.procs.hasKey("rpc.simplepath") - check s.procs.hasKey("rpc.returnint") - check s.procs.hasKey("rpc.returnint") + check s.procs.hasKey("rpc.differentparams") + check s.procs.hasKey("rpc.arrayparam") + check s.procs.hasKey("rpc.seqparam") + check s.procs.hasKey("rpc.objparam") + check s.procs.hasKey("rpc.returntypesimple") + check s.procs.hasKey("rpc.returntypecomplex") test "Array parameters": let r1 = waitfor rpcArrayParam(%[%[1, 2, 3], %"hello"]) @@ -102,4 +101,11 @@ suite "Server types": test "Runtime errors": expect ValueError: + # root param not array + discard waitfor rpcArrayParam(%"test") + expect ValueError: + # too big for array discard waitfor rpcArrayParam(%[%[0, 1, 2, 3, 4, 5, 6], %"hello"]) + expect ValueError: + # wrong sub parameter type + discard waitfor rpcArrayParam(%[%"test", %"hello"]) From ff078478bdd197db552aa439ae5b42856ddf81ea Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 8 May 2018 20:42:16 +0100 Subject: [PATCH 047/116] Remove debug echo --- eth-rpc/server/servertypes.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 1fa750f..079199a 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -118,7 +118,6 @@ macro multiRemove(s: string, values: varargs[string]): string = body.add multiReplaceCall result = newBlockStmt(body) - echo "!!", result.repr macro on*(server: var RpcServer, path: string, body: untyped): untyped = result = newStmtList() From 1ac58430f08e5e82fad9ca32045e4ad8e556a803 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 8 May 2018 20:48:28 +0100 Subject: [PATCH 048/116] Don't call multiRemove macro directly from macro --- eth-rpc/server/servertypes.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 079199a..b09f47a 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -119,13 +119,16 @@ macro multiRemove(s: string, values: varargs[string]): string = body.add multiReplaceCall result = newBlockStmt(body) +proc makeProcName(s: string): string = + s.multiRemove(".", "/") + macro on*(server: var RpcServer, path: string, body: untyped): untyped = result = newStmtList() let parameters = body.findChild(it.kind == nnkFormalParams) paramsIdent = ident"params" pathStr = $path - procName = ident(pathStr.multiRemove(".", "/")) + procName = ident(pathStr.makeProcName) var setup = setupParams(parameters, paramsIdent) procBody: NimNode From e74dfdce62359aadc2f0334db519dd7d4bae42fa Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 9 May 2018 09:51:51 +0100 Subject: [PATCH 049/116] Remove macro for replacing strings in proc name --- eth-rpc/server/servertypes.nim | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index b09f47a..cb49d3d 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -103,24 +103,8 @@ proc setupParams(parameters, paramsIdent: NimNode): NimNode = var `paramName` = `unpackArg`(`pos`, `paramNameStr`, `paramType`, `paramsIdent`) ) -macro multiRemove(s: string, values: varargs[string]): string = - ## Wrapper for multiReplace - var - body = newStmtList() - multiReplaceCall = newCall(ident"multiReplace", s) - - body.add(newVarStmt(ident"eStr", newStrLitNode(""))) - let emptyStr = ident"eStr" - for item in values: - # generate tuples of values with the empty string `eStr` - let sItem = $item - multiReplaceCall.add(newPar(newStrLitNode(sItem), emptyStr)) - - body.add multiReplaceCall - result = newBlockStmt(body) - proc makeProcName(s: string): string = - s.multiRemove(".", "/") + s.multiReplace((".", ""), ("/", "")) macro on*(server: var RpcServer, path: string, body: untyped): untyped = result = newStmtList() From aa99f1076d794e6ba102ffe845bfbbf0a9b7efa6 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 9 May 2018 13:07:32 +0100 Subject: [PATCH 050/116] Add converters to and from common `stint` types and `byte` --- eth-rpc/server/jsonconverters.nim | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 eth-rpc/server/jsonconverters.nim diff --git a/eth-rpc/server/jsonconverters.nim b/eth-rpc/server/jsonconverters.nim new file mode 100644 index 0000000..43d5889 --- /dev/null +++ b/eth-rpc/server/jsonconverters.nim @@ -0,0 +1,25 @@ +import json, stint + +iterator bytes*(i: UInt256|Int256): byte = + let b = cast[ptr array[32, byte]](i.unsafeaddr) + var pos = 0 + while pos < 32: + yield b[pos] + pos += 1 + +proc `%`*(n: UInt256): JsonNode = + ## Generic constructor for JSON data. Creates a new `JInt JsonNode`. + result = newJArray() + for elem in n.bytes: + result.add(%int(elem)) + +proc `%`*(n: Int256): JsonNode = + ## Generic constructor for JSON data. Creates a new `JInt JsonNode`. + result = newJArray() + for elem in n.bytes: + result.add(%int(elem)) + +proc `%`*(n: byte): JsonNode = + ## Generic constructor for JSON data. Creates a new `JInt JsonNode`. + result = newJInt(int(n)) + From 9fd13fd121d953bf5a6da1dbb59a7f47bdb68e6d Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 9 May 2018 13:08:15 +0100 Subject: [PATCH 051/116] Use json converters, special case for rpc's returning JsonNode --- eth-rpc/server/servertypes.nim | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index cb49d3d..b14383d 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -1,5 +1,5 @@ -import asyncdispatch, asyncnet, json, tables, macros, strutils -export asyncdispatch, asyncnet, json +import asyncdispatch, asyncnet, json, tables, macros, strutils, jsonconverters +export asyncdispatch, asyncnet, json, jsonconverters type RpcProc* = proc (params: JsonNode): Future[JsonNode] @@ -121,9 +121,10 @@ macro on*(server: var RpcServer, path: string, body: untyped): untyped = if body.kind == nnkStmtList: procBody = body else: procBody = body.body - if parameters.len > 0 and parameters[0] != nil: + if parameters.len > 0 and parameters[0] != nil and parameters[0] != ident"JsonNode": # when a return type is specified, shadow async's result - # and pass it back jsonified + # and pass it back jsonified - of course, we don't want to do this + # if a JsonNode is explicitly declared as the return type let returnType = parameters[0] res = ident"result" From fd835749ea5f73b605a903879dab1f41b6478e44 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 9 May 2018 14:26:28 +0100 Subject: [PATCH 052/116] Refactor errors --- eth-rpc/server/servertypes.nim | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index b14383d..321f496 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -33,26 +33,29 @@ proc sharedRpcServer*(): RpcServer = if sharedServer.isNil: sharedServer = newRpcServer("") result = sharedServer +template expect*(actual, expected: JsonNodeKind, argName: string) = + if actual != expected: raise newException(ValueError, "Parameter \"" & argName & "\" expected " & $expected & " but got " & $actual) + proc fromJson(n: JsonNode, argName: string, result: var bool) = - if n.kind != JBool: raise newException(ValueError, "Parameter \"" & argName & "\" expected JBool but got " & $n.kind) + n.kind.expect(JBool, argName) result = n.getBool() proc fromJson(n: JsonNode, argName: string, result: var int) = - if n.kind != JInt: raise newException(ValueError, "Parameter \"" & argName & "\" expected JInt but got " & $n.kind) + n.kind.expect(JInt, argName) result = n.getInt() proc fromJson(n: JsonNode, argName: string, result: var byte) = - if n.kind != JInt: raise newException(ValueError, "Parameter \"" & argName & "\" expected JInt but got " & $n.kind) + n.kind.expect(JInt, argName) let v = n.getInt() if v > 255 or v < 0: raise newException(ValueError, "Parameter \"" & argName & "\" value out of range for byte: " & $v) result = byte(v) proc fromJson(n: JsonNode, argName: string, result: var float) = - if n.kind != JFloat: raise newException(ValueError, "Parameter \"" & argName & "\" expected JFloat but got " & $n.kind) + n.kind.expect(JFloat, argName) result = n.getFloat() proc fromJson(n: JsonNode, argName: string, result: var string) = - if n.kind != JString: raise newException(ValueError, "Parameter \"" & argName & "\" expected JString but got " & $n.kind) + n.kind.expect(JString, argName) result = n.getStr() proc fromJson[T](n: JsonNode, argName: string, result: var seq[T]) = @@ -70,18 +73,19 @@ proc fromJson[T: object](n: JsonNode, argName: string, result: var T) = proc unpackArg[T](argIdx: int, argName: string, argtype: typedesc[T], args: JsonNode): T = when argType is array or argType is seq: - if args[argIdx].kind != JArray: raise newException(ValueError, "Parameter \"" & argName & "\" expected JArray but got " & $args[argIdx].kind) + args[argIdx].kind.expect(JArray, argName) when argType is array: if args[argIdx].len > result.len: raise newException(ValueError, "Parameter \"" & argName & "\" item count is too big for array") when argType is object: - if args[argIdx].kind != JObject: raise newException(ValueError, "Parameter \"" & argName & "\" expected JObject but got " & $args[argIdx].kind) + args[argIdx].kind.expect(JObject, argName) fromJson(args[argIdx], argName, result) proc expectArrayLen(node: NimNode, paramsIdent: untyped, length: int) = - let expectedStr = "Expected " & $length & " Json parameter(s) but got " + let + identStr = paramsIdent.repr + expectedStr = "Expected " & $length & " Json parameter(s) but got " node.add(quote do: - if `paramsIdent`.kind != JArray: - raise newException(ValueError, "Parameter params expected JArray but got " & $`paramsIdent`.kind) + `paramsIdent`.kind.expect(JArray, `identStr`) if `paramsIdent`.len != `length`: raise newException(ValueError, `expectedStr` & $`paramsIdent`.len) ) From 7e1fe1800acc5f12a39a42f082f35ff4afcc965b Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 9 May 2018 14:51:19 +0100 Subject: [PATCH 053/116] Use rpcserver import --- tests/testrpcmacro.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testrpcmacro.nim b/tests/testrpcmacro.nim index da5f299..0f83916 100644 --- a/tests/testrpcmacro.nim +++ b/tests/testrpcmacro.nim @@ -1,4 +1,4 @@ -import ../ eth-rpc / server / servertypes, unittest, asyncdispatch, json, tables +import ../ rpcserver, unittest, asyncdispatch, json, tables type # some nested types to check object parsing From 50924d1c12eef961f7ed86222b179317db31ae89 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 9 May 2018 14:57:45 +0100 Subject: [PATCH 054/116] Change inline proc to template --- eth-rpc/client/clientdispatch.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth-rpc/client/clientdispatch.nim b/eth-rpc/client/clientdispatch.nim index 34c1c69..ef34c42 100644 --- a/eth-rpc/client/clientdispatch.nim +++ b/eth-rpc/client/clientdispatch.nim @@ -91,7 +91,7 @@ macro generateCalls: untyped = for callName in ETHEREUM_RPC_CALLS: let nameLit = ident(callName) result.add(quote do: - proc `nameLit`*(client: RpcClient, params: JsonNode): Future[Response] {.inline.} = client.call(`callName`, params) # TODO: Back to template + template `nameLit`*(client: RpcClient, params: JsonNode): Future[Response] = client.call(`callName`, params) # TODO: Back to template ) # generate all client ethereum rpc calls From ab77d3a3711326a4bcef57676eef89ac2d57507f Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 9 May 2018 14:58:21 +0100 Subject: [PATCH 055/116] Don't export `bytes` iterator for `stint` types --- eth-rpc/server/jsonconverters.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/eth-rpc/server/jsonconverters.nim b/eth-rpc/server/jsonconverters.nim index 43d5889..6f5e2c7 100644 --- a/eth-rpc/server/jsonconverters.nim +++ b/eth-rpc/server/jsonconverters.nim @@ -1,6 +1,6 @@ import json, stint -iterator bytes*(i: UInt256|Int256): byte = +iterator bytes(i: UInt256|Int256): byte = let b = cast[ptr array[32, byte]](i.unsafeaddr) var pos = 0 while pos < 32: @@ -19,7 +19,6 @@ proc `%`*(n: Int256): JsonNode = for elem in n.bytes: result.add(%int(elem)) -proc `%`*(n: byte): JsonNode = - ## Generic constructor for JSON data. Creates a new `JInt JsonNode`. +proc `%`*(n: byte{not lit}): JsonNode = result = newJInt(int(n)) From b6db0f151e86e238f3cd1f326c5112164409d2fa Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 9 May 2018 14:58:58 +0100 Subject: [PATCH 056/116] Fix `undeclared identifier: result` in `myProc` custom rpc --- tests/testserverclient.nim | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index 1a8da47..e23ccb4 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -1,7 +1,14 @@ -import ../ rpcclient, ../ rpcserver, - asyncdispatch, json, unittest, tables +import ../ rpcserver, ../ rpcclient, unittest, asyncdispatch, json, tables +#[ + TODO: Importing client before server causes the error: + Error: undeclared identifier: 'result' for the custom procedure. + This is because the rpc procs created by clientdispatch clash with ethprocs. + Currently, easiest solution is to import rpcserver (and therefore generate + ethprocs) before rpcclient. +]# # TODO: dummy implementations of RPC calls handled in async fashion. +# TODO: check required json parameters like version are being raised var srv = sharedRpcServer() srv.address = "localhost" srv.port = Port(8545) @@ -16,7 +23,6 @@ suite "RPC": var client = newRpcClient() await client.connect("localhost", Port(8545)) var response: Response - test "Version": response = waitFor client.web3_clientVersion(newJNull()) check response.result == %"Nimbus-RPC-Test" @@ -28,7 +34,5 @@ suite "RPC": response = waitFor client.call("myProc", %[%"abc", %[1, 2, 3, 4]]) check response.result.getStr == "Hello abc data: [1, 2, 3, 4]" - - waitFor main() \ No newline at end of file From ae182907f2c8a375721b7bc77ae45fa95e60bc00 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 9 May 2018 15:08:03 +0100 Subject: [PATCH 057/116] Add `stint` requirement --- eth_rpc.nimble | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eth_rpc.nimble b/eth_rpc.nimble index 2ba4177..de66dba 100644 --- a/eth_rpc.nimble +++ b/eth_rpc.nimble @@ -7,7 +7,8 @@ srcDir = "src" ### Dependencies requires "nim >= 0.17.3", - "nimcrypto" + "nimcrypto", + "stint" proc configForTests() = --hints: off From 50479cbbdcd7e6d86f6894a3d1b62ce58c4138e6 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 9 May 2018 16:17:33 +0100 Subject: [PATCH 058/116] Update comment to be clearer --- tests/testserverclient.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index e23ccb4..0b68cee 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -2,8 +2,8 @@ import ../ rpcserver, ../ rpcclient, unittest, asyncdispatch, json, tables #[ TODO: Importing client before server causes the error: - Error: undeclared identifier: 'result' for the custom procedure. - This is because the rpc procs created by clientdispatch clash with ethprocs. + Error: undeclared identifier: 'result' for the `myProc` RPC. + This is because the RPC procs created by clientdispatch clash with ethprocs. Currently, easiest solution is to import rpcserver (and therefore generate ethprocs) before rpcclient. ]# From a9715050ae7727eaa0439623ef77ad5946aba482 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 9 May 2018 17:31:28 +0100 Subject: [PATCH 059/116] Added some more simple tests --- tests/testrpcmacro.nim | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/testrpcmacro.nim b/tests/testrpcmacro.nim index 0f83916..d56ca54 100644 --- a/tests/testrpcmacro.nim +++ b/tests/testrpcmacro.nim @@ -33,12 +33,10 @@ s.on("rpc.simplepath"): result = %1 s.on("rpc.differentparams") do(a: int, b: string): - result = %"test" + result = %[%a, %b] s.on("rpc.arrayparam") do(arr: array[0..5, byte], b: string): - var res = newJArray() - for item in arr: - res.add %int(item) + var res = %arr res.add %b result = %res @@ -71,6 +69,16 @@ suite "Server types": check s.procs.hasKey("rpc.returntypesimple") check s.procs.hasKey("rpc.returntypecomplex") + test "Simple paths": + let r = waitFor rpcSimplePath(%[]) + check r == %1 + + test "Different param types": + let + inp = %[%1, %"abc"] + r = waitFor rpcDifferentParams(inp) + check r == inp + test "Array parameters": let r1 = waitfor rpcArrayParam(%[%[1, 2, 3], %"hello"]) var ckR1 = %[1, 2, 3, 0, 0, 0] @@ -109,3 +117,9 @@ suite "Server types": expect ValueError: # wrong sub parameter type discard waitfor rpcArrayParam(%[%"test", %"hello"]) + expect ValueError: + # wrong param type + let res = waitFor rpcDifferentParams(%[%"abc", %1]) + # TODO: When errors are proper return values, check error for param name + + From 7d93ca61688754a4d3cfa25d54f7f73a631a31fd Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 10 May 2018 14:09:44 +0100 Subject: [PATCH 060/116] Fixed parsing issue with forward decls --- eth-rpc/server/servertypes.nim | 38 ++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 321f496..fffe50e 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -44,12 +44,46 @@ proc fromJson(n: JsonNode, argName: string, result: var int) = n.kind.expect(JInt, argName) result = n.getInt() +# TODO: Why does compiler complain that result cannot be assigned to when using result: var int|var int64 +# TODO: Compiler requires forward decl when processing out of module +proc fromJson(n: JsonNode, argName: string, result: var uint64) +proc fromJson(n: JsonNode, argName: string, result: var byte) +proc fromJson(n: JsonNode, argName: string, result: var float) +proc fromJson(n: JsonNode, argName: string, result: var string) +proc fromJson[T](n: JsonNode, argName: string, result: var seq[T]) +proc fromJson[N, T](n: JsonNode, argName: string, result: var array[N, T]) + +# TODO: Why can't this be forward declared? Complains of lack of definition +proc fromJson[T: enum](n: JsonNode, argName: string, result: var T) = + n.kind.expect(JInt, argName) + result = n.getInt().T + +# TODO: Why can't this be forward declared? Complains of lack of definition +proc fromJson[T: object](n: JsonNode, argName: string, result: var T) = + for k, v in fieldpairs(result): + fromJson(n[k], k, v) + +proc fromJson(n: JsonNode, argName: string, result: var int64) = + n.kind.expect(JInt, argName) + result = n.getInt() + +proc fromJson(n: JsonNode, argName: string, result: var uint64) = + n.kind.expect(JInt, argName) + result = n.getInt().uint64 + proc fromJson(n: JsonNode, argName: string, result: var byte) = n.kind.expect(JInt, argName) let v = n.getInt() if v > 255 or v < 0: raise newException(ValueError, "Parameter \"" & argName & "\" value out of range for byte: " & $v) result = byte(v) +# TODO: Alow string input for UInt256? +#[ +proc fromJson(n: JsonNode, argName: string, result: var UInt256) = + n.kind.expect(JString, argName) + result = n.getStr().parse(Stint[256]) # TODO: Requires error checking? +]# + proc fromJson(n: JsonNode, argName: string, result: var float) = n.kind.expect(JFloat, argName) result = n.getFloat() @@ -67,10 +101,6 @@ proc fromJson[N, T](n: JsonNode, argName: string, result: var array[N, T]) = for i in 0 ..< n.len: fromJson(n[i], argName, result[i]) -proc fromJson[T: object](n: JsonNode, argName: string, result: var T) = - for k, v in fieldpairs(result): - fromJson(n[k], k, v) - proc unpackArg[T](argIdx: int, argName: string, argtype: typedesc[T], args: JsonNode): T = when argType is array or argType is seq: args[argIdx].kind.expect(JArray, argName) From 1eb53246cee5a5faad90dbec866d06cb9ab1d563 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 10 May 2018 14:57:07 +0100 Subject: [PATCH 061/116] Add type constraints and comments --- eth-rpc/server/ethprocs.nim | 552 ++++++++++++++++++++++++++++++------ 1 file changed, 472 insertions(+), 80 deletions(-) diff --git a/eth-rpc/server/ethprocs.nim b/eth-rpc/server/ethprocs.nim index 4647c64..75ad2b6 100644 --- a/eth-rpc/server/ethprocs.nim +++ b/eth-rpc/server/ethprocs.nim @@ -1,184 +1,576 @@ -import servertypes, cryptoutils, json, macros +import servertypes, cryptoutils, json, stint + +#[ + For details on available RPC calls, see: https://github.com/ethereum/wiki/wiki/JSON-RPC + Note that many of the calls return hashes and even 'ints' as hex strings. + This module will likely have to be split into smaller sections for ease of use. + + Information: + Default block parameter: https://github.com/ethereum/wiki/wiki/JSON-RPC#the-default-block-parameter + + Parameter types + Changes might be required for parameter types. + For example: + * String might be more appropriate than seq[byte], for example for addresses, although would need length constraints. + * It might be worth replacing array[X, byte] with the equivilent stInt/stUInt. + * Int return values might actually be more hex string than int. + * UInt256/Int256 + * Objects such as BlockObject and TransactionObject might be better as the existing Nimbus objects + + NOTE: + * as `from` is a keyword, this has been replaced with `source` for variable names. + + TODO: + * Check UInt256 is being converted correctly as input + +]# var server = sharedRpcServer() server.on("web3_clientVersion"): + ## Returns the current client version. result = %"Nimbus-RPC-Test" -server.on("web3_sha3") do(data: string): - let kres = k256(data) - result = %kres +server.on("web3_sha3") do(data: string) -> string: + ## Returns Keccak-256 (not the standardized SHA3-256) of the given data. + ## + ## data: the data to convert into a SHA3 hash. + ## Returns the SHA3 result of the given string. + result = k256(data) server.on("net_version"): - #[ See: + ## Returns string of the current network id: + ## "1": Ethereum Mainnet + ## "2": Morden Testnet (deprecated) + ## "3": Ropsten Testnet + ## "4": Rinkeby Testnet + ## "42": Kovan Testnet + #[ Note, See: https://github.com/ethereum/interfaces/issues/6 https://github.com/ethereum/EIPs/issues/611 ]# discard -server.on("net_listening"): - return %"true" +server.on("net_listening") do() -> bool: + ## Returns boolean true when listening, otherwise false. + result = true -server.on("net_peerCount"): - # TODO: Discovery integration +server.on("net_peerCount") do() -> int: + ## Returns integer of the number of connected peers. discard -server.on("eth_protocolVersion"): +server.on("eth_protocolVersion") do() -> string: + ## Returns string of the current ethereum protocol version. discard -server.on("eth_syncing"): +type + SyncObject = object + startingBlock: int + currentBlock: int + highestBlock: int + +server.on("eth_syncing") do() -> JsonNode: + ## Returns SyncObject or false when not syncing. + var + res: JsonNode + sync: SyncObject + if true: res = %sync + else: res = newJBool(false) + result = %res + +server.on("eth_coinbase") do() -> string: + ## Returns the current coinbase address. discard -server.on("eth_coinbase"): +server.on("eth_mining") do() -> bool: + ## Returns true of the client is mining, otherwise false. discard -server.on("eth_mining"): +server.on("eth_hashrate") do() -> int: + ## Returns the number of hashes per second that the node is mining with. discard -server.on("eth_hashrate"): +server.on("eth_gasPrice") do() -> int64: + ## Returns an integer of the current gas price in wei. discard -server.on("eth_gasPrice"): +server.on("eth_accounts") do() -> seq[array[20, byte]]: + ## Returns a list of addresses owned by client. + # TODO: this might be easier to use as seq[string] + # This is what's expected: "result": ["0x407d73d8a49eeb85d32cf465507dd71d507100c1"] discard -server.on("eth_accounts"): +server.on("eth_blockNumber") do() -> int: + ## Returns integer of the current block number the client is on. discard -server.on("eth_blockNumber"): +server.on("eth_getBalance") do(data: array[20, byte], quantityTag: string) -> int: + ## Returns the balance of the account of given address. + ## + ## data: address to check for balance. + ## quantityTag: integer block number, or the string "latest", "earliest" or "pending", see the default block parameter. + ## Returns integer of the current balance in wei. discard -server.on("eth_getBalance") do(data: array[20, byte], quantityTag: string): - discard - -server.on("eth_getStorageAt") do(data: array[20, byte], quantity: int, quantityTag: string): - discard +server.on("eth_getStorageAt") do(data: array[20, byte], quantity: int, quantityTag: string) -> seq[byte]: + ## Returns the value from a storage position at a given address. + ## + ## data: address of the storage. + ## quantity: integer of the position in the storage. + ## quantityTag: integer block number, or the string "latest", "earliest" or "pending", see the default block parameter. + ## Returns: the value at this storage position. + # TODO: More appropriate return type? + # For more details, see: https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getstorageat + result = @[] server.on("eth_getTransactionCount") do(data: array[20, byte], quantityTag: string): + ## Returns the number of transactions sent from an address. + ## + ## data: address. + ## quantityTag: integer block number, or the string "latest", "earliest" or "pending", see the default block parameter. + ## Returns integer of the number of transactions send from this address. discard -server.on("eth_getBlockTransactionCountByHash") do(data: array[32, byte]): +server.on("eth_getBlockTransactionCountByHash") do(data: array[32, byte]) -> int: + ## Returns the number of transactions in a block from a block matching the given block hash. + ## + ## data: hash of a block + ## Returns integer of the number of transactions in this block. discard -server.on("eth_getBlockTransactionCountByNumber") do(quantityTag: string): +server.on("eth_getBlockTransactionCountByNumber") do(quantityTag: string) -> int: + ## Returns the number of transactions in a block matching the given block number. + ## + ## data: integer of a block number, or the string "earliest", "latest" or "pending", as in the default block parameter. + ## Returns integer of the number of transactions in this block. discard server.on("eth_getUncleCountByBlockHash") do(data: array[32, byte]): + ## Returns the number of uncles in a block from a block matching the given block hash. + ## + ## data: hash of a block. + ## Returns integer of the number of uncles in this block. discard server.on("eth_getUncleCountByBlockNumber") do(quantityTag: string): + ## Returns the number of uncles in a block from a block matching the given block number. + ## + ## quantityTag: integer of a block number, or the string "latest", "earliest" or "pending", see the default block parameter. + ## Returns integer of uncles in this block. discard -server.on("eth_getCode") do(data: array[20, byte], quantityTag: string): +server.on("eth_getCode") do(data: array[20, byte], quantityTag: string) -> seq[byte]: + ## Returns code at a given address. + ## + ## data: address + ## quantityTag: integer block number, or the string "latest", "earliest" or "pending", see the default block parameter. + ## Returns the code from the given address. + result = @[] + +server.on("eth_sign") do(data: array[20, byte], message: seq[byte]) -> seq[byte]: + ## The sign method calculates an Ethereum specific signature with: sign(keccak256("\x19Ethereum Signed Message:\n" + len(message) + message))). + ## By adding a prefix to the message makes the calculated signature recognisable as an Ethereum specific signature. + ## This prevents misuse where a malicious DApp can sign arbitrary data (e.g. transaction) and use the signature to impersonate the victim. + ## Note the address to sign with must be unlocked. + ## + ## data: address. + ## message: message to sign. + ## Returns signature. discard -server.on("eth_sign") do(data: array[20, byte], message: seq[byte]): +type EthSend = object + source: array[20, byte] # the address the transaction is send from. + to: array[20, byte] # (optional when creating new contract) the address the transaction is directed to. + gas: int # (optional, default: 90000) integer of the gas provided for the transaction execution. It will return unused gas. + gasPrice: int # (optional, default: To-Be-Determined) integer of the gasPrice used for each paid gas. + value: int # (optional) integer of the value sent with this transaction. + data: int # the compiled code of a contract OR the hash of the invoked method signature and encoded parameters. For details see Ethereum Contract ABI. + nonce: int # (optional) integer of a nonce. This allows to overwrite your own pending transactions that use the same nonce. + +server.on("eth_sendTransaction") do(obj: EthSend) -> UInt256: + ## Creates new message call transaction or a contract creation, if the data field contains code. + ## + ## obj: the transaction object. + ## Returns the transaction hash, or the zero hash if the transaction is not yet available. + ## Note: Use eth_getTransactionReceipt to get the contract address, after the transaction was mined, when you created a contract. discard -server.on("eth_sendTransaction"): # TODO: Object +server.on("eth_sendRawTransaction") do(data: string, quantityTag: int) -> UInt256: # TODO: string or array of byte? + ## Creates new message call transaction or a contract creation for signed transactions. + ## + ## data: the signed transaction data. + ## Returns the transaction hash, or the zero hash if the transaction is not yet available. + ## Note: Use eth_getTransactionReceipt to get the contract address, after the transaction was mined, when you created a contract. discard -server.on("eth_sendRawTransaction") do(data: string): # TODO: string or array of byte? +type EthCall = object + source: array[20, byte] # (optional) The address the transaction is send from. + to: array[20, byte] # The address the transaction is directed to. + gas: int # (optional) Integer of the gas provided for the transaction execution. eth_call consumes zero gas, but this parameter may be needed by some executions. + gasPrice: int # (optional) Integer of the gasPrice used for each paid gas. + value: int # (optional) Integer of the value sent with this transaction. + data: int # (optional) Hash of the method signature and encoded parameters. For details see Ethereum Contract ABI. + +server.on("eth_call") do(call: EthCall, quantityTag: string) -> UInt256: + ## Executes a new message call immediately without creating a transaction on the block chain. + ## + ## call: the transaction call object. + ## quantityTag: integer block number, or the string "latest", "earliest" or "pending", see the default block parameter. + ## Returns the return value of executed contract. + # TODO: Should return value be UInt256 or seq[byte] or string? discard -server.on("eth_call"): # TODO: Object +server.on("eth_estimateGas") do(call: EthCall, quantityTag: string) -> UInt256: # TODO: Int or U/Int256? + ## Generates and returns an estimate of how much gas is necessary to allow the transaction to complete. + ## The transaction will not be added to the blockchain. Note that the estimate may be significantly more than + ## the amount of gas actually used by the transaction, for a variety of reasons including EVM mechanics and node performance. + ## + ## call: the transaction call object. + ## quantityTag: integer block number, or the string "latest", "earliest" or "pending", see the default block parameter. + ## Returns the amount of gas used. discard -server.on("eth_estimateGas"): # TODO: Object +type + ## A block object, or null when no block was found + BlockObject = ref object + number: int # the block number. null when its pending block. + hash: UInt256 # hash of the block. null when its pending block. + parentHash: UInt256 # hash of the parent block. + nonce: int64 # hash of the generated proof-of-work. null when its pending block. + sha3Uncles: UInt256 # SHA3 of the uncles data in the block. + logsBloom: array[256, byte] # the bloom filter for the logs of the block. null when its pending block. + transactionsRoot: UInt256 # the root of the transaction trie of the block. + stateRoot: UInt256 # the root of the final state trie of the block. + receiptsRoot: UInt256 # the root of the receipts trie of the block. + miner: array[20, byte] # the address of the beneficiary to whom the mining rewards were given. + difficulty: int # integer of the difficulty for this block. + totalDifficulty: int # integer of the total difficulty of the chain until this block. + extraData: string # the "extra data" field of this block. + size: int # integer the size of this block in bytes. + gasLimit: int # the maximum gas allowed in this block. + gasUsed: int # the total used gas by all transactions in this block. + timestamp: int # the unix timestamp for when the block was collated. + transactions: seq[Uint256] # list of transaction objects, or 32 Bytes transaction hashes depending on the last given parameter. + uncles: seq[Uint256] # list of uncle hashes. + +server.on("eth_getBlockByHash") do(data: array[32, byte], fullTransactions: bool) -> BlockObject: + ## Returns information about a block by hash. + ## + ## data: Hash of a block. + ## fullTransactions: If true it returns the full transaction objects, if false only the hashes of the transactions. + ## Returns BlockObject or nil when no block was found. discard -server.on("eth_getBlockByHash"): +server.on("eth_getBlockByNumber") do(quantityTag: string, fullTransactions: bool) -> BlockObject: + ## Returns information about a block by block number. + ## + ## quantityTag: integer of a block number, or the string "earliest", "latest" or "pending", as in the default block parameter. + ## fullTransactions: If true it returns the full transaction objects, if false only the hashes of the transactions. + ## Returns BlockObject or nil when no block was found. discard -server.on("eth_getBlockByNumber"): +type + TransactionObject = object # A transaction object, or null when no transaction was found: + hash: UInt256 # hash of the transaction. + nonce: int64 # TODO: Is int? the number of transactions made by the sender prior to this one. + blockHash: UInt256 # hash of the block where this transaction was in. null when its pending. + blockNumber: int64 # block number where this transaction was in. null when its pending. + transactionIndex: int64 # integer of the transactions index position in the block. null when its pending. + source: array[20, byte] # address of the sender. + to: array[20, byte] # address of the receiver. null when its a contract creation transaction. + value: int64 # value transferred in Wei. + gasPrice: int64 # gas price provided by the sender in Wei. + gas: int64 # gas provided by the sender. + input: seq[byte] # the data send along with the transaction. + +server.on("eth_getTransactionByHash") do(data: Uint256) -> TransactionObject: + ## Returns the information about a transaction requested by transaction hash. + ## + ## data: hash of a transaction. + ## Returns requested transaction information. discard -server.on("eth_getTransactionByHash"): +server.on("eth_getTransactionByBlockHashAndIndex") do(data: UInt256, quantity: int) -> TransactionObject: + ## Returns information about a transaction by block hash and transaction index position. + ## + ## data: hash of a block. + ## quantity: integer of the transaction index position. + ## Returns requested transaction information. discard -server.on("eth_getTransactionByBlockHashAndIndex"): +server.on("eth_getTransactionByBlockNumberAndIndex") do(quantityTag: string, quantity: int) -> TransactionObject: + ## Returns information about a transaction by block number and transaction index position. + ## + ## quantityTag: a block number, or the string "earliest", "latest" or "pending", as in the default block parameter. + ## quantity: the transaction index position. discard -server.on("eth_getTransactionByBlockNumberAndIndex"): +type + ReceiptKind = enum rkRoot, rkStatus + ReceiptObject = object + # A transaction receipt object, or null when no receipt was found: + transactionHash: UInt256 # hash of the transaction. + transactionIndex: int # integer of the transactions index position in the block. + blockHash: UInt256 # hash of the block where this transaction was in. + blockNumber: int # block number where this transaction was in. + cumulativeGasUsed: int # the total amount of gas used when this transaction was executed in the block. + gasUsed: int # the amount of gas used by this specific transaction alone. + contractAddress: array[20, byte] # the contract address created, if the transaction was a contract creation, otherwise null. + logs: seq[string] # TODO: See Wiki for details. list of log objects, which this transaction generated. + logsBloom: array[256, byte] # bloom filter for light clients to quickly retrieve related logs. + case kind: ReceiptKind + of rkRoot: root: UInt256 # post-transaction stateroot (pre Byzantium). + of rkStatus: status: int # 1 = success, 0 = failure. + +server.on("eth_getTransactionReceipt") do(data: UInt256) -> ReceiptObject: + ## Returns the receipt of a transaction by transaction hash. + ## + ## data: hash of a transaction. + ## Returns transaction receipt. discard -server.on("eth_getTransactionReceipt"): +server.on("eth_getUncleByBlockHashAndIndex") do(data: UInt256, quantity: int64) -> BlockObject: + ## Returns information about a uncle of a block by hash and uncle index position. + ## + ## data: hash a block. + ## quantity: the uncle's index position. + ## Returns BlockObject or nil when no block was found. discard -server.on("eth_getUncleByBlockHashAndIndex"): +server.on("eth_getUncleByBlockNumberAndIndex") do(quantityTag: string, quantity: int64) -> BlockObject: + # Returns information about a uncle of a block by number and uncle index position. + ## + ## quantityTag: a block number, or the string "earliest", "latest" or "pending", as in the default block parameter. + ## quantity: the uncle's index position. + ## Returns BlockObject or nil when no block was found. discard -server.on("eth_getUncleByBlockNumberAndIndex"): +server.on("eth_getCompilers") do() -> seq[string]: + ## Returns a list of available compilers in the client. + ## + ## Returns a list of available compilers. + result = @[] + +server.on("eth_compileSolidity") do(sourceCode: string) -> seq[byte]: + ## Returns compiled solidity code. + ## + ## sourceCode: source code as string. + ## Returns compiles source code. + result = @[] + +server.on("eth_compileLLL") do(sourceCode: string) -> seq[byte]: + ## Returns compiled LLL code. + ## + ## sourceCode: source code as string. + ## Returns compiles source code. + result = @[] + +server.on("eth_compileSerpent") do(sourceCode: string) -> seq[byte]: + ## Returns compiled serpent code. + ## + ## sourceCode: source code as string. + ## Returns compiles source code. + result = @[] + +type + FilterDataKind = enum fkItem, fkList + FilterData = object + # Difficult to process variant objects in input data, as kind is immutable. + # TODO: This might need more work to handle "or" options + kind: FilterDataKind + items: seq[FilterData] + item: UInt256 + # TODO: I don't think this will work as input, need only one value that is either UInt256 or seq[UInt256] + + FilterOptions = object + fromBlock: string # (optional, default: "latest") integer block number, or "latest" for the last mined block or "pending", "earliest" for not yet mined transactions. + toBlock: string # (optional, default: "latest") integer block number, or "latest" for the last mined block or "pending", "earliest" for not yet mined transactions. + address: seq[array[20, byte]] # (optional) contract address or a list of addresses from which logs should originate. + topics: seq[FilterData] # (optional) list of DATA topics. Topics are order-dependent. Each topic can also be a list of DATA with "or" options. + +server.on("eth_newFilter") do(filterOptions: FilterOptions) -> int: + ## Creates a filter object, based on filter options, to notify when the state changes (logs). + ## To check if the state has changed, call eth_getFilterChanges. + ## Topics are order-dependent. A transaction with a log with topics [A, B] will be matched by the following topic filters: + ## [] "anything" + ## [A] "A in first position (and anything after)" + ## [null, B] "anything in first position AND B in second position (and anything after)" + ## [A, B] "A in first position AND B in second position (and anything after)" + ## [[A, B], [A, B]] "(A OR B) in first position AND (A OR B) in second position (and anything after)" + ## + ## filterOptions: settings for this filter. + ## Returns integer filter id. discard -server.on("eth_getCompilers"): +server.on("eth_newBlockFilter") do() -> int: + ## Creates a filter in the node, to notify when a new block arrives. + ## To check if the state has changed, call eth_getFilterChanges. + ## + ## Returns integer filter id. discard -server.on("eth_compileLLL"): +server.on("eth_newPendingTransactionFilter") do() -> int: + ## Creates a filter in the node, to notify when a new block arrives. + ## To check if the state has changed, call eth_getFilterChanges. + ## + ## Returns integer filter id. discard -server.on("eth_compileSolidity"): +server.on("eth_uninstallFilter") do(filterId: int) -> bool: + ## Uninstalls a filter with given id. Should always be called when watch is no longer needed. + ## Additonally Filters timeout when they aren't requested with eth_getFilterChanges for a period of time. + ## + ## filterId: The filter id. + ## Returns true if the filter was successfully uninstalled, otherwise false. discard -server.on("eth_compileSerpent"): +type + LogObject = object + removed: bool # true when the log was removed, due to a chain reorganization. false if its a valid log. + logIndex: int # integer of the log index position in the block. null when its pending log. + transactionIndex: ref int # integer of the transactions index position log was created from. null when its pending log. + transactionHash: UInt256 # hash of the transactions this log was created from. null when its pending log. + blockHash: ref UInt256 # hash of the block where this log was in. null when its pending. null when its pending log. + blockNumber: ref int64 # the block number where this log was in. null when its pending. null when its pending log. + address: array[20, byte] # address from which this log originated. + data: seq[UInt256] # contains one or more 32 Bytes non-indexed arguments of the log. + topics: array[4, UInt256] # array of 0 to 4 32 Bytes DATA of indexed log arguments. + # (In solidity: The first topic is the hash of the signature of the event. + # (e.g. Deposit(address,bytes32,uint256)), except you declared the event with the anonymous specifier.) + +server.on("eth_getFilterChanges") do(filterId: int) -> seq[LogObject]: + ## Polling method for a filter, which returns an list of logs which occurred since last poll. + ## + ## filterId: the filter id. + result = @[] + +server.on("eth_getFilterLogs") do(filterId: int) -> seq[LogObject]: + ## filterId: the filter id. + ## Returns a list of all logs matching filter with given id. + result = @[] + +server.on("eth_getLogs") do(filterOptions: FilterOptions) -> seq[LogObject]: + ## filterOptions: settings for this filter. + ## Returns a list of all logs matching a given filter object. + result = @[] + +server.on("eth_getWork") do() -> seq[UInt256]: + ## Returns the hash of the current block, the seedHash, and the boundary condition to be met ("target"). + ## Returned list has the following properties: + ## DATA, 32 Bytes - current block header pow-hash. + ## DATA, 32 Bytes - the seed hash used for the DAG. + ## DATA, 32 Bytes - the boundary condition ("target"), 2^256 / difficulty. + result = @[] + +server.on("eth_submitWork") do(nonce: int64, powHash: Uint256, mixDigest: Uint256) -> bool: + ## Used for submitting a proof-of-work solution. + ## + ## nonce: the nonce found. + ## headerPow: the header's pow-hash. + ## mixDigest: the mix digest. + ## Returns true if the provided solution is valid, otherwise false. discard -server.on("eth_newFilter"): +server.on("eth_submitHashrate") do(hashRate: UInt256, id: Uint256) -> bool: + ## Used for submitting mining hashrate. + ## + ## hashRate: a hexadecimal string representation (32 bytes) of the hash rate. + ## id: a random hexadecimal(32 bytes) ID identifying the client. + ## Returns true if submitting went through succesfully and false otherwise. discard -server.on("eth_newBlockFilter"): +server.on("shh_version") do() -> string: + ## Returns string of the current whisper protocol version. discard -server.on("eth_newPendingTransactionFilter"): +type + WhisperPost = object + # The whisper post object: + source: array[60, byte] # (optional) the identity of the sender. + to: array[60, byte] # (optional) the identity of the receiver. When present whisper will encrypt the message so that only the receiver can decrypt it. + topics: seq[UInt256] # TODO: Correct type? list of DATA topics, for the receiver to identify messages. + payload: UInt256 # TODO: Correct type - maybe string? the payload of the message. + priority: int # integer of the priority in a rang from ... (?). + ttl: int # integer of the time to live in seconds. + +server.on("shh_post") do(message: WhisperPost) -> bool: + ## Sends a whisper message. + ## + ## message: Whisper message to post. + ## Returns true if the message was send, otherwise false. discard -server.on("eth_uninstallFilter"): +server.on("shh_newIdentity") do() -> array[60, byte]: + ## Creates new whisper identity in the client. + ## + ## Returns the address of the new identiy. discard -server.on("eth_getFilterChanges"): +server.on("shh_hasIdentity") do(identity: array[60, byte]) -> bool: + ## Checks if the client holds the private keys for a given identity. + ## + ## identity: the identity address to check. + ## Returns true if the client holds the privatekey for that identity, otherwise false. discard -server.on("eth_getFilterLogs"): +server.on("shh_newGroup") do() -> array[60, byte]: + ## (?) - This has no description information in the RPC wiki. + ## + ## Returns the address of the new group. (?) discard -server.on("eth_getLogs"): +server.on("shh_addToGroup") do(identity: array[60, byte]) -> bool: + ## (?) - This has no description information in the RPC wiki. + ## + ## identity: the identity address to add to a group (?). + ## Returns true if the identity was successfully added to the group, otherwise false (?). discard -server.on("eth_getWork"): +server.on("shh_newFilter") do(filterOptions: FilterOptions, to: array[60, byte], topics: seq[UInt256]) -> int: # TODO: Is topic of right type? + ## Creates filter to notify, when client receives whisper message matching the filter options. + ## + ## filterOptions: The filter options: + ## to: DATA, 60 Bytes - (optional) identity of the receiver. When present it will try to decrypt any incoming message if the client holds the private key to this identity. + ## topics: Array of DATA - list of DATA topics which the incoming message's topics should match. You can use the following combinations: + ## [A, B] = A && B + ## [A, [B, C]] = A && (B || C) + ## [null, A, B] = ANYTHING && A && B null works as a wildcard + ## Returns the newly created filter. discard -server.on("eth_submitWork"): +server.on("shh_uninstallFilter") do(id: int) -> bool: + ## Uninstalls a filter with given id. + ## Should always be called when watch is no longer needed. + ## Additonally Filters timeout when they aren't requested with shh_getFilterChanges for a period of time. + ## + ## id: the filter id. + ## Returns true if the filter was successfully uninstalled, otherwise false. discard -server.on("eth_submitHashrate"): +type + WhisperMessage = object + # (?) are from the RPC Wiki, indicating uncertainty in type format. + hash: UInt256 # (?) the hash of the message. + source: array[60, byte] # the sender of the message, if a sender was specified. + to: array[60, byte] # the receiver of the message, if a receiver was specified. + expiry: int # integer of the time in seconds when this message should expire (?). + ttl: int # integer of the time the message should float in the system in seconds (?). + sent: int # integer of the unix timestamp when the message was sent. + topics: seq[UInt256] # list of DATA topics the message contained. + payload: string # TODO: Correct type? the payload of the message. + workProved: int # integer of the work this message required before it was send (?). + +server.on("shh_getFilterChanges") do(id: int) -> seq[WhisperMessage]: + ## Polling method for whisper filters. Returns new messages since the last call of this method. + ## Note: calling the shh_getMessages method, will reset the buffer for this method, so that you won't receive duplicate messages. + ## + ## id: the filter id. discard -server.on("shh_post"): - discard - -server.on("shh_version"): - discard - -server.on("shh_newIdentity"): - discard - -server.on("shh_hasIdentity"): - discard - -server.on("shh_newGroup"): - discard - -server.on("shh_addToGroup"): - discard - -server.on("shh_newFilter"): - discard - -server.on("shh_uninstallFilter"): - discard - -server.on("shh_getFilterChanges"): - discard - -server.on("shh_getMessages"): +server.on("shh_getMessages") do(id: int) -> seq[WhisperMessage]: + ## Get all messages matching a filter. Unlike shh_getFilterChanges this returns all messages. + ## + ## id: the filter id. + ## Returns a list of messages received since last poll. discard From d8b68e14f74dc4806d8aba7e4b44d4414e005b2f Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 10 May 2018 21:52:02 +0100 Subject: [PATCH 062/116] add ref int --- eth-rpc/server/jsonconverters.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eth-rpc/server/jsonconverters.nim b/eth-rpc/server/jsonconverters.nim index 6f5e2c7..5c8de6b 100644 --- a/eth-rpc/server/jsonconverters.nim +++ b/eth-rpc/server/jsonconverters.nim @@ -8,13 +8,11 @@ iterator bytes(i: UInt256|Int256): byte = pos += 1 proc `%`*(n: UInt256): JsonNode = - ## Generic constructor for JSON data. Creates a new `JInt JsonNode`. result = newJArray() for elem in n.bytes: result.add(%int(elem)) proc `%`*(n: Int256): JsonNode = - ## Generic constructor for JSON data. Creates a new `JInt JsonNode`. result = newJArray() for elem in n.bytes: result.add(%int(elem)) @@ -22,3 +20,5 @@ proc `%`*(n: Int256): JsonNode = proc `%`*(n: byte{not lit}): JsonNode = result = newJInt(int(n)) +proc `%`*(n: ref int|ref int64): JsonNode = + result = newJInt(int(n[])) From 4d1d82e5d82b6bdcd3db786b23f99f5c72044ccd Mon Sep 17 00:00:00 2001 From: coffeepots Date: Mon, 14 May 2018 16:30:14 +0100 Subject: [PATCH 063/116] Add StUInt256 json handling, explicitly pass `type` in param setup. --- eth-rpc/server/servertypes.nim | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index fffe50e..cc9f71e 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -1,4 +1,4 @@ -import asyncdispatch, asyncnet, json, tables, macros, strutils, jsonconverters +import asyncdispatch, asyncnet, json, tables, macros, strutils, jsonconverters, stint export asyncdispatch, asyncnet, json, jsonconverters type @@ -46,12 +46,12 @@ proc fromJson(n: JsonNode, argName: string, result: var int) = # TODO: Why does compiler complain that result cannot be assigned to when using result: var int|var int64 # TODO: Compiler requires forward decl when processing out of module -proc fromJson(n: JsonNode, argName: string, result: var uint64) proc fromJson(n: JsonNode, argName: string, result: var byte) proc fromJson(n: JsonNode, argName: string, result: var float) proc fromJson(n: JsonNode, argName: string, result: var string) proc fromJson[T](n: JsonNode, argName: string, result: var seq[T]) proc fromJson[N, T](n: JsonNode, argName: string, result: var array[N, T]) +proc fromJson(n: JsonNode, argName: string, result: var UInt256) # TODO: Why can't this be forward declared? Complains of lack of definition proc fromJson[T: enum](n: JsonNode, argName: string, result: var T) = @@ -67,22 +67,15 @@ proc fromJson(n: JsonNode, argName: string, result: var int64) = n.kind.expect(JInt, argName) result = n.getInt() -proc fromJson(n: JsonNode, argName: string, result: var uint64) = - n.kind.expect(JInt, argName) - result = n.getInt().uint64 - proc fromJson(n: JsonNode, argName: string, result: var byte) = n.kind.expect(JInt, argName) let v = n.getInt() if v > 255 or v < 0: raise newException(ValueError, "Parameter \"" & argName & "\" value out of range for byte: " & $v) result = byte(v) -# TODO: Alow string input for UInt256? -#[ proc fromJson(n: JsonNode, argName: string, result: var UInt256) = n.kind.expect(JString, argName) - result = n.getStr().parse(Stint[256]) # TODO: Requires error checking? -]# + result = parse(StUint[256], n.getStr()) # TODO: Requires error checking? proc fromJson(n: JsonNode, argName: string, result: var float) = n.kind.expect(JFloat, argName) @@ -134,7 +127,7 @@ proc setupParams(parameters, paramsIdent: NimNode): NimNode = paramNameStr = $paramName paramType = parameters[i][1] result.add(quote do: - var `paramName` = `unpackArg`(`pos`, `paramNameStr`, `paramType`, `paramsIdent`) + var `paramName` = `unpackArg`(`pos`, `paramNameStr`, type(`paramType`), `paramsIdent`) ) proc makeProcName(s: string): string = From ffb80ac04ddbf9293c18eea717038abff936151d Mon Sep 17 00:00:00 2001 From: coffeepots Date: Mon, 14 May 2018 16:31:32 +0100 Subject: [PATCH 064/116] fix `result not found` by changing import order... --- tests/testrpcmacro.nim | 2 +- tests/testserverclient.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testrpcmacro.nim b/tests/testrpcmacro.nim index d56ca54..a0681da 100644 --- a/tests/testrpcmacro.nim +++ b/tests/testrpcmacro.nim @@ -1,4 +1,4 @@ -import ../ rpcserver, unittest, asyncdispatch, json, tables +import unittest, asyncdispatch, json, tables, ../ rpcserver type # some nested types to check object parsing diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index 0b68cee..139c5aa 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -1,4 +1,4 @@ -import ../ rpcserver, ../ rpcclient, unittest, asyncdispatch, json, tables +import ../ rpcclient, ../ rpcserver, unittest, asyncdispatch, json, tables #[ TODO: Importing client before server causes the error: From 68333bd30a0410b8d23dda44f9663cc3f9030113 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Mon, 14 May 2018 16:42:05 +0100 Subject: [PATCH 065/116] Update stint parse to use latest parse interface --- eth-rpc/server/servertypes.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index cc9f71e..d2e790b 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -75,7 +75,7 @@ proc fromJson(n: JsonNode, argName: string, result: var byte) = proc fromJson(n: JsonNode, argName: string, result: var UInt256) = n.kind.expect(JString, argName) - result = parse(StUint[256], n.getStr()) # TODO: Requires error checking? + result = n.getStr().parse(StUint[256]) # TODO: Requires error checking? proc fromJson(n: JsonNode, argName: string, result: var float) = n.kind.expect(JFloat, argName) From efb71fce2c472f22ad7fa9344a6bde28f286cb00 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 15 May 2018 17:58:16 +0100 Subject: [PATCH 066/116] Extra checking for client receiving malformed & incorrect json --- eth-rpc/client/clientdispatch.nim | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/eth-rpc/client/clientdispatch.nim b/eth-rpc/client/clientdispatch.nim index ef34c42..36dacfa 100644 --- a/eth-rpc/client/clientdispatch.nim +++ b/eth-rpc/client/clientdispatch.nim @@ -33,8 +33,9 @@ proc call*(self: RpcClient, name: string, params: JsonNode): Future[Response] {. macro checkGet(node: JsonNode, fieldName: string, jKind: static[JsonNodeKind]): untyped = result = quote do: - if not node.hasKey(`fieldName`): raise newException(ValueError, "Message is missing required field \"" & `fieldName` & "\"") - if `node`[`fieldName`].kind != `jKind`.JsonNodeKind: raise newException(ValueError, "Expected " & $(`jKind`.JsonNodeKind) & ", got " & $`node`[`fieldName`].kind) + let n = `node`{`fieldName`} + if n.isNil: raise newException(ValueError, "Message is missing required field \"" & `fieldName` & "\"") + if n.kind != `jKind`.JsonNodeKind: raise newException(ValueError, "Expected " & $(`jKind`.JsonNodeKind) & ", got " & $`node`[`fieldName`].kind) case jKind of JBool: result.add(quote do: `node`[`fieldName`].getBool) of JInt: result.add(quote do: `node`[`fieldName`].getInt) @@ -49,14 +50,19 @@ proc processMessage(self: RpcClient, line: string) = # TODO: Use more appropriate exception objects let version = checkGet(node, "jsonrpc", JString) if version != "2.0": raise newException(ValueError, "Unsupported version of JSON, expected 2.0, received \"" & version & "\"") + let id = checkGet(node, "id", JString) if not self.awaiting.hasKey(id): raise newException(ValueError, "Cannot find message id \"" & node["id"].str & "\"") - if node["error"].kind == JNull: - self.awaiting[id].complete((false, node["result"])) + let errorNode = node{"error"} + if errorNode.isNil or errorNode.kind == JNull: + var res = node{"result"} + if not res.isNil: + self.awaiting[id].complete((false, res)) self.awaiting.del(id) + # TODO: actions on unable find result node else: - self.awaiting[id].complete((true, node["error"])) + self.awaiting[id].complete((true, errorNode)) self.awaiting.del(id) proc connect*(self: RpcClient, address: string, port: Port): Future[void] From 2154047fd5c3338422fcaad4c8611326120f0ae2 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 15 May 2018 18:16:55 +0100 Subject: [PATCH 067/116] Base bytes on bytePairs. --- eth-rpc/server/private/transportutils.nim | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/eth-rpc/server/private/transportutils.nim b/eth-rpc/server/private/transportutils.nim index bdd636d..e2c58e2 100644 --- a/eth-rpc/server/private/transportutils.nim +++ b/eth-rpc/server/private/transportutils.nim @@ -2,16 +2,9 @@ from asyncdispatch import Port proc `$`*(port: Port): string = $int(port) -# REVIEW: you can base one of the iterators on the other to avoid the code duplication -# e.g. implement `bytes` in terms of `bytePairs`. iterator bytes*[T: SomeUnsignedInt](value: T): byte {.inline.} = ## Traverse the bytes of a value in little endian - let argSize = sizeOf(T) - for bIdx in 0 ..< argSize: - let - shift = bIdx.uint * 8 - mask = 0xff'u64 shl shift - yield byte((value and mask) shr shift) + yield value.bytePairs[1] iterator bytePairs*[T: SomeUnsignedInt](value: T): tuple[key: int, val: byte] {.inline.} = let argSize = sizeOf(T) From ebacff63a695e0d8f3bf7f099a5be277dd37bfd3 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 15 May 2018 18:49:27 +0100 Subject: [PATCH 068/116] Remove transport utils module as it's unused --- eth-rpc/server/private/transportutils.nim | 39 ----------------------- eth-rpc/server/serverdispatch.nim | 2 +- eth-rpc/server/servertypes.nim | 2 ++ tests/all.nim | 2 +- tests/testutils.nim | 15 --------- 5 files changed, 4 insertions(+), 56 deletions(-) delete mode 100644 eth-rpc/server/private/transportutils.nim delete mode 100644 tests/testutils.nim diff --git a/eth-rpc/server/private/transportutils.nim b/eth-rpc/server/private/transportutils.nim deleted file mode 100644 index e2c58e2..0000000 --- a/eth-rpc/server/private/transportutils.nim +++ /dev/null @@ -1,39 +0,0 @@ -from asyncdispatch import Port - -proc `$`*(port: Port): string = $int(port) - -iterator bytes*[T: SomeUnsignedInt](value: T): byte {.inline.} = - ## Traverse the bytes of a value in little endian - yield value.bytePairs[1] - -iterator bytePairs*[T: SomeUnsignedInt](value: T): tuple[key: int, val: byte] {.inline.} = - let argSize = sizeOf(T) - for bIdx in 0 ..< argSize: - let - shift = bIdx.uint * 8 - mask = 0xff'u64 shl shift - yield (bIdx, byte((value and mask) shr shift)) - -template stripLeadingZeros(value: string): string = - var cidx = 0 - # ignore the last character so we retain '0' on zero value - while cidx < value.len - 1 and value[cidx] == '0': - cidx.inc - value[cidx .. ^1] - -proc encodeQuantity*(value: SomeUnsignedInt): string = - var hValue = value.toHex.stripLeadingZeros - result = "0x" & hValue - -# REVIEW: I think Mamy has now introduced a similar proc in the `byteutils` package -proc encodeData*[T: SomeUnsignedInt](values: seq[T]): string = - ## Translates seq of values to hex string - let argSize = sizeOf(T) - result = newString((values.len * argSize) * 2 + 2) # reserve 2 bytes for "0x" - result[0..1] = "0x" - var cPos = 0 - for idx, value in values: - for bValue in values[idx].bytes: - result[cPos .. cPos + 1] = bValue.int.toHex(2) - cPos = cPos + 2 - diff --git a/eth-rpc/server/serverdispatch.nim b/eth-rpc/server/serverdispatch.nim index 692f1f3..637ac63 100644 --- a/eth-rpc/server/serverdispatch.nim +++ b/eth-rpc/server/serverdispatch.nim @@ -1,5 +1,5 @@ import asyncdispatch, asyncnet, json, tables, strutils, - servertypes, rpcconsts, private / [transportutils, debugutils], jsonutils, asyncutils, + servertypes, rpcconsts, private / debugutils, jsonutils, asyncutils, options proc processMessage(server: RpcServer, client: AsyncSocket, line: string) {.async.} = diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index d2e790b..c9fb6e0 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -32,6 +32,8 @@ var sharedServer: RpcServer proc sharedRpcServer*(): RpcServer = if sharedServer.isNil: sharedServer = newRpcServer("") result = sharedServer + +proc `$`*(port: Port): string = $int(port) template expect*(actual, expected: JsonNodeKind, argName: string) = if actual != expected: raise newException(ValueError, "Parameter \"" & argName & "\" expected " & $expected & " but got " & $actual) diff --git a/tests/all.nim b/tests/all.nim index 50cec98..c61d327 100644 --- a/tests/all.nim +++ b/tests/all.nim @@ -1,3 +1,3 @@ import - testutils, testrpcmacro, testserverclient + testrpcmacro, testserverclient diff --git a/tests/testutils.nim b/tests/testutils.nim deleted file mode 100644 index cb27129..0000000 --- a/tests/testutils.nim +++ /dev/null @@ -1,15 +0,0 @@ -import strutils, ../eth-rpc/server/private/transportutils, unittest - -suite "Encoding": - test "Encode quantity": - check 0.encodeQuantity == "0x0" - check 0x1000.encodeQuantity == "0x1000" - test "Encode data": - var i = 0 - for b in bytes(0x07_06_05_04_03_02_01_00'u64): - check b == i.byte - i.inc - test "Encode data pairs": - for i, b in bytePairs(0x07_06_05_04_03_02_01_00'u64): - check b == i.byte - From 9c1944977e04e3bd54fce27c49cfdf8f8c5a0533 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 15 May 2018 19:06:05 +0100 Subject: [PATCH 069/116] rpcserver must be before asyncdispatch for correct parsing --- tests/testrpcmacro.nim | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/testrpcmacro.nim b/tests/testrpcmacro.nim index a0681da..d4d2665 100644 --- a/tests/testrpcmacro.nim +++ b/tests/testrpcmacro.nim @@ -1,4 +1,4 @@ -import unittest, asyncdispatch, json, tables, ../ rpcserver +import unittest, ../ rpcserver, asyncdispatch, json, tables type # some nested types to check object parsing @@ -121,5 +121,3 @@ suite "Server types": # wrong param type let res = waitFor rpcDifferentParams(%[%"abc", %1]) # TODO: When errors are proper return values, check error for param name - - From 414c7dd1f2ec40ce58d1b29a34abfc5c3d721099 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 15 May 2018 19:06:51 +0100 Subject: [PATCH 070/116] process `stint`s to json strings rather than arrays --- eth-rpc/server/jsonconverters.nim | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/eth-rpc/server/jsonconverters.nim b/eth-rpc/server/jsonconverters.nim index 5c8de6b..9cc1d2a 100644 --- a/eth-rpc/server/jsonconverters.nim +++ b/eth-rpc/server/jsonconverters.nim @@ -1,21 +1,14 @@ -import json, stint +import json, stint, strutils -iterator bytes(i: UInt256|Int256): byte = - let b = cast[ptr array[32, byte]](i.unsafeaddr) - var pos = 0 - while pos < 32: - yield b[pos] - pos += 1 +template stintStr(n: UInt256|Int256): JsonNode = + var s = n.toString + if s.len mod 2 != 0: s = "0" & s + s = "0x" & s + %s -proc `%`*(n: UInt256): JsonNode = - result = newJArray() - for elem in n.bytes: - result.add(%int(elem)) +proc `%`*(n: UInt256): JsonNode = n.stintStr -proc `%`*(n: Int256): JsonNode = - result = newJArray() - for elem in n.bytes: - result.add(%int(elem)) +proc `%`*(n: Int256): JsonNode = n.stintStr proc `%`*(n: byte{not lit}): JsonNode = result = newJInt(int(n)) From 67779bf4282d28afb02b6ee449b01ff471a5704f Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 16 May 2018 23:14:43 +0100 Subject: [PATCH 071/116] Allow use of `return` in `on` macro rpcs --- eth-rpc/server/servertypes.nim | 67 +++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 25 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index c9fb6e0..f0c3f84 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -133,47 +133,64 @@ proc setupParams(parameters, paramsIdent: NimNode): NimNode = ) proc makeProcName(s: string): string = - s.multiReplace((".", ""), ("/", "")) + # only alpha + result = "" + for c in s: + if c.isAlphaAscii: result.add c + +proc hasReturnType(params: NimNode): bool = + if params.len > 0 and params[0] != nil and params[0].kind != nnkEmpty: + result = true macro on*(server: var RpcServer, path: string, body: untyped): untyped = result = newStmtList() let parameters = body.findChild(it.kind == nnkFormalParams) - paramsIdent = ident"params" - pathStr = $path + paramsIdent = ident"params" # all remote calls have a single parameter: `params: JsonNode` + pathStr = $path # procs are generated from the stripped path procName = ident(pathStr.makeProcName) + doMain = genSym(nskProc) # proc that contains our rpc body + res = ident"result" # async result var setup = setupParams(parameters, paramsIdent) procBody: NimNode - bodyWrapper = newStmtList() if body.kind == nnkStmtList: procBody = body else: procBody = body.body - if parameters.len > 0 and parameters[0] != nil and parameters[0] != ident"JsonNode": - # when a return type is specified, shadow async's result - # and pass it back jsonified - of course, we don't want to do this - # if a JsonNode is explicitly declared as the return type - let - returnType = parameters[0] - res = ident"result" - template doMain(body: untyped): untyped = - # create a new scope to allow shadowing result - block: - body - bodyWrapper = quote do: - `res` = `doMain`: - var `res`: `returnType` + if parameters.hasReturnType: + let returnType = parameters[0] + + # `doMain` is outside of async transformation, + # allowing natural `return` + result.add(quote do: + proc `doMain`(`paramsIdent`: JsonNode): `returnType` {.inline.} = + `setup` `procBody` - %`res` + ) + + # Note ``res` =` (which becomes `result = `) will be transformed by {.async.} to `complete` + if returnType == ident"JsonNode": + # `JsonNode` results don't need conversion + result.add( quote do: + proc `procName`*(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = + `res` = `doMain`(`paramsIdent`) + ) + else: + result.add( quote do: + proc `procName`*(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = + `res` = %`doMain`(`paramsIdent`) + ) else: - bodyWrapper = quote do: `procBody` - - # async proc wrapper around body - result = quote do: + # no return types, inline contents + result.add( quote do: proc `procName`*(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = `setup` - `bodyWrapper` - `server`.register(`path`, `procName`) + `procBody` + ) + result.add( quote do: + `server`.register(`path`, `procName`) + ) + when defined(nimDumpRpcs): echo "\n", pathStr, ": ", result.repr From 39e987c0827c24e29a5494ae8633a3feccd64748 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 16 May 2018 23:15:46 +0100 Subject: [PATCH 072/116] fix double json conversion --- eth-rpc/server/ethprocs.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth-rpc/server/ethprocs.nim b/eth-rpc/server/ethprocs.nim index 75ad2b6..7731ab7 100644 --- a/eth-rpc/server/ethprocs.nim +++ b/eth-rpc/server/ethprocs.nim @@ -76,7 +76,7 @@ server.on("eth_syncing") do() -> JsonNode: sync: SyncObject if true: res = %sync else: res = newJBool(false) - result = %res + result = res server.on("eth_coinbase") do() -> string: ## Returns the current coinbase address. From 2fef44ff884ba961558ca178e6561d65e3fcb49a Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 16 May 2018 23:16:20 +0100 Subject: [PATCH 073/116] add test for `return` statement within `on` macro --- tests/testrpcmacro.nim | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/testrpcmacro.nim b/tests/testrpcmacro.nim index d4d2665..232282b 100644 --- a/tests/testrpcmacro.nim +++ b/tests/testrpcmacro.nim @@ -29,6 +29,7 @@ let var s = newRpcServer("localhost") # RPC definitions + s.on("rpc.simplepath"): result = %1 @@ -57,6 +58,9 @@ s.on("rpc.returntypecomplex") do(i: int) -> Test2: result.x = [1, i, 3] result.y = "test" +s.on("rpc.testreturns") do() -> int: + return 1234 + # Tests suite "Server types": @@ -68,6 +72,7 @@ suite "Server types": check s.procs.hasKey("rpc.objparam") check s.procs.hasKey("rpc.returntypesimple") check s.procs.hasKey("rpc.returntypecomplex") + check s.procs.hasKey("rpc.testreturns") test "Simple paths": let r = waitFor rpcSimplePath(%[]) @@ -107,6 +112,10 @@ suite "Server types": r1 = waitfor rpcReturnTypeComplex(%[%inp]) check r1 == %*{"x": %[1, inp, 3], "y": "test"} + test "Return statement": + let r = waitFor rpcTestReturns(%[]) + check r == %1234 + test "Runtime errors": expect ValueError: # root param not array From 36cd28d077c91fe196da851b7043a94b4f9e2a31 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 17 May 2018 15:23:49 +0100 Subject: [PATCH 074/116] output proc names changed from alpha -> alphanumeric --- eth-rpc/server/servertypes.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index f0c3f84..0bb3458 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -133,10 +133,10 @@ proc setupParams(parameters, paramsIdent: NimNode): NimNode = ) proc makeProcName(s: string): string = - # only alpha + # only alphanumeric result = "" for c in s: - if c.isAlphaAscii: result.add c + if c.isAlphaNumeric: result.add c proc hasReturnType(params: NimNode): bool = if params.len > 0 and params[0] != nil and params[0].kind != nnkEmpty: From 4e4b853365f42547523837d591a5f0441b3fcec3 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 17 May 2018 19:11:47 +0100 Subject: [PATCH 075/116] fix stint to json to output hex rather than decimal --- eth-rpc/server/jsonconverters.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth-rpc/server/jsonconverters.nim b/eth-rpc/server/jsonconverters.nim index 9cc1d2a..54b18ef 100644 --- a/eth-rpc/server/jsonconverters.nim +++ b/eth-rpc/server/jsonconverters.nim @@ -1,7 +1,7 @@ import json, stint, strutils template stintStr(n: UInt256|Int256): JsonNode = - var s = n.toString + var s = n.toHex if s.len mod 2 != 0: s = "0" & s s = "0x" & s %s From 182dd02d80c962aa10b623c5c79696be1c614f5f Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 17 May 2018 19:13:12 +0100 Subject: [PATCH 076/116] Changed stint values to take and output strings --- eth-rpc/server/servertypes.nim | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 0bb3458..711f01f 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -62,6 +62,7 @@ proc fromJson[T: enum](n: JsonNode, argName: string, result: var T) = # TODO: Why can't this be forward declared? Complains of lack of definition proc fromJson[T: object](n: JsonNode, argName: string, result: var T) = + n.kind.expect(JObject, argName) for k, v in fieldpairs(result): fromJson(n[k], k, v) @@ -76,8 +77,9 @@ proc fromJson(n: JsonNode, argName: string, result: var byte) = result = byte(v) proc fromJson(n: JsonNode, argName: string, result: var UInt256) = + # expects base 16 string, starting with "0x" n.kind.expect(JString, argName) - result = n.getStr().parse(StUint[256]) # TODO: Requires error checking? + result = n.getStr().parse(StUint[256], 16) # TODO: Handle errors proc fromJson(n: JsonNode, argName: string, result: var float) = n.kind.expect(JFloat, argName) @@ -88,21 +90,18 @@ proc fromJson(n: JsonNode, argName: string, result: var string) = result = n.getStr() proc fromJson[T](n: JsonNode, argName: string, result: var seq[T]) = + n.kind.expect(JArray, argName) result = newSeq[T](n.len) for i in 0 ..< n.len: fromJson(n[i], argName, result[i]) proc fromJson[N, T](n: JsonNode, argName: string, result: var array[N, T]) = + n.kind.expect(JArray, argName) + if n.len > result.len: raise newException(ValueError, "Parameter \"" & argName & "\" item count is too big for array") for i in 0 ..< n.len: fromJson(n[i], argName, result[i]) -proc unpackArg[T](argIdx: int, argName: string, argtype: typedesc[T], args: JsonNode): T = - when argType is array or argType is seq: - args[argIdx].kind.expect(JArray, argName) - when argType is array: - if args[argIdx].len > result.len: raise newException(ValueError, "Parameter \"" & argName & "\" item count is too big for array") - when argType is object: - args[argIdx].kind.expect(JObject, argName) +proc unpackArg[T](args: JsonNode, argIdx: int, argName: string, argtype: typedesc[T]): T = fromJson(args[argIdx], argName, result) proc expectArrayLen(node: NimNode, paramsIdent: untyped, length: int) = @@ -129,7 +128,7 @@ proc setupParams(parameters, paramsIdent: NimNode): NimNode = paramNameStr = $paramName paramType = parameters[i][1] result.add(quote do: - var `paramName` = `unpackArg`(`pos`, `paramNameStr`, type(`paramType`), `paramsIdent`) + var `paramName` = `unpackArg`(`paramsIdent`, `pos`, `paramNameStr`, type(`paramType`)) ) proc makeProcName(s: string): string = From 55449b119346bff5aee3302d766b754d84facc01 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 17 May 2018 19:13:44 +0100 Subject: [PATCH 077/116] Fix for zero parameter test --- tests/testserverclient.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index 139c5aa..ecc0870 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -24,7 +24,7 @@ suite "RPC": await client.connect("localhost", Port(8545)) var response: Response test "Version": - response = waitFor client.web3_clientVersion(newJNull()) + response = waitFor client.web3_clientVersion(%[]) check response.result == %"Nimbus-RPC-Test" test "SHA3": response = waitFor client.web3_sha3(%["abc"]) @@ -34,5 +34,5 @@ suite "RPC": response = waitFor client.call("myProc", %[%"abc", %[1, 2, 3, 4]]) check response.result.getStr == "Hello abc data: [1, 2, 3, 4]" - waitFor main() + waitFor main() # TODO: When an error occurs during a test, stop the server \ No newline at end of file From 338aa589d987c8ceaebb9eee0c53c18206fb8feb Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 17 May 2018 19:14:31 +0100 Subject: [PATCH 078/116] Add tests for stint return values and parameters --- tests/testrpcmacro.nim | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/testrpcmacro.nim b/tests/testrpcmacro.nim index 232282b..372ab36 100644 --- a/tests/testrpcmacro.nim +++ b/tests/testrpcmacro.nim @@ -1,4 +1,4 @@ -import unittest, ../ rpcserver, asyncdispatch, json, tables +import unittest, ../ rpcserver, asyncdispatch, json, tables, stint type # some nested types to check object parsing @@ -51,6 +51,10 @@ s.on("rpc.seqparam") do(a: string, s: seq[int]): s.on("rpc.objparam") do(a: string, obj: MyObject): result = %obj +s.on("rpc.uint256param") do(i: UInt256): + let r = i + 1.stUint(256) + result = %r + s.on("rpc.returntypesimple") do(i: int) -> int: result = i @@ -61,6 +65,10 @@ s.on("rpc.returntypecomplex") do(i: int) -> Test2: s.on("rpc.testreturns") do() -> int: return 1234 +s.on("rpc.testreturnuint256") do() -> UInt256: + let r: UInt256 = "0x1234567890abcdef".parse(UInt256, 16) + return r + # Tests suite "Server types": @@ -100,6 +108,10 @@ suite "Server types": let r = waitfor rpcObjParam(%[%"abc", testObj]) check r == testObj + test "UInt256 param": + let r = waitFor rpcUInt256Param(%[%"0x1234567890"]) + check r == %"0x1234567891" + test "Simple return types": let inp = %99 @@ -116,6 +128,10 @@ suite "Server types": let r = waitFor rpcTestReturns(%[]) check r == %1234 + test "Return UInt256": + let r = waitFor rpcTestReturnUInt256(%[]) + check r == %"0x1234567890abcdef" + test "Runtime errors": expect ValueError: # root param not array From e19aa96755e163117de13579235f524b810967df Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 17 May 2018 19:36:45 +0100 Subject: [PATCH 079/116] Remove finished UInt256 TODO, add string return for web3_client --- eth-rpc/server/ethprocs.nim | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/eth-rpc/server/ethprocs.nim b/eth-rpc/server/ethprocs.nim index 7731ab7..ae20b35 100644 --- a/eth-rpc/server/ethprocs.nim +++ b/eth-rpc/server/ethprocs.nim @@ -20,16 +20,13 @@ import servertypes, cryptoutils, json, stint NOTE: * as `from` is a keyword, this has been replaced with `source` for variable names. - TODO: - * Check UInt256 is being converted correctly as input - ]# var server = sharedRpcServer() -server.on("web3_clientVersion"): +server.on("web3_clientVersion") do() -> string: ## Returns the current client version. - result = %"Nimbus-RPC-Test" + result = "Nimbus-RPC-Test" server.on("web3_sha3") do(data: string) -> string: ## Returns Keccak-256 (not the standardized SHA3-256) of the given data. @@ -38,7 +35,7 @@ server.on("web3_sha3") do(data: string) -> string: ## Returns the SHA3 result of the given string. result = k256(data) -server.on("net_version"): +server.on("net_version") do() -> string: ## Returns string of the current network id: ## "1": Ethereum Mainnet ## "2": Morden Testnet (deprecated) @@ -49,7 +46,7 @@ server.on("net_version"): https://github.com/ethereum/interfaces/issues/6 https://github.com/ethereum/EIPs/issues/611 ]# - discard + result = "" server.on("net_listening") do() -> bool: ## Returns boolean true when listening, otherwise false. @@ -80,7 +77,7 @@ server.on("eth_syncing") do() -> JsonNode: server.on("eth_coinbase") do() -> string: ## Returns the current coinbase address. - discard + result = "" server.on("eth_mining") do() -> bool: ## Returns true of the client is mining, otherwise false. From 1a062f0946d79c25eb4ccc1bde55fc54d787deb4 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Thu, 17 May 2018 19:51:33 +0100 Subject: [PATCH 080/116] Add check for UInt256 strings being too long --- eth-rpc/server/servertypes.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 711f01f..263d2f3 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -79,7 +79,10 @@ proc fromJson(n: JsonNode, argName: string, result: var byte) = proc fromJson(n: JsonNode, argName: string, result: var UInt256) = # expects base 16 string, starting with "0x" n.kind.expect(JString, argName) - result = n.getStr().parse(StUint[256], 16) # TODO: Handle errors + let hexStr = n.getStr() + if hexStr.len > 64 + 2: # including "0x" + raise newException(ValueError, "Parameter \"" & argName & "\" value too long for UInt256: " & $hexStr.len) + result = hexStr.parse(StUint[256], 16) # TODO: Handle errors proc fromJson(n: JsonNode, argName: string, result: var float) = n.kind.expect(JFloat, argName) From 5bd3670aefcf249cebb41feadb5bb4630c7ae6ea Mon Sep 17 00:00:00 2001 From: coffeepots Date: Fri, 18 May 2018 17:55:13 +0100 Subject: [PATCH 081/116] Refactor Ethereum rpc types to separate module --- eth-rpc/ethtypes.nim | 124 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 eth-rpc/ethtypes.nim diff --git a/eth-rpc/ethtypes.nim b/eth-rpc/ethtypes.nim new file mode 100644 index 0000000..8ce5e0f --- /dev/null +++ b/eth-rpc/ethtypes.nim @@ -0,0 +1,124 @@ +import stint + +type + SyncObject* = object + startingBlock*: int + currentBlock*: int + highestBlock*: int + + EthSend* = object + source*: array[20, byte] # the address the transaction is send from. + to*: array[20, byte] # (optional when creating new contract) the address the transaction is directed to. + gas*: int # (optional, default: 90000) integer of the gas provided for the transaction execution. It will return unused gas. + gasPrice*: int # (optional, default: To-Be-Determined) integer of the gasPrice used for each paid gas. + value*: int # (optional) integer of the value sent with this transaction. + data*: int # the compiled code of a contract OR the hash of the invoked method signature and encoded parameters. For details see Ethereum Contract ABI. + nonce*: int # (optional) integer of a nonce. This allows to overwrite your own pending transactions that use the same nonce + + EthCall* = object + source*: array[20, byte] # (optional) The address the transaction is send from. + to*: array[20, byte] # The address the transaction is directed to. + gas*: int # (optional) Integer of the gas provided for the transaction execution. eth_call consumes zero gas, but this parameter may be needed by some executions. + gasPrice*: int # (optional) Integer of the gasPrice used for each paid gas. + value*: int # (optional) Integer of the value sent with this transaction. + data*: int # (optional) Hash of the method signature and encoded parameters. For details see Ethereum Contract ABI. + + ## A block object, or null when no block was found + BlockObject* = ref object + number*: int # the block number. null when its pending block. + hash*: UInt256 # hash of the block. null when its pending block. + parentHash*: UInt256 # hash of the parent block. + nonce*: int64 # hash of the generated proof-of-work. null when its pending block. + sha3Uncles*: UInt256 # SHA3 of the uncles data in the block. + logsBloom*: array[256, byte] # the bloom filter for the logs of the block. null when its pending block. + transactionsRoot*: UInt256 # the root of the transaction trie of the block. + stateRoot*: UInt256 # the root of the final state trie of the block. + receiptsRoot*: UInt256 # the root of the receipts trie of the block. + miner*: array[20, byte] # the address of the beneficiary to whom the mining rewards were given. + difficulty*: int # integer of the difficulty for this block. + totalDifficulty*: int # integer of the total difficulty of the chain until this block. + extraData*: string # the "extra data" field of this block. + size*: int # integer the size of this block in bytes. + gasLimit*: int # the maximum gas allowed in this block. + gasUsed*: int # the total used gas by all transactions in this block. + timestamp*: int # the unix timestamp for when the block was collated. + transactions*: seq[Uint256] # list of transaction objects, or 32 Bytes transaction hashes depending on the last given parameter. + uncles*: seq[Uint256] # list of uncle hashes. + + TransactionObject* = object # A transaction object, or null when no transaction was found: + hash*: UInt256 # hash of the transaction. + nonce*: int64 # TODO: Is int? the number of transactions made by the sender prior to this one. + blockHash*: UInt256 # hash of the block where this transaction was in. null when its pending. + blockNumber*: int64 # block number where this transaction was in. null when its pending. + transactionIndex*: int64 # integer of the transactions index position in the block. null when its pending. + source*: array[20, byte] # address of the sender. + to*: array[20, byte] # address of the receiver. null when its a contract creation transaction. + value*: int64 # value transferred in Wei. + gasPrice*: int64 # gas price provided by the sender in Wei. + gas*: int64 # gas provided by the sender. + input*: seq[byte] # the data send along with the transaction. + + ReceiptKind* = enum rkRoot, rkStatus + ReceiptObject* = object + # A transaction receipt object, or null when no receipt was found: + transactionHash*: UInt256 # hash of the transaction. + transactionIndex*: int # integer of the transactions index position in the block. + blockHash*: UInt256 # hash of the block where this transaction was in. + blockNumber*: int # block number where this transaction was in. + cumulativeGasUsed*: int # the total amount of gas used when this transaction was executed in the block. + gasUsed*: int # the amount of gas used by this specific transaction alone. + contractAddress*: array[20, byte] # the contract address created, if the transaction was a contract creation, otherwise null. + logs*: seq[string] # TODO: See Wiki for details. list of log objects, which this transaction generated. + logsBloom*: array[256, byte] # bloom filter for light clients to quickly retrieve related logs. + case kind*: ReceiptKind + of rkRoot: root*: UInt256 # post-transaction stateroot (pre Byzantium). + of rkStatus: status*: int # 1 = success, 0 = failure. + + FilterDataKind* = enum fkItem, fkList + FilterData* = object + # Difficult to process variant objects in input data, as kind is immutable. + # TODO: This might need more work to handle "or" options + kind*: FilterDataKind + items*: seq[FilterData] + item*: UInt256 + # TODO: I don't think this will work as input, need only one value that is either UInt256 or seq[UInt256] + + FilterOptions* = object + fromBlock*: string # (optional, default: "latest") integer block number, or "latest" for the last mined block or "pending", "earliest" for not yet mined transactions. + toBlock*: string # (optional, default: "latest") integer block number, or "latest" for the last mined block or "pending", "earliest" for not yet mined transactions. + address*: seq[array[20, byte]] # (optional) contract address or a list of addresses from which logs should originate. + topics*: seq[FilterData] # (optional) list of DATA topics. Topics are order-dependent. Each topic can also be a list of DATA with "or" options. + + LogObject* = object + removed*: bool # true when the log was removed, due to a chain reorganization. false if its a valid log. + logIndex*: int # integer of the log index position in the block. null when its pending log. + transactionIndex*: ref int # integer of the transactions index position log was created from. null when its pending log. + transactionHash*: UInt256 # hash of the transactions this log was created from. null when its pending log. + blockHash*: ref UInt256 # hash of the block where this log was in. null when its pending. null when its pending log. + blockNumber*: ref int64 # the block number where this log was in. null when its pending. null when its pending log. + address*: array[20, byte] # address from which this log originated. + data*: seq[UInt256] # contains one or more 32 Bytes non-indexed arguments of the log. + topics*: array[4, UInt256] # array of 0 to 4 32 Bytes DATA of indexed log arguments. + # (In solidity: The first topic is the hash of the signature of the event. + # (e.g. Deposit(address,bytes32,uint256)), except you declared the event with the anonymous specifier.) + + WhisperPost* = object + # The whisper post object: + source*: array[60, byte] # (optional) the identity of the sender. + to*: array[60, byte] # (optional) the identity of the receiver. When present whisper will encrypt the message so that only the receiver can decrypt it. + topics*: seq[UInt256] # TODO: Correct type? list of DATA topics, for the receiver to identify messages. + payload*: UInt256 # TODO: Correct type - maybe string? the payload of the message. + priority*: int # integer of the priority in a rang from ... (?). + ttl*: int # integer of the time to live in seconds. + + WhisperMessage* = object + # (?) are from the RPC Wiki, indicating uncertainty in type format. + hash*: UInt256 # (?) the hash of the message. + source*: array[60, byte] # the sender of the message, if a sender was specified. + to*: array[60, byte] # the receiver of the message, if a receiver was specified. + expiry*: int # integer of the time in seconds when this message should expire (?). + ttl*: int # integer of the time the message should float in the system in seconds (?). + sent*: int # integer of the unix timestamp when the message was sent. + topics*: seq[UInt256] # list of DATA topics the message contained. + payload*: string # TODO: Correct type? the payload of the message. + workProved*: int # integer of the work this message required before it was send (?). From befd6019c7cf5d4b98da38aa7bc492969d666b96 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Fri, 18 May 2018 18:06:31 +0100 Subject: [PATCH 082/116] Refactor Ethereum types to separate module --- eth-rpc/ethtypes.nim | 2 +- eth-rpc/server/ethprocs.nim | 132 +----------------------------------- 2 files changed, 2 insertions(+), 132 deletions(-) diff --git a/eth-rpc/ethtypes.nim b/eth-rpc/ethtypes.nim index 8ce5e0f..5fa6099 100644 --- a/eth-rpc/ethtypes.nim +++ b/eth-rpc/ethtypes.nim @@ -121,4 +121,4 @@ type sent*: int # integer of the unix timestamp when the message was sent. topics*: seq[UInt256] # list of DATA topics the message contained. payload*: string # TODO: Correct type? the payload of the message. - workProved*: int # integer of the work this message required before it was send (?). + workProved*: int # integer of the work this message required before it was send (?). \ No newline at end of file diff --git a/eth-rpc/server/ethprocs.nim b/eth-rpc/server/ethprocs.nim index ae20b35..9e11d55 100644 --- a/eth-rpc/server/ethprocs.nim +++ b/eth-rpc/server/ethprocs.nim @@ -1,4 +1,4 @@ -import servertypes, cryptoutils, json, stint +import servertypes, cryptoutils, json, stint, ../ethtypes.nim #[ For details on available RPC calls, see: https://github.com/ethereum/wiki/wiki/JSON-RPC @@ -60,12 +60,6 @@ server.on("eth_protocolVersion") do() -> string: ## Returns string of the current ethereum protocol version. discard -type - SyncObject = object - startingBlock: int - currentBlock: int - highestBlock: int - server.on("eth_syncing") do() -> JsonNode: ## Returns SyncObject or false when not syncing. var @@ -175,15 +169,6 @@ server.on("eth_sign") do(data: array[20, byte], message: seq[byte]) -> seq[byte] ## Returns signature. discard -type EthSend = object - source: array[20, byte] # the address the transaction is send from. - to: array[20, byte] # (optional when creating new contract) the address the transaction is directed to. - gas: int # (optional, default: 90000) integer of the gas provided for the transaction execution. It will return unused gas. - gasPrice: int # (optional, default: To-Be-Determined) integer of the gasPrice used for each paid gas. - value: int # (optional) integer of the value sent with this transaction. - data: int # the compiled code of a contract OR the hash of the invoked method signature and encoded parameters. For details see Ethereum Contract ABI. - nonce: int # (optional) integer of a nonce. This allows to overwrite your own pending transactions that use the same nonce. - server.on("eth_sendTransaction") do(obj: EthSend) -> UInt256: ## Creates new message call transaction or a contract creation, if the data field contains code. ## @@ -200,14 +185,6 @@ server.on("eth_sendRawTransaction") do(data: string, quantityTag: int) -> UInt25 ## Note: Use eth_getTransactionReceipt to get the contract address, after the transaction was mined, when you created a contract. discard -type EthCall = object - source: array[20, byte] # (optional) The address the transaction is send from. - to: array[20, byte] # The address the transaction is directed to. - gas: int # (optional) Integer of the gas provided for the transaction execution. eth_call consumes zero gas, but this parameter may be needed by some executions. - gasPrice: int # (optional) Integer of the gasPrice used for each paid gas. - value: int # (optional) Integer of the value sent with this transaction. - data: int # (optional) Hash of the method signature and encoded parameters. For details see Ethereum Contract ABI. - server.on("eth_call") do(call: EthCall, quantityTag: string) -> UInt256: ## Executes a new message call immediately without creating a transaction on the block chain. ## @@ -227,29 +204,6 @@ server.on("eth_estimateGas") do(call: EthCall, quantityTag: string) -> UInt256: ## Returns the amount of gas used. discard -type - ## A block object, or null when no block was found - BlockObject = ref object - number: int # the block number. null when its pending block. - hash: UInt256 # hash of the block. null when its pending block. - parentHash: UInt256 # hash of the parent block. - nonce: int64 # hash of the generated proof-of-work. null when its pending block. - sha3Uncles: UInt256 # SHA3 of the uncles data in the block. - logsBloom: array[256, byte] # the bloom filter for the logs of the block. null when its pending block. - transactionsRoot: UInt256 # the root of the transaction trie of the block. - stateRoot: UInt256 # the root of the final state trie of the block. - receiptsRoot: UInt256 # the root of the receipts trie of the block. - miner: array[20, byte] # the address of the beneficiary to whom the mining rewards were given. - difficulty: int # integer of the difficulty for this block. - totalDifficulty: int # integer of the total difficulty of the chain until this block. - extraData: string # the "extra data" field of this block. - size: int # integer the size of this block in bytes. - gasLimit: int # the maximum gas allowed in this block. - gasUsed: int # the total used gas by all transactions in this block. - timestamp: int # the unix timestamp for when the block was collated. - transactions: seq[Uint256] # list of transaction objects, or 32 Bytes transaction hashes depending on the last given parameter. - uncles: seq[Uint256] # list of uncle hashes. - server.on("eth_getBlockByHash") do(data: array[32, byte], fullTransactions: bool) -> BlockObject: ## Returns information about a block by hash. ## @@ -266,20 +220,6 @@ server.on("eth_getBlockByNumber") do(quantityTag: string, fullTransactions: bool ## Returns BlockObject or nil when no block was found. discard -type - TransactionObject = object # A transaction object, or null when no transaction was found: - hash: UInt256 # hash of the transaction. - nonce: int64 # TODO: Is int? the number of transactions made by the sender prior to this one. - blockHash: UInt256 # hash of the block where this transaction was in. null when its pending. - blockNumber: int64 # block number where this transaction was in. null when its pending. - transactionIndex: int64 # integer of the transactions index position in the block. null when its pending. - source: array[20, byte] # address of the sender. - to: array[20, byte] # address of the receiver. null when its a contract creation transaction. - value: int64 # value transferred in Wei. - gasPrice: int64 # gas price provided by the sender in Wei. - gas: int64 # gas provided by the sender. - input: seq[byte] # the data send along with the transaction. - server.on("eth_getTransactionByHash") do(data: Uint256) -> TransactionObject: ## Returns the information about a transaction requested by transaction hash. ## @@ -302,23 +242,6 @@ server.on("eth_getTransactionByBlockNumberAndIndex") do(quantityTag: string, qua ## quantity: the transaction index position. discard -type - ReceiptKind = enum rkRoot, rkStatus - ReceiptObject = object - # A transaction receipt object, or null when no receipt was found: - transactionHash: UInt256 # hash of the transaction. - transactionIndex: int # integer of the transactions index position in the block. - blockHash: UInt256 # hash of the block where this transaction was in. - blockNumber: int # block number where this transaction was in. - cumulativeGasUsed: int # the total amount of gas used when this transaction was executed in the block. - gasUsed: int # the amount of gas used by this specific transaction alone. - contractAddress: array[20, byte] # the contract address created, if the transaction was a contract creation, otherwise null. - logs: seq[string] # TODO: See Wiki for details. list of log objects, which this transaction generated. - logsBloom: array[256, byte] # bloom filter for light clients to quickly retrieve related logs. - case kind: ReceiptKind - of rkRoot: root: UInt256 # post-transaction stateroot (pre Byzantium). - of rkStatus: status: int # 1 = success, 0 = failure. - server.on("eth_getTransactionReceipt") do(data: UInt256) -> ReceiptObject: ## Returns the receipt of a transaction by transaction hash. ## @@ -369,22 +292,6 @@ server.on("eth_compileSerpent") do(sourceCode: string) -> seq[byte]: ## Returns compiles source code. result = @[] -type - FilterDataKind = enum fkItem, fkList - FilterData = object - # Difficult to process variant objects in input data, as kind is immutable. - # TODO: This might need more work to handle "or" options - kind: FilterDataKind - items: seq[FilterData] - item: UInt256 - # TODO: I don't think this will work as input, need only one value that is either UInt256 or seq[UInt256] - - FilterOptions = object - fromBlock: string # (optional, default: "latest") integer block number, or "latest" for the last mined block or "pending", "earliest" for not yet mined transactions. - toBlock: string # (optional, default: "latest") integer block number, or "latest" for the last mined block or "pending", "earliest" for not yet mined transactions. - address: seq[array[20, byte]] # (optional) contract address or a list of addresses from which logs should originate. - topics: seq[FilterData] # (optional) list of DATA topics. Topics are order-dependent. Each topic can also be a list of DATA with "or" options. - server.on("eth_newFilter") do(filterOptions: FilterOptions) -> int: ## Creates a filter object, based on filter options, to notify when the state changes (logs). ## To check if the state has changed, call eth_getFilterChanges. @@ -421,20 +328,6 @@ server.on("eth_uninstallFilter") do(filterId: int) -> bool: ## Returns true if the filter was successfully uninstalled, otherwise false. discard -type - LogObject = object - removed: bool # true when the log was removed, due to a chain reorganization. false if its a valid log. - logIndex: int # integer of the log index position in the block. null when its pending log. - transactionIndex: ref int # integer of the transactions index position log was created from. null when its pending log. - transactionHash: UInt256 # hash of the transactions this log was created from. null when its pending log. - blockHash: ref UInt256 # hash of the block where this log was in. null when its pending. null when its pending log. - blockNumber: ref int64 # the block number where this log was in. null when its pending. null when its pending log. - address: array[20, byte] # address from which this log originated. - data: seq[UInt256] # contains one or more 32 Bytes non-indexed arguments of the log. - topics: array[4, UInt256] # array of 0 to 4 32 Bytes DATA of indexed log arguments. - # (In solidity: The first topic is the hash of the signature of the event. - # (e.g. Deposit(address,bytes32,uint256)), except you declared the event with the anonymous specifier.) - server.on("eth_getFilterChanges") do(filterId: int) -> seq[LogObject]: ## Polling method for a filter, which returns an list of logs which occurred since last poll. ## @@ -480,16 +373,6 @@ server.on("shh_version") do() -> string: ## Returns string of the current whisper protocol version. discard -type - WhisperPost = object - # The whisper post object: - source: array[60, byte] # (optional) the identity of the sender. - to: array[60, byte] # (optional) the identity of the receiver. When present whisper will encrypt the message so that only the receiver can decrypt it. - topics: seq[UInt256] # TODO: Correct type? list of DATA topics, for the receiver to identify messages. - payload: UInt256 # TODO: Correct type - maybe string? the payload of the message. - priority: int # integer of the priority in a rang from ... (?). - ttl: int # integer of the time to live in seconds. - server.on("shh_post") do(message: WhisperPost) -> bool: ## Sends a whisper message. ## @@ -544,19 +427,6 @@ server.on("shh_uninstallFilter") do(id: int) -> bool: ## Returns true if the filter was successfully uninstalled, otherwise false. discard -type - WhisperMessage = object - # (?) are from the RPC Wiki, indicating uncertainty in type format. - hash: UInt256 # (?) the hash of the message. - source: array[60, byte] # the sender of the message, if a sender was specified. - to: array[60, byte] # the receiver of the message, if a receiver was specified. - expiry: int # integer of the time in seconds when this message should expire (?). - ttl: int # integer of the time the message should float in the system in seconds (?). - sent: int # integer of the unix timestamp when the message was sent. - topics: seq[UInt256] # list of DATA topics the message contained. - payload: string # TODO: Correct type? the payload of the message. - workProved: int # integer of the work this message required before it was send (?). - server.on("shh_getFilterChanges") do(id: int) -> seq[WhisperMessage]: ## Polling method for whisper filters. Returns new messages since the last call of this method. ## Note: calling the shh_getMessages method, will reset the buffer for this method, so that you won't receive duplicate messages. From fa113d1a84b6a656bf2ce2dc7d82d5f29f304e80 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Fri, 18 May 2018 18:10:21 +0100 Subject: [PATCH 083/116] Client forward decls for checking client calls --- eth-rpc/client/ethcallsigs.nim | 61 ++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 eth-rpc/client/ethcallsigs.nim diff --git a/eth-rpc/client/ethcallsigs.nim b/eth-rpc/client/ethcallsigs.nim new file mode 100644 index 0000000..75b6fbd --- /dev/null +++ b/eth-rpc/client/ethcallsigs.nim @@ -0,0 +1,61 @@ +## This module contains signatures for the Ethereum client RPCs. +import json, stint, ../ ethtypes + +proc web3_clientVersion(): string +proc web3_sha3(): string +proc net_version(): string +proc net_peerCount(): int +proc net_listening(): bool +proc eth_protocolVersion(): string +proc eth_syncing(): JsonNode +proc eth_coinbase(): string +proc eth_mining(): bool +proc eth_hashrate(): int +proc eth_gasPrice(): int64 +proc eth_accounts(): seq[array[20, byte]] +proc eth_blockNumber(): int +proc eth_getBalance(data: array[20, byte], quantityTag: string): int +proc eth_getStorageAt(data: array[20, byte], quantity: int, quantityTag: string): seq[byte] +proc eth_getTransactionCount(data: array[20, byte], quantityTag: string) +proc eth_getBlockTransactionCountByHash(data: array[32, byte]) +proc eth_getBlockTransactionCountByNumber(quantityTag: string) +proc eth_getUncleCountByBlockHash(data: array[32, byte]) +proc eth_getUncleCountByBlockNumber(quantityTag: string) +proc eth_getCode(data: array[20, byte], quantityTag: string): seq[byte] +proc eth_sign(data: array[20, byte], message: seq[byte]): seq[byte] +proc eth_sendTransaction(obj: EthSend): UInt256 +proc eth_sendRawTransaction(data: string, quantityTag: int): UInt256 +proc eth_call(call: EthCall, quantityTag: string): UInt256 +proc eth_estimateGas(call: EthCall, quantityTag: string): UInt256 +proc eth_getBlockByHash(data: array[32, byte], fullTransactions: bool): BlockObject +proc eth_getBlockByNumber(quantityTag: string, fullTransactions: bool): BlockObject +proc eth_getTransactionByHash(data: Uint256): TransactionObject +proc eth_getTransactionByBlockHashAndIndex(data: UInt256, quantity: int): TransactionObject +proc eth_getTransactionByBlockNumberAndIndex(quantityTag: string, quantity: int): TransactionObject +proc eth_getTransactionReceipt(data: UInt256): ReceiptObject +proc eth_getUncleByBlockHashAndIndex(data: UInt256, quantity: int64): BlockObject +proc eth_getUncleByBlockNumberAndIndex(quantityTag: string, quantity: int64): BlockObject +proc eth_getCompilers(): seq[string] +proc eth_compileLLL(): seq[byte] +proc eth_compileSolidity(): seq[byte] +proc eth_compileSerpent(): seq[byte] +proc eth_newFilter(filterOptions: FilterOptions): int +proc eth_newBlockFilter(): int +proc eth_newPendingTransactionFilter(): int +proc eth_uninstallFilter(filterId: int): bool +proc eth_getFilterChanges(filterId: int): seq[LogObject] +proc eth_getFilterLogs(filterId: int): seq[LogObject] +proc eth_getLogs(filterOptions: FilterOptions): seq[LogObject] +proc eth_getWork(): seq[UInt256] +proc eth_submitWork(nonce: int64, powHash: Uint256, mixDigest: Uint256): bool +proc eth_submitHashrate(hashRate: UInt256, id: Uint256): bool +proc shh_post(): string +proc shh_version(message: WhisperPost): bool +proc shh_newIdentity(): array[60, byte] +proc shh_hasIdentity(identity: array[60, byte]): bool +proc shh_newGroup(): array[60, byte] +proc shh_addToGroup(identity: array[60, byte]): bool +proc shh_newFilter(filterOptions: FilterOptions, to: array[60, byte], topics: seq[UInt256]): int +proc shh_uninstallFilter(id: int): bool +proc shh_getFilterChanges(id: int): seq[WhisperMessage] +proc shh_getMessages(id: int): seq[WhisperMessage] From 31246edbd4545e700fd2d7ee358565f642db90e7 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 22 May 2018 00:13:00 +0100 Subject: [PATCH 084/116] `on` is now `rpc` for now to fix compilation issue --- eth-rpc/server/ethprocs.nim | 116 ++++++++++++++++----------------- eth-rpc/server/servertypes.nim | 34 +++++----- tests/testrpcmacro.nim | 20 +++--- tests/testserverclient.nim | 2 +- 4 files changed, 84 insertions(+), 88 deletions(-) diff --git a/eth-rpc/server/ethprocs.nim b/eth-rpc/server/ethprocs.nim index 9e11d55..c06ea07 100644 --- a/eth-rpc/server/ethprocs.nim +++ b/eth-rpc/server/ethprocs.nim @@ -24,18 +24,18 @@ import servertypes, cryptoutils, json, stint, ../ethtypes.nim var server = sharedRpcServer() -server.on("web3_clientVersion") do() -> string: +server.rpc("web3_clientVersion") do() -> string: ## Returns the current client version. result = "Nimbus-RPC-Test" -server.on("web3_sha3") do(data: string) -> string: +server.rpc("web3_sha3") do(data: string) -> string: ## Returns Keccak-256 (not the standardized SHA3-256) of the given data. ## ## data: the data to convert into a SHA3 hash. ## Returns the SHA3 result of the given string. result = k256(data) -server.on("net_version") do() -> string: +server.rpc("net_version") do() -> string: ## Returns string of the current network id: ## "1": Ethereum Mainnet ## "2": Morden Testnet (deprecated) @@ -48,19 +48,19 @@ server.on("net_version") do() -> string: ]# result = "" -server.on("net_listening") do() -> bool: +server.rpc("net_listening") do() -> bool: ## Returns boolean true when listening, otherwise false. result = true -server.on("net_peerCount") do() -> int: +server.rpc("net_peerCount") do() -> int: ## Returns integer of the number of connected peers. discard -server.on("eth_protocolVersion") do() -> string: +server.rpc("eth_protocolVersion") do() -> string: ## Returns string of the current ethereum protocol version. discard -server.on("eth_syncing") do() -> JsonNode: +server.rpc("eth_syncing") do() -> JsonNode: ## Returns SyncObject or false when not syncing. var res: JsonNode @@ -69,33 +69,33 @@ server.on("eth_syncing") do() -> JsonNode: else: res = newJBool(false) result = res -server.on("eth_coinbase") do() -> string: +server.rpc("eth_coinbase") do() -> string: ## Returns the current coinbase address. result = "" -server.on("eth_mining") do() -> bool: +server.rpc("eth_mining") do() -> bool: ## Returns true of the client is mining, otherwise false. discard -server.on("eth_hashrate") do() -> int: +server.rpc("eth_hashrate") do() -> int: ## Returns the number of hashes per second that the node is mining with. discard -server.on("eth_gasPrice") do() -> int64: +server.rpc("eth_gasPrice") do() -> int64: ## Returns an integer of the current gas price in wei. discard -server.on("eth_accounts") do() -> seq[array[20, byte]]: +server.rpc("eth_accounts") do() -> seq[array[20, byte]]: ## Returns a list of addresses owned by client. # TODO: this might be easier to use as seq[string] # This is what's expected: "result": ["0x407d73d8a49eeb85d32cf465507dd71d507100c1"] discard -server.on("eth_blockNumber") do() -> int: +server.rpc("eth_blockNumber") do() -> int: ## Returns integer of the current block number the client is on. discard -server.on("eth_getBalance") do(data: array[20, byte], quantityTag: string) -> int: +server.rpc("eth_getBalance") do(data: array[20, byte], quantityTag: string) -> int: ## Returns the balance of the account of given address. ## ## data: address to check for balance. @@ -103,7 +103,7 @@ server.on("eth_getBalance") do(data: array[20, byte], quantityTag: string) -> in ## Returns integer of the current balance in wei. discard -server.on("eth_getStorageAt") do(data: array[20, byte], quantity: int, quantityTag: string) -> seq[byte]: +server.rpc("eth_getStorageAt") do(data: array[20, byte], quantity: int, quantityTag: string) -> seq[byte]: ## Returns the value from a storage position at a given address. ## ## data: address of the storage. @@ -114,7 +114,7 @@ server.on("eth_getStorageAt") do(data: array[20, byte], quantity: int, quantityT # For more details, see: https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getstorageat result = @[] -server.on("eth_getTransactionCount") do(data: array[20, byte], quantityTag: string): +server.rpc("eth_getTransactionCount") do(data: array[20, byte], quantityTag: string): ## Returns the number of transactions sent from an address. ## ## data: address. @@ -122,35 +122,35 @@ server.on("eth_getTransactionCount") do(data: array[20, byte], quantityTag: stri ## Returns integer of the number of transactions send from this address. discard -server.on("eth_getBlockTransactionCountByHash") do(data: array[32, byte]) -> int: +server.rpc("eth_getBlockTransactionCountByHash") do(data: array[32, byte]) -> int: ## Returns the number of transactions in a block from a block matching the given block hash. ## ## data: hash of a block ## Returns integer of the number of transactions in this block. discard -server.on("eth_getBlockTransactionCountByNumber") do(quantityTag: string) -> int: +server.rpc("eth_getBlockTransactionCountByNumber") do(quantityTag: string) -> int: ## Returns the number of transactions in a block matching the given block number. ## ## data: integer of a block number, or the string "earliest", "latest" or "pending", as in the default block parameter. ## Returns integer of the number of transactions in this block. discard -server.on("eth_getUncleCountByBlockHash") do(data: array[32, byte]): +server.rpc("eth_getUncleCountByBlockHash") do(data: array[32, byte]): ## Returns the number of uncles in a block from a block matching the given block hash. ## ## data: hash of a block. ## Returns integer of the number of uncles in this block. discard -server.on("eth_getUncleCountByBlockNumber") do(quantityTag: string): +server.rpc("eth_getUncleCountByBlockNumber") do(quantityTag: string): ## Returns the number of uncles in a block from a block matching the given block number. ## ## quantityTag: integer of a block number, or the string "latest", "earliest" or "pending", see the default block parameter. ## Returns integer of uncles in this block. discard -server.on("eth_getCode") do(data: array[20, byte], quantityTag: string) -> seq[byte]: +server.rpc("eth_getCode") do(data: array[20, byte], quantityTag: string) -> seq[byte]: ## Returns code at a given address. ## ## data: address @@ -158,7 +158,7 @@ server.on("eth_getCode") do(data: array[20, byte], quantityTag: string) -> seq[b ## Returns the code from the given address. result = @[] -server.on("eth_sign") do(data: array[20, byte], message: seq[byte]) -> seq[byte]: +server.rpc("eth_sign") do(data: array[20, byte], message: seq[byte]) -> seq[byte]: ## The sign method calculates an Ethereum specific signature with: sign(keccak256("\x19Ethereum Signed Message:\n" + len(message) + message))). ## By adding a prefix to the message makes the calculated signature recognisable as an Ethereum specific signature. ## This prevents misuse where a malicious DApp can sign arbitrary data (e.g. transaction) and use the signature to impersonate the victim. @@ -169,7 +169,7 @@ server.on("eth_sign") do(data: array[20, byte], message: seq[byte]) -> seq[byte] ## Returns signature. discard -server.on("eth_sendTransaction") do(obj: EthSend) -> UInt256: +server.rpc("eth_sendTransaction") do(obj: EthSend) -> UInt256: ## Creates new message call transaction or a contract creation, if the data field contains code. ## ## obj: the transaction object. @@ -177,7 +177,7 @@ server.on("eth_sendTransaction") do(obj: EthSend) -> UInt256: ## Note: Use eth_getTransactionReceipt to get the contract address, after the transaction was mined, when you created a contract. discard -server.on("eth_sendRawTransaction") do(data: string, quantityTag: int) -> UInt256: # TODO: string or array of byte? +server.rpc("eth_sendRawTransaction") do(data: string, quantityTag: int) -> UInt256: # TODO: string or array of byte? ## Creates new message call transaction or a contract creation for signed transactions. ## ## data: the signed transaction data. @@ -185,7 +185,7 @@ server.on("eth_sendRawTransaction") do(data: string, quantityTag: int) -> UInt25 ## Note: Use eth_getTransactionReceipt to get the contract address, after the transaction was mined, when you created a contract. discard -server.on("eth_call") do(call: EthCall, quantityTag: string) -> UInt256: +server.rpc("eth_call") do(call: EthCall, quantityTag: string) -> UInt256: ## Executes a new message call immediately without creating a transaction on the block chain. ## ## call: the transaction call object. @@ -194,7 +194,7 @@ server.on("eth_call") do(call: EthCall, quantityTag: string) -> UInt256: # TODO: Should return value be UInt256 or seq[byte] or string? discard -server.on("eth_estimateGas") do(call: EthCall, quantityTag: string) -> UInt256: # TODO: Int or U/Int256? +server.rpc("eth_estimateGas") do(call: EthCall, quantityTag: string) -> UInt256: # TODO: Int or U/Int256? ## Generates and returns an estimate of how much gas is necessary to allow the transaction to complete. ## The transaction will not be added to the blockchain. Note that the estimate may be significantly more than ## the amount of gas actually used by the transaction, for a variety of reasons including EVM mechanics and node performance. @@ -204,7 +204,7 @@ server.on("eth_estimateGas") do(call: EthCall, quantityTag: string) -> UInt256: ## Returns the amount of gas used. discard -server.on("eth_getBlockByHash") do(data: array[32, byte], fullTransactions: bool) -> BlockObject: +server.rpc("eth_getBlockByHash") do(data: array[32, byte], fullTransactions: bool) -> BlockObject: ## Returns information about a block by hash. ## ## data: Hash of a block. @@ -212,7 +212,7 @@ server.on("eth_getBlockByHash") do(data: array[32, byte], fullTransactions: bool ## Returns BlockObject or nil when no block was found. discard -server.on("eth_getBlockByNumber") do(quantityTag: string, fullTransactions: bool) -> BlockObject: +server.rpc("eth_getBlockByNumber") do(quantityTag: string, fullTransactions: bool) -> BlockObject: ## Returns information about a block by block number. ## ## quantityTag: integer of a block number, or the string "earliest", "latest" or "pending", as in the default block parameter. @@ -220,14 +220,14 @@ server.on("eth_getBlockByNumber") do(quantityTag: string, fullTransactions: bool ## Returns BlockObject or nil when no block was found. discard -server.on("eth_getTransactionByHash") do(data: Uint256) -> TransactionObject: +server.rpc("eth_getTransactionByHash") do(data: Uint256) -> TransactionObject: ## Returns the information about a transaction requested by transaction hash. ## ## data: hash of a transaction. ## Returns requested transaction information. discard -server.on("eth_getTransactionByBlockHashAndIndex") do(data: UInt256, quantity: int) -> TransactionObject: +server.rpc("eth_getTransactionByBlockHashAndIndex") do(data: UInt256, quantity: int) -> TransactionObject: ## Returns information about a transaction by block hash and transaction index position. ## ## data: hash of a block. @@ -235,21 +235,21 @@ server.on("eth_getTransactionByBlockHashAndIndex") do(data: UInt256, quantity: i ## Returns requested transaction information. discard -server.on("eth_getTransactionByBlockNumberAndIndex") do(quantityTag: string, quantity: int) -> TransactionObject: +server.rpc("eth_getTransactionByBlockNumberAndIndex") do(quantityTag: string, quantity: int) -> TransactionObject: ## Returns information about a transaction by block number and transaction index position. ## ## quantityTag: a block number, or the string "earliest", "latest" or "pending", as in the default block parameter. ## quantity: the transaction index position. discard -server.on("eth_getTransactionReceipt") do(data: UInt256) -> ReceiptObject: +server.rpc("eth_getTransactionReceipt") do(data: UInt256) -> ReceiptObject: ## Returns the receipt of a transaction by transaction hash. ## ## data: hash of a transaction. ## Returns transaction receipt. discard -server.on("eth_getUncleByBlockHashAndIndex") do(data: UInt256, quantity: int64) -> BlockObject: +server.rpc("eth_getUncleByBlockHashAndIndex") do(data: UInt256, quantity: int64) -> BlockObject: ## Returns information about a uncle of a block by hash and uncle index position. ## ## data: hash a block. @@ -257,7 +257,7 @@ server.on("eth_getUncleByBlockHashAndIndex") do(data: UInt256, quantity: int64) ## Returns BlockObject or nil when no block was found. discard -server.on("eth_getUncleByBlockNumberAndIndex") do(quantityTag: string, quantity: int64) -> BlockObject: +server.rpc("eth_getUncleByBlockNumberAndIndex") do(quantityTag: string, quantity: int64) -> BlockObject: # Returns information about a uncle of a block by number and uncle index position. ## ## quantityTag: a block number, or the string "earliest", "latest" or "pending", as in the default block parameter. @@ -265,34 +265,34 @@ server.on("eth_getUncleByBlockNumberAndIndex") do(quantityTag: string, quantity: ## Returns BlockObject or nil when no block was found. discard -server.on("eth_getCompilers") do() -> seq[string]: +server.rpc("eth_getCompilers") do() -> seq[string]: ## Returns a list of available compilers in the client. ## ## Returns a list of available compilers. result = @[] -server.on("eth_compileSolidity") do(sourceCode: string) -> seq[byte]: +server.rpc("eth_compileSolidity") do(sourceCode: string) -> seq[byte]: ## Returns compiled solidity code. ## ## sourceCode: source code as string. ## Returns compiles source code. result = @[] -server.on("eth_compileLLL") do(sourceCode: string) -> seq[byte]: +server.rpc("eth_compileLLL") do(sourceCode: string) -> seq[byte]: ## Returns compiled LLL code. ## ## sourceCode: source code as string. ## Returns compiles source code. result = @[] -server.on("eth_compileSerpent") do(sourceCode: string) -> seq[byte]: +server.rpc("eth_compileSerpent") do(sourceCode: string) -> seq[byte]: ## Returns compiled serpent code. ## ## sourceCode: source code as string. ## Returns compiles source code. result = @[] -server.on("eth_newFilter") do(filterOptions: FilterOptions) -> int: +server.rpc("eth_newFilter") do(filterOptions: FilterOptions) -> int: ## Creates a filter object, based on filter options, to notify when the state changes (logs). ## To check if the state has changed, call eth_getFilterChanges. ## Topics are order-dependent. A transaction with a log with topics [A, B] will be matched by the following topic filters: @@ -306,21 +306,21 @@ server.on("eth_newFilter") do(filterOptions: FilterOptions) -> int: ## Returns integer filter id. discard -server.on("eth_newBlockFilter") do() -> int: +server.rpc("eth_newBlockFilter") do() -> int: ## Creates a filter in the node, to notify when a new block arrives. ## To check if the state has changed, call eth_getFilterChanges. ## ## Returns integer filter id. discard -server.on("eth_newPendingTransactionFilter") do() -> int: +server.rpc("eth_newPendingTransactionFilter") do() -> int: ## Creates a filter in the node, to notify when a new block arrives. ## To check if the state has changed, call eth_getFilterChanges. ## ## Returns integer filter id. discard -server.on("eth_uninstallFilter") do(filterId: int) -> bool: +server.rpc("eth_uninstallFilter") do(filterId: int) -> bool: ## Uninstalls a filter with given id. Should always be called when watch is no longer needed. ## Additonally Filters timeout when they aren't requested with eth_getFilterChanges for a period of time. ## @@ -328,23 +328,23 @@ server.on("eth_uninstallFilter") do(filterId: int) -> bool: ## Returns true if the filter was successfully uninstalled, otherwise false. discard -server.on("eth_getFilterChanges") do(filterId: int) -> seq[LogObject]: +server.rpc("eth_getFilterChanges") do(filterId: int) -> seq[LogObject]: ## Polling method for a filter, which returns an list of logs which occurred since last poll. ## ## filterId: the filter id. result = @[] -server.on("eth_getFilterLogs") do(filterId: int) -> seq[LogObject]: +server.rpc("eth_getFilterLogs") do(filterId: int) -> seq[LogObject]: ## filterId: the filter id. ## Returns a list of all logs matching filter with given id. result = @[] -server.on("eth_getLogs") do(filterOptions: FilterOptions) -> seq[LogObject]: +server.rpc("eth_getLogs") do(filterOptions: FilterOptions) -> seq[LogObject]: ## filterOptions: settings for this filter. ## Returns a list of all logs matching a given filter object. result = @[] -server.on("eth_getWork") do() -> seq[UInt256]: +server.rpc("eth_getWork") do() -> seq[UInt256]: ## Returns the hash of the current block, the seedHash, and the boundary condition to be met ("target"). ## Returned list has the following properties: ## DATA, 32 Bytes - current block header pow-hash. @@ -352,7 +352,7 @@ server.on("eth_getWork") do() -> seq[UInt256]: ## DATA, 32 Bytes - the boundary condition ("target"), 2^256 / difficulty. result = @[] -server.on("eth_submitWork") do(nonce: int64, powHash: Uint256, mixDigest: Uint256) -> bool: +server.rpc("eth_submitWork") do(nonce: int64, powHash: Uint256, mixDigest: Uint256) -> bool: ## Used for submitting a proof-of-work solution. ## ## nonce: the nonce found. @@ -361,7 +361,7 @@ server.on("eth_submitWork") do(nonce: int64, powHash: Uint256, mixDigest: Uint25 ## Returns true if the provided solution is valid, otherwise false. discard -server.on("eth_submitHashrate") do(hashRate: UInt256, id: Uint256) -> bool: +server.rpc("eth_submitHashrate") do(hashRate: UInt256, id: Uint256) -> bool: ## Used for submitting mining hashrate. ## ## hashRate: a hexadecimal string representation (32 bytes) of the hash rate. @@ -369,44 +369,44 @@ server.on("eth_submitHashrate") do(hashRate: UInt256, id: Uint256) -> bool: ## Returns true if submitting went through succesfully and false otherwise. discard -server.on("shh_version") do() -> string: +server.rpc("shh_version") do() -> string: ## Returns string of the current whisper protocol version. discard -server.on("shh_post") do(message: WhisperPost) -> bool: +server.rpc("shh_post") do(message: WhisperPost) -> bool: ## Sends a whisper message. ## ## message: Whisper message to post. ## Returns true if the message was send, otherwise false. discard -server.on("shh_newIdentity") do() -> array[60, byte]: +server.rpc("shh_newIdentity") do() -> array[60, byte]: ## Creates new whisper identity in the client. ## ## Returns the address of the new identiy. discard -server.on("shh_hasIdentity") do(identity: array[60, byte]) -> bool: +server.rpc("shh_hasIdentity") do(identity: array[60, byte]) -> bool: ## Checks if the client holds the private keys for a given identity. ## ## identity: the identity address to check. ## Returns true if the client holds the privatekey for that identity, otherwise false. discard -server.on("shh_newGroup") do() -> array[60, byte]: +server.rpc("shh_newGroup") do() -> array[60, byte]: ## (?) - This has no description information in the RPC wiki. ## ## Returns the address of the new group. (?) discard -server.on("shh_addToGroup") do(identity: array[60, byte]) -> bool: +server.rpc("shh_addToGroup") do(identity: array[60, byte]) -> bool: ## (?) - This has no description information in the RPC wiki. ## ## identity: the identity address to add to a group (?). ## Returns true if the identity was successfully added to the group, otherwise false (?). discard -server.on("shh_newFilter") do(filterOptions: FilterOptions, to: array[60, byte], topics: seq[UInt256]) -> int: # TODO: Is topic of right type? +server.rpc("shh_newFilter") do(filterOptions: FilterOptions, to: array[60, byte], topics: seq[UInt256]) -> int: # TODO: Is topic of right type? ## Creates filter to notify, when client receives whisper message matching the filter options. ## ## filterOptions: The filter options: @@ -418,7 +418,7 @@ server.on("shh_newFilter") do(filterOptions: FilterOptions, to: array[60, byte], ## Returns the newly created filter. discard -server.on("shh_uninstallFilter") do(id: int) -> bool: +server.rpc("shh_uninstallFilter") do(id: int) -> bool: ## Uninstalls a filter with given id. ## Should always be called when watch is no longer needed. ## Additonally Filters timeout when they aren't requested with shh_getFilterChanges for a period of time. @@ -427,14 +427,14 @@ server.on("shh_uninstallFilter") do(id: int) -> bool: ## Returns true if the filter was successfully uninstalled, otherwise false. discard -server.on("shh_getFilterChanges") do(id: int) -> seq[WhisperMessage]: +server.rpc("shh_getFilterChanges") do(id: int) -> seq[WhisperMessage]: ## Polling method for whisper filters. Returns new messages since the last call of this method. ## Note: calling the shh_getMessages method, will reset the buffer for this method, so that you won't receive duplicate messages. ## ## id: the filter id. discard -server.on("shh_getMessages") do(id: int) -> seq[WhisperMessage]: +server.rpc("shh_getMessages") do(id: int) -> seq[WhisperMessage]: ## Get all messages matching a filter. Unlike shh_getFilterChanges this returns all messages. ## ## id: the filter id. diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 263d2f3..25d33d9 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -141,51 +141,47 @@ proc makeProcName(s: string): string = if c.isAlphaNumeric: result.add c proc hasReturnType(params: NimNode): bool = - if params.len > 0 and params[0] != nil and params[0].kind != nnkEmpty: + if params != nil and params.len > 0 and params[0] != nil and params[0].kind != nnkEmpty: result = true -macro on*(server: var RpcServer, path: string, body: untyped): untyped = +macro rpc*(server: var RpcServer, path: string, body: untyped): untyped = result = newStmtList() let parameters = body.findChild(it.kind == nnkFormalParams) - paramsIdent = ident"params" # all remote calls have a single parameter: `params: JsonNode` - pathStr = $path # procs are generated from the stripped path - procName = ident(pathStr.makeProcName) - doMain = genSym(nskProc) # proc that contains our rpc body - res = ident"result" # async result + paramsIdent = newIdentNode"params" # all remote calls have a single parameter: `params: JsonNode` + pathStr = $path # procs are generated from the stripped path + procNameStr = pathStr.makeProcName # strip non alphanumeric + procName = newIdentNode(procNameStr) # public rpc proc + doMain = newIdentNode(procNameStr & "DoMain") # when parameters: proc that contains our rpc body + res = newIdentNode("result") # async result var setup = setupParams(parameters, paramsIdent) - procBody: NimNode - - if body.kind == nnkStmtList: procBody = body - else: procBody = body.body + procBody = if body.kind == nnkStmtList: body else: body.body if parameters.hasReturnType: let returnType = parameters[0] - # `doMain` is outside of async transformation, - # allowing natural `return` + # delgate async proc allows return and setting of result as native type result.add(quote do: - proc `doMain`(`paramsIdent`: JsonNode): `returnType` {.inline.} = + proc `doMain`(`paramsIdent`: JsonNode): Future[`returnType`] {.async.} = `setup` `procBody` ) - # Note ``res` =` (which becomes `result = `) will be transformed by {.async.} to `complete` if returnType == ident"JsonNode": # `JsonNode` results don't need conversion result.add( quote do: proc `procName`*(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = - `res` = `doMain`(`paramsIdent`) + `res` = await `doMain`(`paramsIdent`) ) else: - result.add( quote do: + result.add(quote do: proc `procName`*(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = - `res` = %`doMain`(`paramsIdent`) + `res` = %await `doMain`(`paramsIdent`) ) else: # no return types, inline contents - result.add( quote do: + result.add(quote do: proc `procName`*(`paramsIdent`: JsonNode): Future[JsonNode] {.async.} = `setup` `procBody` diff --git a/tests/testrpcmacro.nim b/tests/testrpcmacro.nim index 372ab36..b1e8e7f 100644 --- a/tests/testrpcmacro.nim +++ b/tests/testrpcmacro.nim @@ -30,42 +30,42 @@ var s = newRpcServer("localhost") # RPC definitions -s.on("rpc.simplepath"): +s.rpc("rpc.simplepath"): result = %1 -s.on("rpc.differentparams") do(a: int, b: string): +s.rpc("rpc.differentparams") do(a: int, b: string): result = %[%a, %b] -s.on("rpc.arrayparam") do(arr: array[0..5, byte], b: string): +s.rpc("rpc.arrayparam") do(arr: array[0..5, byte], b: string): var res = %arr res.add %b result = %res -s.on("rpc.seqparam") do(a: string, s: seq[int]): +s.rpc("rpc.seqparam") do(a: string, s: seq[int]): var res = newJArray() res.add %a for item in s: res.add %int(item) result = res -s.on("rpc.objparam") do(a: string, obj: MyObject): +s.rpc("rpc.objparam") do(a: string, obj: MyObject): result = %obj -s.on("rpc.uint256param") do(i: UInt256): +s.rpc("rpc.uint256param") do(i: UInt256): let r = i + 1.stUint(256) result = %r -s.on("rpc.returntypesimple") do(i: int) -> int: +s.rpc("rpc.returntypesimple") do(i: int) -> int: result = i -s.on("rpc.returntypecomplex") do(i: int) -> Test2: +s.rpc("rpc.returntypecomplex") do(i: int) -> Test2: result.x = [1, i, 3] result.y = "test" -s.on("rpc.testreturns") do() -> int: +s.rpc("rpc.testreturns") do() -> int: return 1234 -s.on("rpc.testreturnuint256") do() -> UInt256: +s.rpc("rpc.testreturnuint256") do() -> UInt256: let r: UInt256 = "0x1234567890abcdef".parse(UInt256, 16) return r diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index ecc0870..fb1691e 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -13,7 +13,7 @@ var srv = sharedRpcServer() srv.address = "localhost" srv.port = Port(8545) -srv.on("myProc") do(input: string, data: array[0..3, int]): +srv.rpc("myProc") do(input: string, data: array[0..3, int]): result = %("Hello " & input & " data: " & $data) asyncCheck srv.serve From abb88c1abc507fe48be38aa891010427c87aeb74 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 22 May 2018 20:40:50 +0100 Subject: [PATCH 085/116] Export ethtypes --- rpcclient.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rpcclient.nim b/rpcclient.nim index 4c36611..db6a851 100644 --- a/rpcclient.nim +++ b/rpcclient.nim @@ -1,3 +1,3 @@ -import eth-rpc / client / clientdispatch -export clientdispatch +import eth-rpc / client / clientdispatch, eth-rpc / ethtypes +export clientdispatch, ethtypes From 4a6db6fbe397ba7896399097b88312481201588d Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 22 May 2018 20:41:31 +0100 Subject: [PATCH 086/116] Moved jsonconverters for easier use by client --- eth-rpc/{server => }/jsonconverters.nim | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename eth-rpc/{server => }/jsonconverters.nim (100%) diff --git a/eth-rpc/server/jsonconverters.nim b/eth-rpc/jsonconverters.nim similarity index 100% rename from eth-rpc/server/jsonconverters.nim rename to eth-rpc/jsonconverters.nim From 5ea1451f23df56dc62349e74612cb1684e5a93cb Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 22 May 2018 20:42:14 +0100 Subject: [PATCH 087/116] Update .nimble so install directory matches --- eth_rpc.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth_rpc.nimble b/eth_rpc.nimble index de66dba..71a35dd 100644 --- a/eth_rpc.nimble +++ b/eth_rpc.nimble @@ -3,7 +3,7 @@ version = "0.0.1" author = "Status Research & Development GmbH" description = "Ethereum remote procedure calls" license = "Apache License 2.0" -srcDir = "src" +skipDirs = @["tests"] ### Dependencies requires "nim >= 0.17.3", From 1f27ea8645295064136ba4c398fdf91eb3726ff1 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 22 May 2018 20:43:08 +0100 Subject: [PATCH 088/116] export ethtypes in server --- rpcserver.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rpcserver.nim b/rpcserver.nim index 6764a93..83c15d2 100644 --- a/rpcserver.nim +++ b/rpcserver.nim @@ -2,5 +2,6 @@ import eth-rpc / server / servertypes, eth-rpc / server / rpcconsts, eth-rpc / server / serverdispatch, - eth-rpc / server / ethprocs -export servertypes, rpcconsts, serverdispatch, ethprocs + eth-rpc / server / ethprocs, + eth-rpc / ethtypes +export servertypes, rpcconsts, serverdispatch, ethprocs, ethtypes From 57259ca15683b3cdc2ce5b3989d926cc82ced5e4 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 22 May 2018 20:45:52 +0100 Subject: [PATCH 089/116] Updated sha3 sig & fix relative import issue (requires nimble install) --- eth-rpc/client/ethcallsigs.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eth-rpc/client/ethcallsigs.nim b/eth-rpc/client/ethcallsigs.nim index 75b6fbd..d10ef72 100644 --- a/eth-rpc/client/ethcallsigs.nim +++ b/eth-rpc/client/ethcallsigs.nim @@ -1,8 +1,10 @@ ## This module contains signatures for the Ethereum client RPCs. -import json, stint, ../ ethtypes +## The signatures are not imported directly, but read and processed with parseStmt, +## then a procedure body is generated to marshal native Nim parameters to json and visa versa. +import json, stint, eth-rpc / ethtypes proc web3_clientVersion(): string -proc web3_sha3(): string +proc web3_sha3(data: string): string proc net_version(): string proc net_peerCount(): int proc net_listening(): bool From 234e17801b81c0aa1d520e00b189389573ebb44e Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 22 May 2018 20:46:19 +0100 Subject: [PATCH 090/116] Refactored processing of params for client --- eth-rpc/client/jsonmarshal.nim | 125 +++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 eth-rpc/client/jsonmarshal.nim diff --git a/eth-rpc/client/jsonmarshal.nim b/eth-rpc/client/jsonmarshal.nim new file mode 100644 index 0000000..80f8314 --- /dev/null +++ b/eth-rpc/client/jsonmarshal.nim @@ -0,0 +1,125 @@ +import macros, json, ../ jsonconverters, stint + +template expect*(actual, expected: JsonNodeKind, argName: string) = + if actual != expected: raise newException(ValueError, "Parameter \"" & argName & "\" expected " & $expected & " but got " & $actual) + +proc fromJson(n: JsonNode, argName: string, result: var bool) = + n.kind.expect(JBool, argName) + result = n.getBool() + +proc fromJson(n: JsonNode, argName: string, result: var int) = + n.kind.expect(JInt, argName) + result = n.getInt() + +# TODO: Why does compiler complain that result cannot be assigned to when using result: var int|var int64 +# TODO: Compiler requires forward decl when processing out of module +proc fromJson(n: JsonNode, argName: string, result: var byte) +proc fromJson(n: JsonNode, argName: string, result: var float) +proc fromJson(n: JsonNode, argName: string, result: var string) +proc fromJson[T](n: JsonNode, argName: string, result: var seq[T]) +proc fromJson[N, T](n: JsonNode, argName: string, result: var array[N, T]) +proc fromJson(n: JsonNode, argName: string, result: var UInt256) +proc fromJson(n: JsonNode, argName: string, result: var int64) +proc fromJson(n: JsonNode, argName: string, result: var ref int64) +proc fromJson(n: JsonNode, argName: string, result: var ref int) +proc fromJson(n: JsonNode, argName: string, result: var ref UInt256) + +# TODO: Why can't this be forward declared? Complains of lack of definition +proc fromJson[T: enum](n: JsonNode, argName: string, result: var T) = + n.kind.expect(JInt, argName) + result = n.getInt().T + +# TODO: Why can't this be forward declared? Complains of lack of definition +proc fromJson[T: object](n: JsonNode, argName: string, result: var T) = + n.kind.expect(JObject, argName) + for k, v in fieldpairs(result): + fromJson(n[k], k, v) + +proc fromJson[T: ref object](n: JsonNode, argName: string, result: var T) = + n.kind.expect(JObject, argName) + result = new T + for k, v in fieldpairs(result[]): + fromJson(n[k], k, v) + +proc fromJson(n: JsonNode, argName: string, result: var int64) = + n.kind.expect(JInt, argName) + result = n.getInt() + +proc fromJson(n: JsonNode, argName: string, result: var ref int64) = + n.kind.expect(JInt, argName) + new result + result[] = n.getInt() + +proc fromJson(n: JsonNode, argName: string, result: var ref int) = + n.kind.expect(JInt, argName) + new result + result[] = n.getInt() + +proc fromJson(n: JsonNode, argName: string, result: var byte) = + n.kind.expect(JInt, argName) + let v = n.getInt() + if v > 255 or v < 0: raise newException(ValueError, "Parameter \"" & argName & "\" value out of range for byte: " & $v) + result = byte(v) + +proc fromJson(n: JsonNode, argName: string, result: var UInt256) = + # expects base 16 string, starting with "0x" + n.kind.expect(JString, argName) + let hexStr = n.getStr() + if hexStr.len > 64 + 2: # including "0x" + raise newException(ValueError, "Parameter \"" & argName & "\" value too long for UInt256: " & $hexStr.len) + result = hexStr.parse(StUint[256], 16) # TODO: Handle errors + +proc fromJson(n: JsonNode, argName: string, result: var ref UInt256) = + # expects base 16 string, starting with "0x" + n.kind.expect(JString, argName) + let hexStr = n.getStr() + if hexStr.len > 64 + 2: # including "0x" + raise newException(ValueError, "Parameter \"" & argName & "\" value too long for UInt256: " & $hexStr.len) + new result + result[] = hexStr.parse(StUint[256], 16) # TODO: Handle errors + +proc fromJson(n: JsonNode, argName: string, result: var float) = + n.kind.expect(JFloat, argName) + result = n.getFloat() + +proc fromJson(n: JsonNode, argName: string, result: var string) = + n.kind.expect(JString, argName) + result = n.getStr() + +proc fromJson[T](n: JsonNode, argName: string, result: var seq[T]) = + n.kind.expect(JArray, argName) + result = newSeq[T](n.len) + for i in 0 ..< n.len: + fromJson(n[i], argName, result[i]) + +proc fromJson[N, T](n: JsonNode, argName: string, result: var array[N, T]) = + n.kind.expect(JArray, argName) + if n.len > result.len: raise newException(ValueError, "Parameter \"" & argName & "\" item count is too big for array") + for i in 0 ..< n.len: + fromJson(n[i], argName, result[i]) + +import typetraits +proc unpackArg[T](args: JsonNode, argName: string, argtype: typedesc[T]): T = + fromJson(args, argName, result) + +proc expectArrayLen(node: NimNode, paramsIdent: untyped, length: int) = + let + identStr = paramsIdent.repr + expectedStr = "Expected " & $length & " Json parameter(s) but got " + node.add(quote do: + `paramsIdent`.kind.expect(JArray, `identStr`) + if `paramsIdent`.len != `length`: + raise newException(ValueError, `expectedStr` & $`paramsIdent`.len) + ) + +proc setupParamFromJson*(assignIdent, paramType, jsonIdent: NimNode): NimNode = + # Add code to verify input and load json parameters into provided Nim type + result = newStmtList() + # initial parameter array length check + # TODO: do this higher up + #result.expectArrayLen(jsonIdent, nimParameters.len - 1) + # unpack each parameter and provide assignments + let paramNameStr = $assignIdent + result.add(quote do: + `assignIdent` = `unpackArg`(`jsonIdent`, `paramNameStr`, type(`paramType`)) + ) \ No newline at end of file From ff93b182968c07ca65cc0259624999f8db12d370 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 22 May 2018 20:47:22 +0100 Subject: [PATCH 091/116] Updated path for jsonconverters --- eth-rpc/server/servertypes.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 25d33d9..ff042cb 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -1,4 +1,4 @@ -import asyncdispatch, asyncnet, json, tables, macros, strutils, jsonconverters, stint +import asyncdispatch, asyncnet, json, tables, macros, strutils, ../ jsonconverters, stint export asyncdispatch, asyncnet, json, jsonconverters type @@ -192,3 +192,5 @@ macro rpc*(server: var RpcServer, path: string, body: untyped): untyped = when defined(nimDumpRpcs): echo "\n", pathStr, ": ", result.repr + +# TODO: Check supplied rpc do signatures with ethcallsigs From 3878ee0ad0a710f47b1ca9d5fc84983d3ef233d1 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 22 May 2018 20:48:12 +0100 Subject: [PATCH 092/116] Implemented signature parsing to create client rpc procs based on params --- eth-rpc/client/clientdispatch.nim | 119 +++++++++++++++++++++++++++--- 1 file changed, 108 insertions(+), 11 deletions(-) diff --git a/eth-rpc/client/clientdispatch.nim b/eth-rpc/client/clientdispatch.nim index 36dacfa..59888ef 100644 --- a/eth-rpc/client/clientdispatch.nim +++ b/eth-rpc/client/clientdispatch.nim @@ -1,4 +1,5 @@ import asyncnet, asyncdispatch, tables, json, oids, ethcalls, macros +import ../ ethtypes, stint, ../ jsonconverters type RpcClient* = ref object @@ -88,17 +89,113 @@ proc connect*(self: RpcClient, address: string, port: Port) {.async.} = self.port = port asyncCheck processData(self) -macro generateCalls: untyped = - ## Generate templates for client calls so that: - ## client.call("web3_clientVersion", params) - ## can be written as: - ## client.web3_clientVersion(params) +import jsonmarshal + +proc createRpcFromSig*(rpcDecl: NimNode): NimNode = + let procNameState = rpcDecl[0] + var + parameters = rpcDecl.findChild(it.kind == nnkFormalParams).copy + path: NimNode + pathStr: string + + # get proc signature's name. This becomes the path we send to the server + if procNameState.kind == nnkPostFix: + path = rpcDecl[0][1] + else: + path = rpcDecl[0] + pathStr = $path + + if parameters.isNil or parameters.kind == nnkEmpty: + parameters = newNimNode(nnkFormalParams) + + # if no return parameters specified (parameters[0]) + if parameters.len == 0: parameters.add(newEmptyNode()) + # insert rpc client as first parameter + let + clientParam = + nnkIdentDefs.newTree( + ident"client", + ident"RpcClient", + newEmptyNode() + ) + parameters.insert(1, clientParam) + + # For each input parameter we need to + # take the Nim type and translate to json with `%`. + # For return types, we need to take the json and + # convert it to the Nim type. + var + callBody = newStmtList() + returnType: NimNode + let jsonParamIdent = genSym(nskVar, "jsonParam") + callBody.add(quote do: + var `jsonParamIdent` = newJArray() + ) + if parameters.len > 2: + # skip return type and the inserted rpc client parameter + # add the rest to json node via `%` + for i in 2 ..< parameters.len: + let curParam = parameters[i][0] + if curParam.kind != nnkEmpty: + callBody.add(quote do: + `jsonParamIdent`.add(%`curParam`) + ) + if parameters[0].kind != nnkEmpty: + returnType = parameters[0] + else: + returnType = ident"JsonNode" + + # convert return type to Future + parameters[0] = nnkBracketExpr.newTree(ident"Future", returnType) + + # client call to server using json params + var updatedParams = newSeq[NimNode]() + # convert parameter tree to seq + for p in parameters: updatedParams.add(p) + # create new proc + result = newProc(path, updatedParams, callBody) + # convert this proc to async + result.addPragma ident"async" + # export this proc + result[0] = nnkPostFix.newTree(ident"*", ident(pathStr)) + # add rpc call to proc body + var callResult = genSym(nskVar, "res") + + callBody.add(quote do: + let res = await client.call(`pathStr`, `jsonParamIdent`) + if res.error: raise newException(ValueError, $res.result) + var `callResult` = res.result + ) + let + procRes = ident"result" + # now we need to extract the response and build it into the expected type + if returnType != ident"JsonNode": + let setup = setupParamFromJson(procRes, returnType, callResult) + callBody.add(quote do: `setup`) + else: + callBody.add(quote do: + `procRes` = `callResult` + ) + when defined(nimDumpRpcs): + echo pathStr, ":\n", result.repr + +from os import getCurrentDir, DirSep +from strutils import rsplit + +macro processRpcSigs(): untyped = result = newStmtList() - for callName in ETHEREUM_RPC_CALLS: - let nameLit = ident(callName) - result.add(quote do: - template `nameLit`*(client: RpcClient, params: JsonNode): Future[Response] = client.call(`callName`, params) # TODO: Back to template - ) + const + codePath = currentSourcePath.rsplit(DirSep, 1)[0] & DirSep & "ethcallsigs.nim" + code = staticRead(codePath) + + let parsedCode = parseStmt(code) + var line = 0 + while line < parsedCode.len: + if parsedCode[line].kind == nnkProcDef: break + line += 1 + for curLine in line ..< parsedCode.len: + var procDef = createRpcFromSig(parsedCode[curLine]) + result.add(procDef) # generate all client ethereum rpc calls -generateCalls() +processRpcSigs() From 592acb20c227aa46fb990198d959128811a5cd7b Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 22 May 2018 20:49:23 +0100 Subject: [PATCH 093/116] Temporarily hide variant object part in lue of improving marshal macro --- eth-rpc/ethtypes.nim | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/eth-rpc/ethtypes.nim b/eth-rpc/ethtypes.nim index 5fa6099..beabdce 100644 --- a/eth-rpc/ethtypes.nim +++ b/eth-rpc/ethtypes.nim @@ -70,9 +70,10 @@ type contractAddress*: array[20, byte] # the contract address created, if the transaction was a contract creation, otherwise null. logs*: seq[string] # TODO: See Wiki for details. list of log objects, which this transaction generated. logsBloom*: array[256, byte] # bloom filter for light clients to quickly retrieve related logs. - case kind*: ReceiptKind - of rkRoot: root*: UInt256 # post-transaction stateroot (pre Byzantium). - of rkStatus: status*: int # 1 = success, 0 = failure. + # TODO: + #case kind*: ReceiptKind + #of rkRoot: root*: UInt256 # post-transaction stateroot (pre Byzantium). + #of rkStatus: status*: int # 1 = success, 0 = failure. FilterDataKind* = enum fkItem, fkList FilterData* = object From dd5db585717cbee0cac257e03cd98b00ea5dc570 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 22 May 2018 20:49:54 +0100 Subject: [PATCH 094/116] Updated test to use new generated client procs --- tests/testserverclient.nim | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index fb1691e..45bd8a7 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -22,16 +22,16 @@ suite "RPC": proc main {.async.} = var client = newRpcClient() await client.connect("localhost", Port(8545)) - var response: Response + test "Version": - response = waitFor client.web3_clientVersion(%[]) - check response.result == %"Nimbus-RPC-Test" + var response = waitFor client.web3_clientVersion() + check response == "Nimbus-RPC-Test" test "SHA3": - response = waitFor client.web3_sha3(%["abc"]) - check response.result.getStr == "3A985DA74FE225B2045C172D6BD390BD855F086E3E9D525B46BFE24511431532" + var response = waitFor client.web3_sha3("abc") + check response == "3A985DA74FE225B2045C172D6BD390BD855F086E3E9D525B46BFE24511431532" test "Custom RPC": # Custom async RPC call - response = waitFor client.call("myProc", %[%"abc", %[1, 2, 3, 4]]) + var response = waitFor client.call("myProc", %[%"abc", %[1, 2, 3, 4]]) check response.result.getStr == "Hello abc data: [1, 2, 3, 4]" waitFor main() # TODO: When an error occurs during a test, stop the server From 198fb76f4295d0d046417e88d67bbea4b4f34cd5 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 23 May 2018 14:00:37 +0100 Subject: [PATCH 095/116] Refactor marshalling and proc creation, better rpc sig parsing --- eth-rpc/client/clientdispatch.nim | 99 ++++++++++++++----------------- 1 file changed, 45 insertions(+), 54 deletions(-) diff --git a/eth-rpc/client/clientdispatch.nim b/eth-rpc/client/clientdispatch.nim index 59888ef..2fcc10e 100644 --- a/eth-rpc/client/clientdispatch.nim +++ b/eth-rpc/client/clientdispatch.nim @@ -91,55 +91,58 @@ proc connect*(self: RpcClient, address: string, port: Port) {.async.} = import jsonmarshal +proc createRpcProc(procName, parameters, callBody: NimNode): NimNode = + # parameters come as a tree + var paramList = newSeq[NimNode]() + for p in parameters: paramList.add(p) + + result = newProc(procName, paramList, callBody) # build proc + result.addPragma ident"async" # make proc async + result[0] = nnkPostFix.newTree(ident"*", newIdentNode($procName)) # export this proc + +proc toJsonNode(parameters: NimNode): NimNode = + # outputs an array of jsonified parameters + # ie; %[%a, %b, %c] + parameters.expectKind nnkFormalParams + var items = newNimNode(nnkBracket) + for i in 2 ..< parameters.len: + let curParam = parameters[i][0] + if curParam.kind != nnkEmpty: + items.add(nnkPrefix.newTree(ident"%", curParam)) + result = nnkPrefix.newTree(newIdentNode("%"), items) + proc createRpcFromSig*(rpcDecl: NimNode): NimNode = - let procNameState = rpcDecl[0] var parameters = rpcDecl.findChild(it.kind == nnkFormalParams).copy - path: NimNode - pathStr: string + procName = rpcDecl.name + pathStr = $procName - # get proc signature's name. This becomes the path we send to the server - if procNameState.kind == nnkPostFix: - path = rpcDecl[0][1] - else: - path = rpcDecl[0] - pathStr = $path + # ensure we have at least space for a return parameter + if parameters.isNil or parameters.kind == nnkEmpty or parameters.len == 0: + parameters = nnkFormalParams.newTree(newEmptyNode()) - if parameters.isNil or parameters.kind == nnkEmpty: - parameters = newNimNode(nnkFormalParams) - - # if no return parameters specified (parameters[0]) - if parameters.len == 0: parameters.add(newEmptyNode()) # insert rpc client as first parameter - let - clientParam = - nnkIdentDefs.newTree( - ident"client", - ident"RpcClient", - newEmptyNode() - ) - parameters.insert(1, clientParam) + parameters.insert(1, + nnkIdentDefs.newTree( + ident"client", + ident"RpcClient", + newEmptyNode() + ) + ) # For each input parameter we need to # take the Nim type and translate to json with `%`. # For return types, we need to take the json and # convert it to the Nim type. + let + jsonParamIdent = genSym(nskVar, "jsonParam") + jsonArrayInit = parameters.toJsonNode() var - callBody = newStmtList() returnType: NimNode - let jsonParamIdent = genSym(nskVar, "jsonParam") - callBody.add(quote do: - var `jsonParamIdent` = newJArray() - ) - if parameters.len > 2: - # skip return type and the inserted rpc client parameter - # add the rest to json node via `%` - for i in 2 ..< parameters.len: - let curParam = parameters[i][0] - if curParam.kind != nnkEmpty: - callBody.add(quote do: - `jsonParamIdent`.add(%`curParam`) - ) + callBody = newStmtList().add(quote do: + var `jsonParamIdent` = `jsonArrayInit` + ) + if parameters[0].kind != nnkEmpty: returnType = parameters[0] else: @@ -148,19 +151,10 @@ proc createRpcFromSig*(rpcDecl: NimNode): NimNode = # convert return type to Future parameters[0] = nnkBracketExpr.newTree(ident"Future", returnType) - # client call to server using json params - var updatedParams = newSeq[NimNode]() - # convert parameter tree to seq - for p in parameters: updatedParams.add(p) - # create new proc - result = newProc(path, updatedParams, callBody) - # convert this proc to async - result.addPragma ident"async" - # export this proc - result[0] = nnkPostFix.newTree(ident"*", ident(pathStr)) - # add rpc call to proc body + result = createRpcProc(procName, parameters, callBody) var callResult = genSym(nskVar, "res") + # create client call to server using json params callBody.add(quote do: let res = await client.call(`pathStr`, `jsonParamIdent`) if res.error: raise newException(ValueError, $res.result) @@ -189,13 +183,10 @@ macro processRpcSigs(): untyped = code = staticRead(codePath) let parsedCode = parseStmt(code) - var line = 0 - while line < parsedCode.len: - if parsedCode[line].kind == nnkProcDef: break - line += 1 - for curLine in line ..< parsedCode.len: - var procDef = createRpcFromSig(parsedCode[curLine]) - result.add(procDef) + for line in parsedCode: + if line.kind == nnkProcDef: + var procDef = createRpcFromSig(line) + result.add(procDef) # generate all client ethereum rpc calls processRpcSigs() From 28aa77582b593eb2a314bee13c6d88279c3837bb Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 23 May 2018 16:12:00 +0100 Subject: [PATCH 096/116] General refactoring and tidying up --- eth-rpc/client/clientdispatch.nim | 79 +++++++++++++++---------------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/eth-rpc/client/clientdispatch.nim b/eth-rpc/client/clientdispatch.nim index 2fcc10e..d880011 100644 --- a/eth-rpc/client/clientdispatch.nim +++ b/eth-rpc/client/clientdispatch.nim @@ -100,7 +100,7 @@ proc createRpcProc(procName, parameters, callBody: NimNode): NimNode = result.addPragma ident"async" # make proc async result[0] = nnkPostFix.newTree(ident"*", newIdentNode($procName)) # export this proc -proc toJsonNode(parameters: NimNode): NimNode = +proc toJsonArray(parameters: NimNode): NimNode = # outputs an array of jsonified parameters # ie; %[%a, %b, %c] parameters.expectKind nnkFormalParams @@ -112,63 +112,62 @@ proc toJsonNode(parameters: NimNode): NimNode = result = nnkPrefix.newTree(newIdentNode("%"), items) proc createRpcFromSig*(rpcDecl: NimNode): NimNode = - var - parameters = rpcDecl.findChild(it.kind == nnkFormalParams).copy - procName = rpcDecl.name - pathStr = $procName + # Each input parameter in the rpc signature is converted + # to json with `%`. + # Return types are then converted back to native Nim types. + let iJsonNode = newIdentNode("JsonNode") + var parameters = rpcDecl.findChild(it.kind == nnkFormalParams).copy # ensure we have at least space for a return parameter if parameters.isNil or parameters.kind == nnkEmpty or parameters.len == 0: - parameters = nnkFormalParams.newTree(newEmptyNode()) + parameters = nnkFormalParams.newTree(iJsonNode) + + let + procName = rpcDecl.name + pathStr = $procName + returnType = + # if no return type specified, defaults to JsonNode + if parameters[0].kind == nnkEmpty: iJsonNode + else: parameters[0] + customReturnType = returnType != iJsonNode # insert rpc client as first parameter - parameters.insert(1, - nnkIdentDefs.newTree( - ident"client", - ident"RpcClient", - newEmptyNode() - ) - ) + parameters.insert(1, nnkIdentDefs.newTree(ident"client", ident"RpcClient", newEmptyNode())) - # For each input parameter we need to - # take the Nim type and translate to json with `%`. - # For return types, we need to take the json and - # convert it to the Nim type. let - jsonParamIdent = genSym(nskVar, "jsonParam") - jsonArrayInit = parameters.toJsonNode() + jsonParamIdent = genSym(nskVar, "jsonParam") # variable used to send json to the server + jsonParamArray = parameters.toJsonArray() # json array of marshalled parameters var - returnType: NimNode + # populate json params - even rpcs with no parameters have an empty json array node sent callBody = newStmtList().add(quote do: - var `jsonParamIdent` = `jsonArrayInit` + var `jsonParamIdent` = `jsonParamArray` ) - if parameters[0].kind != nnkEmpty: - returnType = parameters[0] - else: - returnType = ident"JsonNode" - # convert return type to Future parameters[0] = nnkBracketExpr.newTree(ident"Future", returnType) - + # create rpc proc result = createRpcProc(procName, parameters, callBody) - var callResult = genSym(nskVar, "res") - # create client call to server using json params - callBody.add(quote do: - let res = await client.call(`pathStr`, `jsonParamIdent`) - if res.error: raise newException(ValueError, $res.result) - var `callResult` = res.result - ) let - procRes = ident"result" - # now we need to extract the response and build it into the expected type - if returnType != ident"JsonNode": - let setup = setupParamFromJson(procRes, returnType, callResult) - callBody.add(quote do: `setup`) + rpcResult = genSym(nskLet, "res") # temporary variable to hold `Response` from rpc call + procRes = ident"result" # proc return variable + jsonRpcResult = # actual return value, `rpcResult`.result + nnkDotExpr.newTree(rpcResult, newIdentNode("result")) + + # perform rpc call + callBody.add(quote do: + let `rpcResult` = await client.call(`pathStr`, `jsonParamIdent`) # `rpcResult` is of type `Response` + if `rpcResult`.error: raise newException(ValueError, $`rpcResult`.result) # TODO: is raise suitable here? + ) + + if customReturnType: + # marshal json to native Nim type + let setup = setupParamFromJson(procRes, returnType, jsonRpcResult) + callBody.add(setup) else: + # native json expected so no work callBody.add(quote do: - `procRes` = `callResult` + `procRes` = `rpcResult`.result ) when defined(nimDumpRpcs): echo pathStr, ":\n", result.repr From 43ccd28d4f727a6fbb460cb760d2d99721c5277f Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 23 May 2018 18:00:30 +0100 Subject: [PATCH 097/116] Unify jsonmarshal to support both client and server --- eth-rpc/client/clientdispatch.nim | 5 +- eth-rpc/{client => }/jsonmarshal.nim | 67 +++++++++++------ eth-rpc/server/servertypes.nim | 105 +-------------------------- 3 files changed, 48 insertions(+), 129 deletions(-) rename eth-rpc/{client => }/jsonmarshal.nim (74%) diff --git a/eth-rpc/client/clientdispatch.nim b/eth-rpc/client/clientdispatch.nim index d880011..3b6a9b5 100644 --- a/eth-rpc/client/clientdispatch.nim +++ b/eth-rpc/client/clientdispatch.nim @@ -89,7 +89,7 @@ proc connect*(self: RpcClient, address: string, port: Port) {.async.} = self.port = port asyncCheck processData(self) -import jsonmarshal +import ../ jsonmarshal proc createRpcProc(procName, parameters, callBody: NimNode): NimNode = # parameters come as a tree @@ -162,8 +162,7 @@ proc createRpcFromSig*(rpcDecl: NimNode): NimNode = if customReturnType: # marshal json to native Nim type - let setup = setupParamFromJson(procRes, returnType, jsonRpcResult) - callBody.add(setup) + callBody.add(jsonToNim(procRes, returnType, jsonRpcResult, "result")) else: # native json expected so no work callBody.add(quote do: diff --git a/eth-rpc/client/jsonmarshal.nim b/eth-rpc/jsonmarshal.nim similarity index 74% rename from eth-rpc/client/jsonmarshal.nim rename to eth-rpc/jsonmarshal.nim index 80f8314..bbea015 100644 --- a/eth-rpc/client/jsonmarshal.nim +++ b/eth-rpc/jsonmarshal.nim @@ -1,18 +1,11 @@ -import macros, json, ../ jsonconverters, stint +import macros, json, jsonconverters, stint template expect*(actual, expected: JsonNodeKind, argName: string) = if actual != expected: raise newException(ValueError, "Parameter \"" & argName & "\" expected " & $expected & " but got " & $actual) -proc fromJson(n: JsonNode, argName: string, result: var bool) = - n.kind.expect(JBool, argName) - result = n.getBool() - -proc fromJson(n: JsonNode, argName: string, result: var int) = - n.kind.expect(JInt, argName) - result = n.getInt() - -# TODO: Why does compiler complain that result cannot be assigned to when using result: var int|var int64 -# TODO: Compiler requires forward decl when processing out of module +# Compiler requires forward decl when processing out of module +proc fromJson(n: JsonNode, argName: string, result: var bool) +proc fromJson(n: JsonNode, argName: string, result: var int) proc fromJson(n: JsonNode, argName: string, result: var byte) proc fromJson(n: JsonNode, argName: string, result: var float) proc fromJson(n: JsonNode, argName: string, result: var string) @@ -24,6 +17,14 @@ proc fromJson(n: JsonNode, argName: string, result: var ref int64) proc fromJson(n: JsonNode, argName: string, result: var ref int) proc fromJson(n: JsonNode, argName: string, result: var ref UInt256) +proc fromJson(n: JsonNode, argName: string, result: var bool) = + n.kind.expect(JBool, argName) + result = n.getBool() + +proc fromJson(n: JsonNode, argName: string, result: var int) = + n.kind.expect(JInt, argName) + result = n.getInt() + # TODO: Why can't this be forward declared? Complains of lack of definition proc fromJson[T: enum](n: JsonNode, argName: string, result: var T) = n.kind.expect(JInt, argName) @@ -98,28 +99,46 @@ proc fromJson[N, T](n: JsonNode, argName: string, result: var array[N, T]) = for i in 0 ..< n.len: fromJson(n[i], argName, result[i]) -import typetraits proc unpackArg[T](args: JsonNode, argName: string, argtype: typedesc[T]): T = fromJson(args, argName, result) -proc expectArrayLen(node: NimNode, paramsIdent: untyped, length: int) = +proc expectArrayLen(node: NimNode, jsonIdent: untyped, length: int) = let - identStr = paramsIdent.repr + identStr = jsonIdent.repr expectedStr = "Expected " & $length & " Json parameter(s) but got " node.add(quote do: - `paramsIdent`.kind.expect(JArray, `identStr`) - if `paramsIdent`.len != `length`: - raise newException(ValueError, `expectedStr` & $`paramsIdent`.len) + `jsonIdent`.kind.expect(JArray, `identStr`) + if `jsonIdent`.len != `length`: + raise newException(ValueError, `expectedStr` & $`jsonIdent`.len) ) -proc setupParamFromJson*(assignIdent, paramType, jsonIdent: NimNode): NimNode = - # Add code to verify input and load json parameters into provided Nim type +proc jsonToNim*(assignIdent, paramType, jsonIdent: NimNode, paramNameStr: string): NimNode = + # verify input and load a Nim type from json data + # note: does not create `assignIdent` so can be used for `result` variables result = newStmtList() - # initial parameter array length check - # TODO: do this higher up - #result.expectArrayLen(jsonIdent, nimParameters.len - 1) # unpack each parameter and provide assignments - let paramNameStr = $assignIdent result.add(quote do: `assignIdent` = `unpackArg`(`jsonIdent`, `paramNameStr`, type(`paramType`)) - ) \ No newline at end of file + ) + +proc jsonToNim*(parameters, jsonIdent: NimNode): NimNode = + # Add code to verify input and load parameters into Nim types + result = newStmtList() + if not parameters.isNil: + # initial parameter array length check + result.expectArrayLen(jsonIdent, parameters.len - 1) + # unpack each parameter and provide assignments + for i in 1 ..< parameters.len: + let + pos = i - 1 + paramIdent = parameters[i][0] + paramName = $paramIdent + paramType = parameters[i][1] + jsonElement = quote do: + `jsonIdent`.elems[`pos`] + # declare variable before assignment + result.add(quote do: + var `paramIdent`: `paramType` + ) + # unpack Nim type and assign from json + result.add jsonToNim(paramIdent, paramType, jsonElement, paramName) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index ff042cb..866fd0d 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -1,5 +1,5 @@ -import asyncdispatch, asyncnet, json, tables, macros, strutils, ../ jsonconverters, stint -export asyncdispatch, asyncnet, json, jsonconverters +import asyncdispatch, asyncnet, json, tables, macros, strutils, ../ jsonconverters, ../ jsonmarshal, stint +export asyncdispatch, asyncnet, json, jsonconverters, expect type RpcProc* = proc (params: JsonNode): Future[JsonNode] @@ -34,105 +34,6 @@ proc sharedRpcServer*(): RpcServer = result = sharedServer proc `$`*(port: Port): string = $int(port) - -template expect*(actual, expected: JsonNodeKind, argName: string) = - if actual != expected: raise newException(ValueError, "Parameter \"" & argName & "\" expected " & $expected & " but got " & $actual) - -proc fromJson(n: JsonNode, argName: string, result: var bool) = - n.kind.expect(JBool, argName) - result = n.getBool() - -proc fromJson(n: JsonNode, argName: string, result: var int) = - n.kind.expect(JInt, argName) - result = n.getInt() - -# TODO: Why does compiler complain that result cannot be assigned to when using result: var int|var int64 -# TODO: Compiler requires forward decl when processing out of module -proc fromJson(n: JsonNode, argName: string, result: var byte) -proc fromJson(n: JsonNode, argName: string, result: var float) -proc fromJson(n: JsonNode, argName: string, result: var string) -proc fromJson[T](n: JsonNode, argName: string, result: var seq[T]) -proc fromJson[N, T](n: JsonNode, argName: string, result: var array[N, T]) -proc fromJson(n: JsonNode, argName: string, result: var UInt256) - -# TODO: Why can't this be forward declared? Complains of lack of definition -proc fromJson[T: enum](n: JsonNode, argName: string, result: var T) = - n.kind.expect(JInt, argName) - result = n.getInt().T - -# TODO: Why can't this be forward declared? Complains of lack of definition -proc fromJson[T: object](n: JsonNode, argName: string, result: var T) = - n.kind.expect(JObject, argName) - for k, v in fieldpairs(result): - fromJson(n[k], k, v) - -proc fromJson(n: JsonNode, argName: string, result: var int64) = - n.kind.expect(JInt, argName) - result = n.getInt() - -proc fromJson(n: JsonNode, argName: string, result: var byte) = - n.kind.expect(JInt, argName) - let v = n.getInt() - if v > 255 or v < 0: raise newException(ValueError, "Parameter \"" & argName & "\" value out of range for byte: " & $v) - result = byte(v) - -proc fromJson(n: JsonNode, argName: string, result: var UInt256) = - # expects base 16 string, starting with "0x" - n.kind.expect(JString, argName) - let hexStr = n.getStr() - if hexStr.len > 64 + 2: # including "0x" - raise newException(ValueError, "Parameter \"" & argName & "\" value too long for UInt256: " & $hexStr.len) - result = hexStr.parse(StUint[256], 16) # TODO: Handle errors - -proc fromJson(n: JsonNode, argName: string, result: var float) = - n.kind.expect(JFloat, argName) - result = n.getFloat() - -proc fromJson(n: JsonNode, argName: string, result: var string) = - n.kind.expect(JString, argName) - result = n.getStr() - -proc fromJson[T](n: JsonNode, argName: string, result: var seq[T]) = - n.kind.expect(JArray, argName) - result = newSeq[T](n.len) - for i in 0 ..< n.len: - fromJson(n[i], argName, result[i]) - -proc fromJson[N, T](n: JsonNode, argName: string, result: var array[N, T]) = - n.kind.expect(JArray, argName) - if n.len > result.len: raise newException(ValueError, "Parameter \"" & argName & "\" item count is too big for array") - for i in 0 ..< n.len: - fromJson(n[i], argName, result[i]) - -proc unpackArg[T](args: JsonNode, argIdx: int, argName: string, argtype: typedesc[T]): T = - fromJson(args[argIdx], argName, result) - -proc expectArrayLen(node: NimNode, paramsIdent: untyped, length: int) = - let - identStr = paramsIdent.repr - expectedStr = "Expected " & $length & " Json parameter(s) but got " - node.add(quote do: - `paramsIdent`.kind.expect(JArray, `identStr`) - if `paramsIdent`.len != `length`: - raise newException(ValueError, `expectedStr` & $`paramsIdent`.len) - ) - -proc setupParams(parameters, paramsIdent: NimNode): NimNode = - # Add code to verify input and load parameters into Nim types - result = newStmtList() - if not parameters.isNil: - # initial parameter array length check - result.expectArrayLen(paramsIdent, parameters.len - 1) - # unpack each parameter and provide assignments - for i in 1 ..< parameters.len: - let - pos = i - 1 - paramName = parameters[i][0] - paramNameStr = $paramName - paramType = parameters[i][1] - result.add(quote do: - var `paramName` = `unpackArg`(`paramsIdent`, `pos`, `paramNameStr`, type(`paramType`)) - ) proc makeProcName(s: string): string = # only alphanumeric @@ -155,7 +56,7 @@ macro rpc*(server: var RpcServer, path: string, body: untyped): untyped = doMain = newIdentNode(procNameStr & "DoMain") # when parameters: proc that contains our rpc body res = newIdentNode("result") # async result var - setup = setupParams(parameters, paramsIdent) + setup = jsonToNim(parameters, paramsIdent) procBody = if body.kind == nnkStmtList: body else: body.body if parameters.hasReturnType: From 201f31554f11c15e03c1b225a1bedc72c6f4c4ae Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 23 May 2018 18:04:16 +0100 Subject: [PATCH 098/116] Update comments with current issue link --- eth-rpc/jsonmarshal.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eth-rpc/jsonmarshal.nim b/eth-rpc/jsonmarshal.nim index bbea015..f219aed 100644 --- a/eth-rpc/jsonmarshal.nim +++ b/eth-rpc/jsonmarshal.nim @@ -25,12 +25,12 @@ proc fromJson(n: JsonNode, argName: string, result: var int) = n.kind.expect(JInt, argName) result = n.getInt() -# TODO: Why can't this be forward declared? Complains of lack of definition +# This can't be forward declared: https://github.com/nim-lang/Nim/issues/7868 proc fromJson[T: enum](n: JsonNode, argName: string, result: var T) = n.kind.expect(JInt, argName) result = n.getInt().T -# TODO: Why can't this be forward declared? Complains of lack of definition +# This can't be forward declared: https://github.com/nim-lang/Nim/issues/7868 proc fromJson[T: object](n: JsonNode, argName: string, result: var T) = n.kind.expect(JObject, argName) for k, v in fieldpairs(result): From b8bbc25e6add2dbdddacf85d97a42281fef451f7 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 23 May 2018 18:05:38 +0100 Subject: [PATCH 099/116] Changed proc order pending forward decl issue --- eth-rpc/jsonmarshal.nim | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/eth-rpc/jsonmarshal.nim b/eth-rpc/jsonmarshal.nim index f219aed..8464379 100644 --- a/eth-rpc/jsonmarshal.nim +++ b/eth-rpc/jsonmarshal.nim @@ -17,14 +17,6 @@ proc fromJson(n: JsonNode, argName: string, result: var ref int64) proc fromJson(n: JsonNode, argName: string, result: var ref int) proc fromJson(n: JsonNode, argName: string, result: var ref UInt256) -proc fromJson(n: JsonNode, argName: string, result: var bool) = - n.kind.expect(JBool, argName) - result = n.getBool() - -proc fromJson(n: JsonNode, argName: string, result: var int) = - n.kind.expect(JInt, argName) - result = n.getInt() - # This can't be forward declared: https://github.com/nim-lang/Nim/issues/7868 proc fromJson[T: enum](n: JsonNode, argName: string, result: var T) = n.kind.expect(JInt, argName) @@ -36,6 +28,14 @@ proc fromJson[T: object](n: JsonNode, argName: string, result: var T) = for k, v in fieldpairs(result): fromJson(n[k], k, v) +proc fromJson(n: JsonNode, argName: string, result: var bool) = + n.kind.expect(JBool, argName) + result = n.getBool() + +proc fromJson(n: JsonNode, argName: string, result: var int) = + n.kind.expect(JInt, argName) + result = n.getInt() + proc fromJson[T: ref object](n: JsonNode, argName: string, result: var T) = n.kind.expect(JObject, argName) result = new T From 17b8db7b3c48e488baeb8aeac6aecfe94960b150 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 23 May 2018 18:06:32 +0100 Subject: [PATCH 100/116] Missing comma on comment --- eth-rpc/jsonmarshal.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth-rpc/jsonmarshal.nim b/eth-rpc/jsonmarshal.nim index 8464379..5edd5e6 100644 --- a/eth-rpc/jsonmarshal.nim +++ b/eth-rpc/jsonmarshal.nim @@ -114,7 +114,7 @@ proc expectArrayLen(node: NimNode, jsonIdent: untyped, length: int) = proc jsonToNim*(assignIdent, paramType, jsonIdent: NimNode, paramNameStr: string): NimNode = # verify input and load a Nim type from json data - # note: does not create `assignIdent` so can be used for `result` variables + # note: does not create `assignIdent`, so can be used for `result` variables result = newStmtList() # unpack each parameter and provide assignments result.add(quote do: From e4105770aaf7ab96df5235ff282daacf8a87fe13 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 23 May 2018 18:09:18 +0100 Subject: [PATCH 101/116] Remove ethcalls as ethcallsigs supplants it --- eth-rpc/client/clientdispatch.nim | 2 +- eth-rpc/client/ethcalls.nim | 62 ------------------------------- 2 files changed, 1 insertion(+), 63 deletions(-) delete mode 100644 eth-rpc/client/ethcalls.nim diff --git a/eth-rpc/client/clientdispatch.nim b/eth-rpc/client/clientdispatch.nim index 3b6a9b5..6f97e4b 100644 --- a/eth-rpc/client/clientdispatch.nim +++ b/eth-rpc/client/clientdispatch.nim @@ -1,4 +1,4 @@ -import asyncnet, asyncdispatch, tables, json, oids, ethcalls, macros +import asyncnet, asyncdispatch, tables, json, oids, macros import ../ ethtypes, stint, ../ jsonconverters type diff --git a/eth-rpc/client/ethcalls.nim b/eth-rpc/client/ethcalls.nim deleted file mode 100644 index 3d6657a..0000000 --- a/eth-rpc/client/ethcalls.nim +++ /dev/null @@ -1,62 +0,0 @@ -const - ETHEREUM_RPC_CALLS* = [ - "web3_clientVersion", - "web3_sha3", - "net_version", - "net_peerCount", - "net_listening", - "eth_protocolVersion", - "eth_syncing", - "eth_coinbase", - "eth_mining", - "eth_hashrate", - "eth_gasPrice", - "eth_accounts", - "eth_blockNumber", - "eth_getBalance", - "eth_getStorageAt", - "eth_getTransactionCount", - "eth_getBlockTransactionCountByHash", - "eth_getBlockTransactionCountByNumber", - "eth_getUncleCountByBlockHash", - "eth_getUncleCountByBlockNumber", - "eth_getCode", - "eth_sign", - "eth_sendTransaction", - "eth_sendRawTransaction", - "eth_call", - "eth_estimateGas", - "eth_getBlockByHash", - "eth_getBlockByNumber", - "eth_getTransactionByHash", - "eth_getTransactionByBlockHashAndIndex", - "eth_getTransactionByBlockNumberAndIndex", - "eth_getTransactionReceipt", - "eth_getUncleByBlockHashAndIndex", - "eth_getUncleByBlockNumberAndIndex", - "eth_getCompilers", - "eth_compileLLL", - "eth_compileSolidity", - "eth_compileSerpent", - "eth_newFilter", - "eth_newBlockFilter", - "eth_newPendingTransactionFilter", - "eth_uninstallFilter", - "eth_getFilterChanges", - "eth_getFilterLogs", - "eth_getLogs", - "eth_getWork", - "eth_submitWork", - "eth_submitHashrate", - "shh_post", - "shh_version", - "shh_newIdentity", - "shh_hasIdentity", - "shh_newGroup", - "shh_addToGroup", - "shh_newFilter", - "shh_uninstallFilter", - "shh_getFilterChanges", - "shh_getMessages" - ] - From ef53b52c9e41ed089af64eb045e5383b4ddbe1e5 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 23 May 2018 18:37:26 +0100 Subject: [PATCH 102/116] cryptoutils is no longer needed --- eth-rpc/server/cryptoutils.nim | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 eth-rpc/server/cryptoutils.nim diff --git a/eth-rpc/server/cryptoutils.nim b/eth-rpc/server/cryptoutils.nim deleted file mode 100644 index 3d3315c..0000000 --- a/eth-rpc/server/cryptoutils.nim +++ /dev/null @@ -1,9 +0,0 @@ -import nimcrypto - -proc k256*(data: string): string = - # do not convert, assume string is data - # REVIEW: Nimcrypto has a one-liner for the code here: sha3_256.digest(data) - var k = sha3_256() - k.init - k.update(cast[ptr uint8](data[0].unsafeaddr), data.len.uint) - result = $finish(k) From 3ec16e743a18750a3700b2aeb0475707dfc32ca6 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Fri, 25 May 2018 16:36:18 +0100 Subject: [PATCH 103/116] Update web3_sha3 to use nimcrypto --- eth-rpc/server/ethprocs.nim | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/eth-rpc/server/ethprocs.nim b/eth-rpc/server/ethprocs.nim index c06ea07..9c2129e 100644 --- a/eth-rpc/server/ethprocs.nim +++ b/eth-rpc/server/ethprocs.nim @@ -1,4 +1,4 @@ -import servertypes, cryptoutils, json, stint, ../ethtypes.nim +import servertypes, nimcrypto, json, stint, ../ethtypes.nim, strutils #[ For details on available RPC calls, see: https://github.com/ethereum/wiki/wiki/JSON-RPC @@ -12,14 +12,18 @@ import servertypes, cryptoutils, json, stint, ../ethtypes.nim Changes might be required for parameter types. For example: * String might be more appropriate than seq[byte], for example for addresses, although would need length constraints. - * It might be worth replacing array[X, byte] with the equivilent stInt/stUInt. * Int return values might actually be more hex string than int. - * UInt256/Int256 - * Objects such as BlockObject and TransactionObject might be better as the existing Nimbus objects + * array[32, byte] could be UInt256 or Int256, but default to UInt256. + * EthTypes such as BlockObject and TransactionObject might be better as existing Nimbus objects if present. NOTE: - * as `from` is a keyword, this has been replaced with `source` for variable names. + * as `from` is a keyword, this has been replaced with `source` for variable names. TODO: Related - eplace `to` with `dest`? + TODO: + * Some values can be returned as different types (eg, int or bool) + * Currently implemented as variant types, but server macros need to support + initialisation of these types before any use as `kind` can only be + specified once without invoking `reset`. ]# var server = sharedRpcServer() @@ -33,7 +37,14 @@ server.rpc("web3_sha3") do(data: string) -> string: ## ## data: the data to convert into a SHA3 hash. ## Returns the SHA3 result of the given string. - result = k256(data) + # TODO: Capture error on malformed input + var rawData: seq[byte] + if data.len > 2 and data[0] == '0' and data[1] in ['x', 'X']: + rawData = data[2..data.high].fromHex + else: + rawData = data.fromHex + # data will have 0x prefix + result = "0x" & $keccak_256.digest(rawData) server.rpc("net_version") do() -> string: ## Returns string of the current network id: From a963fed33fdf7556bdf458600cb54e558ffcff7b Mon Sep 17 00:00:00 2001 From: coffeepots Date: Fri, 25 May 2018 16:37:23 +0100 Subject: [PATCH 104/116] Update comments and change web3_sha3 test to use example in wiki --- tests/testserverclient.nim | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index 45bd8a7..47bc5bc 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -1,12 +1,5 @@ import ../ rpcclient, ../ rpcserver, unittest, asyncdispatch, json, tables -#[ - TODO: Importing client before server causes the error: - Error: undeclared identifier: 'result' for the `myProc` RPC. - This is because the RPC procs created by clientdispatch clash with ethprocs. - Currently, easiest solution is to import rpcserver (and therefore generate - ethprocs) before rpcclient. -]# # TODO: dummy implementations of RPC calls handled in async fashion. # TODO: check required json parameters like version are being raised var srv = sharedRpcServer() @@ -27,8 +20,8 @@ suite "RPC": var response = waitFor client.web3_clientVersion() check response == "Nimbus-RPC-Test" test "SHA3": - var response = waitFor client.web3_sha3("abc") - check response == "3A985DA74FE225B2045C172D6BD390BD855F086E3E9D525B46BFE24511431532" + var response = waitFor client.web3_sha3("0x68656c6c6f20776f726c64") + check response == "0x47173285A8D7341E5E972FC677286384F802F8EF42A5EC5F03BBFA254CB01FAD" test "Custom RPC": # Custom async RPC call var response = waitFor client.call("myProc", %[%"abc", %[1, 2, 3, 4]]) From 0031068774c6978dbc3f25bd7ba05e32fb240014 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 29 May 2018 18:54:39 +0100 Subject: [PATCH 105/116] Move ethcallsigs to tests to confirm stand alone rpc setup --- {eth-rpc/client => tests}/ethcallsigs.nim | 0 tests/testserverclient.nim | 10 +++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) rename {eth-rpc/client => tests}/ethcallsigs.nim (100%) diff --git a/eth-rpc/client/ethcallsigs.nim b/tests/ethcallsigs.nim similarity index 100% rename from eth-rpc/client/ethcallsigs.nim rename to tests/ethcallsigs.nim diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index 47bc5bc..83d302d 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -1,4 +1,7 @@ -import ../ rpcclient, ../ rpcserver, unittest, asyncdispatch, json, tables +import ../ rpcclient, ../ rpcserver +import unittest, asyncdispatch, json, tables +from os import getCurrentDir, DirSep +from strutils import rsplit # TODO: dummy implementations of RPC calls handled in async fashion. # TODO: check required json parameters like version are being raised @@ -6,6 +9,11 @@ var srv = sharedRpcServer() srv.address = "localhost" srv.port = Port(8545) +import stint + +# generate all client ethereum rpc calls +createRpcSigs(currentSourcePath.rsplit(DirSep, 1)[0] & DirSep & "ethcallsigs.nim") + srv.rpc("myProc") do(input: string, data: array[0..3, int]): result = %("Hello " & input & " data: " & $data) From 21c1247cfe4b69bdcf9561f9f4f5bb247a869f04 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 29 May 2018 18:55:10 +0100 Subject: [PATCH 106/116] inline imports --- rpcserver.nim | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/rpcserver.nim b/rpcserver.nim index 83c15d2..446c5c3 100644 --- a/rpcserver.nim +++ b/rpcserver.nim @@ -1,7 +1,4 @@ import - eth-rpc / server / servertypes, - eth-rpc / server / rpcconsts, - eth-rpc / server / serverdispatch, - eth-rpc / server / ethprocs, + r"eth-rpc/server" / [servertypes, rpcconsts, serverdispatch, ethprocs], eth-rpc / ethtypes export servertypes, rpcconsts, serverdispatch, ethprocs, ethtypes From 457fa4d454d309085c9c785be9291d55faf4ddf9 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 29 May 2018 18:55:57 +0100 Subject: [PATCH 107/116] Reuse symbol on checkGet --- eth-rpc/client/clientdispatch.nim | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/eth-rpc/client/clientdispatch.nim b/eth-rpc/client/clientdispatch.nim index 6f97e4b..a9e8825 100644 --- a/eth-rpc/client/clientdispatch.nim +++ b/eth-rpc/client/clientdispatch.nim @@ -33,10 +33,11 @@ proc call*(self: RpcClient, name: string, params: JsonNode): Future[Response] {. result = await newFut macro checkGet(node: JsonNode, fieldName: string, jKind: static[JsonNodeKind]): untyped = - result = quote do: - let n = `node`{`fieldName`} - if n.isNil: raise newException(ValueError, "Message is missing required field \"" & `fieldName` & "\"") - if n.kind != `jKind`.JsonNodeKind: raise newException(ValueError, "Expected " & $(`jKind`.JsonNodeKind) & ", got " & $`node`[`fieldName`].kind) + let n = genSym(ident = "n") #`node`{`fieldName`} + result = quote: + let `n` = `node`{`fieldname`} + if `n`.isNil: raise newException(ValueError, "Message is missing required field \"" & `fieldName` & "\"") + if `n`.kind != `jKind`.JsonNodeKind: raise newException(ValueError, "Expected " & $(`jKind`.JsonNodeKind) & ", got " & $`node`[`fieldName`].kind) case jKind of JBool: result.add(quote do: `node`[`fieldName`].getBool) of JInt: result.add(quote do: `node`[`fieldName`].getInt) @@ -109,7 +110,7 @@ proc toJsonArray(parameters: NimNode): NimNode = let curParam = parameters[i][0] if curParam.kind != nnkEmpty: items.add(nnkPrefix.newTree(ident"%", curParam)) - result = nnkPrefix.newTree(newIdentNode("%"), items) + result = nnkPrefix.newTree(bindSym("%", brForceOpen), items) proc createRpcFromSig*(rpcDecl: NimNode): NimNode = # Each input parameter in the rpc signature is converted @@ -171,20 +172,17 @@ proc createRpcFromSig*(rpcDecl: NimNode): NimNode = when defined(nimDumpRpcs): echo pathStr, ":\n", result.repr -from os import getCurrentDir, DirSep -from strutils import rsplit - -macro processRpcSigs(): untyped = +proc processRpcSigs(parsedCode: NimNode): NimNode = result = newStmtList() - const - codePath = currentSourcePath.rsplit(DirSep, 1)[0] & DirSep & "ethcallsigs.nim" - code = staticRead(codePath) - let parsedCode = parseStmt(code) for line in parsedCode: if line.kind == nnkProcDef: var procDef = createRpcFromSig(line) result.add(procDef) -# generate all client ethereum rpc calls -processRpcSigs() +macro createRpcSigs*(filePath: static[string]): untyped = + ## Takes a file of forward declarations in Nim and builds them into RPC + ## calls, based on their parameters. + ## Inputs are marshalled to json, and results are put into the signature's + ## Nim type. + result = processRpcSigs(staticRead($filePath).parseStmt()) From 74f3ed4c6a335c078db993635cd2fdf8ebbec4f8 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 29 May 2018 18:56:23 +0100 Subject: [PATCH 108/116] Minor comment updates --- eth-rpc/server/servertypes.nim | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 866fd0d..1921d49 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -36,7 +36,6 @@ proc sharedRpcServer*(): RpcServer = proc `$`*(port: Port): string = $int(port) proc makeProcName(s: string): string = - # only alphanumeric result = "" for c in s: if c.isAlphaNumeric: result.add c @@ -53,7 +52,7 @@ macro rpc*(server: var RpcServer, path: string, body: untyped): untyped = pathStr = $path # procs are generated from the stripped path procNameStr = pathStr.makeProcName # strip non alphanumeric procName = newIdentNode(procNameStr) # public rpc proc - doMain = newIdentNode(procNameStr & "DoMain") # when parameters: proc that contains our rpc body + doMain = newIdentNode(procNameStr & "DoMain") # when parameters present: proc that contains our rpc body res = newIdentNode("result") # async result var setup = jsonToNim(parameters, paramsIdent) @@ -62,7 +61,7 @@ macro rpc*(server: var RpcServer, path: string, body: untyped): untyped = if parameters.hasReturnType: let returnType = parameters[0] - # delgate async proc allows return and setting of result as native type + # delegate async proc allows return and setting of result as native type result.add(quote do: proc `doMain`(`paramsIdent`: JsonNode): Future[`returnType`] {.async.} = `setup` @@ -94,4 +93,4 @@ macro rpc*(server: var RpcServer, path: string, body: untyped): untyped = when defined(nimDumpRpcs): echo "\n", pathStr, ": ", result.repr -# TODO: Check supplied rpc do signatures with ethcallsigs +# TODO: Allow cross checking between client signatures and server calls From f30c4651a1b80c03ac3f81469b52bd0331831712 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 29 May 2018 19:21:30 +0100 Subject: [PATCH 109/116] Move ethprocs and ethtypes into tests, remove external imports --- eth-rpc/client/clientdispatch.nim | 3 ++- rpcclient.nim | 4 ++-- rpcserver.nim | 5 ++--- tests/ethcallsigs.nim | 2 +- {eth-rpc/server => tests}/ethprocs.nim | 2 +- {eth-rpc => tests}/ethtypes.nim | 0 tests/testserverclient.nim | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) rename {eth-rpc/server => tests}/ethprocs.nim (99%) rename {eth-rpc => tests}/ethtypes.nim (100%) diff --git a/eth-rpc/client/clientdispatch.nim b/eth-rpc/client/clientdispatch.nim index a9e8825..1c16f9b 100644 --- a/eth-rpc/client/clientdispatch.nim +++ b/eth-rpc/client/clientdispatch.nim @@ -1,5 +1,5 @@ import asyncnet, asyncdispatch, tables, json, oids, macros -import ../ ethtypes, stint, ../ jsonconverters +import ../jsonconverters type RpcClient* = ref object @@ -186,3 +186,4 @@ macro createRpcSigs*(filePath: static[string]): untyped = ## Inputs are marshalled to json, and results are put into the signature's ## Nim type. result = processRpcSigs(staticRead($filePath).parseStmt()) + echo "**", result.repr diff --git a/rpcclient.nim b/rpcclient.nim index db6a851..4c36611 100644 --- a/rpcclient.nim +++ b/rpcclient.nim @@ -1,3 +1,3 @@ -import eth-rpc / client / clientdispatch, eth-rpc / ethtypes -export clientdispatch, ethtypes +import eth-rpc / client / clientdispatch +export clientdispatch diff --git a/rpcserver.nim b/rpcserver.nim index 446c5c3..ffebe29 100644 --- a/rpcserver.nim +++ b/rpcserver.nim @@ -1,4 +1,3 @@ import - r"eth-rpc/server" / [servertypes, rpcconsts, serverdispatch, ethprocs], - eth-rpc / ethtypes -export servertypes, rpcconsts, serverdispatch, ethprocs, ethtypes + r"eth-rpc/server" / [servertypes, rpcconsts, serverdispatch] +export servertypes, rpcconsts, serverdispatch diff --git a/tests/ethcallsigs.nim b/tests/ethcallsigs.nim index d10ef72..f72d507 100644 --- a/tests/ethcallsigs.nim +++ b/tests/ethcallsigs.nim @@ -1,7 +1,7 @@ ## This module contains signatures for the Ethereum client RPCs. ## The signatures are not imported directly, but read and processed with parseStmt, ## then a procedure body is generated to marshal native Nim parameters to json and visa versa. -import json, stint, eth-rpc / ethtypes +import json, stint, ethtypes proc web3_clientVersion(): string proc web3_sha3(data: string): string diff --git a/eth-rpc/server/ethprocs.nim b/tests/ethprocs.nim similarity index 99% rename from eth-rpc/server/ethprocs.nim rename to tests/ethprocs.nim index 9c2129e..f8a561c 100644 --- a/eth-rpc/server/ethprocs.nim +++ b/tests/ethprocs.nim @@ -1,4 +1,4 @@ -import servertypes, nimcrypto, json, stint, ../ethtypes.nim, strutils +import ../rpcserver, nimcrypto, json, stint, strutils, ethtypes #[ For details on available RPC calls, see: https://github.com/ethereum/wiki/wiki/JSON-RPC diff --git a/eth-rpc/ethtypes.nim b/tests/ethtypes.nim similarity index 100% rename from eth-rpc/ethtypes.nim rename to tests/ethtypes.nim diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index 83d302d..a085c6f 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -9,7 +9,7 @@ var srv = sharedRpcServer() srv.address = "localhost" srv.port = Port(8545) -import stint +import stint, ethtypes, ethprocs # generate all client ethereum rpc calls createRpcSigs(currentSourcePath.rsplit(DirSep, 1)[0] & DirSep & "ethcallsigs.nim") From 5292b0b53e2c12e694070b9975268da0bd171346 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 29 May 2018 21:46:11 +0100 Subject: [PATCH 110/116] Remove debugging echo --- eth-rpc/client/clientdispatch.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/eth-rpc/client/clientdispatch.nim b/eth-rpc/client/clientdispatch.nim index 1c16f9b..a9d3a2b 100644 --- a/eth-rpc/client/clientdispatch.nim +++ b/eth-rpc/client/clientdispatch.nim @@ -186,4 +186,3 @@ macro createRpcSigs*(filePath: static[string]): untyped = ## Inputs are marshalled to json, and results are put into the signature's ## Nim type. result = processRpcSigs(staticRead($filePath).parseStmt()) - echo "**", result.repr From f38dcebe3493322d1849e145cbbf9f16ee79b344 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Tue, 29 May 2018 21:58:27 +0100 Subject: [PATCH 111/116] Split tests into client-server test and ethprocs test --- tests/all.nim | 2 +- tests/testethcalls.nim | 32 ++++++++++++++++++++++++++++++++ tests/testserverclient.nim | 24 +++++------------------- 3 files changed, 38 insertions(+), 20 deletions(-) create mode 100644 tests/testethcalls.nim diff --git a/tests/all.nim b/tests/all.nim index c61d327..43a8c9e 100644 --- a/tests/all.nim +++ b/tests/all.nim @@ -1,3 +1,3 @@ import - testrpcmacro, testserverclient + testrpcmacro, testserverclient, testethcalls diff --git a/tests/testethcalls.nim b/tests/testethcalls.nim new file mode 100644 index 0000000..ea9e063 --- /dev/null +++ b/tests/testethcalls.nim @@ -0,0 +1,32 @@ +import ../ rpcclient, ../ rpcserver +import unittest, asyncdispatch, json, tables + +from os import getCurrentDir, DirSep +from strutils import rsplit +template sourceDir: string = currentSourcePath.rsplit(DirSep, 1)[0] + +var srv = sharedRpcServer() +srv.address = "localhost" +srv.port = Port(8546) + +# importing ethprocs creates the server rpc calls +import stint, ethtypes, ethprocs +# generate all client ethereum rpc calls +createRpcSigs(sourceDir & DirSep & "ethcallsigs.nim") + +asyncCheck srv.serve + +suite "Ethereum RPCs": + proc main {.async.} = + var client = newRpcClient() + await client.connect("localhost", Port(8546)) + + test "Version": + var + response = waitFor client.web3_clientVersion() + check response == "Nimbus-RPC-Test" + test "SHA3": + var response = waitFor client.web3_sha3("0x68656c6c6f20776f726c64") + check response == "0x47173285A8D7341E5E972FC677286384F802F8EF42A5EC5F03BBFA254CB01FAD" + + waitFor main() diff --git a/tests/testserverclient.nim b/tests/testserverclient.nim index a085c6f..b7af85b 100644 --- a/tests/testserverclient.nim +++ b/tests/testserverclient.nim @@ -1,39 +1,25 @@ import ../ rpcclient, ../ rpcserver -import unittest, asyncdispatch, json, tables -from os import getCurrentDir, DirSep -from strutils import rsplit +import unittest, asyncdispatch, json -# TODO: dummy implementations of RPC calls handled in async fashion. -# TODO: check required json parameters like version are being raised -var srv = sharedRpcServer() +var srv = newRpcServer() srv.address = "localhost" srv.port = Port(8545) -import stint, ethtypes, ethprocs - -# generate all client ethereum rpc calls -createRpcSigs(currentSourcePath.rsplit(DirSep, 1)[0] & DirSep & "ethcallsigs.nim") - srv.rpc("myProc") do(input: string, data: array[0..3, int]): result = %("Hello " & input & " data: " & $data) asyncCheck srv.serve -suite "RPC": +suite "Server/Client RPC": proc main {.async.} = var client = newRpcClient() await client.connect("localhost", Port(8545)) - test "Version": - var response = waitFor client.web3_clientVersion() - check response == "Nimbus-RPC-Test" - test "SHA3": - var response = waitFor client.web3_sha3("0x68656c6c6f20776f726c64") - check response == "0x47173285A8D7341E5E972FC677286384F802F8EF42A5EC5F03BBFA254CB01FAD" test "Custom RPC": # Custom async RPC call var response = waitFor client.call("myProc", %[%"abc", %[1, 2, 3, 4]]) check response.result.getStr == "Hello abc data: [1, 2, 3, 4]" - waitFor main() # TODO: When an error occurs during a test, stop the server + # TODO: When an error occurs during a test, stop the server + asyncCheck main() \ No newline at end of file From 243285bb68ea443d57f3cb5c124f58bc6bb4af61 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 30 May 2018 10:46:08 +0100 Subject: [PATCH 112/116] Removing unused import, some tidying --- eth-rpc/client/clientdispatch.nim | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/eth-rpc/client/clientdispatch.nim b/eth-rpc/client/clientdispatch.nim index a9d3a2b..0996360 100644 --- a/eth-rpc/client/clientdispatch.nim +++ b/eth-rpc/client/clientdispatch.nim @@ -1,5 +1,5 @@ -import asyncnet, asyncdispatch, tables, json, oids, macros -import ../jsonconverters +import asyncnet, asyncdispatch, tables, json, macros +import ".." / [jsonconverters, jsonmarshal] type RpcClient* = ref object @@ -90,8 +90,6 @@ proc connect*(self: RpcClient, address: string, port: Port) {.async.} = self.port = port asyncCheck processData(self) -import ../ jsonmarshal - proc createRpcProc(procName, parameters, callBody: NimNode): NimNode = # parameters come as a tree var paramList = newSeq[NimNode]() From fe87e3c386980e079b0681d715793aaea0bbf459 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 30 May 2018 10:51:27 +0100 Subject: [PATCH 113/116] Use proper error const --- eth-rpc/server/serverdispatch.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth-rpc/server/serverdispatch.nim b/eth-rpc/server/serverdispatch.nim index 637ac63..392cfea 100644 --- a/eth-rpc/server/serverdispatch.nim +++ b/eth-rpc/server/serverdispatch.nim @@ -40,7 +40,7 @@ proc processClient(server: RpcServer, client: AsyncSocket) {.async.} = let err = future.readError.RpcProcError await client.sendError(err.code, err.msg, err.data) else: - await client.sendError(-32000, "Error", %getCurrentExceptionMsg()) + await client.sendError(SERVER_ERROR, "Error", %getCurrentExceptionMsg()) proc serve*(server: RpcServer) {.async.} = server.socket.bindAddr(server.port, server.address) From 6ef3861c20ba73a0750494698c0604405d62da32 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 30 May 2018 16:04:17 +0100 Subject: [PATCH 114/116] Remove jsonconverters and internal references to stint --- eth-rpc/client/clientdispatch.nim | 2 +- eth-rpc/jsonconverters.nim | 17 -------------- eth-rpc/jsonmarshal.nim | 38 ++++++++++++++----------------- eth-rpc/server/servertypes.nim | 4 ++-- 4 files changed, 20 insertions(+), 41 deletions(-) delete mode 100644 eth-rpc/jsonconverters.nim diff --git a/eth-rpc/client/clientdispatch.nim b/eth-rpc/client/clientdispatch.nim index 0996360..06020dd 100644 --- a/eth-rpc/client/clientdispatch.nim +++ b/eth-rpc/client/clientdispatch.nim @@ -1,5 +1,5 @@ import asyncnet, asyncdispatch, tables, json, macros -import ".." / [jsonconverters, jsonmarshal] +import ../ jsonmarshal type RpcClient* = ref object diff --git a/eth-rpc/jsonconverters.nim b/eth-rpc/jsonconverters.nim deleted file mode 100644 index 54b18ef..0000000 --- a/eth-rpc/jsonconverters.nim +++ /dev/null @@ -1,17 +0,0 @@ -import json, stint, strutils - -template stintStr(n: UInt256|Int256): JsonNode = - var s = n.toHex - if s.len mod 2 != 0: s = "0" & s - s = "0x" & s - %s - -proc `%`*(n: UInt256): JsonNode = n.stintStr - -proc `%`*(n: Int256): JsonNode = n.stintStr - -proc `%`*(n: byte{not lit}): JsonNode = - result = newJInt(int(n)) - -proc `%`*(n: ref int|ref int64): JsonNode = - result = newJInt(int(n[])) diff --git a/eth-rpc/jsonmarshal.nim b/eth-rpc/jsonmarshal.nim index 5edd5e6..426f0ef 100644 --- a/eth-rpc/jsonmarshal.nim +++ b/eth-rpc/jsonmarshal.nim @@ -1,8 +1,17 @@ -import macros, json, jsonconverters, stint +import macros, json template expect*(actual, expected: JsonNodeKind, argName: string) = if actual != expected: raise newException(ValueError, "Parameter \"" & argName & "\" expected " & $expected & " but got " & $actual) +proc `%`*(n: byte{not lit}): JsonNode = + result = newJInt(int(n)) + +proc `%`*(n: uint64{not lit}): JsonNode = + result = newJInt(int(n)) + +proc `%`*(n: ref int|ref int64): JsonNode = + result = newJInt(int(n[])) + # Compiler requires forward decl when processing out of module proc fromJson(n: JsonNode, argName: string, result: var bool) proc fromJson(n: JsonNode, argName: string, result: var int) @@ -11,11 +20,10 @@ proc fromJson(n: JsonNode, argName: string, result: var float) proc fromJson(n: JsonNode, argName: string, result: var string) proc fromJson[T](n: JsonNode, argName: string, result: var seq[T]) proc fromJson[N, T](n: JsonNode, argName: string, result: var array[N, T]) -proc fromJson(n: JsonNode, argName: string, result: var UInt256) proc fromJson(n: JsonNode, argName: string, result: var int64) +proc fromJson(n: JsonNode, argName: string, result: var uint64) proc fromJson(n: JsonNode, argName: string, result: var ref int64) proc fromJson(n: JsonNode, argName: string, result: var ref int) -proc fromJson(n: JsonNode, argName: string, result: var ref UInt256) # This can't be forward declared: https://github.com/nim-lang/Nim/issues/7868 proc fromJson[T: enum](n: JsonNode, argName: string, result: var T) = @@ -25,7 +33,7 @@ proc fromJson[T: enum](n: JsonNode, argName: string, result: var T) = # This can't be forward declared: https://github.com/nim-lang/Nim/issues/7868 proc fromJson[T: object](n: JsonNode, argName: string, result: var T) = n.kind.expect(JObject, argName) - for k, v in fieldpairs(result): + for k, v in fieldPairs(result): fromJson(n[k], k, v) proc fromJson(n: JsonNode, argName: string, result: var bool) = @@ -46,6 +54,10 @@ proc fromJson(n: JsonNode, argName: string, result: var int64) = n.kind.expect(JInt, argName) result = n.getInt() +proc fromJson(n: JsonNode, argName: string, result: var uint64) = + n.kind.expect(JInt, argName) + result = n.getInt().uint64 + proc fromJson(n: JsonNode, argName: string, result: var ref int64) = n.kind.expect(JInt, argName) new result @@ -62,23 +74,6 @@ proc fromJson(n: JsonNode, argName: string, result: var byte) = if v > 255 or v < 0: raise newException(ValueError, "Parameter \"" & argName & "\" value out of range for byte: " & $v) result = byte(v) -proc fromJson(n: JsonNode, argName: string, result: var UInt256) = - # expects base 16 string, starting with "0x" - n.kind.expect(JString, argName) - let hexStr = n.getStr() - if hexStr.len > 64 + 2: # including "0x" - raise newException(ValueError, "Parameter \"" & argName & "\" value too long for UInt256: " & $hexStr.len) - result = hexStr.parse(StUint[256], 16) # TODO: Handle errors - -proc fromJson(n: JsonNode, argName: string, result: var ref UInt256) = - # expects base 16 string, starting with "0x" - n.kind.expect(JString, argName) - let hexStr = n.getStr() - if hexStr.len > 64 + 2: # including "0x" - raise newException(ValueError, "Parameter \"" & argName & "\" value too long for UInt256: " & $hexStr.len) - new result - result[] = hexStr.parse(StUint[256], 16) # TODO: Handle errors - proc fromJson(n: JsonNode, argName: string, result: var float) = n.kind.expect(JFloat, argName) result = n.getFloat() @@ -142,3 +137,4 @@ proc jsonToNim*(parameters, jsonIdent: NimNode): NimNode = ) # unpack Nim type and assign from json result.add jsonToNim(paramIdent, paramType, jsonElement, paramName) + diff --git a/eth-rpc/server/servertypes.nim b/eth-rpc/server/servertypes.nim index 1921d49..bf027d3 100644 --- a/eth-rpc/server/servertypes.nim +++ b/eth-rpc/server/servertypes.nim @@ -1,5 +1,5 @@ -import asyncdispatch, asyncnet, json, tables, macros, strutils, ../ jsonconverters, ../ jsonmarshal, stint -export asyncdispatch, asyncnet, json, jsonconverters, expect +import asyncdispatch, asyncnet, json, tables, macros, strutils, ../ jsonmarshal +export asyncdispatch, asyncnet, json, jsonmarshal type RpcProc* = proc (params: JsonNode): Future[JsonNode] From c5957c658838794dcddc2ecb203e8dad46903148 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 30 May 2018 16:04:52 +0100 Subject: [PATCH 115/116] Just putting first import on a single line --- rpcserver.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rpcserver.nim b/rpcserver.nim index ffebe29..1472f75 100644 --- a/rpcserver.nim +++ b/rpcserver.nim @@ -1,3 +1,2 @@ -import - r"eth-rpc/server" / [servertypes, rpcconsts, serverdispatch] +import r"eth-rpc/server" / [servertypes, rpcconsts, serverdispatch] export servertypes, rpcconsts, serverdispatch From b0b0b1372d4f409b1a3d6a73054ecbb1b8604975 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 30 May 2018 16:05:53 +0100 Subject: [PATCH 116/116] Split tests into standard client-server, rpc macro tests and ethtests --- tests/ethprocs.nim | 2 +- tests/stintJsonConverters.nim | 32 ++++++++++++++++++++++++++++++++ tests/testethcalls.nim | 20 ++++++++++++++++++-- tests/testrpcmacro.nim | 17 +---------------- 4 files changed, 52 insertions(+), 19 deletions(-) create mode 100644 tests/stintJsonConverters.nim diff --git a/tests/ethprocs.nim b/tests/ethprocs.nim index f8a561c..6fa1bcb 100644 --- a/tests/ethprocs.nim +++ b/tests/ethprocs.nim @@ -1,4 +1,4 @@ -import ../rpcserver, nimcrypto, json, stint, strutils, ethtypes +import ../rpcserver, nimcrypto, json, stint, strutils, ethtypes, stintjsonconverters #[ For details on available RPC calls, see: https://github.com/ethereum/wiki/wiki/JSON-RPC diff --git a/tests/stintJsonConverters.nim b/tests/stintJsonConverters.nim new file mode 100644 index 0000000..ac9af61 --- /dev/null +++ b/tests/stintJsonConverters.nim @@ -0,0 +1,32 @@ +import json, stint +from ../rpcserver import expect + +template stintStr(n: UInt256|Int256): JsonNode = + var s = n.toHex + if s.len mod 2 != 0: s = "0" & s + s = "0x" & s + %s + +proc `%`*(n: UInt256): JsonNode = n.stintStr + +proc `%`*(n: Int256): JsonNode = n.stintStr + +# allows UInt256 to be passed as a json string +proc fromJson*(n: JsonNode, argName: string, result: var UInt256) = + # expects base 16 string, starting with "0x" + n.kind.expect(JString, argName) + let hexStr = n.getStr() + if hexStr.len > 64 + 2: # including "0x" + raise newException(ValueError, "Parameter \"" & argName & "\" value too long for UInt256: " & $hexStr.len) + result = hexStr.parse(StUint[256], 16) # TODO: Handle errors + +# allows ref UInt256 to be passed as a json string +proc fromJson*(n: JsonNode, argName: string, result: var ref UInt256) = + # expects base 16 string, starting with "0x" + n.kind.expect(JString, argName) + let hexStr = n.getStr() + if hexStr.len > 64 + 2: # including "0x" + raise newException(ValueError, "Parameter \"" & argName & "\" value too long for UInt256: " & $hexStr.len) + new result + result[] = hexStr.parse(StUint[256], 16) # TODO: Handle errors + diff --git a/tests/testethcalls.nim b/tests/testethcalls.nim index ea9e063..857762b 100644 --- a/tests/testethcalls.nim +++ b/tests/testethcalls.nim @@ -1,4 +1,4 @@ -import ../ rpcclient, ../ rpcserver +import ../ rpcclient, ../ rpcserver import unittest, asyncdispatch, json, tables from os import getCurrentDir, DirSep @@ -10,10 +10,18 @@ srv.address = "localhost" srv.port = Port(8546) # importing ethprocs creates the server rpc calls -import stint, ethtypes, ethprocs +import stint, ethtypes, ethprocs, stintJsonConverters # generate all client ethereum rpc calls createRpcSigs(sourceDir & DirSep & "ethcallsigs.nim") +srv.rpc("rpc.uint256param") do(i: UInt256): + let r = i + 1.stUint(256) + result = %r + +srv.rpc("rpc.testreturnuint256") do() -> UInt256: + let r: UInt256 = "0x1234567890abcdef".parse(UInt256, 16) + return r + asyncCheck srv.serve suite "Ethereum RPCs": @@ -21,6 +29,14 @@ suite "Ethereum RPCs": var client = newRpcClient() await client.connect("localhost", Port(8546)) + test "UInt256 param": + let r = waitFor rpcUInt256Param(%[%"0x1234567890"]) + check r == %"0x1234567891" + + test "Return UInt256": + let r = waitFor rpcTestReturnUInt256(%[]) + check r == %"0x1234567890abcdef" + test "Version": var response = waitFor client.web3_clientVersion() diff --git a/tests/testrpcmacro.nim b/tests/testrpcmacro.nim index b1e8e7f..4317fc3 100644 --- a/tests/testrpcmacro.nim +++ b/tests/testrpcmacro.nim @@ -1,4 +1,4 @@ -import unittest, ../ rpcserver, asyncdispatch, json, tables, stint +import unittest, ../ rpcserver, asyncdispatch, json, tables type # some nested types to check object parsing @@ -51,9 +51,6 @@ s.rpc("rpc.seqparam") do(a: string, s: seq[int]): s.rpc("rpc.objparam") do(a: string, obj: MyObject): result = %obj -s.rpc("rpc.uint256param") do(i: UInt256): - let r = i + 1.stUint(256) - result = %r s.rpc("rpc.returntypesimple") do(i: int) -> int: result = i @@ -65,10 +62,6 @@ s.rpc("rpc.returntypecomplex") do(i: int) -> Test2: s.rpc("rpc.testreturns") do() -> int: return 1234 -s.rpc("rpc.testreturnuint256") do() -> UInt256: - let r: UInt256 = "0x1234567890abcdef".parse(UInt256, 16) - return r - # Tests suite "Server types": @@ -108,10 +101,6 @@ suite "Server types": let r = waitfor rpcObjParam(%[%"abc", testObj]) check r == testObj - test "UInt256 param": - let r = waitFor rpcUInt256Param(%[%"0x1234567890"]) - check r == %"0x1234567891" - test "Simple return types": let inp = %99 @@ -128,10 +117,6 @@ suite "Server types": let r = waitFor rpcTestReturns(%[]) check r == %1234 - test "Return UInt256": - let r = waitFor rpcTestReturnUInt256(%[]) - check r == %"0x1234567890abcdef" - test "Runtime errors": expect ValueError: # root param not array