Restore the compilation of Nimbus by restoring the support for p2pProtocol decorators

eth/p2p/p2p_backends_helpers.nim

@@ -1,3 +1,10 @@
+var
+  gProtocols: seq[ProtocolInfo]
+
+# The variables above are immutable RTTI information. We need to tell
+# Nim to not consider them GcSafe violations:
+template allProtocols*: auto = {.gcsafe.}: gProtocols
+
 proc getState*(peer: Peer, proto: ProtocolInfo): RootRef =
   peer.protocolStates[proto.index]
 

eth/p2p/p2p_protocol_dsl.nim

@@ -1,5 +1,5 @@
 import
-  macros,
+  macros, options,
   std_shims/macros_shim, chronos
 
 type
@@ -49,9 +49,9 @@ type
     timeoutParam*: NimNode
       ## Cached ident for the timeout parameter
 
-    allDefs*: NimNode
+    extraDefs*: NimNode
-      ## The final definitions that must be become part of the
+      ## The reponse procs have extra templates that must become
-      ## p2pProtocol macro output. May include helper templates.
+      ## part of the generated code
 
   P2PProtocol* = ref object
     # Settings
@@ -102,6 +102,7 @@ type
     NetworkType*: NimNode
     SerializationFormat*: NimNode
     ResponderType*: NimNode
+    ReqIdType*: NimNode
 
     registerProtocol*: NimNode
     setEventHandlers*: NimNode
@@ -113,6 +114,7 @@ type
 
 const
   defaultReqTimeout = 10.seconds
+  tracingEnabled = defined(p2pdump)
 
 let
   # Variable names affecting the public interface of the library:
@@ -133,6 +135,34 @@ let
 template Opt(T): auto = newTree(nnkBracketExpr, Option, T)
 template Fut(T): auto = newTree(nnkBracketExpr, Future, T)
 
+proc initFuture*[T](loc: var Future[T]) =
+  loc = newFuture[T]()
+
+template applyDecorator(p: NimNode, decorator: NimNode) =
+  if decorator.kind != nnkNilLit:
+    p.pragma.insert(0, decorator)
+
+when tracingEnabled:
+  proc logSentMsgFields(peer: NimNode,
+                        protocolInfo: NimNode,
+                        msgName: string,
+                        fields: openarray[NimNode]): NimNode =
+    ## This generates the tracing code inserted in the message sending procs
+    ## `fields` contains all the params that were serialized in the message
+    var tracer = ident("tracer")
+
+    result = quote do:
+      var `tracer` = init StringJsonWriter
+      beginRecord(`tracer`)
+
+    for f in fields:
+      result.add newCall(bindSym"writeField", tracer, newLit($f), f)
+
+    result.add quote do:
+      endRecord(`tracer`)
+      logMsgEventImpl("outgoing_msg", `peer`,
+                      `protocolInfo`, `msgName`, getOutput(`tracer`))
+
 proc createPeerState[Peer, ProtocolState](peer: Peer): RootRef =
   var res = new ProtocolState
   mixin initProtocolState
@@ -218,6 +248,9 @@ proc init*(T: type P2PProtocol, backendFactory: BackendFactory,
   assert(not result.backend.implementProtocolInit.isNil)
   assert(not result.backend.ResponderType.isNil)
 
+  if result.backend.ReqIdType.isNil:
+    result.backend.ReqIdType = ident "int"
+
   result.processProtocolBody body
 
   if not result.backend.afterProtocolInit.isNil:
@@ -242,15 +275,6 @@ proc augmentUserHandler(p: P2PProtocol, userHandlerProc: NimNode, msgId = -1) =
 
   userHandlerProc.body.insert 0, prelude
 
-  when false:
-    # TODO
-    ## Turns a regular proc definition into an async proc and adds
-    ## the helpers for accessing the peer and network protocol states.
-    case msgKind
-    of msgRequest:  userHandlerProc.applyDecorator incomingRequestDecorator
-    of msgResponse: userHandlerProc.applyDecorator incomingResponseDecorator
-    else: discard
-
   # We allow the user handler to use `openarray` params, but we turn
   # those into sequences to make the `async` pragma happy.
   for i in 1 ..< userHandlerProc.params.len:
@@ -351,13 +375,13 @@ proc newMsg(protocol: P2PProtocol, kind: MessageKind, id: int,
   if procDef.body.kind != nnkEmpty:
     var userHandler = copy procDef
 
-    protocol.augmentUserHandler userHandler
+    protocol.augmentUserHandler userHandler, id
     userHandler.name = genSym(nskProc, msgName)
 
     # Request and Response handlers get an extra `reqId` parameter if the
     # protocol uses them:
     if result.hasReqId:
-      userHandler.params.insert(2, newIdentDefs(reqIdVar, ReqIdType))
+      userHandler.params.insert(2, newIdentDefs(reqIdVar, protocol.backend.ReqIdType))
 
     # All request handlers get an automatically inserter `response` variable:
     if kind == msgRequest:
@@ -370,13 +394,17 @@ proc newMsg(protocol: P2PProtocol, kind: MessageKind, id: int,
       if protocol.useRequestIds:
         initResponderCall.add reqIdVar
 
-      userHandler.addPreludeDefs quote do:
+      userHandler.addPreludeDefs newVarStmt(responseVar, initResponderCall)
-        let `responseVar` = `initResponderCall`
 
       result.initResponderCall = initResponderCall
 
-    protocol.outRecvProcs.add userHandler
+    case kind
+    of msgRequest:  userHandler.applyDecorator protocol.incomingRequestDecorator
+    of msgResponse: userHandler.applyDecorator protocol.incomingResponseDecorator
+    else: discard
+
     result.userHandler = userHandler
+    protocol.outRecvProcs.add result.userHandler
 
   protocol.messages.add result
 
@@ -391,7 +419,7 @@ proc createSendProc*(msg: Message,
 
   let
     name = if not isRawSender: msg.identWithExportMarker
-           else: genSym(nskProc, $msg.ident & "RawSender")
+           else: ident($msg.ident & "RawSender")
 
     pragmas = if procType == nnkProcDef: newTree(nnkPragma, ident"gcsafe")
               else: newEmptyNode()
@@ -405,9 +433,12 @@ proc createSendProc*(msg: Message,
     newEmptyNode(),
     newStmtList()) ## body
 
+  if proctype == nnkProcDef:
+    for p in msg.procDef.pragma:
+      def.addPragma p
+
   result.msg = msg
   result.def = def
-  result.allDefs = newStmtList(def)
 
   for param, paramType in def.typedParams():
     if result.peerParam.isNil:
@@ -435,7 +466,7 @@ proc createSendProc*(msg: Message,
 
     # We create a helper that enables the `response.send()` syntax
     # inside the user handler of the request proc:
-    result.allDefs.add quote do:
+    result.extraDefs = quote do:
       template send*(r: `ResponderType`, args: varargs[untyped]): auto =
         `sendProcName`(r, args)
 
@@ -451,33 +482,24 @@ proc createSendProc*(msg: Message,
               else:
                 Fut(Void)
 
-const tracingEnabled = defined(p2pdump)
+proc setBody*(sendProc: SendProc, body: NimNode) =
+  var
+    msg = sendProc.msg
+    protocol = msg.protocol
+    def = sendProc.def
 
-when tracingEnabled:
+  # TODO: macros.body triggers an assertion error when the proc type is nnkMacroDef
-  proc logSentMsgFields(peer: NimNode,
+  def[6] = body
-                        protocolInfo: NimNode,
-                        msgName: string,
-                        fields: openarray[NimNode]): NimNode =
-    ## This generates the tracing code inserted in the message sending procs
-    ## `fields` contains all the params that were serialized in the message
-    var tracer = ident("tracer")
 
-    result = quote do:
+  if msg.kind == msgRequest:
-      var `tracer` = init StringJsonWriter
+    def.applyDecorator protocol.outgoingRequestDecorator
-      beginRecord(`tracer`)
 
-    for f in fields:
+  msg.protocol.outSendProcs.add def
-      result.add newCall(bindSym"writeField", tracer, newLit($f), f)
 
-    result.add quote do:
+  if sendProc.extraDefs != nil:
-      endRecord(`tracer`)
+    msg.protocol.outSendProcs.add sendProc.extraDefs
-      logMsgEventImpl("outgoing_msg", `peer`,
-                      `protocolInfo`, `msgName`, getOutput(`tracer`))
 
-proc initFuture*[T](loc: var Future[T]) =
+proc useStandardBody*(sendProc: SendProc,
-  loc = newFuture[T]()
-
-proc implementBody*(sendProc: SendProc,
                       preludeGenerator: proc(stream: NimNode): NimNode,
                       sendCallGenerator: proc (peer, bytes: NimNode): NimNode) =
   let
@@ -510,7 +532,7 @@ proc implementBody*(sendProc: SendProc,
   for param in sendProc.msgParams:
     appendParams.add newCall(writeField, writer, newLit($param), param)
 
-  sendProc.def.body = quote do:
+  sendProc.setBody quote do:
     mixin init, WriterType, beginRecord, endRecord, getOutput
 
     `initResultFuture`
@@ -524,6 +546,16 @@ proc implementBody*(sendProc: SendProc,
     let `msgBytes` = getOutput(`outputStream`)
     `sendCall`
 
+proc defineThunk*(msg: Message, thunk: NimNode) =
+  let protocol = msg.protocol
+
+  case msg.kind
+  of msgRequest:  thunk.applyDecorator protocol.incomingRequestThunkDecorator
+  of msgResponse: thunk.applyDecorator protocol.incomingResponseThunkDecorator
+  else: discard
+
+  protocol.outRecvProcs.add thunk
+
 proc genAwaitUserHandler*(msg: Message, receivedMsg: NimNode,
                           leadingParams: varargs[NimNode]): NimNode =
   if msg.userHandler == nil:
@@ -551,8 +583,9 @@ proc netInit*(p: P2PProtocol): NimNode =
                             p.backend.NetworkType,
                             p.NetworkStateType)
 
-proc genHandshakeTemplate*(msg: Message,
+proc createHandshakeTemplate*(msg: Message,
-                           rawSendProc, handshakeImpl, nextMsg: NimNode): NimNode =
+                              rawSendProc, handshakeImpl,
+                              nextMsg: NimNode): SendProc =
   let
     handshakeExchanger = msg.createSendProc(procType = nnkTemplateDef)
     forwardCall = newCall(rawSendProc).appendAllParams(handshakeExchanger.def)
@@ -564,7 +597,7 @@ proc genHandshakeTemplate*(msg: Message,
   forwardCall[1] = peerVarSym
   forwardCall.del(forwardCall.len - 1)
 
-  handshakeExchanger.def.body = quote do:
+  handshakeExchanger.setBody quote do:
     let `peerVarSym` = `peerValue`
     let sendingFuture = `forwardCall`
     `handshakeImpl`(`peerVarSym`,
@@ -572,7 +605,7 @@ proc genHandshakeTemplate*(msg: Message,
                     `nextMsg`(`peerVarSym`, `msgRecName`),
                     `timeoutValue`)
 
-  return handshakeExchanger.def
+  return handshakeExchanger
 
 proc peerInit*(p: P2PProtocol): NimNode =
   if p.PeerStateType == nil:

eth/p2p/rlpx.nim

@@ -8,7 +8,7 @@ when useSnappy:
   const devp2pSnappyVersion* = 5
 
 export
-  p2pProtocol
+  options, p2pProtocol
 
 logScope:
   topics = "rlpx"
@@ -36,15 +36,6 @@ when tracingEnabled:
     # See a more detailed comment in p2p_tracing.nim
     init, writeValue, getOutput
 
-var
-  gProtocols: seq[ProtocolInfo]
-  gDevp2pInfo: ProtocolInfo
-
-# The variables above are immutable RTTI information. We need to tell
-# Nim to not consider them GcSafe violations:
-template allProtocols*: auto = {.gcsafe.}: gProtocols
-template devp2pInfo: auto = {.gcsafe.}: gDevp2pInfo
-
 proc init*[MsgName](T: type ResponderWithId[MsgName],
                     peer: Peer, reqId: int): T =
   T(peer: peer, reqId: reqId)
@@ -71,6 +62,9 @@ proc disconnectAndRaise(peer: Peer,
 
 include p2p_backends_helpers
 
+var gDevp2pInfo: ProtocolInfo
+template devp2pInfo: auto = {.gcsafe.}: gDevp2pInfo
+
 # Dispatcher
 #
 
@@ -548,9 +542,6 @@ proc dispatchMessages*(peer: Peer) {.async.} =
         return
       peer.awaitedMessages[msgId] = nil
 
-template applyDecorator(p: NimNode, decorator: NimNode) =
-  if decorator.kind != nnkNilLit: p.addPragma decorator
-
 proc p2pProtocolBackendImpl*(protocol: P2PProtocol): Backend =
   let
     resultIdent = ident "result"
@@ -692,10 +683,11 @@ proc p2pProtocolBackendImpl*(protocol: P2PProtocol): Backend =
     var userHandlerParams = @[peerVar]
     if hasReqId: userHandlerParams.add reqIdVar
 
-    let awaitUserHandler = msg.genAwaitUserHandler(receivedMsg, userHandlerParams)
+    let
+      awaitUserHandler = msg.genAwaitUserHandler(receivedMsg, userHandlerParams)
+      thunkName = ident(msgName & "_thunk")
 
-    let thunkName = ident(msgName & "_thunk")
+    msg.defineThunk quote do:
-    var thunkProc = quote do:
       proc `thunkName`(`peerVar`: `Peer`, _: int, data: Rlp) {.async, gcsafe.} =
         var `receivedRlp` = data
         var `receivedMsg` {.noinit.}: `msgRecName`
@@ -704,21 +696,9 @@ proc p2pProtocolBackendImpl*(protocol: P2PProtocol): Backend =
         `awaitUserHandler`
         `callResolvedResponseFuture`
 
-    case msg.kind
-    of msgRequest:  thunkProc.applyDecorator protocol.incomingRequestThunkDecorator
-    of msgResponse: thunkProc.applyDecorator protocol.incomingResponseThunkDecorator
-    else: discard
-
-    protocol.outRecvProcs.add thunkProc
-
     var sendProc = msg.createSendProc(isRawSender = (msg.kind == msgHandshake))
     sendProc.def.params[1][0] = peerOrResponder
 
-    protocol.outSendProcs.add sendProc.allDefs
-
-    if msg.kind == msgHandshake:
-      protocol.outSendProcs.add msg.genHandshakeTemplate(sendProc.def.name,
-                                                         handshakeImpl, nextMsg)
     let
       msgBytes = ident"msgBytes"
       finalizeRequest = quote do:
@@ -762,15 +742,15 @@ proc p2pProtocolBackendImpl*(protocol: P2PProtocol): Backend =
       appendParams.add logSentMsgFields(peerVar, protocol, msgId, paramsToWrite)
 
     # let paramCountNode = newLit(paramCount)
-    sendProc.def.body = quote do:
+    sendProc.setBody quote do:
       let `peerVar` = getPeer(`peerOrResponder`)
       `initWriter`
       `appendParams`
       `finalizeRequest`
       `senderEpilogue`
 
-    if msg.kind == msgRequest:
+    if msg.kind == msgHandshake:
-      sendProc.def.applyDecorator protocol.outgoingRequestDecorator
+      discard msg.createHandshakeTemplate(sendProc.def.name, handshakeImpl, nextMsg)
 
     protocol.outProcRegistrations.add(
       newCall(registerMsg,

eth/p2p/rlpx_protocols/les_protocol.nim

@@ -9,7 +9,7 @@
 #
 
 import
-  times, tables, options, sets, hashes, strutils, macros,
+  times, tables, options, sets, hashes, strutils, std_shims/macros_shim,
   chronicles, chronos, nimcrypto/[keccak, hash],
   eth/[rlp, keys], eth/common/eth_types,
   ../rlpx, ../kademlia, ../private/p2p_types, ../blockchain_utils,
@@ -19,7 +19,7 @@ export
   les_types
 
 const
-  lesVersion = 2'u
+  lesVersion = 2
   maxHeadersFetch = 192
   maxBodiesFetch = 32
   maxReceiptsFetch = 128
@@ -92,11 +92,11 @@ template costQuantity(quantityExpr, max: untyped) {.pragma.}
 proc getCostQuantity(fn: NimNode): tuple[quantityExpr, maxQuantity: NimNode] =
   # XXX: `getCustomPragmaVal` doesn't work yet on regular nnkProcDef nodes
   # (TODO: file as an issue)
-  let p = fn.pragma
+  let costQuantity = fn.pragma.findPragma(bindSym"costQuantity")
-  doAssert p.kind == nnkPragma and p.len > 0 and $p[0][0] == "costQuantity"
+  doAssert costQuantity != nil
 
-  result.quantityExpr = p[0][1]
+  result.quantityExpr = costQuantity[1]
-  result.maxQuantity= p[0][2]
+  result.maxQuantity= costQuantity[2]
 
   if result.maxQuantity.kind == nnkExprEqExpr:
     result.maxQuantity = result.maxQuantity[1]
@@ -106,8 +106,8 @@ macro outgoingRequestDecorator(n: untyped): untyped =
   let (costQuantity, maxQuantity) = n.getCostQuantity
 
   result.body.add quote do:
-    trackOutgoingRequest(msgRecipient.networkState(les),
+    trackOutgoingRequest(peer.networkState(les),
-                         msgRecipient.state(les),
+                         peer.state(les),
                          perProtocolMsgId, reqId, `costQuantity`)
   # echo result.repr
 
@@ -115,7 +115,7 @@ macro incomingResponseDecorator(n: untyped): untyped =
   result = n
 
   let trackingCall = quote do:
-    trackIncomingResponse(msgSender.state(les), reqId, msg.bufValue)
+    trackIncomingResponse(peer.state(les), reqId, msg.bufValue)
 
   result.body.insert(n.body.len - 1, trackingCall)
   # echo result.repr
@@ -216,7 +216,7 @@ p2pProtocol les(version = lesVersion,
                            "Incompatibility detected! $1 mismatch ($2 != $3)" %
                            [varName, $localVar, $peerVar])
 
-    requireCompatibility(peerLesVersion,  lesVersion,        "les version")
+    requireCompatibility(peerLesVersion,  uint(lesVersion),  "les version")
     requireCompatibility(peerNetworkId,   network.networkId, "network id")
     requireCompatibility(peerGenesisHash, chain.genesisHash, "genesis hash")