Revert #544 "Fix related to nim devel branch" (#551)

This reverts commit:
5d13052dd9
e1bdf1741a
d238693571

eth/keyfile/keyfile.nim

@@ -93,10 +93,7 @@ type
   KfResult*[T] = Result[T, KeyFileError]
 
 proc mapErrTo[T, E](r: Result[T, E], v: static KeyFileError): KfResult[T] =
-  if r.isOk:
+  r.mapErr(proc (e: E): KeyFileError = v)
-    ok(r.value)
-  else:
-    err(v)
 
 const
   SupportedHashes = [

eth/p2p.nim

@@ -106,7 +106,7 @@ proc newEthereumNode*(
     result.protocolVersion = if useCompression: devp2pSnappyVersion
                              else: devp2pVersion
 
-  result.protocolStates.newSeq protocolCount()
+  result.protocolStates.newSeq allProtocols.len
 
   result.peerPool = newPeerPool(
     result, networkId, keys, nil, clientId, minPeers = minPeers)
@@ -114,8 +114,8 @@ proc newEthereumNode*(
   result.peerPool.discovery = result.discovery
 
   if addAllCapabilities:
-    for cap in protocols():
+    for p in allProtocols:
-      result.addCapability(cap)
+      result.addCapability(p)
 
 proc processIncoming(server: StreamServer,
                      remote: StreamTransport): Future[void] {.async, gcsafe.} =

eth/p2p/discovery.nim

@@ -132,7 +132,6 @@ proc sendNeighbours*(d: DiscoveryProtocol, node: Node, neighbours: seq[Node]) =
   const MAX_NEIGHBOURS_PER_PACKET = 12 # TODO: Implement a smarter way to compute it
   type Neighbour = tuple[ip: IpAddress, udpPort, tcpPort: Port, pk: PublicKey]
   var nodes = newSeqOfCap[Neighbour](MAX_NEIGHBOURS_PER_PACKET)
-  when not defined(nimSeqsV2):
   shallow(nodes)
 
   template flush() =

eth/p2p/discoveryv5/protocol.nim

@@ -211,7 +211,7 @@ proc randomNodes*(d: Protocol, maxAmount: int): seq[Node] =
   d.routingTable.randomNodes(maxAmount)
 
 proc randomNodes*(d: Protocol, maxAmount: int,
-    pred: proc(x: Node): bool {.gcsafe, noSideEffect, raises:[Defect].}): seq[Node] =
+    pred: proc(x: Node): bool {.gcsafe, noSideEffect.}): seq[Node] =
   ## Get a `maxAmount` of random nodes from the local routing table with the
   ## `pred` predicate function applied as filter on the nodes selected.
   d.routingTable.randomNodes(maxAmount, pred)

eth/p2p/discoveryv5/routing_table.nim

@@ -482,10 +482,10 @@ proc len*(r: RoutingTable): int =
 proc moveRight[T](arr: var openArray[T], a, b: int) =
   ## In `arr` move elements in range [a, b] right by 1.
   var t: T
-  t = system.move(arr[b + 1])
+  shallowCopy(t, arr[b + 1])
   for i in countdown(b, a):
-    arr[i + 1] = system.move(arr[i])
+    shallowCopy(arr[i + 1], arr[i])
-  arr[a] = system.move(t)
+  shallowCopy(arr[a], t)
 
 proc setJustSeen*(r: RoutingTable, n: Node) =
   ## Move `n` to the head (most recently seen) of its bucket.
@@ -512,7 +512,7 @@ proc nodeToRevalidate*(r: RoutingTable): Node =
       return b.nodes[^1]
 
 proc randomNodes*(r: RoutingTable, maxAmount: int,
-    pred: proc(x: Node): bool {.gcsafe, noSideEffect, raises:[Defect].} = nil): seq[Node] =
+    pred: proc(x: Node): bool {.gcsafe, noSideEffect.} = nil): seq[Node] =
   ## Get a `maxAmount` of random nodes from the routing table with the `pred`
   ## predicate function applied as filter on the nodes selected.
   var maxAmount = maxAmount

eth/p2p/p2p_backends_helpers.nim

@@ -1,33 +1,9 @@
-let protocolManager = ProtocolManager()
+var
+  gProtocols: seq[ProtocolInfo]
 
 # The variables above are immutable RTTI information. We need to tell
 # Nim to not consider them GcSafe violations:
-
+template allProtocols*: auto = {.gcsafe.}: gProtocols
-proc registerProtocol*(proto: ProtocolInfo) {.gcsafe.} =
-  {.gcsafe.}:
-    proto.index = protocolManager.protocols.len
-    if proto.name == "p2p":
-      doAssert(proto.index == 0)
-    protocolManager.protocols.add proto
-
-proc protocolCount*(): int {.gcsafe.} =
-  {.gcsafe.}:
-    protocolManager.protocols.len
-
-proc getProtocol*(index: int): ProtocolInfo {.gcsafe.} =
-  {.gcsafe.}:
-    protocolManager.protocols[index]
-
-iterator protocols*(): ProtocolInfo {.gcsafe.} =
-  {.gcsafe.}:
-    for x in protocolManager.protocols:
-      yield x
-
-template getProtocol*(Protocol: type): ProtocolInfo =
-  getProtocol(Protocol.index)
-
-template devp2pInfo*(): ProtocolInfo =
-  getProtocol(0)
 
 proc getState*(peer: Peer, proto: ProtocolInfo): RootRef =
   peer.protocolStates[proto.index]
@@ -59,8 +35,9 @@ proc initProtocolState*[T](state: T, x: Peer|EthereumNode)
 proc initProtocolStates(peer: Peer, protocols: openArray[ProtocolInfo])
     {.raises: [Defect].} =
   # Initialize all the active protocol states
-  newSeq(peer.protocolStates, protocolCount())
+  newSeq(peer.protocolStates, allProtocols.len)
   for protocol in protocols:
     let peerStateInit = protocol.peerStateInitializer
     if peerStateInit != nil:
       peer.protocolStates[protocol.index] = peerStateInit(peer)
+

eth/p2p/p2p_protocol_dsl.nim

@@ -1,7 +1,7 @@
 {.push raises: [Defect].}
 
 import
-  std/[options, sequtils, macrocache],
+  std/[options, sequtils],
   stew/shims/macros, chronos, faststreams/outputs
 
 type
@@ -76,7 +76,7 @@ type
 
     # Cached properties
     nameIdent*: NimNode
-    protocolInfo*: NimNode
+    protocolInfoVar*: NimNode
 
     # All messages
     messages*: seq[Message]
@@ -146,9 +146,6 @@ let
   PROTO                 {.compileTime.} = ident "PROTO"
   MSG                   {.compileTime.} = ident "MSG"
 
-const
-  protocolCounter = CacheCounter"protocolCounter"
-
 template Opt(T): auto = newTree(nnkBracketExpr, Option, T)
 template Fut(T): auto = newTree(nnkBracketExpr, Future, T)
 
@@ -314,7 +311,7 @@ proc init*(T: type P2PProtocol, backendFactory: BackendFactory,
     PeerStateType: verifyStateType peerState,
     NetworkStateType: verifyStateType networkState,
     nameIdent: ident(name),
-    protocolInfo: newCall(ident("protocolInfo"), ident(name)),
+    protocolInfoVar: ident(name & "Protocol"),
     outSendProcs: newStmtList(),
     outRecvProcs: newStmtList(),
     outProcRegistrations: newStmtList())
@@ -346,7 +343,7 @@ proc augmentUserHandler(p: P2PProtocol, userHandlerProc: NimNode, msgId = -1) =
   var
     getState = ident"getState"
     getNetworkState = ident"getNetworkState"
-    protocolInfo = p.protocolInfo
+    protocolInfoVar = p.protocolInfoVar
     protocolNameIdent = p.nameIdent
     PeerType = p.backend.PeerType
     PeerStateType = p.PeerStateType
@@ -373,12 +370,12 @@ proc augmentUserHandler(p: P2PProtocol, userHandlerProc: NimNode, msgId = -1) =
   if PeerStateType != nil:
     prelude.add quote do:
       template state(`peerVar`: `PeerType`): `PeerStateType` =
-        `PeerStateType`(`getState`(`peerVar`, `protocolInfo`))
+        cast[`PeerStateType`](`getState`(`peerVar`, `protocolInfoVar`))
 
   if NetworkStateType != nil:
     prelude.add quote do:
       template networkState(`peerVar`: `PeerType`): `NetworkStateType` =
-        `NetworkStateType`(`getNetworkState`(`peerVar`.network, `protocolInfo`))
+        cast[`NetworkStateType`](`getNetworkState`(`peerVar`.network, `protocolInfoVar`))
 
 proc addPreludeDefs*(userHandlerProc: NimNode, definitions: NimNode) =
   userHandlerProc.body[0].add definitions
@@ -702,7 +699,7 @@ proc useStandardBody*(sendProc: SendProc,
                 newStmtList()
               else:
                 logSentMsgFields(recipient,
-                                 msg.protocol.protocolInfo,
+                                 msg.protocol.protocolInfoVar,
                                  $msg.ident,
                                  sendProc.msgParams)
 
@@ -898,24 +895,16 @@ proc processProtocolBody*(p: P2PProtocol, protocolBody: NimNode) =
 
 proc genTypeSection*(p: P2PProtocol): NimNode =
   var
-    protocolIdx = protocolCounter.value
     protocolName = p.nameIdent
     peerState = p.PeerStateType
     networkState= p.NetworkStateType
 
-  protocolCounter.inc
   result = newStmtList()
   result.add quote do:
     # Create a type acting as a pseudo-object representing the protocol
     # (e.g. p2p)
     type `protocolName`* = object
 
-    # The protocol run-time index is available as a pseudo-field
-    # (e.g. `p2p.index`)
-    template index*(`PROTO`: type `protocolName`): auto = `protocolIdx`
-    template protocolInfo*(`PROTO`: type `protocolName`): auto =
-      getProtocol(`protocolIdx`)
-
   if peerState != nil:
     result.add quote do:
       template State*(`PROTO`: type `protocolName`): type = `peerState`
@@ -960,29 +949,33 @@ proc genCode*(p: P2PProtocol): NimNode =
   result.add p.genTypeSection()
 
   let
+    protocolInfoVar = p.protocolInfoVar
+    protocolInfoVarObj = ident($protocolInfoVar & "Obj")
+    protocolName = p.nameIdent
     protocolInit = p.backend.implementProtocolInit(p)
-    protocolReg  = ident($p.nameIdent & "Registration")
+
-    regBody      = newStmtList()
+  result.add quote do:
+    # One global variable per protocol holds the protocol run-time data
+    var `protocolInfoVarObj` = `protocolInit`
+    var `protocolInfoVar` = addr `protocolInfoVarObj`
+
+    # The protocol run-time data is available as a pseudo-field
+    # (e.g. `p2p.protocolInfo`)
+    template protocolInfo*(`PROTO`: type `protocolName`): auto = `protocolInfoVar`
 
   result.add p.outSendProcs,
-             p.outRecvProcs
+             p.outRecvProcs,
+             p.outProcRegistrations
 
   if p.onPeerConnected != nil: result.add p.onPeerConnected
   if p.onPeerDisconnected != nil: result.add p.onPeerDisconnected
 
-  regBody.add newCall(p.backend.setEventHandlers,
+  result.add newCall(p.backend.setEventHandlers,
-                      protocolVar,
+                     protocolInfoVar,
                      nameOrNil p.onPeerConnected,
                      nameOrNil p.onPeerDisconnected)
 
-  regBody.add p.outProcRegistrations
+  result.add newCall(p.backend.registerProtocol, protocolInfoVar)
-  regBody.add newCall(p.backend.registerProtocol, protocolVar)
-
-  result.add quote do:
-    proc `protocolReg`() {.raises: [RlpError, Defect].} =
-      let `protocolVar` = `protocolInit`
-      `regBody`
-    `protocolReg`()
 
 macro emitForSingleBackend(
     name: static[string],

eth/p2p/private/p2p_types.nim

@@ -93,10 +93,7 @@ type
   ## Quasy-private types. Use at your own risk.
   ##
 
-  ProtocolManager* = ref object
+  ProtocolInfoObj* = object
-    protocols*: seq[ProtocolInfo]
-
-  ProtocolInfo* = ref object
     name*: string
     version*: int
     messages*: seq[MessageInfo]
@@ -109,7 +106,9 @@ type
     handshake*: HandshakeStep
     disconnectHandler*: DisconnectionHandler
 
-  MessageInfo* = ref object
+  ProtocolInfo* = ptr ProtocolInfoObj
+
+  MessageInfo* = object
     id*: int
     name*: string
 
@@ -133,7 +132,7 @@ type
     # `messages` holds a mapping from valid message IDs to their handler procs.
     #
     protocolOffsets*: seq[int]
-    messages*: seq[MessageInfo]
+    messages*: seq[ptr MessageInfo]
     activeProtocols*: seq[ProtocolInfo]
 
   ##

eth/p2p/rlpx.nim

@@ -192,6 +192,9 @@ proc handshakeImpl[T](peer: Peer,
   else:
     return responseFut.read
 
+var gDevp2pInfo: ProtocolInfo
+template devp2pInfo: auto = {.gcsafe.}: gDevp2pInfo
+
 # Dispatcher
 #
 
@@ -217,7 +220,7 @@ proc getDispatcher(node: EthereumNode,
   # We should be able to find an existing dispatcher without allocating a new one
 
   new result
-  newSeq(result.protocolOffsets, protocolCount())
+  newSeq(result.protocolOffsets, allProtocols.len)
   result.protocolOffsets.fill -1
 
   var nextUserMsgId = 0x10
@@ -234,9 +237,9 @@ proc getDispatcher(node: EthereumNode,
 
   template copyTo(src, dest; index: int) =
     for i in 0 ..< src.len:
-      dest[index + i] = src[i]
+      dest[index + i] = addr src[i]
 
-  result.messages = newSeq[MessageInfo](nextUserMsgId)
+  result.messages = newSeq[ptr MessageInfo](nextUserMsgId)
   devp2pInfo.messages.copyTo(result.messages, 0)
 
   for localProtocol in node.protocols:
@@ -259,35 +262,30 @@ proc getMsgName*(peer: Peer, msgId: int): string =
            of 3: "pong"
            else: $msgId
 
-proc getMsgMetadata*(peer: Peer, msgId: int): (ProtocolInfo, MessageInfo) =
+proc getMsgMetadata*(peer: Peer, msgId: int): (ProtocolInfo, ptr MessageInfo) =
   doAssert msgId >= 0
 
-  let dpInfo = devp2pInfo()
+  if msgId <= devp2pInfo.messages[^1].id:
-  if msgId <= dpInfo.messages[^1].id:
+    return (devp2pInfo, addr devp2pInfo.messages[msgId])
-    return (dpInfo, dpInfo.messages[msgId])
 
   if msgId < peer.dispatcher.messages.len:
-    let numProtocol = protocolCount()
+    for i in 0 ..< allProtocols.len:
-    for i in 0 ..< numProtocol:
-      let protocol = getProtocol(i)
       let offset = peer.dispatcher.protocolOffsets[i]
       if offset != -1 and
-         offset + protocol.messages[^1].id >= msgId:
+         offset + allProtocols[i].messages[^1].id >= msgId:
-        return (protocol, peer.dispatcher.messages[msgId])
+        return (allProtocols[i], peer.dispatcher.messages[msgId])
 
 # Protocol info objects
 #
 
 proc initProtocol(name: string, version: int,
                   peerInit: PeerStateInitializer,
-                  networkInit: NetworkStateInitializer): ProtocolInfo =
+                  networkInit: NetworkStateInitializer): ProtocolInfoObj =
-  ProtocolInfo(
+  result.name = name
-    name    : name,
+  result.version = version
-    version : version,
+  result.messages = @[]
-    messages: @[],
+  result.peerStateInitializer = peerInit
-    peerStateInitializer: peerInit,
+  result.networkStateInitializer = networkInit
-    networkStateInitializer: networkInit
-  )
 
 proc setEventHandlers(p: ProtocolInfo,
                       handshake: HandshakeStep,
@@ -323,6 +321,16 @@ proc registerMsg(protocol: ProtocolInfo,
                                       requestResolver: requestResolver,
                                       nextMsgResolver: nextMsgResolver)
 
+proc registerProtocol(protocol: ProtocolInfo) =
+  # TODO: This can be done at compile-time in the future
+  if protocol.name != "p2p":
+    let pos = lowerBound(gProtocols, protocol)
+    gProtocols.insert(protocol, pos)
+    for i in 0 ..< gProtocols.len:
+      gProtocols[i].index = i
+  else:
+    gDevp2pInfo = protocol
+
 # Message composition and encryption
 #
 
@@ -965,7 +973,7 @@ proc p2pProtocolBackendImpl*(protocol: P2PProtocol): Backend =
       quote: return `sendCall`
 
     let perPeerMsgIdValue = if isSubprotocol:
-      newCall(perPeerMsgIdImpl, peerVar, protocol.protocolInfo, newLit(msgId))
+      newCall(perPeerMsgIdImpl, peerVar, protocol.protocolInfoVar, newLit(msgId))
     else:
       newLit(msgId)
 
@@ -1001,7 +1009,7 @@ proc p2pProtocolBackendImpl*(protocol: P2PProtocol): Backend =
 
     protocol.outProcRegistrations.add(
       newCall(registerMsg,
-              protocolVar,
+              protocol.protocolInfoVar,
               newLit(msgId),
               newLit(msgName),
               thunkName,
@@ -1055,7 +1063,7 @@ proc removePeer(network: EthereumNode, peer: Peer) =
 
 proc callDisconnectHandlers(peer: Peer, reason: DisconnectionReason):
     Future[void] {.async.} =
-  var futures = newSeqOfCap[Future[void]](protocolCount())
+  var futures = newSeqOfCap[Future[void]](allProtocols.len)
 
   for protocol in peer.dispatcher.activeProtocols:
     if protocol.disconnectHandler != nil:
@@ -1136,7 +1144,7 @@ proc postHelloSteps(peer: Peer, h: DevP2P.hello) {.async.} =
   # chance to send any initial packages they might require over
   # the network and to yield on their `nextMsg` waits.
   #
-  var subProtocolsHandshakes = newSeqOfCap[Future[void]](protocolCount())
+  var subProtocolsHandshakes = newSeqOfCap[Future[void]](allProtocols.len)
   for protocol in peer.dispatcher.activeProtocols:
     if protocol.handshake != nil:
       subProtocolsHandshakes.add((protocol.handshake)(peer))

eth/utp/growable_buffer.nim

@@ -21,9 +21,6 @@ type
     items: seq[Option[A]]
     mask: uint32
 
-when not defined(nimHasEffectsOfs):
-  template effectsOf(f: untyped) {.pragma.}
-
 # provided size will always be adjusted to next power of two
 proc init*[A](T: type GrowableCircularBuffer[A], size: uint32 = 16): T =
   let powOfTwoSize = nextPowerOfTwo(int(size))
@@ -47,12 +44,10 @@ proc delete*[A](buff: var GrowableCircularBuffer[A], i: uint32) =
 proc hasKey*[A](buff: GrowableCircularBuffer[A], i: uint32): bool =
   buff.get(i).isSome()
 
-proc exists*[A](buff: GrowableCircularBuffer[A], i: uint32,
+proc exists*[A](buff: GrowableCircularBuffer[A], i: uint32, check: proc (x: A): bool): bool =
-                check: proc (x: A): bool): bool {.gcsafe, effectsOf: check.} =
   let maybeElem = buff.get(i)
   if (maybeElem.isSome()):
     let elem = maybeElem.unsafeGet()
-    {.gcsafe.}:
     check(elem)
   else:
     false

tests/p2p/eth_protocol.nim

@@ -6,7 +6,7 @@ import
 # real eth protocol implementation is in nimbus-eth1 repo
 
 type
-  PeerState = ref object of RootRef
+  PeerState = ref object
     initialized*: bool
 
 p2pProtocol eth(version = 63,

tests/p2p/test_protocol_handlers.nim

@@ -16,12 +16,9 @@ import
   ./p2p_test_helper
 
 type
-  network = ref object of RootRef
+  network = ref object
     count*: int
 
-  PeerState = ref object of RootRef
-    status*: string
-
 p2pProtocol abc(version = 1,
                 rlpxName = "abc",
                 networkState = network):
@@ -36,18 +33,15 @@ p2pProtocol abc(version = 1,
 
 p2pProtocol xyz(version = 1,
                 rlpxName = "xyz",
-                networkState = network,
+                networkState = network):
-                peerState = PeerState):
 
   onPeerConnected do (peer: Peer):
     peer.networkState.count += 1
-    peer.state.status = "connected"
 
   onPeerDisconnected do (peer: Peer, reason: DisconnectionReason) {.gcsafe.}:
     peer.networkState.count -= 1
     if true:
       raise newException(CatchableError, "Fake xyz exception")
-    peer.state.status = "disconnected"
 
 p2pProtocol hah(version = 1,
                 rlpxName = "hah",
@@ -73,7 +67,6 @@ suite "Testing protocol handlers":
     let peer = await node1.rlpxConnect(newNode(node2.toENode()))
     check:
       peer.isNil == false
-      peer.state(xyz).status == "connected"
 
     await peer.disconnect(SubprotocolReason, true)
     check:
@@ -81,7 +74,6 @@ suite "Testing protocol handlers":
       # handlers, each handler still ran
       node1.protocolState(abc).count == 0
       node1.protocolState(xyz).count == 0
-      peer.state(xyz).status == "connected"
 
   asyncTest "Failing connection handler":
     let rng = newRng()