more temporary changes

.gitmodules

@@ -184,3 +184,8 @@
 	url = https://github.com/logos-messaging/waku-rlnv2-contract.git
 	ignore = untracked
 	branch = master
+[submodule "vendor/nim-ffi"]
+	path = vendor/nim-ffi
+	url = https://github.com/waku-org/nim-ffi/
+	ignore = untracked
+	branch = main

library/alloc.nim

@@ -1,42 +0,0 @@
-## Can be shared safely between threads
-type SharedSeq*[T] = tuple[data: ptr UncheckedArray[T], len: int]
-
-proc alloc*(str: cstring): cstring =
-  # Byte allocation from the given address.
-  # There should be the corresponding manual deallocation with deallocShared !
-  if str.isNil():
-    var ret = cast[cstring](allocShared(1)) # Allocate memory for the null terminator
-    ret[0] = '\0' # Set the null terminator
-    return ret
-
-  let ret = cast[cstring](allocShared(len(str) + 1))
-  copyMem(ret, str, len(str) + 1)
-  return ret
-
-proc alloc*(str: string): cstring =
-  ## Byte allocation from the given address.
-  ## There should be the corresponding manual deallocation with deallocShared !
-  var ret = cast[cstring](allocShared(str.len + 1))
-  let s = cast[seq[char]](str)
-  for i in 0 ..< str.len:
-    ret[i] = s[i]
-  ret[str.len] = '\0'
-  return ret
-
-proc allocSharedSeq*[T](s: seq[T]): SharedSeq[T] =
-  let data = allocShared(sizeof(T) * s.len)
-  if s.len != 0:
-    copyMem(data, unsafeAddr s[0], s.len)
-  return (cast[ptr UncheckedArray[T]](data), s.len)
-
-proc deallocSharedSeq*[T](s: var SharedSeq[T]) =
-  deallocShared(s.data)
-  s.len = 0
-
-proc toSeq*[T](s: SharedSeq[T]): seq[T] =
-  ## Creates a seq[T] from a SharedSeq[T]. No explicit dealloc is required
-  ## as req[T] is a GC managed type.
-  var ret = newSeq[T]()
-  for i in 0 ..< s.len:
-    ret.add(s.data[i])
-  return ret

library/ffi_types.nim

@@ -1,30 +0,0 @@
-################################################################################
-### Exported types
-
-type WakuCallBack* = proc(
-  callerRet: cint, msg: ptr cchar, len: csize_t, userData: pointer
-) {.cdecl, gcsafe, raises: [].}
-
-const RET_OK*: cint = 0
-const RET_ERR*: cint = 1
-const RET_MISSING_CALLBACK*: cint = 2
-
-### End of exported types
-################################################################################
-
-################################################################################
-### FFI utils
-
-template foreignThreadGc*(body: untyped) =
-  when declared(setupForeignThreadGc):
-    setupForeignThreadGc()
-
-  body
-
-  when declared(tearDownForeignThreadGc):
-    tearDownForeignThreadGc()
-
-type onDone* = proc()
-
-### End of FFI utils
-################################################################################

library/waku_context.nim

@@ -3,39 +3,24 @@
 {.passc: "-fPIC".}
 
 import std/[options, atomics, os, net, locks]
-import chronicles, chronos, chronos/threadsync, taskpools/channels_spsc_single, results
+import
+  chronicles, chronos, chronos/threadsync, taskpools/channels_spsc_single, results, ffi
 import
   waku/common/logging,
   waku/factory/waku,
-  waku/node/peer_manager,
+  # waku/node/peer_manager,
   waku/waku_relay/[protocol, topic_health],
   waku/waku_core/[topics/pubsub_topic, message],
   ./waku_thread_requests/[waku_thread_request, requests/debug_node_request],
-  ./ffi_types,
   ./events/[
     json_message_event, json_topic_health_change_event, json_connection_change_event,
     json_waku_not_responding_event,
   ]
 
-type WakuContext* = object
-  wakuThread: Thread[(ptr WakuContext)]
-  watchdogThread: Thread[(ptr WakuContext)]
-    # monitors the Waku thread and notifies the Waku SDK consumer if it hangs
-  lock: Lock
-  reqChannel: ChannelSPSCSingle[ptr WakuThreadRequest]
-  reqSignal: ThreadSignalPtr
-    # to inform The Waku Thread (a.k.a TWT) that a new request is sent
-  reqReceivedSignal: ThreadSignalPtr
-    # to inform the main thread that the request is rx by TWT
-  userData*: pointer
-  eventCallback*: pointer
-  eventUserdata*: pointer
-  running: Atomic[bool] # To control when the threads are running
-
 const git_version* {.strdefine.} = "n/a"
 const versionString = "version / git commit hash: " & waku.git_version
 
-template callEventCallback(ctx: ptr WakuContext, eventName: string, body: untyped) =
+template callEventCallback(ctx: ptr FFIContext, eventName: string, body: untyped) =
   if isNil(ctx[].eventCallback):
     error eventName & " - eventCallback is nil"
     return
@@ -43,41 +28,41 @@ template callEventCallback(ctx: ptr WakuContext, eventName: string, body: untype
   foreignThreadGc:
     try:
       let event = body
-      cast[WakuCallBack](ctx[].eventCallback)(
+      cast[FFICallBack](ctx[].eventCallback)(
         RET_OK, unsafeAddr event[0], cast[csize_t](len(event)), ctx[].eventUserData
       )
     except Exception, CatchableError:
       let msg =
         "Exception " & eventName & " when calling 'eventCallBack': " &
         getCurrentExceptionMsg()
-      cast[WakuCallBack](ctx[].eventCallback)(
+      cast[FFICallBack](ctx[].eventCallback)(
         RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), ctx[].eventUserData
       )
 
-proc onConnectionChange*(ctx: ptr WakuContext): ConnectionChangeHandler =
+# proc onConnectionChange*(ctx: ptr FFIContext): ConnectionChangeHandler =
-  return proc(peerId: PeerId, peerEvent: PeerEventKind) {.async.} =
+#   return proc(peerId: PeerId, peerEvent: PeerEventKind) {.async.} =
-    callEventCallback(ctx, "onConnectionChange"):
+#     callEventCallback(ctx, "onConnectionChange"):
-      $JsonConnectionChangeEvent.new($peerId, peerEvent)
+#       $JsonConnectionChangeEvent.new($peerId, peerEvent)
 
-proc onReceivedMessage*(ctx: ptr WakuContext): WakuRelayHandler =
+proc onReceivedMessage*(ctx: ptr FFIContext): WakuRelayHandler =
   return proc(pubsubTopic: PubsubTopic, msg: WakuMessage) {.async.} =
     callEventCallback(ctx, "onReceivedMessage"):
       $JsonMessageEvent.new(pubsubTopic, msg)
 
-proc onTopicHealthChange*(ctx: ptr WakuContext): TopicHealthChangeHandler =
+proc onTopicHealthChange*(ctx: ptr FFIContext): TopicHealthChangeHandler =
   return proc(pubsubTopic: PubsubTopic, topicHealth: TopicHealth) {.async.} =
     callEventCallback(ctx, "onTopicHealthChange"):
       $JsonTopicHealthChangeEvent.new(pubsubTopic, topicHealth)
 
-proc onWakuNotResponding*(ctx: ptr WakuContext) =
+proc onWakuNotResponding*(ctx: ptr FFIContext) =
   callEventCallback(ctx, "onWakuNotResponsive"):
     $JsonWakuNotRespondingEvent.new()
 
 proc sendRequestToWakuThread*(
-    ctx: ptr WakuContext,
+    ctx: ptr FFIContext,
     reqType: RequestType,
     reqContent: pointer,
-    callback: WakuCallBack,
+    callback: FFICallBack,
     userData: pointer,
     timeout = InfiniteDuration,
 ): Result[void, string] =
@@ -113,10 +98,10 @@ proc sendRequestToWakuThread*(
   ## process proc. See the 'waku_thread_request.nim' module for more details.
   ok()
 
-proc watchdogThreadBody(ctx: ptr WakuContext) {.thread.} =
+proc watchdogThreadBody(ctx: ptr FFIContext) {.thread.} =
   ## Watchdog thread that monitors the Waku thread and notifies the library user if it hangs.
 
-  let watchdogRun = proc(ctx: ptr WakuContext) {.async.} =
+  let watchdogRun = proc(ctx: ptr FFIContext) {.async.} =
     const WatchdogStartDelay = 10.seconds
     const WatchdogTimeinterval = 1.seconds
     const WakuNotRespondingTimeout = 3.seconds
@@ -127,7 +112,7 @@ proc watchdogThreadBody(ctx: ptr WakuContext) {.thread.} =
       await sleepAsync(WatchdogTimeinterval)
 
       if ctx.running.load == false:
-        info "Watchdog thread exiting because WakuContext is not running"
+        info "Watchdog thread exiting because FFIContext is not running"
         break
 
       let wakuCallback = proc(
@@ -151,12 +136,12 @@ proc watchdogThreadBody(ctx: ptr WakuContext) {.thread.} =
 
   waitFor watchdogRun(ctx)
 
-proc wakuThreadBody(ctx: ptr WakuContext) {.thread.} =
+proc wakuThreadBody(ctx: ptr FFIContext) {.thread.} =
   ## Waku thread that attends library user requests (stop, connect_to, etc.)
 
   logging.setupLog(logging.LogLevel.DEBUG, logging.LogFormat.TEXT)
 
-  let wakuRun = proc(ctx: ptr WakuContext) {.async.} =
+  let wakuRun = proc(ctx: ptr FFIContext) {.async.} =
     var waku: Waku
     while true:
       await ctx.reqSignal.wait()
@@ -179,10 +164,10 @@ proc wakuThreadBody(ctx: ptr WakuContext) {.thread.} =
 
   waitFor wakuRun(ctx)
 
-proc createWakuContext*(): Result[ptr WakuContext, string] =
+proc createWakuContext*[T](): Result[ptr FFIContext, string] =
   ## This proc is called from the main thread and it creates
   ## the Waku working thread.
-  var ctx = createShared(WakuContext, 1)
+  var ctx = createShared(FFIContext, 1)
   ctx.reqSignal = ThreadSignalPtr.new().valueOr:
     return err("couldn't create reqSignal ThreadSignalPtr")
   ctx.reqReceivedSignal = ThreadSignalPtr.new().valueOr:
@@ -205,7 +190,7 @@ proc createWakuContext*(): Result[ptr WakuContext, string] =
 
   return ok(ctx)
 
-proc destroyWakuContext*(ctx: ptr WakuContext): Result[void, string] =
+proc destroyWakuContext*(ctx: ptr FFIContext): Result[void, string] =
   ctx.running.store(false)
 
   let signaledOnTime = ctx.reqSignal.fireSync().valueOr:

library/waku_thread_requests/requests/discovery_request.nim

@@ -1,13 +1,12 @@
 import std/json
-import chronos, chronicles, results, strutils, libp2p/multiaddress
+import chronos, chronicles, results, strutils, libp2p/multiaddress, ffi
 import
   ../../../waku/factory/waku,
   ../../../waku/discovery/waku_dnsdisc,
   ../../../waku/discovery/waku_discv5,
   ../../../waku/waku_core/peers,
-  ../../../waku/node/waku_node,
+  ../../../waku/node/waku_node
   ../../../waku/node/kernel_api,
-  ../../alloc
 
 type DiscoveryMsgType* = enum
   GET_BOOTSTRAP_NODES

library/waku_thread_requests/requests/node_lifecycle_request.nim

@@ -1,5 +1,5 @@
 import std/[options, json, strutils, net]
-import chronos, chronicles, results, confutils, confutils/std/net
+import chronos, chronicles, results, confutils, confutils/std/net, ffi
 
 import
   waku/node/peer_manager/peer_manager,
@@ -8,10 +8,9 @@ import
   waku/factory/node_factory,
   waku/factory/networks_config,
   waku/factory/app_callbacks,
-  waku/rest_api/endpoint/builder
+  waku/waku_api/rest/builder
 
 import 
-  ../../alloc
 
 type NodeLifecycleMsgType* = enum
   CREATE_NODE
@@ -30,7 +29,6 @@ proc createShared*(
     appCallbacks: AppCallbacks = nil,
 ): ptr type T =
   var ret = createShared(T)
-  ret[].operation = op
   ret[].appCallbacks = appCallbacks
   ret[].configJson = configJson.alloc()
   return ret

library/waku_thread_requests/requests/peer_manager_request.nim

@@ -1,9 +1,8 @@
 import std/[sequtils, strutils, tables]
-import chronicles, chronos, results, options, json
+import chronicles, chronos, results, options, json, ffi
 import
   ../../../waku/factory/waku,
   ../../../waku/node/waku_node,
-  ../../alloc,
   ../../../waku/node/peer_manager
 
 type PeerManagementMsgType* {.pure.} = enum
@@ -57,7 +56,7 @@ proc destroyShared(self: ptr PeerManagementRequest) =
   deallocShared(self)
 
 proc process*(
-    self: ptr PeerManagementRequest, waku: Waku
+    self: ptr PeerManagementRequest, waku: ptr Waku
 ): Future[Result[string, string]] {.async.} =
   defer:
     destroyShared(self)
@@ -65,18 +64,18 @@ proc process*(
   case self.operation
   of CONNECT_TO:
     let peers = ($self[].peerMultiAddr).split(",").mapIt(strip(it))
-    await waku.node.connectToNodes(peers, source = "static")
+    await waku[].node.connectToNodes(peers, source = "static")
     return ok("")
   of GET_ALL_PEER_IDS:
     ## returns a comma-separated string of peerIDs
     let peerIDs =
-      waku.node.peerManager.switch.peerStore.peers().mapIt($it.peerId).join(",")
+      waku[].node.peerManager.switch.peerStore.peers().mapIt($it.peerId).join(",")
     return ok(peerIDs)
   of GET_CONNECTED_PEERS_INFO:
     ## returns a JSON string mapping peerIDs to objects with protocols and addresses
 
     var peersMap = initTable[string, PeerInfo]()
-    let peers = waku.node.peerManager.switch.peerStore.peers().filterIt(
+    let peers = waku[].node.peerManager.switch.peerStore.peers().filterIt(
         it.connectedness == Connected
       )
 
@@ -92,7 +91,7 @@ proc process*(
     return ok(jsonStr)
   of GET_PEER_IDS_BY_PROTOCOL:
     ## returns a comma-separated string of peerIDs that mount the given protocol
-    let connectedPeers = waku.node.peerManager.switch.peerStore
+    let connectedPeers = waku[].node.peerManager.switch.peerStore
       .peers($self[].protocol)
       .filterIt(it.connectedness == Connected)
       .mapIt($it.peerId)
@@ -102,16 +101,16 @@ proc process*(
     let peerId = PeerId.init($self[].peerId).valueOr:
       error "DISCONNECT_PEER_BY_ID failed", error = $error
       return err($error)
-    await waku.node.peerManager.disconnectNode(peerId)
+    await waku[].node.peerManager.disconnectNode(peerId)
     return ok("")
   of DISCONNECT_ALL_PEERS:
-    await waku.node.peerManager.disconnectAllPeers()
+    await waku[].node.peerManager.disconnectAllPeers()
     return ok("")
   of DIAL_PEER:
     let remotePeerInfo = parsePeerInfo($self[].peerMultiAddr).valueOr:
       error "DIAL_PEER failed", error = $error
       return err($error)
-    let conn = await waku.node.peerManager.dialPeer(remotePeerInfo, $self[].protocol)
+    let conn = await waku[].node.peerManager.dialPeer(remotePeerInfo, $self[].protocol)
     if conn.isNone():
       let msg = "failed dialing peer"
       error "DIAL_PEER failed", error = msg, peerId = $remotePeerInfo.peerId
@@ -120,7 +119,7 @@ proc process*(
     let peerId = PeerId.init($self[].peerId).valueOr:
       error "DIAL_PEER_BY_ID failed", error = $error
       return err($error)
-    let conn = await waku.node.peerManager.dialPeer(peerId, $self[].protocol)
+    let conn = await waku[].node.peerManager.dialPeer(peerId, $self[].protocol)
     if conn.isNone():
       let msg = "failed dialing peer"
       error "DIAL_PEER_BY_ID failed", error = msg, peerId = $peerId
@@ -128,7 +127,7 @@ proc process*(
   of GET_CONNECTED_PEERS:
     ## returns a comma-separated string of peerIDs
     let
-      (inPeerIds, outPeerIds) = waku.node.peerManager.connectedPeers()
+      (inPeerIds, outPeerIds) = waku[].node.peerManager.connectedPeers()
       connectedPeerids = concat(inPeerIds, outPeerIds)
     return ok(connectedPeerids.mapIt($it).join(","))
 

library/waku_thread_requests/requests/ping_request.nim

@@ -1,7 +1,7 @@
 import std/[json, strutils]
-import chronos, results
+import chronos, results, ffi
 import libp2p/[protocols/ping, switch, multiaddress, multicodec]
-import ../../../waku/[factory/waku, waku_core/peers, node/waku_node], ../../alloc
+import ../../../waku/[factory/waku, waku_core/peers, node/waku_node]
 
 type PingRequest* = object
   peerAddr: cstring

library/waku_thread_requests/requests/protocols/filter_request.nim

@@ -1,5 +1,5 @@
 import options, std/[strutils, sequtils]
-import chronicles, chronos, results
+import chronicles, chronos, results, ffi
 import
   ../../../../waku/waku_filter_v2/client,
   ../../../../waku/waku_core/message/message,
@@ -10,8 +10,7 @@ import
   ../../../../waku/node/waku_node,
   ../../../../waku/node/kernel_api,
   ../../../../waku/waku_core/topics/pubsub_topic,
-  ../../../../waku/waku_core/topics/content_topic,
+  ../../../../waku/waku_core/topics/content_topic
-  ../../../alloc
 
 type FilterMsgType* = enum
   SUBSCRIBE

library/waku_thread_requests/requests/protocols/lightpush_request.nim

@@ -1,5 +1,5 @@
 import options
-import chronicles, chronos, results
+import chronicles, chronos, results, ffi
 import
   ../../../../waku/waku_core/message/message,
   ../../../../waku/waku_core/codecs,
@@ -9,8 +9,7 @@ import
   ../../../../waku/waku_core/topics/pubsub_topic,
   ../../../../waku/waku_lightpush_legacy/client,
   ../../../../waku/waku_lightpush_legacy/common,
-  ../../../../waku/node/peer_manager/peer_manager,
+  ../../../../waku/node/peer_manager/peer_manager
-  ../../../alloc
 
 type LightpushMsgType* = enum
   PUBLISH

library/waku_thread_requests/requests/protocols/relay_request.nim

@@ -1,5 +1,5 @@
 import std/[net, sequtils, strutils]
-import chronicles, chronos, stew/byteutils, results
+import chronicles, chronos, stew/byteutils, results, ffi
 import
   waku/waku_core/message/message,
   waku/factory/[validator_signed, waku],
@@ -13,7 +13,6 @@ import
   waku/node/peer_manager
   
 import
-  ../../../alloc
 
 type RelayMsgType* = enum
   SUBSCRIBE

library/waku_thread_requests/requests/protocols/store_request.nim

@@ -1,8 +1,7 @@
 import std/[json, sugar, strutils, options]
-import chronos, chronicles, results, stew/byteutils
+import chronos, chronicles, results, stew/byteutils, ffi
 import
   ../../../../waku/factory/waku,
-  ../../../alloc,
   ../../../utils,
   ../../../../waku/waku_core/peers,
   ../../../../waku/waku_core/time,

library/waku_thread_requests/waku_thread_request.nim

@@ -3,10 +3,9 @@
 ## the Waku Thread.
 
 import std/json, results
-import chronos, chronos/threadsync
+import chronos, chronos/threadsync, ffi
 import
   ../../waku/factory/waku,
-  ../ffi_types,
   ./requests/node_lifecycle_request,
   ./requests/peer_manager_request,
   ./requests/protocols/relay_request,
@@ -31,14 +30,14 @@ type RequestType* {.pure.} = enum
 type WakuThreadRequest* = object
   reqType: RequestType
   reqContent: pointer
-  callback: WakuCallBack
+  callback: FFICallBack
   userData: pointer
 
 proc createShared*(
     T: type WakuThreadRequest,
     reqType: RequestType,
     reqContent: pointer,
-    callback: WakuCallBack,
+    callback: FFICallBack,
     userData: pointer,
 ): ptr type T =
   var ret = createShared(T)
@@ -78,7 +77,7 @@ proc process*(
     T: type WakuThreadRequest, request: ptr WakuThreadRequest, waku: ptr Waku
 ) {.async.} =
   let retFut =
-    case request[].reqType
+    case request[].reqId
     of LIFECYCLE:
       cast[ptr NodeLifecycleRequest](request[].reqContent).process(waku)
     of PEER_MANAGER:

vendor/nim-ffi

@@ -0,0 +1 @@
+Subproject commit fc743fbbdd2423d6f541e78cbd5c278cac8137ce

waku.nimble

@@ -31,6 +31,7 @@ requires "nim >= 2.2.4",
   "results",
   "db_connector",
   "minilru"
+  "ffi"
 
 ### Helper functions
 proc buildModule(filePath, params = "", lang = "c"): bool =