logos-messaging-nim/waku.nimble

#!fmt: off

import os
mode = ScriptMode.Verbose

### Package
version = "0.38.1"
author = "Status Research & Development GmbH"
description = "Waku, Private P2P Messaging for Resource-Restricted Devices"
license = "MIT or Apache License 2.0"

const RequiredNimVersion = "2.2.4"
  ## This is the nim compiler version that we are working on. Other versions may behave differently.
const RequiredNimbleVersion = "0.22.3"
  ## Enforced nimble version to ensure a reproducible flow

### Dependencies
requires "nim >= 2.2.4",
  "chronos >= 4.2.0",
  "taskpools",
  # Logging & Configuration
  "chronicles",
  "confutils",
  # Serialization
  "serialization",
  "json_serialization",
  "toml_serialization",
  "faststreams",
  # Networking & P2P
  "https://github.com/vacp2p/nim-libp2p.git#ff8d51857b4b79a68468e7bcc27b2026cca02996",
  "eth",
  "nat_traversal",
  "dnsdisc",
  "dnsclient",
  "httputils >= 0.4.1",
  "websock >= 0.3.0",
  # Cryptography
  "nimcrypto == 0.6.4", # 0.6.4 used in libp2p. Version 0.7.3 makes test to crash on Ubuntu.
  "secp256k1",
  "bearssl",
  # RPC & APIs
  "https://github.com/status-im/nim-json-rpc.git#43bbf499143eb45046c83ac9794c9e3280a2b8e7",
  "presto",
  "web3",
  # Database
  "db_connector",
  "sqlite3_abi",
  # Utilities
  "stew",
  "stint",
  "metrics",
  "regex",
  "unicodedb",
  "results",
  "minilru",
  "zlib",
  # Debug & Testing
  "testutils",
  "unittest2"

# Packages not on nimble (use git URLs)
requires "https://github.com/logos-messaging/nim-ffi"

requires "https://github.com/logos-messaging/nim-sds.git#2e9a7683f0e180bf112135fae3a3803eed8490d4"

# brokers: pinned by URL+commit rather than the bare `brokers >= 2.0.1`
# form because the nim-lang/packages registry entry for `brokers` only
# carries metadata for the original v0.1.0 publication. Until that
# registry entry is refreshed, the local SAT solver enumerates "0.1.0"
# as the only available version and cannot satisfy `>= 2.0.1`. The URL
# pin below bypasses the registry and locks the exact commit of the
# v2.0.1 tag. Revert to the bare form once nim-lang/packages is
# updated.
requires "https://github.com/NagyZoltanPeter/nim-brokers.git#v2.0.1"

requires "https://github.com/vacp2p/nim-lsquic"
requires "https://github.com/vacp2p/nim-jwt.git#057ec95eb5af0eea9c49bfe9025b3312c95dc5f2"

proc getMyCPU(): string =
  ## Need to set cpu more explicit manner to avoid arch issues between dependencies
  when defined(macosx) and defined(arm64):
    return " --cpu:arm64 --passC:\"-arch arm64\" --passL:\"-arch arm64\" "
  elif defined(macosx) and defined(amd64):
    return " --cpu:amd64 --passC:\"-arch x86_64\" --passL:\"-arch x86_64\" "
  elif defined(arm64):
    return " --cpu:arm64 "
  elif defined(amd64):
    return " --cpu:amd64 "

proc getNimParams(): string =
  return " " & getEnv("NIM_PARAMS") & " "

### Helper functions
proc buildModule(filePath, params = ""): bool =
  if not dirExists "build":
    mkDir "build"

  if not fileExists(filePath):
    echo "File to build not found: " & filePath
    return false

  exec "nim c --out:build/" & filepath & ".bin --mm:refc " & getMyCPU() & getNimParams() & " " & params &
    " " & filePath

  # exec will raise exception if anything goes wrong
  return true

proc buildBinary(name: string, srcDir = "./", params = "") =
  if not dirExists "build":
    mkDir "build"
  exec "nim c --out:build/" & name & " --mm:refc " & getMyCPU() & getNimParams() & " " & params & " " &
    srcDir & name & ".nim"

proc buildLibrary(lib_name: string, srcDir = "./", params = "", `type` = "static", srcFile = "libwaku.nim", mainPrefix = "libwaku") =
  if not dirExists "build":
    mkDir "build"

  if `type` == "static":
    exec "nim c" & " --out:build/" & lib_name &
      " --threads:on --app:staticlib --opt:speed --noMain --mm:refc --header -d:metrics --nimMainPrefix:" & mainPrefix & " --skipParentCfg:on -d:discv5_protocol_id=d5waku " &
      getMyCPU() & getNimParams() & srcDir & "/" & srcFile
  else:
    exec "nim c" & " --out:build/" & lib_name &
      " --threads:on --app:lib --opt:speed --noMain --mm:refc --header -d:metrics --nimMainPrefix:" & mainPrefix & " --skipParentCfg:off -d:discv5_protocol_id=d5waku " &
      getMyCPU() & getNimParams() & " " & srcDir & "/" & srcFile

proc buildLibDynamicWindows(libName: string, folderName: string) =
  buildLibrary libName & ".dll", folderName,
    """-d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE """,
    "dynamic", libName & ".nim", libname

proc buildLibDynamicLinux(libName: string, folderName: string) =
  buildLibrary libName & ".so", folderName,
    """-d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE """,
    "dynamic", libName & ".nim", libname

proc buildLibDynamicMac(libName: string, folderName: string) =
  let sdkPath = staticExec("xcrun --show-sdk-path").strip()
  when defined(arm64):
    let archFlags = "--cpu:arm64 --passC:\"-arch arm64\" --passL:\"-arch arm64\" --passC:\"-isysroot " & sdkPath & "\" --passL:\"-isysroot " & sdkPath & "\""
  elif defined(amd64):
    let archFlags = "--cpu:amd64 --passC:\"-arch x86_64\" --passL:\"-arch x86_64\" --passC:\"-isysroot " & sdkPath & "\" --passL:\"-isysroot " & sdkPath & "\""
  else:
    {.error: "Unsupported macOS architecture".}
  buildLibrary libName & ".dylib", folderName,
    archFlags & " -d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE",
    "dynamic", libName & ".nim", libname

proc buildLibStaticWindows(libName: string, folderName: string) =
  buildLibrary libName & ".lib", folderName,
    """-d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE """,
    "static", libName & ".nim", libname

proc buildLibStaticLinux(libName: string, folderName: string) =
  buildLibrary libName & ".a", folderName,
    """-d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE """,
    "static", libName & ".nim", libname

proc buildLibStaticMac(libName: string, folderName: string) =
  let sdkPath = staticExec("xcrun --show-sdk-path").strip()
  when defined(arm64):
    let archFlags = "--cpu:arm64 --passC:\"-arch arm64\" --passL:\"-arch arm64\" --passC:\"-isysroot " & sdkPath & "\" --passL:\"-isysroot " & sdkPath & "\""
  elif defined(amd64):
    let archFlags = "--cpu:amd64 --passC:\"-arch x86_64\" --passL:\"-arch x86_64\" --passC:\"-isysroot " & sdkPath & "\" --passL:\"-isysroot " & sdkPath & "\""
  else:
    {.error: "Unsupported macOS architecture".}
  buildLibrary libName & ".a", folderName,
    archFlags & " -d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE",
    "static", libName & ".nim", libname

### Mobile Android

proc buildMobileAndroid(srcDir = ".", params = "") =
  let cpu = getEnv("CPU")
  let abiDir = getEnv("ABIDIR")

  let outDir = "build/android/" & abiDir
  if not dirExists outDir:
    mkDir outDir

  exec "nim c" & " --out:" & outDir &
    "/liblogosdelivery.so --threads:on --app:lib --opt:speed --noMain --mm:refc -d:chronicles_sinks=textlines[dynamic] --header -d:chronosEventEngine=epoll --passL:-L" &
    outdir & " --passL:-lrln --passL:-llog --cpu:" & cpu & " --nimMainPrefix:liblogosdelivery --os:android -d:androidNDK " & params &
    getNimParams() & " " & srcDir & "/liblogosdelivery.nim"

task libLogosDeliveryAndroid, "Build the mobile bindings for Android":
  let srcDir = "./library"
  buildMobileAndroid srcDir, "-d:chronicles_log_level=ERROR"

### Mobile iOS

import std/sequtils

proc buildMobileIOS(srcDir = ".", params = "") =
  echo "Building iOS liblogosdelivery library"

  let iosArch = getEnv("IOS_ARCH")
  let iosSdk = getEnv("IOS_SDK")
  let sdkPath = getEnv("IOS_SDK_PATH")

  if sdkPath.len == 0:
    quit "Error: IOS_SDK_PATH not set. Set it to the path of the iOS SDK"

  # Get nimble package paths
  let bearsslPath = gorge("nimble path bearssl").strip()
  let secp256k1Path = gorge("nimble path secp256k1").strip()
  let natTraversalPath = gorge("nimble path nat_traversal").strip()

  # Get Nim standard library path
  let nimPath = gorge("nim --fullhelp 2>&1 | head -1 | sed 's/.*\\[//' | sed 's/\\].*//'").strip()
  let nimLibPath = nimPath.parentDir.parentDir / "lib"

  # Use SDK name in path to differentiate device vs simulator
  let outDir = "build/ios/" & iosSdk & "-" & iosArch
  if not dirExists outDir:
    mkDir outDir

  var extra_params = params
  let args = commandLineParams()
  for arg in args:
    extra_params &= " " & arg

  let cpu = if iosArch == "arm64": "arm64" else: "amd64"

  # The output static library
  let nimcacheDir = outDir & "/nimcache"
  let objDir = outDir & "/obj"
  let vendorObjDir = outDir & "/vendor_obj"
  let aFile = outDir & "/liblogosdelivery.a"

  if not dirExists objDir:
    mkDir objDir
  if not dirExists vendorObjDir:
    mkDir vendorObjDir

  let clangBase = "clang -arch " & iosArch & " -isysroot " & sdkPath &
      " -mios-version-min=18.0 -fembed-bitcode -fPIC -O2"

  # Generate C sources from Nim (no linking)
  exec "nim c" &
      " --nimcache:" & nimcacheDir &
      " --os:ios --cpu:" & cpu &
      " --compileOnly:on" &
      " --noMain --mm:refc" &
      " --threads:on --opt:size --header" &
      " -d:metrics -d:discv5_protocol_id=d5waku" &
      " --nimMainPrefix:liblogosdelivery --skipParentCfg:on" &
      " --cc:clang" &
      " " & extra_params &
      " " & srcDir & "/liblogosdelivery.nim"

  # Compile vendor C libraries for iOS

  # --- BearSSL ---
  echo "Compiling BearSSL for iOS..."
  let bearSslSrcDir = bearsslPath / "bearssl/csources/src"
  let bearSslIncDir = bearsslPath / "bearssl/csources/inc"
  for path in walkDirRec(bearSslSrcDir):
    if path.endsWith(".c"):
      let relPath = path.replace(bearSslSrcDir & "/", "").replace("/", "_")
      let baseName = relPath.changeFileExt("o")
      let oFile = vendorObjDir / ("bearssl_" & baseName)
      if not fileExists(oFile):
        exec clangBase & " -I" & bearSslIncDir & " -I" & bearSslSrcDir & " -c " & path & " -o " & oFile

  # --- secp256k1 ---
  echo "Compiling secp256k1 for iOS..."
  let secp256k1Dir = secp256k1Path / "vendor/secp256k1"
  let secp256k1Flags = " -I" & secp256k1Dir & "/include" &
        " -I" & secp256k1Dir & "/src" &
        " -I" & secp256k1Dir &
        " -DENABLE_MODULE_RECOVERY=1" &
        " -DENABLE_MODULE_ECDH=1" &
        " -DECMULT_WINDOW_SIZE=15" &
        " -DECMULT_GEN_PREC_BITS=4"

  # Main secp256k1 source
  let secp256k1Obj = vendorObjDir / "secp256k1.o"
  if not fileExists(secp256k1Obj):
    exec clangBase & secp256k1Flags & " -c " & secp256k1Dir & "/src/secp256k1.c -o " & secp256k1Obj

  # Precomputed tables (required for ecmult operations)
  let secp256k1PreEcmultObj = vendorObjDir / "secp256k1_precomputed_ecmult.o"
  if not fileExists(secp256k1PreEcmultObj):
    exec clangBase & secp256k1Flags & " -c " & secp256k1Dir & "/src/precomputed_ecmult.c -o " & secp256k1PreEcmultObj

  let secp256k1PreEcmultGenObj = vendorObjDir / "secp256k1_precomputed_ecmult_gen.o"
  if not fileExists(secp256k1PreEcmultGenObj):
    exec clangBase & secp256k1Flags & " -c " & secp256k1Dir & "/src/precomputed_ecmult_gen.c -o " & secp256k1PreEcmultGenObj

  # --- miniupnpc ---
  echo "Compiling miniupnpc for iOS..."
  let miniupnpcSrcDir = natTraversalPath / "vendor/miniupnp/miniupnpc/src"
  let miniupnpcIncDir = natTraversalPath / "vendor/miniupnp/miniupnpc/include"
  let miniupnpcBuildDir = natTraversalPath / "vendor/miniupnp/miniupnpc/build"
  let miniupnpcFiles = @[
    "addr_is_reserved.c", "connecthostport.c", "igd_desc_parse.c",
    "minisoap.c", "minissdpc.c", "miniupnpc.c", "miniwget.c",
    "minixml.c", "portlistingparse.c", "receivedata.c", "upnpcommands.c",
    "upnpdev.c", "upnperrors.c", "upnpreplyparse.c"
  ]
  for fileName in miniupnpcFiles:
    let srcPath = miniupnpcSrcDir / fileName
    let oFile = vendorObjDir / ("miniupnpc_" & fileName.changeFileExt("o"))
    if fileExists(srcPath) and not fileExists(oFile):
      exec clangBase &
          " -I" & miniupnpcIncDir &
          " -I" & miniupnpcSrcDir &
          " -I" & miniupnpcBuildDir &
          " -DMINIUPNPC_SET_SOCKET_TIMEOUT" &
          " -D_BSD_SOURCE -D_DEFAULT_SOURCE" &
          " -c " & srcPath & " -o " & oFile

  # --- libnatpmp ---
  echo "Compiling libnatpmp for iOS..."
  let natpmpSrcDir = natTraversalPath / "vendor/libnatpmp-upstream"
  # Only compile natpmp.c - getgateway.c uses net/route.h which is not available on iOS
  let natpmpObj = vendorObjDir / "natpmp_natpmp.o"
  if not fileExists(natpmpObj):
    exec clangBase &
        " -I" & natpmpSrcDir &
        " -DENABLE_STRNATPMPERR" &
        " -c " & natpmpSrcDir & "/natpmp.c -o " & natpmpObj

 # Use iOS-specific stub for getgateway
  let getgatewayStubSrc = "./library/ios_natpmp_stubs.c"
  let getgatewayStubObj = vendorObjDir / "natpmp_getgateway_stub.o"
  if fileExists(getgatewayStubSrc) and not fileExists(getgatewayStubObj):
    exec clangBase & " -c " & getgatewayStubSrc & " -o " & getgatewayStubObj

  # --- BearSSL stubs (for tools functions not in main library) ---
  echo "Compiling BearSSL stubs for iOS..."
  let bearSslStubsSrc = "./library/ios_bearssl_stubs.c"
  let bearSslStubsObj = vendorObjDir / "bearssl_stubs.o"
  if fileExists(bearSslStubsSrc) and not fileExists(bearSslStubsObj):
    exec clangBase & " -c " & bearSslStubsSrc & " -o " & bearSslStubsObj

  # Compile all Nim-generated C files to object files
  echo "Compiling Nim-generated C files for iOS..."
  var cFiles: seq[string] = @[]
  for kind, path in walkDir(nimcacheDir):
    if kind == pcFile and path.endsWith(".c"):
      cFiles.add(path)

  for cFile in cFiles:
    let baseName = extractFilename(cFile).changeFileExt("o")
    let oFile = objDir / baseName
    exec clangBase &
        " -DENABLE_STRNATPMPERR" &
        " -I" & nimLibPath &
        " -I" & bearsslPath & "/bearssl/csources/inc/" &
        " -I" & bearsslPath & "/bearssl/csources/tools/" &
        " -I" & bearsslPath & "/bearssl/abi/" &
        " -I" & secp256k1Path & "/vendor/secp256k1/include/" &
        " -I" & natTraversalPath & "/vendor/miniupnp/miniupnpc/include/" &
        " -I" & natTraversalPath & "/vendor/libnatpmp-upstream/" &
        " -I" & nimcacheDir &
        " -c " & cFile &
        " -o " & oFile

  # Create static library from all object files
  echo "Creating static library..."
  var objFiles: seq[string] = @[]
  for kind, path in walkDir(objDir):
    if kind == pcFile and path.endsWith(".o"):
      objFiles.add(path)
  for kind, path in walkDir(vendorObjDir):
    if kind == pcFile and path.endsWith(".o"):
      objFiles.add(path)

  exec "libtool -static -o " & aFile & " " & objFiles.join(" ")

  echo "iOS library created: " & aFile

task libWakuIOS, "Build the mobile bindings for iOS":
  let srcDir = "./library"
  let extraParams = "-d:chronicles_log_level=ERROR"
  buildMobileIOS srcDir, extraParams

proc test(name: string, params = "-d:chronicles_log_level=DEBUG") =
  # XXX: When running `> NIM_PARAMS="-d:chronicles_log_level=INFO" make test2`
  # I expect compiler flag to be overridden, however it stays with whatever is
  # specified here.
  buildBinary name, "tests/", params
  exec "build/" & name

### Waku common tasks
task testcommon, "Build & run common tests":
  test "all_tests_common", "-d:chronicles_log_level=DEBUG -d:chronosStrictException"

### Waku tasks
task wakunode2, "Build Waku v2 cli node":
  let name = "wakunode2"
  buildBinary name, "apps/wakunode2/", " -d:chronicles_log_level=TRACE "

task benchmarks, "Some benchmarks":
  let name = "benchmarks"
  buildBinary name, "apps/benchmarks/", "-p:../.."

task wakucanary, "Build waku-canary tool":
  let name = "wakucanary"
  buildBinary name, "apps/wakucanary/"

task networkmonitor, "Build network monitor tool":
  let name = "networkmonitor"
  buildBinary name, "apps/networkmonitor/"

task rln_db_inspector, "Build the rln db inspector":
  let name = "rln_db_inspector"
  buildBinary name, "tools/rln_db_inspector/"

task test, "Build & run Waku tests":
  test "all_tests_waku"

task testwakunode2, "Build & run wakunode2 app tests":
  test "all_tests_wakunode2"

task example2, "Build Waku examples":
  buildBinary "api_example", "examples/api_example/"
  buildBinary "publisher", "examples/"
  buildBinary "subscriber", "examples/"
  buildBinary "filter_subscriber", "examples/"
  buildBinary "lightpush_publisher", "examples/"

task chat2, "Build example Waku chat usage":
  # NOTE For debugging, set debug level. For chat usage we want minimal log
  # output to STDOUT. Can be fixed by redirecting logs to file (e.g.)
  #buildBinary name, "examples/", "-d:chronicles_log_level=WARN"

  let name = "chat2"
  buildBinary name,
    "apps/chat2/",
    "-d:chronicles_sinks=textlines[file] -d:chronicles_log_level=TRACE "
  #  -d:ssl - cause unlisted exception error in libp2p/utility...

task chat2mix, "Build example Waku chat mix usage":
  # NOTE For debugging, set debug level. For chat usage we want minimal log
  # output to STDOUT. Can be fixed by redirecting logs to file (e.g.)
  #buildBinary name, "examples/", "-d:chronicles_log_level=WARN"

  let name = "chat2mix"
  buildBinary name,
    "apps/chat2mix/",
    "-d:chronicles_sinks=textlines[file] -d:chronicles_log_level=TRACE "
  #  -d:ssl - cause unlisted exception error in libp2p/utility...

task chat2bridge, "Build chat2bridge":
  let name = "chat2bridge"
  buildBinary name, "apps/chat2bridge/"

task liteprotocoltester, "Build liteprotocoltester":
  let name = "liteprotocoltester"
  buildBinary name, "apps/liteprotocoltester/", "-d:chronicles_log_level=TRACE"

task lightpushwithmix, "Build lightpushwithmix":
  let name = "lightpush_publisher_mix"
  buildBinary name, "examples/lightpush_mix/"

task buildTest, "Test custom target":
  let args = commandLineParams()
  if args.len == 0:
    quit "Missing test file"

  let filepath = args[^1]
  discard buildModule(filepath)

import std/strutils

task execTest, "Run test":
  let args = commandLineParams()
  if args.len == 0:
    quit "Missing arguments"
  # expects: <file> "<test name>"
  let filepath =
    if args.len >= 2: args[^2]
    else: args[^1]
  var testSuite =
    if args.len >= 1: args[^1].strip(chars = {'\"'})
    else: ""
  if testSuite != "":
    testSuite = " \"" & testSuite & "\""
  exec "build/" & filepath & ".bin " & testSuite

### C Bindings
let chroniclesParams =
  "-d:chronicles_line_numbers " & "-d:chronicles_runtime_filtering=on " &
  """-d:chronicles_sinks="textlines,json" """ &
  "-d:chronicles_default_output_device=Dynamic " &
  """-d:chronicles_disabled_topics="eth,dnsdisc.client" """ & "--warning:Deprecated:off " &
  "--warning:UnusedImport:on " & "-d:chronicles_log_level=TRACE"

## Libwaku build tasks

task libwakuDynamicWindows, "Generate bindings":
  buildLibDynamicWindows("libwaku", "library")

task libwakuDynamicLinux, "Generate bindings":
  buildLibDynamicLinux("libwaku", "library")

task libwakuDynamicMac, "Generate bindings":
  buildLibDynamicMac("libwaku", "library")

task libwakuStaticWindows, "Generate bindings":
  buildLibStaticWindows("libwaku", "library")

task libwakuStaticLinux, "Generate bindings":
  buildLibStaticLinux("libwaku", "library")

task libwakuStaticMac, "Generate bindings":
  buildLibStaticMac("libwaku", "library")

## Liblogosdelivery build tasks

task liblogosdeliveryDynamicWindows, "Generate bindings":
  buildLibDynamicWindows("liblogosdelivery", "liblogosdelivery")

task liblogosdeliveryDynamicLinux, "Generate bindings":
  buildLibDynamicLinux("liblogosdelivery", "liblogosdelivery")

task liblogosdeliveryDynamicMac, "Generate bindings":
  buildLibDynamicMac("liblogosdelivery", "liblogosdelivery")

task liblogosdeliveryStaticWindows, "Generate bindings":
  buildLibStaticWindows("liblogosdelivery", "liblogosdelivery")

task liblogosdeliveryStaticLinux, "Generate bindings":
  buildLibStaticLinux("liblogosdelivery", "liblogosdelivery")

task liblogosdeliveryStaticMac, "Generate bindings":
  buildLibStaticMac("liblogosdelivery", "liblogosdelivery")