Merge branch 'master' into deprecate_sync_strategy

.gitmodules

@@ -168,7 +168,7 @@
 	path = vendor/db_connector
 	url = https://github.com/nim-lang/db_connector.git
 	ignore = untracked
-	branch = master
+	branch = devel
 [submodule "vendor/nph"]
 	ignore = untracked
 	branch = master
@@ -179,16 +179,6 @@
 	url = https://github.com/status-im/nim-minilru.git
 	ignore = untracked
 	branch = master
-[submodule "vendor/nim-quic"]
-	path = vendor/nim-quic
-	url = https://github.com/status-im/nim-quic.git
-	ignore = untracked
-	branch = master
-[submodule "vendor/nim-ngtcp2"]
-	path = vendor/nim-ngtcp2
-	url = https://github.com/vacp2p/nim-ngtcp2.git
-	ignore = untracked
-	branch = master
 [submodule "vendor/waku-rlnv2-contract"]
 	path = vendor/waku-rlnv2-contract
 	url = https://github.com/waku-org/waku-rlnv2-contract.git

Makefile

@@ -4,8 +4,8 @@
 # - MIT license
 # at your option. This file may not be copied, modified, or distributed except
 # according to those terms.
-BUILD_SYSTEM_DIR := vendor/nimbus-build-system
-EXCLUDED_NIM_PACKAGES := vendor/nim-dnsdisc/vendor
+export BUILD_SYSTEM_DIR := vendor/nimbus-build-system
+export EXCLUDED_NIM_PACKAGES := vendor/nim-dnsdisc/vendor
 LINK_PCRE := 0
 FORMAT_MSG := "\\x1B[95mFormatting:\\x1B[39m"
 # we don't want an error here, so we can handle things later, in the ".DEFAULT" target
@@ -152,6 +152,12 @@ endif
 
 clean: | clean-libbacktrace
 
+### Create nimble links (used when building with Nix)
+
+nimbus-build-system-nimble-dir:
+	NIMBLE_DIR="$(CURDIR)/$(NIMBLE_DIR)" \
+	PWD_CMD="$(PWD)" \
+	$(CURDIR)/scripts/generate_nimble_links.sh
 
 ##################
 ##     RLN      ##

README.md

@@ -21,6 +21,13 @@ The standard developer tools, including a C compiler, GNU Make, Bash, and Git. M
 
 > In some distributions (Fedora linux for example), you may need to install `which` utility separately. Nimbus build system is relying on it.
 
+You'll also need an installation of Rust and its toolchain (specifically `rustc` and `cargo`).
+The easiest way to install these, is using `rustup`:
+
+```bash
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
+```
+
 ### Wakunode
 
 ```bash

flake.lock

@@ -0,0 +1,49 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1740603184,
+        "narHash": "sha256-t+VaahjQAWyA+Ctn2idyo1yxRIYpaDxMgHkgCNiMJa4=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "f44bd8ca21e026135061a0a57dcf3d0775b67a49",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "f44bd8ca21e026135061a0a57dcf3d0775b67a49",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs",
+        "zerokit": "zerokit"
+      }
+    },
+    "zerokit": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1743756626,
+        "narHash": "sha256-SvhfEl0bJcRsCd79jYvZbxQecGV2aT+TXjJ57WVv7Aw=",
+        "owner": "vacp2p",
+        "repo": "zerokit",
+        "rev": "c60e0c33fc6350a4b1c20e6b6727c44317129582",
+        "type": "github"
+      },
+      "original": {
+        "owner": "vacp2p",
+        "repo": "zerokit",
+        "rev": "c60e0c33fc6350a4b1c20e6b6727c44317129582",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,63 @@
+{
+  description = "NWaku build flake";
+
+  nixConfig = {
+    extra-substituters = [ "https://nix-cache.status.im/" ];
+    extra-trusted-public-keys = [ "nix-cache.status.im-1:x/93lOfLU+duPplwMSBR+OlY4+mo+dCN7n0mr4oPwgY=" ];
+  };
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs?rev=f44bd8ca21e026135061a0a57dcf3d0775b67a49";
+    zerokit = {
+      url = "github:vacp2p/zerokit?rev=c60e0c33fc6350a4b1c20e6b6727c44317129582";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+  };
+
+  outputs = { self, nixpkgs, zerokit }:
+    let
+      stableSystems = [
+        "x86_64-linux" "aarch64-linux"
+        "x86_64-darwin" "aarch64-darwin"
+        "x86_64-windows" "i686-linux"
+        "i686-windows"
+      ];
+
+      forAllSystems = f: nixpkgs.lib.genAttrs stableSystems (system: f system);
+
+      pkgsFor = forAllSystems (
+        system: import nixpkgs {
+          inherit system;
+          config = {
+            android_sdk.accept_license = true;
+            allowUnfree = true;
+          };
+          overlays =  [
+            (final: prev: {
+              androidEnvCustom = prev.callPackage ./nix/pkgs/android-sdk { };
+              androidPkgs = final.androidEnvCustom.pkgs;
+              androidShell = final.androidEnvCustom.shell;
+            })
+          ];
+        }
+      );
+
+    in rec {
+      packages = forAllSystems (system: let
+        pkgs = pkgsFor.${system};
+      in rec {
+        libwaku-android-arm64 = pkgs.callPackage ./nix/default.nix {
+          inherit stableSystems;
+          src = self;
+          targets = ["libwaku-android-arm64"]; 
+          androidArch = "aarch64-linux-android";
+          zerokitPkg = zerokit.packages.${system}.zerokit-android-arm64;
+        };
+        default = libwaku-android-arm64;
+      });
+
+      devShells = forAllSystems (system: {
+        default = pkgsFor.${system}.callPackage ./nix/shell.nix {};
+      });
+    };
+}

library/libwaku.nim

@@ -52,10 +52,6 @@ template callEventCallback(ctx: ptr WakuContext, eventName: string, body: untype
     error eventName & " - eventCallback is nil"
     return
 
-  if isNil(ctx[].eventUserData):
-    error eventName & " - eventUserData is nil"
-    return
-
   foreignThreadGc:
     try:
       let event = body

nix/README.md

@@ -0,0 +1,35 @@
+# Usage
+
+## Shell
+
+A development shell can be started using:
+```sh
+nix develop
+```
+
+## Building
+
+To build a Codex you can use:
+```sh
+nix build '.?submodules=1#default'
+```
+The `?submodules=1` part should eventually not be necessary.
+For more details see:
+https://github.com/NixOS/nix/issues/4423
+
+It can be also done without even cloning the repo:
+```sh
+nix build 'git+https://github.com/waku-org/nwaku?submodules=1#'
+```
+
+## Running
+
+```sh
+nix run 'git+https://github.com/waku-org/nwaku?submodules=1#''
+```
+
+## Testing
+
+```sh
+nix flake check ".?submodules=1#"
+```

nix/atlas.nix

@@ -0,0 +1,12 @@
+{ pkgs ? import <nixpkgs> { } }:
+
+let
+  tools = pkgs.callPackage ./tools.nix {};
+  sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim;
+in pkgs.fetchFromGitHub {
+  owner = "nim-lang";
+  repo = "atlas";
+  rev = tools.findKeyValue "^ +AtlasStableCommit = \"([a-f0-9]+)\"$" sourceFile;
+  # WARNING: Requires manual updates when Nim compiler version changes.
+  hash = "sha256-G1TZdgbRPSgxXZ3VsBP2+XFCLHXVb3an65MuQx67o/k=";
+}

nix/checksums.nix

@@ -0,0 +1,12 @@
+{ pkgs ? import <nixpkgs> { } }:
+
+let
+  tools = pkgs.callPackage ./tools.nix {};
+  sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim;
+in pkgs.fetchFromGitHub {
+  owner = "nim-lang";
+  repo = "checksums";
+  rev = tools.findKeyValue "^ +ChecksumsStableCommit = \"([a-f0-9]+)\"$" sourceFile;
+  # WARNING: Requires manual updates when Nim compiler version changes.
+  hash = "sha256-Bm5iJoT2kAvcTexiLMFBa9oU5gf7d4rWjo3OiN7obWQ=";
+}

nix/csources.nix

@@ -0,0 +1,12 @@
+{ pkgs ? import <nixpkgs> { } }:
+
+let
+  tools = pkgs.callPackage ./tools.nix {};
+  sourceFile = ../vendor/nimbus-build-system/vendor/Nim/config/build_config.txt;
+in pkgs.fetchFromGitHub {
+  owner = "nim-lang";
+  repo = "csources_v2";
+  rev = tools.findKeyValue "^nim_csourcesHash=([a-f0-9]+)$" sourceFile;
+  # WARNING: Requires manual updates when Nim compiler version changes.
+  hash = "sha256-UCLtoxOcGYjBdvHx7A47x6FjLMi6VZqpSs65MN7fpBs=";
+}

nix/default.nix

@@ -0,0 +1,111 @@
+{
+  config ? {},
+  pkgs ? import <nixpkgs> { },
+  src ? ../.,
+  targets ? ["libwaku-android-arm64"],
+  verbosity ? 2,
+  useSystemNim ? true,
+  quickAndDirty ? true,
+  stableSystems ? [
+    "x86_64-linux" "aarch64-linux"
+  ],
+  androidArch,
+  zerokitPkg,
+}:
+
+assert pkgs.lib.assertMsg ((src.submodules or true) == true)
+  "Unable to build without submodules. Append '?submodules=1#' to the URI.";
+
+let
+  inherit (pkgs) stdenv lib writeScriptBin callPackage;
+
+  revision = lib.substring 0 8 (src.rev or "dirty");
+
+in stdenv.mkDerivation rec {
+
+  pname = "nwaku";
+
+  version = "1.0.0-${revision}";
+
+  inherit src;
+
+  buildInputs = with pkgs; [
+    openssl
+    gmp
+  ];
+
+  # Dependencies that should only exist in the build environment.
+  nativeBuildInputs = let
+    # Fix for Nim compiler calling 'git rev-parse' and 'lsb_release'.
+    fakeGit = writeScriptBin "git" "echo ${version}";
+    # Fix for the zerokit package that is built with cargo/rustup/cross.
+    fakeCargo = writeScriptBin "cargo" "echo ${version}";
+    # Fix for the zerokit package that is built with cargo/rustup/cross.
+    fakeRustup = writeScriptBin "rustup" "echo ${version}";
+    # Fix for the zerokit package that is built with cargo/rustup/cross.
+    fakeCross = writeScriptBin "cross" "echo ${version}";
+  in
+    with pkgs; [
+      cmake
+      which
+      lsb-release
+      zerokitPkg
+      nim-unwrapped-2_0
+      fakeGit
+      fakeCargo
+      fakeRustup
+      fakeCross
+  ];
+
+  # Environment variables required for Android builds
+  ANDROID_SDK_ROOT="${pkgs.androidPkgs.sdk}";
+  ANDROID_NDK_HOME="${pkgs.androidPkgs.ndk}";
+  NIMFLAGS = "-d:disableMarchNative -d:git_revision_override=${revision}";
+  XDG_CACHE_HOME = "/tmp";
+
+  makeFlags = targets ++ [
+    "V=${toString verbosity}"
+    "QUICK_AND_DIRTY_COMPILER=${if quickAndDirty then "1" else "0"}"
+    "QUICK_AND_DIRTY_NIMBLE=${if quickAndDirty then "1" else "0"}"
+    "USE_SYSTEM_NIM=${if useSystemNim then "1" else "0"}"
+  ];
+
+  configurePhase = ''
+    patchShebangs . vendor/nimbus-build-system > /dev/null
+    make nimbus-build-system-paths
+    make nimbus-build-system-nimble-dir
+  '';
+
+  preBuild = ''
+    ln -s waku.nimble waku.nims
+    pushd vendor/nimbus-build-system/vendor/Nim
+    mkdir dist
+    cp -r ${callPackage ./nimble.nix {}}    dist/nimble
+    chmod 777 -R dist/nimble
+    mkdir -p dist/nimble/dist
+    cp -r ${callPackage ./checksums.nix {}} dist/checksums  # need both
+    cp -r ${callPackage ./checksums.nix {}} dist/nimble/dist/checksums
+    cp -r ${callPackage ./atlas.nix {}}     dist/atlas
+    chmod 777 -R dist/atlas
+    mkdir dist/atlas/dist
+    cp -r ${callPackage ./sat.nix {}}       dist/nimble/dist/sat
+    cp -r ${callPackage ./sat.nix {}}       dist/atlas/dist/sat
+    cp -r ${callPackage ./csources.nix {}}  csources_v2
+    chmod 777 -R dist/nimble csources_v2
+    popd
+    mkdir -p vendor/zerokit/target/${androidArch}/release
+    cp ${zerokitPkg}/librln.so vendor/zerokit/target/${androidArch}/release/
+  '';
+
+  installPhase = ''
+    mkdir -p $out/build/android
+    cp -r ./build/android/* $out/build/android/
+  '';
+
+  meta = with pkgs.lib; {
+    description = "NWaku derivation to build libwaku for mobile targets using Android NDK and Rust.";
+    homepage = "https://github.com/status-im/nwaku";
+    license = licenses.mit;
+    platforms = stableSystems;
+  };
+}

nix/nimble.nix

@@ -0,0 +1,12 @@
+{ pkgs ? import <nixpkgs> { } }:
+
+let
+  tools = pkgs.callPackage ./tools.nix {};
+  sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim;
+in pkgs.fetchFromGitHub {
+  owner = "nim-lang";
+  repo = "nimble";
+  rev = tools.findKeyValue "^ +NimbleStableCommit = \"([a-f0-9]+)\".+" sourceFile;
+  # WARNING: Requires manual updates when Nim compiler version changes.
+  hash = "sha256-MVHf19UbOWk8Zba2scj06PxdYYOJA6OXrVyDQ9Ku6Us=";
+}

nix/pkgs/android-sdk/compose.nix

@@ -0,0 +1,26 @@
+#
+# This Nix expression centralizes the configuration
+# for the Android development environment.
+#
+
+{ androidenv, lib, stdenv }:
+
+assert lib.assertMsg (stdenv.system != "aarch64-darwin")
+  "aarch64-darwin not supported for Android SDK. Use: NIXPKGS_SYSTEM_OVERRIDE=x86_64-darwin";
+
+# The "android-sdk-license" license is accepted
+# by setting android_sdk.accept_license = true.
+androidenv.composeAndroidPackages {
+  cmdLineToolsVersion = "9.0";
+  toolsVersion = "26.1.1";
+  platformToolsVersion = "33.0.3";
+  buildToolsVersions = [ "34.0.0" ];
+  platformVersions = [ "34" ];
+  cmakeVersions = [ "3.22.1" ];
+  ndkVersion = "25.2.9519653";
+  includeNDK = true;
+  includeExtras = [
+    "extras;android;m2repository"
+    "extras;google;m2repository"
+  ];
+}

nix/pkgs/android-sdk/default.nix

@@ -0,0 +1,14 @@
+#
+# This Nix expression centralizes the configuration
+# for the Android development environment.
+#
+
+{ callPackage }:
+
+let
+  compose = callPackage ./compose.nix { };
+  pkgs = callPackage ./pkgs.nix { inherit compose; };
+  shell = callPackage ./shell.nix { androidPkgs = pkgs; };
+in {
+  inherit compose pkgs shell;
+}

nix/pkgs/android-sdk/pkgs.nix

@@ -0,0 +1,17 @@
+{ stdenv, compose }:
+
+#
+# This derivation simply symlinks some stuff to get
+# shorter paths as libexec/android-sdk is quite the mouthful.
+# With this you can just do `androidPkgs.sdk` and `androidPkgs.ndk`.
+#
+stdenv.mkDerivation {
+  name = "${compose.androidsdk.name}-mod";
+  phases = [ "symlinkPhase" ];
+  outputs = [ "out" "sdk" "ndk" ];
+  symlinkPhase = ''
+    ln -s ${compose.androidsdk} $out
+    ln -s ${compose.androidsdk}/libexec/android-sdk $sdk
+    ln -s ${compose.androidsdk}/libexec/android-sdk/ndk-bundle $ndk
+  '';
+}

nix/pkgs/android-sdk/shell.nix

@@ -0,0 +1,19 @@
+{ mkShell, openjdk, androidPkgs }:
+
+mkShell {
+  name = "android-sdk-shell";
+  buildInputs = [ openjdk ];
+
+  shellHook = ''
+    export ANDROID_HOME="${androidPkgs.sdk}"
+    export ANDROID_NDK_ROOT="${androidPkgs.ndk}"
+    export ANDROID_SDK_ROOT="$ANDROID_HOME"
+    export ANDROID_NDK_HOME="${androidPkgs.ndk}"
+
+    export PATH="$ANDROID_NDK_ROOT:$PATH"
+    export PATH="$ANDROID_SDK_ROOT/tools:$PATH"
+    export PATH="$ANDROID_SDK_ROOT/tools/bin:$PATH"
+    export PATH="$(echo $ANDROID_SDK_ROOT/cmdline-tools/*/bin):$PATH"
+    export PATH="$ANDROID_SDK_ROOT/platform-tools:$PATH"
+  '';
+}

nix/sat.nix

@@ -0,0 +1,12 @@
+{ pkgs ? import <nixpkgs> { } }:
+
+let
+  tools = pkgs.callPackage ./tools.nix {};
+  sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim;
+in pkgs.fetchFromGitHub {
+  owner = "nim-lang";
+  repo = "sat";
+  rev = tools.findKeyValue "^ +SatStableCommit = \"([a-f0-9]+)\"$" sourceFile;
+  # WARNING: Requires manual updates when Nim compiler version changes.
+  hash = "sha256-JFrrSV+mehG0gP7NiQ8hYthL0cjh44HNbXfuxQNhq7c=";
+}

nix/shell.nix

@@ -0,0 +1,26 @@
+{ 
+  pkgs ? import <nixpkgs> { },
+}:
+let
+  optionalDarwinDeps = pkgs.lib.optionals pkgs.stdenv.isDarwin [
+    pkgs.libiconv
+    pkgs.darwin.apple_sdk.frameworks.Security
+  ];
+in
+pkgs.mkShell {
+  inputsFrom = [
+    pkgs.androidShell
+  ] ++ optionalDarwinDeps;
+
+  buildInputs = with pkgs; [
+    git
+    cargo
+    rustup
+    cmake
+    nim-unwrapped-2_0
+  ];
+
+  LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
+    pkgs.pcre
+  ];
+}

nix/tools.nix

@@ -0,0 +1,15 @@
+{ pkgs ? import <nixpkgs> { } }:
+
+let
+
+  inherit (pkgs.lib) fileContents last splitString flatten remove;
+  inherit (builtins) map match;
+in {
+  findKeyValue = regex: sourceFile:
+    let
+      linesFrom = file: splitString "\n" (fileContents file);
+      matching = regex: lines: map (line: match regex line) lines;
+      extractMatch = matches: last (flatten (remove null matches));
+    in
+      extractMatch (matching regex (linesFrom sourceFile));
+}

scripts/generate_nimble_links.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# This script is used for building Nix derivation which doesn't allow Git commands.
+# It implements similar logic as $(NIMBLE_DIR) target in nimbus-build-system Makefile.
+
+create_nimble_link_script_path="$(pwd)/${BUILD_SYSTEM_DIR}/scripts/create_nimble_link.sh"
+
+process_gitmodules() {
+  local gitmodules_file="$1"
+  local gitmodules_dir=$(dirname "$gitmodules_file")
+  
+  # Extract all submodule paths from the .gitmodules file
+  grep "path" $gitmodules_file | awk '{print $3}' | while read submodule_path; do
+    # Change pwd to the submodule dir and execute script
+    pushd "$gitmodules_dir/$submodule_path" > /dev/null
+    NIMBLE_DIR=$NIMBLE_DIR PWD_CMD=$PWD_CMD EXCLUDED_NIM_PACKAGES=$EXCLUDED_NIM_PACKAGES \
+    "$create_nimble_link_script_path" "$submodule_path"
+    popd  > /dev/null
+  done
+}
+
+# Create the base directory if it doesn't exist
+mkdir -p "${NIMBLE_DIR}/pkgs"
+
+# Find all .gitmodules files and process them
+for gitmodules_file in $(find . -name '.gitmodules'); do
+  echo "Processing .gitmodules file: $gitmodules_file"
+  process_gitmodules "$gitmodules_file"
+done

shell.nix

@@ -1,22 +0,0 @@
-{ pkgs ? import (builtins.fetchTarball {
-  url = "https://github.com/NixOS/nixpkgs/archive/dbf1d73cd1a17276196afeee169b4cf7834b7a96.tar.gz";
-  sha256 = "sha256:1k5nvn2yzw370cqsfh62lncsgydq2qkbjrx34cprzf0k6b93v7ch";
-}) {} }:
-
-pkgs.mkShell {
-  name = "nim-waku-build-shell";
-
-  # Versions dependent on nixpkgs commit. Update manually.
-  buildInputs = with pkgs; [
-    git   # 2.37.3
-    which # 2.21
-    rustc # 1.63.0
-  ] ++ lib.optionals stdenv.isDarwin [ 
-    libiconv
-    darwin.apple_sdk.frameworks.Security
-  ];
-
-  LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
-    pkgs.pcre
-  ];
-}