nix: create nix flake and libwaku-android-arm64 target

* android-ndk is added
* in the derivation, system nim is default but one can change it to
  nimbus-build-system
* special script for creating nimble links, necessary for the
  compilation to succeed.

Referenced issue:
* https://github.com/waku-org/nwaku/issues/3232

.gitmodules

@@ -168,7 +168,7 @@
 	path = vendor/db_connector
 	url = https://github.com/nim-lang/db_connector.git
 	ignore = untracked
-	branch = master
+	branch = devel
 [submodule "vendor/nph"]
 	ignore = untracked
 	branch = master

flake.lock

@@ -0,0 +1,49 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1740603184,
+        "narHash": "sha256-t+VaahjQAWyA+Ctn2idyo1yxRIYpaDxMgHkgCNiMJa4=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "f44bd8ca21e026135061a0a57dcf3d0775b67a49",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "f44bd8ca21e026135061a0a57dcf3d0775b67a49",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs",
+        "zerokit": "zerokit"
+      }
+    },
+    "zerokit": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1743756626,
+        "narHash": "sha256-SvhfEl0bJcRsCd79jYvZbxQecGV2aT+TXjJ57WVv7Aw=",
+        "owner": "vacp2p",
+        "repo": "zerokit",
+        "rev": "c60e0c33fc6350a4b1c20e6b6727c44317129582",
+        "type": "github"
+      },
+      "original": {
+        "owner": "vacp2p",
+        "repo": "zerokit",
+        "rev": "c60e0c33fc6350a4b1c20e6b6727c44317129582",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,63 @@
+{
+  description = "NWaku build flake";
+
+  nixConfig = {
+    extra-substituters = [ "https://nix-cache.status.im/" ];
+    extra-trusted-public-keys = [ "nix-cache.status.im-1:x/93lOfLU+duPplwMSBR+OlY4+mo+dCN7n0mr4oPwgY=" ];
+  };
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs?rev=f44bd8ca21e026135061a0a57dcf3d0775b67a49";
+    zerokit = {
+      url = "github:vacp2p/zerokit?rev=c60e0c33fc6350a4b1c20e6b6727c44317129582";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+  };
+
+  outputs = { self, nixpkgs, zerokit }:
+    let
+      stableSystems = [
+        "x86_64-linux" "aarch64-linux"
+        "x86_64-darwin" "aarch64-darwin"
+        "x86_64-windows" "i686-linux"
+        "i686-windows"
+      ];
+
+      forAllSystems = f: nixpkgs.lib.genAttrs stableSystems (system: f system);
+
+      pkgsFor = forAllSystems (
+        system: import nixpkgs {
+          inherit system;
+          config = {
+            android_sdk.accept_license = true;
+            allowUnfree = true;
+          };
+          overlays =  [
+            (final: prev: {
+              androidEnvCustom = prev.callPackage ./nix/pkgs/android-sdk { };
+              androidPkgs = final.androidEnvCustom.pkgs;
+              androidShell = final.androidEnvCustom.shell;
+            })
+          ];
+        }
+      );
+
+    in rec {
+      packages = forAllSystems (system: let
+        pkgs = pkgsFor.${system};
+      in rec {
+        libwaku-android-arm64 = pkgs.callPackage ./nix/default.nix {
+          inherit stableSystems;
+          src = self;
+          targets = ["libwaku-android-arm64"]; 
+          androidArch = "aarch64-linux-android";
+          zerokitPkg = zerokit.packages.${system}.zerokit-android-arm64;
+        };
+        default = libwaku-android-arm64;
+      });
+
+      devShells = forAllSystems (system: {
+        default = pkgsFor.${system}.callPackage ./nix/shell.nix {};
+      });
+    };
+}

nix/README.md

@@ -0,0 +1,35 @@
+# Usage
+
+## Shell
+
+A development shell can be started using:
+```sh
+nix develop
+```
+
+## Building
+
+To build a Codex you can use:
+```sh
+nix build '.?submodules=1#default'
+```
+The `?submodules=1` part should eventually not be necessary.
+For more details see:
+https://github.com/NixOS/nix/issues/4423
+
+It can be also done without even cloning the repo:
+```sh
+nix build 'git+https://github.com/waku-org/nwaku?submodules=1#'
+```
+
+## Running
+
+```sh
+nix run 'git+https://github.com/waku-org/nwaku?submodules=1#''
+```
+
+## Testing
+
+```sh
+nix flake check ".?submodules=1#"
+```

nix/atlas.nix

@@ -0,0 +1,12 @@
+{ pkgs ? import <nixpkgs> { } }:
+
+let
+  tools = pkgs.callPackage ./tools.nix {};
+  sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim;
+in pkgs.fetchFromGitHub {
+  owner = "nim-lang";
+  repo = "atlas";
+  rev = tools.findKeyValue "^ +AtlasStableCommit = \"([a-f0-9]+)\"$" sourceFile;
+  # WARNING: Requires manual updates when Nim compiler version changes.
+  hash = "sha256-G1TZdgbRPSgxXZ3VsBP2+XFCLHXVb3an65MuQx67o/k=";
+}

nix/checksums.nix

@@ -0,0 +1,12 @@
+{ pkgs ? import <nixpkgs> { } }:
+
+let
+  tools = pkgs.callPackage ./tools.nix {};
+  sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim;
+in pkgs.fetchFromGitHub {
+  owner = "nim-lang";
+  repo = "checksums";
+  rev = tools.findKeyValue "^ +ChecksumsStableCommit = \"([a-f0-9]+)\"$" sourceFile;
+  # WARNING: Requires manual updates when Nim compiler version changes.
+  hash = "sha256-Bm5iJoT2kAvcTexiLMFBa9oU5gf7d4rWjo3OiN7obWQ=";
+}

nix/create-nimble-link.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+# This script generates `.nimble-link` files and the folder structure typically created by `make update`
+# within this repository. It is an alternative to `vendor/nimbus-build-system/scripts/create_nimble_link.sh`,
+# designed for execution inside a Nix derivation, where Git commands are not available.
+
+output_file="submodule_paths.txt"
+
+# The EXCLUDED_NIM_PACKAGES variable (defined in the Makefile) contains a colon-separated list of
+# submodule paths that should be ignored. We split it into an array for easier matching.
+IFS=':' read -ra EXCLUDED_PATTERNS <<< "$EXCLUDED_NIM_PACKAGES"
+
+# Function to check if a given submodule path should be excluded
+should_exclude() {
+    local path="$1"
+    for pattern in "${EXCLUDED_PATTERNS[@]}"; do
+        if [[ "$path" == *"$pattern"* ]]; then
+            return 0 # Match found, exclude this submodule
+        fi
+    done
+    return 1  # No match, include this submodule
+}
+
+# Locate all `.gitmodules` files and extract submodule paths
+find . -name .gitmodules | while read -r gitmodules_file; do
+    module_dir=$(dirname "$(realpath "$gitmodules_file")")
+
+    while IFS= read -r line; do
+        # Extract the submodule path from lines matching `path = /some/path`
+        if [[ $line =~ path[[:space:]]*=[[:space:]]*(.*) ]]; then
+            submodule_path="${BASH_REMATCH[1]}"
+            abs_path="$module_dir/$submodule_path"
+            
+            # Skip if the submodule is in the excluded list
+            if should_exclude "$abs_path"; then
+                continue
+            fi
+            
+             # If the submodule contains a `src/` folder, use it as the path
+            if [[ -d "$abs_path/src" ]]; then
+                abs_path="$abs_path/src"
+            fi
+
+            echo "$abs_path" >> "$output_file"
+        fi
+    done < "$gitmodules_file"
+done
+
+echo "Submodule paths collected in $output_file"
+
+# Directory where Nimble packages will be linked
+nimble_pkgs_dir="./vendor/.nimble/pkgs"
+
+mkdir -p "$nimble_pkgs_dir"
+
+# Process each submodule path collected earlier
+while IFS= read -r submodule_path; do
+    # Determine the submodule name from its path
+    if [[ "$submodule_path" == */src ]]; then
+        submodule_name=$(basename "$(dirname "$submodule_path")")
+    else
+        submodule_name=$(basename "$submodule_path")
+    fi
+
+    # Check if the submodule contains at least one `.nimble` file
+    base_dir="${submodule_path%/src}"
+    nimble_files_count=$(find "$base_dir" -maxdepth 1 -type f -name "*.nimble" | wc -l)
+
+    if [ "$nimble_files_count" -gt 0 ]; then
+        submodule_dir="$nimble_pkgs_dir/${submodule_name}-#head"
+        mkdir -p "$submodule_dir"
+
+        nimble_link_file="$submodule_dir/${submodule_name}.nimble-link"
+        # `.nimble-link` files require two identical lines for Nimble to recognize them properly
+        echo "$submodule_path" > "$nimble_link_file"
+        echo "$submodule_path" >> "$nimble_link_file"
+    fi
+done < "$output_file"
+
+echo "Nimble packages prepared in $nimble_pkgs_dir"
+
+rm "$output_file"

nix/csources.nix

@@ -0,0 +1,12 @@
+{ pkgs ? import <nixpkgs> { } }:
+
+let
+  tools = pkgs.callPackage ./tools.nix {};
+  sourceFile = ../vendor/nimbus-build-system/vendor/Nim/config/build_config.txt;
+in pkgs.fetchFromGitHub {
+  owner = "nim-lang";
+  repo = "csources_v2";
+  rev = tools.findKeyValue "^nim_csourcesHash=([a-f0-9]+)$" sourceFile;
+  # WARNING: Requires manual updates when Nim compiler version changes.
+  hash = "sha256-UCLtoxOcGYjBdvHx7A47x6FjLMi6VZqpSs65MN7fpBs=";
+}

nix/default.nix

@@ -0,0 +1,112 @@
+{
+  config ? {},
+  pkgs ? import <nixpkgs> { },
+  src ? ../.,
+  targets ? ["libwaku-android-arm64"],
+  verbosity ? 2,
+  useSystemNim ? true,
+  quickAndDirty ? true,
+  stableSystems ? [
+    "x86_64-linux" "aarch64-linux"
+  ],
+  androidArch,
+  zerokitPkg,
+}:
+
+assert pkgs.lib.assertMsg ((src.submodules or true) == true)
+  "Unable to build without submodules. Append '?submodules=1#' to the URI.";
+
+let
+  inherit (pkgs) stdenv lib writeScriptBin callPackage;
+
+  revision = lib.substring 0 8 (src.rev or "dirty");
+
+in stdenv.mkDerivation rec {
+
+  pname = "nwaku";
+
+  version = "1.0.0-${revision}";
+
+  inherit src;
+
+  buildInputs = with pkgs; [
+    openssl
+    gmp
+  ];
+
+  # Dependencies that should only exist in the build environment.
+  nativeBuildInputs = let
+    # Fix for Nim compiler calling 'git rev-parse' and 'lsb_release'.
+    fakeGit = writeScriptBin "git" "echo ${version}";
+    # Fix for the zerokit package that is built with cargo/rustup/cross.
+    fakeCargo = writeScriptBin "cargo" "echo ${version}";
+    # Fix for the zerokit package that is built with cargo/rustup/cross.
+    fakeRustup = writeScriptBin "rustup" "echo ${version}";
+    # Fix for the zerokit package that is built with cargo/rustup/cross.
+    fakeCross = writeScriptBin "cross" "echo ${version}";
+  in
+    with pkgs; [
+      cmake
+      which
+      lsb-release
+      zerokitPkg
+      nim-unwrapped-2_0
+      fakeGit
+      fakeCargo
+      fakeRustup
+      fakeCross
+  ];
+
+  # Environment variables required for Android builds
+  ANDROID_SDK_ROOT="${pkgs.androidPkgs.sdk}";
+  ANDROID_NDK_HOME="${pkgs.androidPkgs.ndk}";
+  NIMFLAGS = "-d:disableMarchNative -d:git_revision_override=${revision}";
+  XDG_CACHE_HOME = "/tmp";
+  EXCLUDED_NIM_PACKAGES="vendor/nim-dnsdisc/vendor";
+
+  makeFlags = targets ++ [
+    "V=${toString verbosity}"
+    "QUICK_AND_DIRTY_COMPILER=${if quickAndDirty then "1" else "0"}"
+    "QUICK_AND_DIRTY_NIMBLE=${if quickAndDirty then "1" else "0"}"
+    "USE_SYSTEM_NIM=${if useSystemNim then "1" else "0"}"
+  ];
+
+  configurePhase = ''
+    patchShebangs . vendor/nimbus-build-system > /dev/null
+    make nimbus-build-system-paths
+    ./nix/create-nimble-link.sh
+  '';
+
+  preBuild = ''
+    ln -s waku.nimble waku.nims
+    pushd vendor/nimbus-build-system/vendor/Nim
+    mkdir dist
+    cp -r ${callPackage ./nimble.nix {}}    dist/nimble
+    chmod 777 -R dist/nimble
+    mkdir -p dist/nimble/dist
+    cp -r ${callPackage ./checksums.nix {}} dist/checksums  # need both
+    cp -r ${callPackage ./checksums.nix {}} dist/nimble/dist/checksums
+    cp -r ${callPackage ./atlas.nix {}}     dist/atlas
+    chmod 777 -R dist/atlas
+    mkdir dist/atlas/dist
+    cp -r ${callPackage ./sat.nix {}}       dist/nimble/dist/sat
+    cp -r ${callPackage ./sat.nix {}}       dist/atlas/dist/sat
+    cp -r ${callPackage ./csources.nix {}}  csources_v2
+    chmod 777 -R dist/nimble csources_v2
+    popd
+    mkdir -p vendor/zerokit/target/${androidArch}/release
+    cp ${zerokitPkg}/librln.so vendor/zerokit/target/${androidArch}/release/
+  '';
+
+  installPhase = ''
+    mkdir -p $out/build/android
+    cp -r ./build/android/* $out/build/android/
+  '';
+
+  meta = with pkgs.lib; {
+    description = "NWaku derivation to build libwaku for mobile targets using Android NDK and Rust.";
+    homepage = "https://github.com/status-im/nwaku";
+    license = licenses.mit;
+    platforms = stableSystems;
+  };
+}

nix/nimble.nix

@@ -0,0 +1,12 @@
+{ pkgs ? import <nixpkgs> { } }:
+
+let
+  tools = pkgs.callPackage ./tools.nix {};
+  sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim;
+in pkgs.fetchFromGitHub {
+  owner = "nim-lang";
+  repo = "nimble";
+  rev = tools.findKeyValue "^ +NimbleStableCommit = \"([a-f0-9]+)\".+" sourceFile;
+  # WARNING: Requires manual updates when Nim compiler version changes.
+  hash = "sha256-MVHf19UbOWk8Zba2scj06PxdYYOJA6OXrVyDQ9Ku6Us=";
+}

nix/pkgs/android-sdk/compose.nix

@@ -0,0 +1,26 @@
+#
+# This Nix expression centralizes the configuration
+# for the Android development environment.
+#
+
+{ androidenv, lib, stdenv }:
+
+assert lib.assertMsg (stdenv.system != "aarch64-darwin")
+  "aarch64-darwin not supported for Android SDK. Use: NIXPKGS_SYSTEM_OVERRIDE=x86_64-darwin";
+
+# The "android-sdk-license" license is accepted
+# by setting android_sdk.accept_license = true.
+androidenv.composeAndroidPackages {
+  cmdLineToolsVersion = "9.0";
+  toolsVersion = "26.1.1";
+  platformToolsVersion = "33.0.3";
+  buildToolsVersions = [ "34.0.0" ];
+  platformVersions = [ "34" ];
+  cmakeVersions = [ "3.22.1" ];
+  ndkVersion = "25.2.9519653";
+  includeNDK = true;
+  includeExtras = [
+    "extras;android;m2repository"
+    "extras;google;m2repository"
+  ];
+}

nix/pkgs/android-sdk/default.nix

@@ -0,0 +1,14 @@
+#
+# This Nix expression centralizes the configuration
+# for the Android development environment.
+#
+
+{ callPackage }:
+
+let
+  compose = callPackage ./compose.nix { };
+  pkgs = callPackage ./pkgs.nix { inherit compose; };
+  shell = callPackage ./shell.nix { androidPkgs = pkgs; };
+in {
+  inherit compose pkgs shell;
+}

nix/pkgs/android-sdk/pkgs.nix

@@ -0,0 +1,17 @@
+{ stdenv, compose }:
+
+#
+# This derivation simply symlinks some stuff to get
+# shorter paths as libexec/android-sdk is quite the mouthful.
+# With this you can just do `androidPkgs.sdk` and `androidPkgs.ndk`.
+#
+stdenv.mkDerivation {
+  name = "${compose.androidsdk.name}-mod";
+  phases = [ "symlinkPhase" ];
+  outputs = [ "out" "sdk" "ndk" ];
+  symlinkPhase = ''
+    ln -s ${compose.androidsdk} $out
+    ln -s ${compose.androidsdk}/libexec/android-sdk $sdk
+    ln -s ${compose.androidsdk}/libexec/android-sdk/ndk-bundle $ndk
+  '';
+}

nix/pkgs/android-sdk/shell.nix

@@ -0,0 +1,19 @@
+{ mkShell, openjdk, androidPkgs }:
+
+mkShell {
+  name = "android-sdk-shell";
+  buildInputs = [ openjdk ];
+
+  shellHook = ''
+    export ANDROID_HOME="${androidPkgs.sdk}"
+    export ANDROID_NDK_ROOT="${androidPkgs.ndk}"
+    export ANDROID_SDK_ROOT="$ANDROID_HOME"
+    export ANDROID_NDK_HOME="${androidPkgs.ndk}"
+
+    export PATH="$ANDROID_NDK_ROOT:$PATH"
+    export PATH="$ANDROID_SDK_ROOT/tools:$PATH"
+    export PATH="$ANDROID_SDK_ROOT/tools/bin:$PATH"
+    export PATH="$(echo $ANDROID_SDK_ROOT/cmdline-tools/*/bin):$PATH"
+    export PATH="$ANDROID_SDK_ROOT/platform-tools:$PATH"
+  '';
+}

nix/sat.nix

@@ -0,0 +1,12 @@
+{ pkgs ? import <nixpkgs> { } }:
+
+let
+  tools = pkgs.callPackage ./tools.nix {};
+  sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim;
+in pkgs.fetchFromGitHub {
+  owner = "nim-lang";
+  repo = "sat";
+  rev = tools.findKeyValue "^ +SatStableCommit = \"([a-f0-9]+)\"$" sourceFile;
+  # WARNING: Requires manual updates when Nim compiler version changes.
+  hash = "sha256-JFrrSV+mehG0gP7NiQ8hYthL0cjh44HNbXfuxQNhq7c=";
+}

nix/shell.nix

@@ -0,0 +1,26 @@
+{ 
+  pkgs ? import <nixpkgs> { },
+}:
+let
+  optionalDarwinDeps = pkgs.lib.optionals pkgs.stdenv.isDarwin [
+    pkgs.libiconv
+    pkgs.darwin.apple_sdk.frameworks.Security
+  ];
+in
+pkgs.mkShell {
+  inputsFrom = [
+    pkgs.androidShell
+  ] ++ optionalDarwinDeps;
+
+  buildInputs = with pkgs; [
+    git
+    cargo
+    rustup
+    cmake
+    nim-unwrapped-2_0
+  ];
+
+  LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
+    pkgs.pcre
+  ];
+}

nix/tools.nix

@@ -0,0 +1,15 @@
+{ pkgs ? import <nixpkgs> { } }:
+
+let
+
+  inherit (pkgs.lib) fileContents last splitString flatten remove;
+  inherit (builtins) map match;
+in {
+  findKeyValue = regex: sourceFile:
+    let
+      linesFrom = file: splitString "\n" (fileContents file);
+      matching = regex: lines: map (line: match regex line) lines;
+      extractMatch = matches: last (flatten (remove null matches));
+    in
+      extractMatch (matching regex (linesFrom sourceFile));
+}

scripts/generate_nimble_links.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# This script is used for building Nix derivation which doesn't allow Git commands.
+# It implements similar logic as $(NIMBLE_DIR) target in nimbus-build-system Makefile.
+
+create_nimble_link_script_path="$(pwd)/${BUILD_SYSTEM_DIR}/scripts/create_nimble_link.sh"
+
+process_gitmodules() {
+  local gitmodules_file="$1"
+  local gitmodules_dir=$(dirname "$gitmodules_file")
+  
+  # Extract all submodule paths from the .gitmodules file
+  grep "path" $gitmodules_file | awk '{print $3}' | while read submodule_path; do
+    # Change pwd to the submodule dir and execute script
+    pushd "$gitmodules_dir/$submodule_path" > /dev/null
+    NIMBLE_DIR=$NIMBLE_DIR PWD_CMD=$PWD_CMD EXCLUDED_NIM_PACKAGES=$EXCLUDED_NIM_PACKAGES \
+    "$create_nimble_link_script_path" "$submodule_path"
+    popd  > /dev/null
+  done
+}
+
+# Create the base directory if it doesn't exist
+mkdir -p "${NIMBLE_DIR}/pkgs"
+
+# Find all .gitmodules files and process them
+for gitmodules_file in $(find . -name '.gitmodules'); do
+  echo "Processing .gitmodules file: $gitmodules_file"
+  process_gitmodules "$gitmodules_file"
+done

shell.nix

@@ -1,22 +0,0 @@
-{ pkgs ? import (builtins.fetchTarball {
-  url = "https://github.com/NixOS/nixpkgs/archive/dbf1d73cd1a17276196afeee169b4cf7834b7a96.tar.gz";
-  sha256 = "sha256:1k5nvn2yzw370cqsfh62lncsgydq2qkbjrx34cprzf0k6b93v7ch";
-}) {} }:
-
-pkgs.mkShell {
-  name = "nim-waku-build-shell";
-
-  # Versions dependent on nixpkgs commit. Update manually.
-  buildInputs = with pkgs; [
-    git   # 2.37.3
-    which # 2.21
-    rustc # 1.63.0
-  ] ++ lib.optionals stdenv.isDarwin [ 
-    libiconv
-    darwin.apple_sdk.frameworks.Security
-  ];
-
-  LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
-    pkgs.pcre
-  ];
-}