nix: create initial version of flake

It includes also androidndk and files needed for nimbus-build-system

Referenced issue: https://github.com/waku-org/nwaku/issues/3232

Signed-off-by: markoburcul <marko@status.im>

.gitmodules

@@ -168,7 +168,7 @@
 	path = vendor/db_connector
 	url = https://github.com/nim-lang/db_connector.git
 	ignore = untracked
-	branch = master
+	branch = devel
 [submodule "vendor/nph"]
 	ignore = untracked
 	branch = master

flake.lock

@@ -0,0 +1,49 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1735563628,
+        "narHash": "sha256-OnSAY7XDSx7CtDoqNh8jwVwh4xNL/2HaJxGjryLWzX8=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "b134951a4c9f3c995fd7be05f3243f8ecd65d798",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-24.05",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs",
+        "zerokit": "zerokit"
+      }
+    },
+    "zerokit": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1737544289,
+        "narHash": "sha256-lrSkeUHEkB30sH7VPzLJXoxqLPkj+NzDI2vrmm2L/Gg=",
+        "owner": "vacp2p",
+        "repo": "zerokit",
+        "rev": "7cbb57c2b5b987750db8dbf85fa81275d7c607f7",
+        "type": "github"
+      },
+      "original": {
+        "owner": "vacp2p",
+        "ref": "add-nix-flake-and-derivation",
+        "repo": "zerokit",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,54 @@
+{
+  description = "NWaku build flake";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05";
+    zerokit = {
+      url = "github:vacp2p/zerokit?ref=add-nix-flake-and-derivation";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+  };
+
+  outputs = { self, nixpkgs, zerokit }:
+    let
+      stableSystems = [
+        "x86_64-linux" "aarch64-linux"
+      ];
+      forAllSystems = f: nixpkgs.lib.genAttrs stableSystems (system: f system);
+      pkgsFor = forAllSystems (
+        system: import nixpkgs {
+          inherit system;
+          config = {
+            android_sdk.accept_license = true;
+            allowUnfree = true;
+          };
+          overlays =  [
+            (final: prev: {
+              androidEnvCustom = prev.callPackage ./nix/pkgs/android-sdk { };
+              androidPkgs = final.androidEnvCustom.pkgs;
+              androidShell = final.androidEnvCustom.shell;
+            })
+          ];
+        }
+      );
+    in rec {
+      packages = forAllSystems (system: let
+        zerokitPkg = zerokit.packages.${system}.default;
+        buildTarget = pkgsFor.${system}.callPackage ./nix/default.nix rec {
+          inherit stableSystems zerokitPkg;
+          src = self;
+        };
+        build = targets: buildTarget.override { inherit targets; };
+      in rec {
+        libwaku-android-amd64 = build ["libwaku-android-amd64"];
+        libwaku-android-arm64 = build ["libwaku-android-arm64"];
+        libwaku-android-arm   = build ["libwaku-android-arm"];
+        libwaku-android-x86   = build ["libwaku-android-x86"];
+        default = libwaku-android-amd64;
+      });
+
+      devShells = forAllSystems (system: {
+        default = pkgsFor.${system}.callPackage ./nix/shell.nix { };
+      });
+    };
+}

nix/README.md

@@ -0,0 +1,35 @@
+# Usage
+
+## Shell
+
+A development shell can be started using:
+```sh
+nix develop
+```
+
+## Building
+
+To build a Codex you can use:
+```sh
+nix build '.?submodules=1#default'
+```
+The `?submodules=1` part should eventually not be necessary.
+For more details see:
+https://github.com/NixOS/nix/issues/4423
+
+It can be also done without even cloning the repo:
+```sh
+nix build 'git+https://github.com/waku-org/nwaku?submodules=1#'
+```
+
+## Running
+
+```sh
+nix run 'git+https://github.com/waku-org/nwaku?submodules=1#''
+```
+
+## Testing
+
+```sh
+nix flake check ".?submodules=1#"
+```

nix/atlas.nix

@@ -0,0 +1,12 @@
+{ pkgs ? import <nixpkgs> { } }:
+
+let
+  tools = pkgs.callPackage ./tools.nix {};
+  sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim;
+in pkgs.fetchFromGitHub {
+  owner = "nim-lang";
+  repo = "atlas";
+  rev = tools.findKeyValue "^ +AtlasStableCommit = \"([a-f0-9]+)\"$" sourceFile;
+  # WARNING: Requires manual updates when Nim compiler version changes.
+  hash = "sha256-G1TZdgbRPSgxXZ3VsBP2+XFCLHXVb3an65MuQx67o/k=";
+}

nix/checksums.nix

@@ -0,0 +1,12 @@
+{ pkgs ? import <nixpkgs> { } }:
+
+let
+  tools = pkgs.callPackage ./tools.nix {};
+  sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim;
+in pkgs.fetchFromGitHub {
+  owner = "nim-lang";
+  repo = "checksums";
+  rev = tools.findKeyValue "^ +ChecksumsStableCommit = \"([a-f0-9]+)\"$" sourceFile;
+  # WARNING: Requires manual updates when Nim compiler version changes.
+  hash = "sha256-Bm5iJoT2kAvcTexiLMFBa9oU5gf7d4rWjo3OiN7obWQ=";
+}

nix/csources.nix

@@ -0,0 +1,12 @@
+{ pkgs ? import <nixpkgs> { } }:
+
+let
+  tools = pkgs.callPackage ./tools.nix {};
+  sourceFile = ../vendor/nimbus-build-system/vendor/Nim/config/build_config.txt;
+in pkgs.fetchFromGitHub {
+  owner = "nim-lang";
+  repo = "csources_v2";
+  rev = tools.findKeyValue "^nim_csourcesHash=([a-f0-9]+)$" sourceFile;
+  # WARNING: Requires manual updates when Nim compiler version changes.
+  hash = "sha256-UCLtoxOcGYjBdvHx7A47x6FjLMi6VZqpSs65MN7fpBs=";
+}

nix/default.nix

@@ -0,0 +1,103 @@
+{
+  config ? {},
+  pkgs ? import ./pkgs.nix { inherit config; },
+  src ? ../.,
+  targets ? ["libwaku-android-amd64"],
+  verbosity ? 2,
+  useSystemNim ? false,
+  stableSystems ? [
+    "x86_64-linux" "aarch64-linux"
+  ],
+  zerokitPkg ? (
+    builtins.getFlake "github:vacp2p/zerokit?ref=add-nix-flake-and-derivation"
+  ).packages.${builtins.currentSystem}.default
+}:
+
+assert pkgs.lib.assertMsg ((src.submodules or true) == true)
+  "Unable to build without submodules. Append '?submodules=1#' to the URI.";
+
+let
+  inherit (pkgs) stdenv lib writeScriptBin callPackage;
+
+  revision = lib.substring 0 8 (src.rev or "dirty");
+
+in stdenv.mkDerivation rec {
+
+  pname = "nwaku";
+
+  version = "1.0.0-${revision}";
+
+  inherit src;
+
+  buildInputs = with pkgs; [
+    openssl
+    gmp
+  ];
+
+  # Dependencies that should only exist in the build environment.
+  nativeBuildInputs = let
+    # Fix for Nim compiler calling 'git rev-parse' and 'lsb_release'.
+    fakeGit = writeScriptBin "git" "echo ${version}";
+    # Fix for the zerokit package that is built with cargo/rustup/cross.
+    fakeCargo = writeScriptBin "cargo" "echo ${version}";
+    # Fix for the zerokit package that is built with cargo/rustup/cross.
+    fakeRustup = writeScriptBin "rustup" "echo ${version}";
+    # Fix for the zerokit package that is built with cargo/rustup/cross.
+    fakeCross = writeScriptBin "cross" "echo ${version}";
+  in
+    with pkgs; [
+      cmake
+      which
+      lsb-release
+      zerokitPkg
+      fakeGit
+      fakeCargo
+      fakeRustup
+      fakeCross
+  ];
+
+  # Environment variables required for Android builds
+  ANDROID_SDK_ROOT="${pkgs.androidPkgs.sdk}";
+  ANDROID_NDK_HOME="${pkgs.androidPkgs.ndk}";
+  NIMFLAGS = "-d:disableMarchNative -d:git_revision_override=${revision}";
+  XDG_CACHE_HOME = "/tmp";
+
+  makeFlags = targets ++ [
+    "V=${toString verbosity}"
+  ];
+
+  configurePhase = ''
+    patchShebangs . vendor/nimbus-build-system > /dev/null
+    make nimbus-build-system-paths
+  '';
+
+  preBuild = ''
+    pushd vendor/nimbus-build-system/vendor/Nim
+    mkdir dist
+    cp -r ${callPackage ./nimble.nix {}}    dist/nimble
+    chmod 777 -R dist/nimble
+    mkdir -p dist/nimble/dist
+    cp -r ${callPackage ./checksums.nix {}} dist/checksums  # need both
+    cp -r ${callPackage ./checksums.nix {}} dist/nimble/dist/checksums
+    cp -r ${callPackage ./atlas.nix {}}     dist/atlas
+    chmod 777 -R dist/atlas
+    mkdir dist/atlas/dist
+    cp -r ${callPackage ./sat.nix {}}       dist/nimble/dist/sat
+    cp -r ${callPackage ./sat.nix {}}       dist/atlas/dist/sat
+    cp -r ${callPackage ./csources.nix {}}  csources_v2
+    chmod 777 -R dist/nimble csources_v2
+    popd
+  '';
+
+  installPhase = ''
+    mkdir -p $out/build/android
+    cp -r ./build/android/* $out/build/android/
+  '';
+
+  meta = with pkgs.lib; {
+    description = "NWaku derivation to build libwaku for mobile targets using Android NDK and Rust.";
+    homepage = "https://github.com/status-im/nwaku";
+    license = licenses.mit;
+    platforms = stableSystems;
+  };
+}

nix/nimble.nix

@@ -0,0 +1,12 @@
+{ pkgs ? import <nixpkgs> { } }:
+
+let
+  tools = pkgs.callPackage ./tools.nix {};
+  sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim;
+in pkgs.fetchFromGitHub {
+  owner = "nim-lang";
+  repo = "nimble";
+  rev = tools.findKeyValue "^ +NimbleStableCommit = \"([a-f0-9]+)\".+" sourceFile;
+  # WARNING: Requires manual updates when Nim compiler version changes.
+  hash = "sha256-MVHf19UbOWk8Zba2scj06PxdYYOJA6OXrVyDQ9Ku6Us=";
+}

nix/overlay.nix

@@ -0,0 +1,55 @@
+# Override some packages and utilities in 'pkgs'
+# and make them available globally via callPackage.
+#
+# For more details see:
+# - https://nixos.wiki/wiki/Overlays
+# - https://nixos.org/nixos/nix-pills/callpackage-design-pattern.html
+
+self: super:
+
+let
+  inherit (super) stdenv stdenvNoCC callPackage;
+in {
+  # Fix for MacOS
+  mkShell = super.mkShell.override { stdenv = stdenvNoCC; };
+
+  lib = (super.lib or { }) // (import ./lib {
+    inherit (super) lib;
+  });
+
+  # Checks fail on darwin.
+  git-lfs = super.git-lfs.overrideAttrs (old: {
+    doCheck = false;
+  });
+
+  # Package version adjustments
+  nodejs = super.nodejs_20;
+  ruby = super.ruby_3_1;
+  yarn = super.yarn.override { nodejs = super.nodejs_20; };
+  openjdk = super.openjdk17_headless;
+  go = super.go_1_21;
+  clang = super.clang_15;
+  buildGoPackage = super.buildGo121Package;
+  buildGoModule = super.buildGo121Module;
+  gomobile = (super.gomobile.overrideAttrs (old: {
+    patches = [
+      (self.fetchurl { # https://github.com/golang/mobile/pull/84
+        url = "https://github.com/golang/mobile/commit/f20e966e05b8f7e06bed500fa0da81cf6ebca307.patch";
+        sha256 = "sha256-TZ/Yhe8gMRQUZFAs9G5/cf2b9QGtTHRSObBFD5Pbh7Y=";
+      })
+      (self.fetchurl { # https://github.com/golang/go/issues/58426
+        url = "https://github.com/golang/mobile/commit/406ed3a7b8e44dc32844953647b49696d8847d51.patch";
+        sha256 = "sha256-dqbYukHkQEw8npOkKykOAzMC3ot/Y4DEuh7fE+ptlr8=";
+      })
+    ];
+  })).override {
+    # FIXME: No Android SDK packages for aarch64-darwin.
+    withAndroidPkgs = stdenv.system != "aarch64-darwin";
+    androidPkgs = self.androidEnvCustom.compose;
+  };
+
+  # Android environment
+  androidEnvCustom = callPackage ./pkgs/android-sdk { };
+  androidPkgs = self.androidEnvCustom.pkgs;
+  androidShell = self.androidEnvCustom.shell;
+}

nix/pkgs.nix

@@ -0,0 +1,34 @@
+# This file controls the pinned version of nixpkgs we use for our Nix environment
+# as well as which versions of package we use, including their overrides.
+{ 
+  config ? { },
+  stableSystems ? [
+    "x86_64-linux" "aarch64-linux"
+  ],
+}:
+
+let
+  # For testing local version of nixpkgs
+  #nixpkgsSrc = (import <nixpkgs> { }).lib.cleanSource "/home/jakubgs/work/nixpkgs";
+
+  # We follow release 24-05 of nixpkgs
+  # https://github.com/NixOS/nixpkgs/releases/tag/24.05
+  nixpkgsSrc = builtins.fetchTarball {
+    url = "https://github.com/NixOS/nixpkgs/archive/df27247e6f3e636c119e2610bf12d38b5e98cc79.tar.gz";
+    sha256 = "sha256:0bbvimk7xb7akrx106mmsiwf9nzxnssisqmqffla03zz51d0kz2n";
+  };
+
+  # Status specific configuration defaults
+  defaultConfig = {
+    android_sdk.accept_license = true;
+    allowUnfree = true;
+  };
+
+  # Override some packages and utilities
+  pkgsOverlay = import ./overlay.nix;
+
+  (import nixpkgsSrc) {
+    config = defaultConfig // config;
+    system = stableSystems;
+    overlays = [ pkgsOverlay ];
+  }

nix/pkgs/android-sdk/compose.nix

@@ -0,0 +1,26 @@
+#
+# This Nix expression centralizes the configuration
+# for the Android development environment.
+#
+
+{ androidenv, lib, stdenv }:
+
+assert lib.assertMsg (stdenv.system != "aarch64-darwin")
+  "aarch64-darwin not supported for Android SDK. Use: NIXPKGS_SYSTEM_OVERRIDE=x86_64-darwin";
+
+# The "android-sdk-license" license is accepted
+# by setting android_sdk.accept_license = true.
+androidenv.composeAndroidPackages {
+  cmdLineToolsVersion = "9.0";
+  toolsVersion = "26.1.1";
+  platformToolsVersion = "33.0.3";
+  buildToolsVersions = [ "34.0.0" ];
+  platformVersions = [ "34" ];
+  cmakeVersions = [ "3.22.1" ];
+  ndkVersion = "25.2.9519653";
+  includeNDK = true;
+  includeExtras = [
+    "extras;android;m2repository"
+    "extras;google;m2repository"
+  ];
+}

nix/pkgs/android-sdk/default.nix

@@ -0,0 +1,14 @@
+#
+# This Nix expression centralizes the configuration
+# for the Android development environment.
+#
+
+{ callPackage }:
+
+let
+  compose = callPackage ./compose.nix { };
+  pkgs = callPackage ./pkgs.nix { inherit compose; };
+  shell = callPackage ./shell.nix { androidPkgs = pkgs; };
+in {
+  inherit compose pkgs shell;
+}

nix/pkgs/android-sdk/pkgs.nix

@@ -0,0 +1,17 @@
+{ stdenv, compose }:
+
+#
+# This derivation simply symlinks some stuff to get
+# shorter paths as libexec/android-sdk is quite the mouthful.
+# With this you can just do `androidPkgs.sdk` and `androidPkgs.ndk`.
+#
+stdenv.mkDerivation {
+  name = "${compose.androidsdk.name}-mod";
+  phases = [ "symlinkPhase" ];
+  outputs = [ "out" "sdk" "ndk" ];
+  symlinkPhase = ''
+    ln -s ${compose.androidsdk} $out
+    ln -s ${compose.androidsdk}/libexec/android-sdk $sdk
+    ln -s ${compose.androidsdk}/libexec/android-sdk/ndk-bundle $ndk
+  '';
+}

nix/pkgs/android-sdk/shell.nix

@@ -0,0 +1,18 @@
+{ mkShell, openjdk, androidPkgs }:
+
+mkShell {
+  name = "android-sdk-shell";
+  buildInputs = [ openjdk ];
+
+  shellHook = ''
+    export ANDROID_HOME="${androidPkgs.sdk}"
+    export ANDROID_NDK_ROOT="${androidPkgs.ndk}"
+    export ANDROID_SDK_ROOT="$ANDROID_HOME"
+
+    export PATH="$ANDROID_NDK_ROOT:$PATH"
+    export PATH="$ANDROID_SDK_ROOT/tools:$PATH"
+    export PATH="$ANDROID_SDK_ROOT/tools/bin:$PATH"
+    export PATH="$(echo $ANDROID_SDK_ROOT/cmdline-tools/*/bin):$PATH"
+    export PATH="$ANDROID_SDK_ROOT/platform-tools:$PATH"
+  '';
+}

nix/sat.nix

@@ -0,0 +1,12 @@
+{ pkgs ? import <nixpkgs> { } }:
+
+let
+  tools = pkgs.callPackage ./tools.nix {};
+  sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim;
+in pkgs.fetchFromGitHub {
+  owner = "nim-lang";
+  repo = "sat";
+  rev = tools.findKeyValue "^ +SatStableCommit = \"([a-f0-9]+)\"$" sourceFile;
+  # WARNING: Requires manual updates when Nim compiler version changes.
+  hash = "sha256-JFrrSV+mehG0gP7NiQ8hYthL0cjh44HNbXfuxQNhq7c=";
+}

nix/shell.nix

@@ -0,0 +1,28 @@
+{ 
+  pkgs ? import ./pkgs.nix {},
+}:
+
+pkgs.mkShell {
+  buildInputs = with pkgs; [
+    git
+    cmake
+    openssl
+    which
+    rustup
+    docker
+    cargo
+  ] ++ lib.optionals stdenv.isDarwin [ 
+    libiconv
+    darwin.apple_sdk.frameworks.Security
+  ];
+
+  LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
+    pkgs.pcre
+  ];
+
+  shellHook = ''
+    export ANDROID_SDK_ROOT="${pkgs.androidPkgs.sdk}"
+    export ANDROID_NDK_HOME="${pkgs.androidPkgs.ndk}"
+    rustup default stable
+  '';
+}

nix/tools.nix

@@ -0,0 +1,15 @@
+{ pkgs ? import <nixpkgs> { } }:
+
+let
+
+  inherit (pkgs.lib) fileContents last splitString flatten remove;
+  inherit (builtins) map match;
+in {
+  findKeyValue = regex: sourceFile:
+    let
+      linesFrom = file: splitString "\n" (fileContents file);
+      matching = regex: lines: map (line: match regex line) lines;
+      extractMatch = matches: last (flatten (remove null matches));
+    in
+      extractMatch (matching regex (linesFrom sourceFile));
+}

shell.nix

@@ -1,22 +0,0 @@
-{ pkgs ? import (builtins.fetchTarball {
-  url = "https://github.com/NixOS/nixpkgs/archive/dbf1d73cd1a17276196afeee169b4cf7834b7a96.tar.gz";
-  sha256 = "sha256:1k5nvn2yzw370cqsfh62lncsgydq2qkbjrx34cprzf0k6b93v7ch";
-}) {} }:
-
-pkgs.mkShell {
-  name = "nim-waku-build-shell";
-
-  # Versions dependent on nixpkgs commit. Update manually.
-  buildInputs = with pkgs; [
-    git   # 2.37.3
-    which # 2.21
-    rustc # 1.63.0
-  ] ++ lib.optionals stdenv.isDarwin [ 
-    libiconv
-    darwin.apple_sdk.frameworks.Security
-  ];
-
-  LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
-    pkgs.pcre
-  ];
-}