nix: fix builds of libsds for host and android

For some unknown reason the builds would fail with:

/bin/sh: line 1: clang: command not found

Unless we added Android NDK toolchains/llvm/prebuilt to PATH.

Signed-off-by: Jakub Sokołowski <jakub@status.im>

.gitignore

@@ -6,6 +6,7 @@ docs
 for_reference
 do_not_commit
 build/*
+result
 sds.nims
 /.update.timestamp
 

Makefile

@@ -6,6 +6,11 @@ LINK_PCRE := 0
 # we don't want an error here, so we can handle things later, in the ".DEFAULT" target
 -include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk
 
+nimbus-build-system-nimble-dir:
+	NIMBLE_DIR="$(CURDIR)/$(NIMBLE_DIR)" \
+	PWD_CMD="$(PWD)" \
+	$(CURDIR)/scripts/generate_nimble_links.sh
+
 ifeq ($(NIM_PARAMS),)
 # "variables.mk" was not included, so we update the submodules.
 GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
@@ -90,6 +95,8 @@ ifeq ($(detected_OS),Darwin)
 else
 	ANDROID_TOOLCHAIN_DIR := $(ANDROID_NDK_HOME)/toolchains/llvm/prebuilt/linux-x86_64
 endif
+# Fixes "clang: not found" errors
+PATH := $(ANDROID_TOOLCHAIN_DIR)/bin:$(PATH)
 
 libsds-android-precheck:
 ifndef ANDROID_NDK_HOME
@@ -161,4 +168,3 @@ else ifeq ($(ARCH),x86)
 else
 	$(error Unsupported ARCH '$(ARCH)'. Please set ARCH to one of: arm64, arm, amd64, x86)
 endif
-

README.md

@@ -3,6 +3,13 @@ Nim implementation of the e2e reliability protocol
 
 ## Building
 
+### Nix
+
+```bash
+nix build --print-out-paths '.?submodules=1#libsds'
+nix build --print-out-paths '.?submodules=1#libsds-android-arm64'
+```
+
 ### Windows, Linux or MacOS
 
 ```code

flake.lock

@@ -2,17 +2,17 @@
   "nodes": {
     "nixpkgs": {
       "locked": {
-        "lastModified": 1740603184,
+        "lastModified": 1757590060,
-        "narHash": "sha256-t+VaahjQAWyA+Ctn2idyo1yxRIYpaDxMgHkgCNiMJa4=",
+        "narHash": "sha256-EWwwdKLMZALkgHFyKW7rmyhxECO74+N+ZO5xTDnY/5c=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "f44bd8ca21e026135061a0a57dcf3d0775b67a49",
+        "rev": "0ef228213045d2cdb5a169a95d63ded38670b293",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "f44bd8ca21e026135061a0a57dcf3d0775b67a49",
+        "rev": "0ef228213045d2cdb5a169a95d63ded38670b293",
         "type": "github"
       }
     },

flake.nix

@@ -7,7 +7,7 @@
   };
 
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs?rev=f44bd8ca21e026135061a0a57dcf3d0775b67a49";
+    nixpkgs.url = "github:NixOS/nixpkgs?rev=0ef228213045d2cdb5a169a95d63ded38670b293";
   };
 
   outputs = { self, nixpkgs }:
@@ -15,8 +15,7 @@
       stableSystems = [
         "x86_64-linux" "aarch64-linux"
         "x86_64-darwin" "aarch64-darwin"
-        "x86_64-windows" "i686-linux"
+        #"x86_64-windows" FIXME: Check if it works.
-        "i686-windows"
       ];
 
       forAllSystems = f: nixpkgs.lib.genAttrs stableSystems (system: f system);
@@ -45,15 +44,20 @@
         libsds-android-arm64 = pkgs.callPackage ./nix/default.nix {
           inherit stableSystems;
           src = self;
-          targets = ["libsds-android-arm64"]; 
+          targets = ["libsds-android-arm64"];
-          androidArch = "aarch64-linux-android";
-          abidir = "arm64-v8a";
         };
-        default = libsds-android-arm64;
+
+        libsds = pkgs.callPackage ./nix/default.nix {
+          inherit stableSystems;
+          src = self;
+          targets = ["libsds"];
+        };
+
+        default = libsds;
       });
 
       devShells = forAllSystems (system: {
         default = pkgsFor.${system}.callPackage ./nix/shell.nix {};
       });
     };
-}
+}

nix/README.md

@@ -0,0 +1,35 @@
+# Usage
+
+## Shell
+
+A development shell can be started using:
+```sh
+nix develop
+```
+
+## Building
+
+To build a Codex you can use:
+```sh
+nix build '.?submodules=1#default'
+```
+The `?submodules=1` part should eventually not be necessary.
+For more details see:
+https://github.com/NixOS/nix/issues/4423
+
+It can be also done without even cloning the repo:
+```sh
+nix build 'git+https://github.com/waku-org/nim-sds?submodules=1#'
+```
+
+## Running
+
+```sh
+nix run 'git+https://github.com/waku-org/nim-sds?submodules=1#''
+```
+
+## Testing
+
+```sh
+nix flake check ".?submodules=1#"
+```

nix/default.nix

@@ -0,0 +1,89 @@
+{
+  config ? {},
+  pkgs ? import <nixpkgs> { },
+  src ? ../.,
+  targets ? ["libsds-android-arm64"],
+  verbosity ? 2,
+  useSystemNim ? true,
+  quickAndDirty ? true,
+  stableSystems ? [
+    "x86_64-linux" "aarch64-linux"
+  ]
+}:
+
+assert pkgs.lib.assertMsg ((src.submodules or true) == true)
+  "Unable to build without submodules. Append '?submodules=1#' to the URI.";
+
+let
+  inherit (pkgs) stdenv lib writeScriptBin callPackage;
+  inherit (lib) any match substring;
+
+  version = substring 0 8 (src.sourceInfo.rev or "dirty");
+
+in stdenv.mkDerivation rec {
+  pname = "nim-sds";
+  inherit src version;
+
+  buildInputs = with pkgs; [
+    openssl
+    gmp
+    zip
+  ];
+
+  # Dependencies that should only exist in the build environment.
+  nativeBuildInputs = let
+    # Fix for Nim compiler calling 'git rev-parse' and 'lsb_release'.
+    fakeGit = writeScriptBin "git" "echo ${version}";
+  in
+    with pkgs; [
+      cmake
+      which
+      lsb-release
+      nim-unwrapped-2_2
+      fakeGit
+  ];
+
+  # Environment variables required for Android builds
+  ANDROID_SDK_ROOT = "${pkgs.androidPkgs.sdk}";
+  ANDROID_NDK_HOME = "${pkgs.androidPkgs.ndk}";
+  NIMFLAGS = "-d:disableMarchNative -d:git_revision_override=${version}";
+  XDG_CACHE_HOME = "/tmp";
+
+  makeFlags = targets ++ [
+    "V=${toString verbosity}"
+    "USE_SYSTEM_NIM=1"
+  ];
+
+  configurePhase = ''
+    patchShebangs . vendor/nimbus-build-system > /dev/null
+    make nimbus-build-system-paths
+    make nimbus-build-system-nimble-dir
+  '';
+
+  preBuild = ''
+    ln -s sds.nimble sds.nims
+  '';
+
+  installPhase = let
+    androidManifest = ''
+      <manifest xmlns:android=\"http://schemas.android.com/apk/res/android\" package=\"org.waku.${pname}\" />
+    '';
+    containsAndroid = s: (match ".*android.*" s) != null;
+  in if (any containsAndroid targets) then ''
+    mkdir -p $out/jni
+    cp -r build/* $out/jni/
+    echo '${androidManifest}' > $out/jni/AndroidManifest.xml
+    cd $out
+    zip -r libwaku.aar *
+  '' else ''
+    mkdir -p $out/lib
+    cp -r build/* $out/lib
+  '';
+
+  meta = with pkgs.lib; {
+    description = "Nim implementation of the e2e reliability protocol";
+    homepage = "https://github.com/status-im/nim-sds";
+    license = licenses.mit;
+    platforms = stableSystems;
+  };
+}

nix/pkgs/android-sdk/compose.nix

@@ -0,0 +1,26 @@
+#
+# This Nix expression centralizes the configuration
+# for the Android development environment.
+#
+
+{ androidenv, lib, stdenv }:
+
+assert lib.assertMsg (stdenv.system != "aarch64-darwin")
+  "aarch64-darwin not supported for Android SDK. Use: NIXPKGS_SYSTEM_OVERRIDE=x86_64-darwin";
+
+# The "android-sdk-license" license is accepted
+# by setting android_sdk.accept_license = true.
+androidenv.composeAndroidPackages {
+  cmdLineToolsVersion = "9.0";
+  toolsVersion = "26.1.1";
+  platformToolsVersion = "34.0.5";
+  buildToolsVersions = [ "34.0.0" ];
+  platformVersions = [ "34" ];
+  cmakeVersions = [ "3.22.1" ];
+  ndkVersion = "27.2.12479018";
+  includeNDK = true;
+  includeExtras = [
+    "extras;android;m2repository"
+    "extras;google;m2repository"
+  ];
+}

nix/pkgs/android-sdk/default.nix

@@ -0,0 +1,14 @@
+#
+# This Nix expression centralizes the configuration
+# for the Android development environment.
+#
+
+{ callPackage }:
+
+let
+  compose = callPackage ./compose.nix { };
+  pkgs = callPackage ./pkgs.nix { inherit compose; };
+  shell = callPackage ./shell.nix { androidPkgs = pkgs; };
+in {
+  inherit compose pkgs shell;
+}

nix/pkgs/android-sdk/pkgs.nix

@@ -0,0 +1,17 @@
+{ stdenv, compose }:
+
+#
+# This derivation simply symlinks some stuff to get
+# shorter paths as libexec/android-sdk is quite the mouthful.
+# With this you can just do `androidPkgs.sdk` and `androidPkgs.ndk`.
+#
+stdenv.mkDerivation {
+  name = "${compose.androidsdk.name}-mod";
+  phases = [ "symlinkPhase" ];
+  outputs = [ "out" "sdk" "ndk" ];
+  symlinkPhase = ''
+    ln -s ${compose.androidsdk} $out
+    ln -s ${compose.androidsdk}/libexec/android-sdk $sdk
+    ln -s ${compose.androidsdk}/libexec/android-sdk/ndk-bundle $ndk
+  '';
+}

nix/pkgs/android-sdk/shell.nix

@@ -0,0 +1,20 @@
+{ mkShell, openjdk, androidPkgs }:
+
+mkShell {
+  name = "android-sdk-shell";
+  buildInputs = [ openjdk ];
+
+  shellHook = ''
+    export ANDROID_HOME="${androidPkgs.sdk}"
+    export ANDROID_NDK_ROOT="${androidPkgs.ndk}"
+    export ANDROID_SDK_ROOT="$ANDROID_HOME"
+    export ANDROID_NDK_HOME="${androidPkgs.ndk}"
+
+    export PATH="$ANDROID_NDK_ROOT:$PATH"
+    export PATH="$ANDROID_SDK_ROOT/tools:$PATH"
+    export PATH="$ANDROID_SDK_ROOT/tools/bin:$PATH"
+    export PATH="$ANDROID_SDK_ROOT/platform-tools:$PATH"
+    export PATH="$(echo $ANDROID_SDK_ROOT/cmdline-tools/*/bin):$PATH"
+    export PATH="$(echo $ANDROID_NDK_HOME/toolchains/llvm/prebuilt/*/bin):$PATH"
+  '';
+}

nix/shell.nix

@@ -0,0 +1,26 @@
+{
+  pkgs ? import <nixpkgs> { },
+}:
+let
+  optionalDarwinDeps = pkgs.lib.optionals pkgs.stdenv.isDarwin [
+    pkgs.libiconv
+    pkgs.darwin.apple_sdk.frameworks.Security
+  ];
+in
+pkgs.mkShell {
+  inputsFrom = [
+    pkgs.androidShell
+  ] ++ optionalDarwinDeps;
+
+  buildInputs = with pkgs; [
+    which
+    git
+    cmake
+    nim-unwrapped-2_2
+  ];
+
+  # Avoid compiling Nim itself.
+  shellHook = ''
+    export MAKEFLAGS='USE_SYSTEM_NIM=1'
+  '';
+}

scripts/generate_nimble_links.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# This script is used for building Nix derivation which doesn't allow Git commands.
+# It implements similar logic as $(NIMBLE_DIR) target in nimbus-build-system Makefile.
+
+create_nimble_link_script_path="$(pwd)/${BUILD_SYSTEM_DIR}/scripts/create_nimble_link.sh"
+
+process_gitmodules() {
+  local gitmodules_file="$1"
+  local gitmodules_dir=$(dirname "$gitmodules_file")
+  
+  # Extract all submodule paths from the .gitmodules file
+  grep "path" $gitmodules_file | awk '{print $3}' | while read submodule_path; do
+    # Change pwd to the submodule dir and execute script
+    pushd "$gitmodules_dir/$submodule_path" > /dev/null
+    NIMBLE_DIR=$NIMBLE_DIR PWD_CMD=$PWD_CMD EXCLUDED_NIM_PACKAGES=$EXCLUDED_NIM_PACKAGES \
+    "$create_nimble_link_script_path" "$submodule_path"
+    popd  > /dev/null
+  done
+}
+
+# Create the base directory if it doesn't exist
+mkdir -p "${NIMBLE_DIR}/pkgs"
+
+# Find all .gitmodules files and process them
+for gitmodules_file in $(find . -name '.gitmodules'); do
+  echo "Processing .gitmodules file: $gitmodules_file"
+  process_gitmodules "$gitmodules_file"
+done