From 87d5716ac6654fa557e0313a8af39ffc24509afc Mon Sep 17 00:00:00 2001 From: Ivan Folgueira Bande Date: Wed, 15 Oct 2025 11:17:02 +0200 Subject: [PATCH] nix: fix builds of libsds for host and android MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For some unknown reason the builds would fail with: /bin/sh: line 1: clang: command not found Unless we added Android NDK toolchains/llvm/prebuilt to PATH. Signed-off-by: Jakub SokoĊ‚owski --- .gitignore | 1 + Makefile | 8 ++- README.md | 7 +++ flake.lock | 8 +-- flake.nix | 20 ++++--- nix/README.md | 35 +++++++++++++ nix/default.nix | 89 ++++++++++++++++++++++++++++++++ nix/pkgs/android-sdk/compose.nix | 26 ++++++++++ nix/pkgs/android-sdk/default.nix | 14 +++++ nix/pkgs/android-sdk/pkgs.nix | 17 ++++++ nix/pkgs/android-sdk/shell.nix | 20 +++++++ nix/shell.nix | 26 ++++++++++ scripts/generate_nimble_links.sh | 29 +++++++++++ 13 files changed, 287 insertions(+), 13 deletions(-) create mode 100644 nix/README.md create mode 100644 nix/default.nix create mode 100644 nix/pkgs/android-sdk/compose.nix create mode 100644 nix/pkgs/android-sdk/default.nix create mode 100644 nix/pkgs/android-sdk/pkgs.nix create mode 100644 nix/pkgs/android-sdk/shell.nix create mode 100644 nix/shell.nix create mode 100755 scripts/generate_nimble_links.sh diff --git a/.gitignore b/.gitignore index 898171b..a087d35 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ docs for_reference do_not_commit build/* +result sds.nims /.update.timestamp diff --git a/Makefile b/Makefile index 096a0a5..06447c4 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,11 @@ LINK_PCRE := 0 # we don't want an error here, so we can handle things later, in the ".DEFAULT" target -include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk +nimbus-build-system-nimble-dir: + NIMBLE_DIR="$(CURDIR)/$(NIMBLE_DIR)" \ + PWD_CMD="$(PWD)" \ + $(CURDIR)/scripts/generate_nimble_links.sh + ifeq ($(NIM_PARAMS),) # "variables.mk" was not included, so we update the submodules. GIT_SUBMODULE_UPDATE := git submodule update --init --recursive @@ -90,6 +95,8 @@ ifeq ($(detected_OS),Darwin) else ANDROID_TOOLCHAIN_DIR := $(ANDROID_NDK_HOME)/toolchains/llvm/prebuilt/linux-x86_64 endif +# Fixes "clang: not found" errors +PATH := $(ANDROID_TOOLCHAIN_DIR)/bin:$(PATH) libsds-android-precheck: ifndef ANDROID_NDK_HOME @@ -161,4 +168,3 @@ else ifeq ($(ARCH),x86) else $(error Unsupported ARCH '$(ARCH)'. Please set ARCH to one of: arm64, arm, amd64, x86) endif - diff --git a/README.md b/README.md index 0092490..32b11ec 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,13 @@ Nim implementation of the e2e reliability protocol ## Building +### Nix + +```bash +nix build --print-out-paths '.?submodules=1#libsds' +nix build --print-out-paths '.?submodules=1#libsds-android-arm64' +``` + ### Windows, Linux or MacOS ```code diff --git a/flake.lock b/flake.lock index 46a1cbf..1ca4043 100644 --- a/flake.lock +++ b/flake.lock @@ -2,17 +2,17 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1740603184, - "narHash": "sha256-t+VaahjQAWyA+Ctn2idyo1yxRIYpaDxMgHkgCNiMJa4=", + "lastModified": 1757590060, + "narHash": "sha256-EWwwdKLMZALkgHFyKW7rmyhxECO74+N+ZO5xTDnY/5c=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "f44bd8ca21e026135061a0a57dcf3d0775b67a49", + "rev": "0ef228213045d2cdb5a169a95d63ded38670b293", "type": "github" }, "original": { "owner": "NixOS", "repo": "nixpkgs", - "rev": "f44bd8ca21e026135061a0a57dcf3d0775b67a49", + "rev": "0ef228213045d2cdb5a169a95d63ded38670b293", "type": "github" } }, diff --git a/flake.nix b/flake.nix index 3171e22..184ffa5 100644 --- a/flake.nix +++ b/flake.nix @@ -7,7 +7,7 @@ }; inputs = { - nixpkgs.url = "github:NixOS/nixpkgs?rev=f44bd8ca21e026135061a0a57dcf3d0775b67a49"; + nixpkgs.url = "github:NixOS/nixpkgs?rev=0ef228213045d2cdb5a169a95d63ded38670b293"; }; outputs = { self, nixpkgs }: @@ -15,8 +15,7 @@ stableSystems = [ "x86_64-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" - "x86_64-windows" "i686-linux" - "i686-windows" + #"x86_64-windows" FIXME: Check if it works. ]; forAllSystems = f: nixpkgs.lib.genAttrs stableSystems (system: f system); @@ -45,15 +44,20 @@ libsds-android-arm64 = pkgs.callPackage ./nix/default.nix { inherit stableSystems; src = self; - targets = ["libsds-android-arm64"]; - androidArch = "aarch64-linux-android"; - abidir = "arm64-v8a"; + targets = ["libsds-android-arm64"]; }; - default = libsds-android-arm64; + + libsds = pkgs.callPackage ./nix/default.nix { + inherit stableSystems; + src = self; + targets = ["libsds"]; + }; + + default = libsds; }); devShells = forAllSystems (system: { default = pkgsFor.${system}.callPackage ./nix/shell.nix {}; }); }; -} \ No newline at end of file +} diff --git a/nix/README.md b/nix/README.md new file mode 100644 index 0000000..fd481b6 --- /dev/null +++ b/nix/README.md @@ -0,0 +1,35 @@ +# Usage + +## Shell + +A development shell can be started using: +```sh +nix develop +``` + +## Building + +To build a Codex you can use: +```sh +nix build '.?submodules=1#default' +``` +The `?submodules=1` part should eventually not be necessary. +For more details see: +https://github.com/NixOS/nix/issues/4423 + +It can be also done without even cloning the repo: +```sh +nix build 'git+https://github.com/waku-org/nim-sds?submodules=1#' +``` + +## Running + +```sh +nix run 'git+https://github.com/waku-org/nim-sds?submodules=1#'' +``` + +## Testing + +```sh +nix flake check ".?submodules=1#" +``` diff --git a/nix/default.nix b/nix/default.nix new file mode 100644 index 0000000..c3cb29a --- /dev/null +++ b/nix/default.nix @@ -0,0 +1,89 @@ +{ + config ? {}, + pkgs ? import { }, + src ? ../., + targets ? ["libsds-android-arm64"], + verbosity ? 2, + useSystemNim ? true, + quickAndDirty ? true, + stableSystems ? [ + "x86_64-linux" "aarch64-linux" + ] +}: + +assert pkgs.lib.assertMsg ((src.submodules or true) == true) + "Unable to build without submodules. Append '?submodules=1#' to the URI."; + +let + inherit (pkgs) stdenv lib writeScriptBin callPackage; + inherit (lib) any match substring; + + version = substring 0 8 (src.sourceInfo.rev or "dirty"); + +in stdenv.mkDerivation rec { + pname = "nim-sds"; + inherit src version; + + buildInputs = with pkgs; [ + openssl + gmp + zip + ]; + + # Dependencies that should only exist in the build environment. + nativeBuildInputs = let + # Fix for Nim compiler calling 'git rev-parse' and 'lsb_release'. + fakeGit = writeScriptBin "git" "echo ${version}"; + in + with pkgs; [ + cmake + which + lsb-release + nim-unwrapped-2_2 + fakeGit + ]; + + # Environment variables required for Android builds + ANDROID_SDK_ROOT = "${pkgs.androidPkgs.sdk}"; + ANDROID_NDK_HOME = "${pkgs.androidPkgs.ndk}"; + NIMFLAGS = "-d:disableMarchNative -d:git_revision_override=${version}"; + XDG_CACHE_HOME = "/tmp"; + + makeFlags = targets ++ [ + "V=${toString verbosity}" + "USE_SYSTEM_NIM=1" + ]; + + configurePhase = '' + patchShebangs . vendor/nimbus-build-system > /dev/null + make nimbus-build-system-paths + make nimbus-build-system-nimble-dir + ''; + + preBuild = '' + ln -s sds.nimble sds.nims + ''; + + installPhase = let + androidManifest = '' + + ''; + containsAndroid = s: (match ".*android.*" s) != null; + in if (any containsAndroid targets) then '' + mkdir -p $out/jni + cp -r build/* $out/jni/ + echo '${androidManifest}' > $out/jni/AndroidManifest.xml + cd $out + zip -r libwaku.aar * + '' else '' + mkdir -p $out/lib + cp -r build/* $out/lib + ''; + + meta = with pkgs.lib; { + description = "Nim implementation of the e2e reliability protocol"; + homepage = "https://github.com/status-im/nim-sds"; + license = licenses.mit; + platforms = stableSystems; + }; +} diff --git a/nix/pkgs/android-sdk/compose.nix b/nix/pkgs/android-sdk/compose.nix new file mode 100644 index 0000000..90291d4 --- /dev/null +++ b/nix/pkgs/android-sdk/compose.nix @@ -0,0 +1,26 @@ +# +# This Nix expression centralizes the configuration +# for the Android development environment. +# + +{ androidenv, lib, stdenv }: + +assert lib.assertMsg (stdenv.system != "aarch64-darwin") + "aarch64-darwin not supported for Android SDK. Use: NIXPKGS_SYSTEM_OVERRIDE=x86_64-darwin"; + +# The "android-sdk-license" license is accepted +# by setting android_sdk.accept_license = true. +androidenv.composeAndroidPackages { + cmdLineToolsVersion = "9.0"; + toolsVersion = "26.1.1"; + platformToolsVersion = "34.0.5"; + buildToolsVersions = [ "34.0.0" ]; + platformVersions = [ "34" ]; + cmakeVersions = [ "3.22.1" ]; + ndkVersion = "27.2.12479018"; + includeNDK = true; + includeExtras = [ + "extras;android;m2repository" + "extras;google;m2repository" + ]; +} diff --git a/nix/pkgs/android-sdk/default.nix b/nix/pkgs/android-sdk/default.nix new file mode 100644 index 0000000..f3f7952 --- /dev/null +++ b/nix/pkgs/android-sdk/default.nix @@ -0,0 +1,14 @@ +# +# This Nix expression centralizes the configuration +# for the Android development environment. +# + +{ callPackage }: + +let + compose = callPackage ./compose.nix { }; + pkgs = callPackage ./pkgs.nix { inherit compose; }; + shell = callPackage ./shell.nix { androidPkgs = pkgs; }; +in { + inherit compose pkgs shell; +} diff --git a/nix/pkgs/android-sdk/pkgs.nix b/nix/pkgs/android-sdk/pkgs.nix new file mode 100644 index 0000000..645987b --- /dev/null +++ b/nix/pkgs/android-sdk/pkgs.nix @@ -0,0 +1,17 @@ +{ stdenv, compose }: + +# +# This derivation simply symlinks some stuff to get +# shorter paths as libexec/android-sdk is quite the mouthful. +# With this you can just do `androidPkgs.sdk` and `androidPkgs.ndk`. +# +stdenv.mkDerivation { + name = "${compose.androidsdk.name}-mod"; + phases = [ "symlinkPhase" ]; + outputs = [ "out" "sdk" "ndk" ]; + symlinkPhase = '' + ln -s ${compose.androidsdk} $out + ln -s ${compose.androidsdk}/libexec/android-sdk $sdk + ln -s ${compose.androidsdk}/libexec/android-sdk/ndk-bundle $ndk + ''; +} diff --git a/nix/pkgs/android-sdk/shell.nix b/nix/pkgs/android-sdk/shell.nix new file mode 100644 index 0000000..0afbf80 --- /dev/null +++ b/nix/pkgs/android-sdk/shell.nix @@ -0,0 +1,20 @@ +{ mkShell, openjdk, androidPkgs }: + +mkShell { + name = "android-sdk-shell"; + buildInputs = [ openjdk ]; + + shellHook = '' + export ANDROID_HOME="${androidPkgs.sdk}" + export ANDROID_NDK_ROOT="${androidPkgs.ndk}" + export ANDROID_SDK_ROOT="$ANDROID_HOME" + export ANDROID_NDK_HOME="${androidPkgs.ndk}" + + export PATH="$ANDROID_NDK_ROOT:$PATH" + export PATH="$ANDROID_SDK_ROOT/tools:$PATH" + export PATH="$ANDROID_SDK_ROOT/tools/bin:$PATH" + export PATH="$ANDROID_SDK_ROOT/platform-tools:$PATH" + export PATH="$(echo $ANDROID_SDK_ROOT/cmdline-tools/*/bin):$PATH" + export PATH="$(echo $ANDROID_NDK_HOME/toolchains/llvm/prebuilt/*/bin):$PATH" + ''; +} diff --git a/nix/shell.nix b/nix/shell.nix new file mode 100644 index 0000000..d21eb01 --- /dev/null +++ b/nix/shell.nix @@ -0,0 +1,26 @@ +{ + pkgs ? import { }, +}: +let + optionalDarwinDeps = pkgs.lib.optionals pkgs.stdenv.isDarwin [ + pkgs.libiconv + pkgs.darwin.apple_sdk.frameworks.Security + ]; +in +pkgs.mkShell { + inputsFrom = [ + pkgs.androidShell + ] ++ optionalDarwinDeps; + + buildInputs = with pkgs; [ + which + git + cmake + nim-unwrapped-2_2 + ]; + + # Avoid compiling Nim itself. + shellHook = '' + export MAKEFLAGS='USE_SYSTEM_NIM=1' + ''; +} diff --git a/scripts/generate_nimble_links.sh b/scripts/generate_nimble_links.sh new file mode 100755 index 0000000..e01e6db --- /dev/null +++ b/scripts/generate_nimble_links.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# This script is used for building Nix derivation which doesn't allow Git commands. +# It implements similar logic as $(NIMBLE_DIR) target in nimbus-build-system Makefile. + +create_nimble_link_script_path="$(pwd)/${BUILD_SYSTEM_DIR}/scripts/create_nimble_link.sh" + +process_gitmodules() { + local gitmodules_file="$1" + local gitmodules_dir=$(dirname "$gitmodules_file") + + # Extract all submodule paths from the .gitmodules file + grep "path" $gitmodules_file | awk '{print $3}' | while read submodule_path; do + # Change pwd to the submodule dir and execute script + pushd "$gitmodules_dir/$submodule_path" > /dev/null + NIMBLE_DIR=$NIMBLE_DIR PWD_CMD=$PWD_CMD EXCLUDED_NIM_PACKAGES=$EXCLUDED_NIM_PACKAGES \ + "$create_nimble_link_script_path" "$submodule_path" + popd > /dev/null + done +} + +# Create the base directory if it doesn't exist +mkdir -p "${NIMBLE_DIR}/pkgs" + +# Find all .gitmodules files and process them +for gitmodules_file in $(find . -name '.gitmodules'); do + echo "Processing .gitmodules file: $gitmodules_file" + process_gitmodules "$gitmodules_file" +done