From de1642a292a2123030dc30b368c36d8bd3f7f582 Mon Sep 17 00:00:00 2001 From: darshankabariya Date: Mon, 9 Feb 2026 18:22:20 +0530 Subject: [PATCH] chore: Nix + Nimble --- Makefile | 13 ++++ README.md | 75 ++++++++++++++------ env.sh | 11 +++ flake.nix | 2 + nix/README.md | 49 ++++++++----- nix/default.nix | 116 ++++++++++++++++++++++++++----- nix/shell.nix | 24 ++++++- scripts/generate_nimble_links.sh | 79 +++++++++++++++++---- sds.nimble | 29 +++++++- 9 files changed, 326 insertions(+), 72 deletions(-) diff --git a/Makefile b/Makefile index 6f096ca..a06ad22 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,16 @@ +# ============================================================================ +# DEPRECATED: This Makefile is deprecated in favor of direct Nix + Nimble builds. +# +# New workflow: +# nix build '.?submodules=1#libsds' # Build via Nix +# nix develop '.?submodules=1' # Enter dev shell, then: +# nim libsdsDynamicLinux sds.nims # Linux +# nim libsdsDynamicMac sds.nims # macOS +# nimble test # Run tests +# +# This Makefile will be removed in a future release. +# ============================================================================ + .PHONY: libsds deps export BUILD_SYSTEM_DIR := vendor/nimbus-build-system diff --git a/README.md b/README.md index 701d5ee..a495df0 100644 --- a/README.md +++ b/README.md @@ -3,47 +3,82 @@ Nim implementation of the e2e reliability protocol ## Building -### Nix +### Nix (recommended) +Build the shared library: ```bash nix build --print-out-paths '.?submodules=1#libsds' +``` + +Build for Android: +```bash nix build --print-out-paths '.?submodules=1#libsds-android-arm64' +nix build --print-out-paths '.?submodules=1#libsds-android-amd64' +nix build --print-out-paths '.?submodules=1#libsds-android-x86' +nix build --print-out-paths '.?submodules=1#libsds-android-arm' ``` -### Windows, Linux or MacOS +### Development shell -```code -make libsds +Enter the dev shell (sets up vendored dependencies automatically): +```bash +nix develop '.?submodules=1' ``` -### Android +Then build directly with nim: +```bash +# Linux +nim libsdsDynamicLinux sds.nims +nim libsdsStaticLinux sds.nims + +# macOS +nim libsdsDynamicMac sds.nims +nim libsdsStaticMac sds.nims + +# Windows +nim libsdsDynamicWindows sds.nims +nim libsdsStaticWindows sds.nims +``` + +Run tests: +```bash +nimble test +``` + +The built library is output to `build/`. + +### Android (without Nix) Download the latest Android NDK. For example, on Ubuntu with Intel: -```code +```bash cd ~ wget https://dl.google.com/android/repository/android-ndk-r27c-linux.zip -``` -```code unzip android-ndk-r27c-linux.zip ``` -Then, add the following to your ~/.bashrc file: -```code +Then, add the following to your `~/.bashrc` file: +```bash export ANDROID_NDK_ROOT=$HOME/android-ndk-r27c export PATH=$ANDROID_NDK_ROOT/toolchains/llvm/prebuilt/linux-x86_64/bin:$PATH ``` -Then, use one of the following commands, according to the current architecture: - -| Architecture | command | -| ------------ | ------- | -| arm64 | `make libsds-android ARCH=arm64` | -| amd64 | `make libsds-android ARCH=amd64` | -| x86 | `make libsds-android ARCH=x86` | - -At the end of the process, the library will be created in build/libsds.so - +Set up vendored dependencies and build: +```bash +export NIMBLE_DIR="$(pwd)/vendor/.nimble" +bash scripts/generate_nimble_links.sh +ln -s sds.nimble sds.nims +# Set arch-specific env vars, then build +ARCH=arm64 ANDROID_ARCH=aarch64-linux-android ARCH_DIRNAME=aarch64-linux-android \ + nim libsdsAndroid sds.nims +``` +| Architecture | ARCH | ANDROID_ARCH | ARCH_DIRNAME | +| ------------ | ---- | ------------ | ------------ | +| arm64 | `arm64` | `aarch64-linux-android` | `aarch64-linux-android` | +| amd64 | `amd64` | `x86_64-linux-android` | `x86_64-linux-android` | +| x86 | `i386` | `i686-linux-android` | `i686-linux-android` | +| arm | `arm` | `armv7a-linux-androideabi` | `arm-linux-androideabi` | +The library is output to `build/libsds.so`. diff --git a/env.sh b/env.sh index 697a426..b353067 100644 --- a/env.sh +++ b/env.sh @@ -1,5 +1,16 @@ #!/usr/bin/env bash +# ============================================================================ +# DEPRECATED: This script is deprecated in favor of 'nix develop'. +# +# New workflow: +# nix develop '.?submodules=1' # Enter dev shell with Nim + deps ready +# +# This script will be removed in a future release. +# ============================================================================ + +echo "WARNING: env.sh is deprecated. Use 'nix develop' instead." >&2 + # We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file # and we fall back to a Zsh-specific special var to also support Zsh. REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})" diff --git a/flake.nix b/flake.nix index 2fca775..40989bb 100644 --- a/flake.nix +++ b/flake.nix @@ -10,6 +10,8 @@ # We are pinning the commit because ultimately we want to use same commit across different projects. # A commit from nixpkgs 24.11 release : https://github.com/NixOS/nixpkgs/tree/release-24.11 nixpkgs.url = "github:NixOS/nixpkgs?rev=0ef228213045d2cdb5a169a95d63ded38670b293"; + # Used only for its pinned Nim compiler package. + # The build system itself (Makefiles, env.sh) is NOT used — we call nimble directly. # WARNING: Remember to update commit and use 'nix flake update' to update flake.lock. nimbusBuildSystem = { url = "git+file:./vendor/nimbus-build-system?submodules=1"; diff --git a/nix/README.md b/nix/README.md index fd481b6..dcfedcd 100644 --- a/nix/README.md +++ b/nix/README.md @@ -1,35 +1,48 @@ -# Usage +# Nix Build System + +The Nix configuration builds nim-sds by calling nimble tasks directly (no Makefile involved). + +## Architecture + +``` +flake.nix — Entry point, defines packages and dev shell +├── nix/default.nix — Build derivation: configures env, runs nim sds.nims +├── nix/shell.nix — Dev shell: sets up NIMBLE_DIR, nimble-links, NIM_LIB_DIR +└── nix/tools.nix — Helper: extracts version from sds.nimble +``` + +The `nimbusBuildSystem` flake input is used **only** for its pinned Nim compiler. +All build logic lives in `sds.nimble` (nimble tasks). ## Shell A development shell can be started using: ```sh -nix develop +nix develop '.?submodules=1' ``` +This automatically: +- Runs `scripts/generate_nimble_links.sh` to set up vendored dependencies +- Exports `NIMBLE_DIR` and `NIM_LIB_DIR` +- Creates the `sds.nims` symlink + ## Building -To build a Codex you can use: ```sh -nix build '.?submodules=1#default' -``` -The `?submodules=1` part should eventually not be necessary. -For more details see: -https://github.com/NixOS/nix/issues/4423 - -It can be also done without even cloning the repo: -```sh -nix build 'git+https://github.com/waku-org/nim-sds?submodules=1#' +nix build '.?submodules=1#libsds' +nix build '.?submodules=1#libsds-android-arm64' ``` -## Running - -```sh -nix run 'git+https://github.com/waku-org/nim-sds?submodules=1#'' -``` +The `?submodules=1` part is required because vendored dependencies are git submodules. +For more details see: https://github.com/NixOS/nix/issues/4423 ## Testing ```sh -nix flake check ".?submodules=1#" +nix flake check '.?submodules=1' +``` + +Or inside the dev shell: +```sh +nimble test ``` diff --git a/nix/default.nix b/nix/default.nix index 51386a3..8366ae0 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -1,12 +1,12 @@ { pkgs, src ? ../., - # Nimbus-build-system package. + # Nimbus-build-system package (used only for pinned Nim compiler). nim ? null, # Options: 0,1,2 verbosity ? 2, - # Make targets - targets ? ["libsds-android-arm64"], + # Build targets (e.g., ["libsds"], ["libsds-android-arm64"]) + targets ? ["libsds"], # These are the only platforms tested in CI and considered stable. stableSystems ? ["x86_64-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "x86_64-windows"], }: @@ -15,8 +15,8 @@ assert pkgs.lib.assertMsg ((src.submodules or true) == true) "Unable to build without submodules. Append '?submodules=1#' to the URI."; let - inherit (pkgs) stdenv lib writeScriptBin callPackage; - inherit (lib) any match substring optionals optionalString; + inherit (pkgs) stdenv lib callPackage; + inherit (lib) any match optionals optionalString substring; # Check if build is for android platform. containsAndroid = s: (match ".*android.*" s) != null; @@ -27,6 +27,75 @@ let revision = substring 0 8 (src.rev or src.dirtyRev or "00000000"); version = tools.findKeyValue "^version = \"([a-f0-9.-]+)\"$" ../sds.nimble; + # Map target names to nimble task + env vars + targetConfig = { + "libsds" = rec { + nimbleTask = + if stdenv.isDarwin then "libsdsDynamicMac" + else if stdenv.isLinux then "libsdsDynamicLinux" + else "libsdsDynamicWindows"; + envVars = {}; + }; + "libsds-static" = rec { + nimbleTask = + if stdenv.isDarwin then "libsdsStaticMac" + else if stdenv.isLinux then "libsdsStaticLinux" + else "libsdsStaticWindows"; + envVars = {}; + }; + "libsds-android-arm64" = { + nimbleTask = "libsdsAndroid"; + envVars = { + ARCH = "arm64"; + ANDROID_ARCH = "aarch64-linux-android"; + ARCH_DIRNAME = "aarch64-linux-android"; + ABIDIR = "arm64-v8a"; + }; + }; + "libsds-android-amd64" = { + nimbleTask = "libsdsAndroid"; + envVars = { + ARCH = "amd64"; + ANDROID_ARCH = "x86_64-linux-android"; + ARCH_DIRNAME = "x86_64-linux-android"; + ABIDIR = "x86_64"; + }; + }; + "libsds-android-x86" = { + nimbleTask = "libsdsAndroid"; + envVars = { + ARCH = "i386"; + ANDROID_ARCH = "i686-linux-android"; + ARCH_DIRNAME = "i686-linux-android"; + ABIDIR = "x86"; + }; + }; + "libsds-android-arm" = { + nimbleTask = "libsdsAndroid"; + envVars = { + ARCH = "arm"; + ANDROID_ARCH = "armv7a-linux-androideabi"; + ARCH_DIRNAME = "arm-linux-androideabi"; + ABIDIR = "armeabi-v7a"; + }; + }; + }; + + # Git version for NIM_PARAMS + gitVersion = substring 0 8 (src.rev or src.dirtyRev or "00000000"); + + # Build the nim command for a single target + buildCommandForTarget = target: let + cfg = targetConfig.${target}; + nimParams = "-d:git_version=\"${gitVersion}\" -d:release -d:disableMarchNative"; + envExports = lib.concatStringsSep "\n" ( + lib.mapAttrsToList (k: v: "export ${k}=${v}") cfg.envVars + ); + in '' + ${envExports} + nim ${cfg.nimbleTask} ${nimParams} sds.nims + ''; + in stdenv.mkDerivation { pname = "nim-sds"; inherit src; @@ -34,32 +103,45 @@ in stdenv.mkDerivation { env = { NIMFLAGS = "-d:disableMarchNative"; - ANDROID_SDK_ROOT = optionalString isAndroidBuild pkgs.androidPkgs.sdk; - ANDROID_NDK_ROOT = optionalString isAndroidBuild pkgs.androidPkgs.ndk; + } // lib.optionalAttrs isAndroidBuild { + ANDROID_SDK_ROOT = pkgs.androidPkgs.sdk; + ANDROID_NDK_ROOT = pkgs.androidPkgs.ndk; + ANDROID_TARGET = "30"; }; buildInputs = with pkgs; [ openssl gmp zip ]; - # Dependencies that should only exist in the build environment. nativeBuildInputs = with pkgs; [ nim cmake which patchelf ] ++ optionals stdenv.isLinux [ pkgs.lsb-release ]; - makeFlags = targets ++ [ - "V=${toString verbosity}" - # Built from nimbus-build-system via flake. - "USE_SYSTEM_NIM=1" - ]; - configurePhase = '' # Avoid /tmp write errors. export XDG_CACHE_HOME=$TMPDIR/cache - patchShebangs . vendor/nimbus-build-system/scripts - make nimbus-build-system-nimble-dir + + patchShebangs scripts/ + + # Set up nimble-link directory for vendored dependencies + export NIMBLE_DIR="$(pwd)/vendor/.nimble" + scripts/generate_nimble_links.sh + + # Create sds.nims symlink if missing (nimble task runner needs it) + if [ ! -e sds.nims ]; then + ln -s sds.nimble sds.nims + fi + + # Tell Nim where to find its standard library headers (for iOS C compilation) + export NIM_LIB_DIR="$(dirname $(dirname $(which nim)))/lib" + ''; + + buildPhase = '' + export NIMBLE_DIR="$(pwd)/vendor/.nimble" + + ${lib.concatMapStringsSep "\n" buildCommandForTarget targets} ''; installPhase = let @@ -73,7 +155,7 @@ in stdenv.mkDerivation { cd $out zip -r libwaku.aar * '' else '' - mkdir -p $out/lib -p $out/include + mkdir -p $out/lib $out/include cp build/* $out/lib/ cp library/libsds.h $out/include/ ''; diff --git a/nix/shell.nix b/nix/shell.nix index 53b0663..d725257 100644 --- a/nix/shell.nix +++ b/nix/shell.nix @@ -20,8 +20,28 @@ in pkgs.mkShell { pkgs.libiconv ]; - # Avoid compiling Nim itself. shellHook = '' - export USE_SYSTEM_NIM=1 + # Set up nimble-link directory for vendored dependencies + export NIMBLE_DIR="$(pwd)/vendor/.nimble" + if [ -f scripts/generate_nimble_links.sh ]; then + bash scripts/generate_nimble_links.sh + fi + + # Create sds.nims symlink if missing + if [ ! -e sds.nims ]; then + ln -s sds.nimble sds.nims + fi + + # Tell Nim where to find its standard library headers + export NIM_LIB_DIR="$(dirname $(dirname $(which nim)))/lib" + + echo "" + echo "nim-sds dev shell ready. Build commands:" + echo " nim libsdsDynamicLinux sds.nims # Linux shared library" + echo " nim libsdsDynamicMac sds.nims # macOS shared library" + echo " nim libsdsStaticLinux sds.nims # Linux static library" + echo " nim libsdsStaticMac sds.nims # macOS static library" + echo " nimble test # Run tests" + echo "" ''; } diff --git a/scripts/generate_nimble_links.sh b/scripts/generate_nimble_links.sh index e01e6db..3f3f6b6 100755 --- a/scripts/generate_nimble_links.sh +++ b/scripts/generate_nimble_links.sh @@ -1,29 +1,80 @@ #!/usr/bin/env bash -# This script is used for building Nix derivation which doesn't allow Git commands. -# It implements similar logic as $(NIMBLE_DIR) target in nimbus-build-system Makefile. +# This script creates nimble-link files for all vendored Nim submodules. +# It replaces the dependency on nimbus-build-system's create_nimble_link.sh +# by inlining the same logic. +# +# Required env vars: +# NIMBLE_DIR — directory where nimble-link packages are created +# (e.g., vendor/.nimble) +# +# Optional env vars: +# EXCLUDED_NIM_PACKAGES — space-separated list of submodule paths to skip -create_nimble_link_script_path="$(pwd)/${BUILD_SYSTEM_DIR}/scripts/create_nimble_link.sh" +set -euo pipefail + +: "${NIMBLE_DIR:?NIMBLE_DIR must be set}" + +create_nimble_link() { + local submodule_dir="$1" + local module_name + module_name="$(basename "$submodule_dir")" + + # Only process directories that contain a .nimble file + if ! ls "$submodule_dir"/*.nimble &>/dev/null; then + return + fi + + local pkg_dir + pkg_dir="$(cd "$submodule_dir" && pwd)" + + # Check exclusions + for excluded in ${EXCLUDED_NIM_PACKAGES:-}; do + if [[ "$pkg_dir" =~ $excluded ]]; then + return + fi + done + + # If src/ subdir exists, use it as the package directory + if [[ -d "$pkg_dir/src" ]]; then + pkg_dir="$pkg_dir/src" + fi + + local link_dir="${NIMBLE_DIR}/pkgs/${module_name}-#head" + local link_path="${link_dir}/${module_name}.nimble-link" + + mkdir -p "$link_dir" + + if [[ -e "$link_path" ]]; then + echo "ERROR: Nim package already present in '${link_path}': '$(head -n1 "$link_path")'" + echo "Will not replace it with '${pkg_dir}'." + echo "Pick one and put the other's relative path in EXCLUDED_NIM_PACKAGES." + rm -rf "${NIMBLE_DIR}" + exit 1 + fi + + printf '%s\n%s\n' "$pkg_dir" "$pkg_dir" > "$link_path" +} process_gitmodules() { local gitmodules_file="$1" - local gitmodules_dir=$(dirname "$gitmodules_file") - + local gitmodules_dir + gitmodules_dir="$(dirname "$gitmodules_file")" + # Extract all submodule paths from the .gitmodules file - grep "path" $gitmodules_file | awk '{print $3}' | while read submodule_path; do - # Change pwd to the submodule dir and execute script - pushd "$gitmodules_dir/$submodule_path" > /dev/null - NIMBLE_DIR=$NIMBLE_DIR PWD_CMD=$PWD_CMD EXCLUDED_NIM_PACKAGES=$EXCLUDED_NIM_PACKAGES \ - "$create_nimble_link_script_path" "$submodule_path" - popd > /dev/null + grep "path" "$gitmodules_file" | awk '{print $3}' | while read -r submodule_path; do + local full_path="$gitmodules_dir/$submodule_path" + if [[ -d "$full_path" ]]; then + create_nimble_link "$full_path" + fi done } -# Create the base directory if it doesn't exist +# Create the base directory mkdir -p "${NIMBLE_DIR}/pkgs" # Find all .gitmodules files and process them -for gitmodules_file in $(find . -name '.gitmodules'); do +while IFS= read -r -d '' gitmodules_file; do echo "Processing .gitmodules file: $gitmodules_file" process_gitmodules "$gitmodules_file" -done +done < <(find . -name '.gitmodules' -print0) diff --git a/sds.nimble b/sds.nimble index 28716e1..8a55b2c 100644 --- a/sds.nimble +++ b/sds.nimble @@ -139,8 +139,14 @@ proc buildMobileIOS(srcDir = ".", sdkPath = "") = # 2) Compile all generated C files to object files with hidden visibility # This prevents symbol conflicts with other Nim libraries (e.g., libnim_status_client) + # Find Nim's lib directory for C headers (nim.h etc.) + # NIM_LIB_DIR is set by Nix; fallback finds it from nim dump output + let nimLibDir = if existsEnv("NIM_LIB_DIR"): getEnv("NIM_LIB_DIR") + else: gorge("nim dump 2>&1 | grep '/lib/pure$' | sed 's|/pure$||' | head -1").strip() + let nimLibInclude = if nimLibDir.len > 0: " -I" & nimLibDir else: "" + let clangFlags = "-arch " & arch & " -isysroot " & sdkPath & - " -I./vendor/nimbus-build-system/vendor/Nim/lib/" & + nimLibInclude & " -fembed-bitcode -miphoneos-version-min=16.0 -O2" & " -fvisibility=hidden" @@ -180,6 +186,27 @@ proc buildMobileAndroid(srcDir = ".", params = "") = for i in 2 ..< paramCount(): extra_params &= " " & paramStr(i) + # Android NDK toolchain setup — reads env vars set by Nix or the caller + let ndkRoot = getEnv("ANDROID_NDK_ROOT") + let androidArch = getEnv("ANDROID_ARCH") + let androidTarget = getEnv("ANDROID_TARGET", "30") + let archDirname = getEnv("ARCH_DIRNAME") + + if ndkRoot.len > 0 and androidArch.len > 0 and archDirname.len > 0: + let detectedOS = when defined(macosx): "darwin-x86_64" else: "linux-x86_64" + let toolchain = ndkRoot & "/toolchains/llvm/prebuilt/" & detectedOS + let sysroot = toolchain & "/sysroot" + + extra_params &= " --passC:\"--sysroot=" & sysroot & "\"" + extra_params &= " --passL:\"--sysroot=" & sysroot & "\"" + extra_params &= " --passC:\"--target=" & androidArch & androidTarget & "\"" + extra_params &= " --passL:\"--target=" & androidArch & androidTarget & "\"" + extra_params &= " --passC:\"-I" & sysroot & "/usr/include\"" + extra_params &= " --passC:\"-I" & sysroot & "/usr/include/" & archDirname & "\"" + extra_params &= " --passL:\"-L" & sysroot & "/usr/lib/" & archDirname & "/" & androidTarget & "\"" + + putEnv("CC", toolchain & "/bin/" & androidArch & androidTarget & "-clang") + exec "nim c" & " --out:" & outDir & "/libsds.so --threads:on --app:lib --opt:size --noMain --mm:refc --nimMainPrefix:libsds " & "-d:chronicles_sinks=textlines[dynamic] --header --passL:-L" & outdir &