diff --git a/.gitmodules b/.gitmodules index bde56a76e..34a5b88e4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -168,7 +168,7 @@ path = vendor/db_connector url = https://github.com/nim-lang/db_connector.git ignore = untracked - branch = master + branch = devel [submodule "vendor/nph"] ignore = untracked branch = master diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..9d40f285e --- /dev/null +++ b/flake.lock @@ -0,0 +1,49 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1735563628, + "narHash": "sha256-OnSAY7XDSx7CtDoqNh8jwVwh4xNL/2HaJxGjryLWzX8=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "b134951a4c9f3c995fd7be05f3243f8ecd65d798", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-24.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs", + "zerokit": "zerokit" + } + }, + "zerokit": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1737544289, + "narHash": "sha256-lrSkeUHEkB30sH7VPzLJXoxqLPkj+NzDI2vrmm2L/Gg=", + "owner": "vacp2p", + "repo": "zerokit", + "rev": "7cbb57c2b5b987750db8dbf85fa81275d7c607f7", + "type": "github" + }, + "original": { + "owner": "vacp2p", + "ref": "add-nix-flake-and-derivation", + "repo": "zerokit", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..a37a1a3bf --- /dev/null +++ b/flake.nix @@ -0,0 +1,54 @@ +{ + description = "NWaku build flake"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; + zerokit = { + url = "github:vacp2p/zerokit?ref=add-nix-flake-and-derivation"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + }; + + outputs = { self, nixpkgs, zerokit }: + let + stableSystems = [ + "x86_64-linux" "aarch64-linux" + ]; + forAllSystems = f: nixpkgs.lib.genAttrs stableSystems (system: f system); + pkgsFor = forAllSystems ( + system: import nixpkgs { + inherit system; + config = { + android_sdk.accept_license = true; + allowUnfree = true; + }; + overlays = [ + (final: prev: { + androidEnvCustom = prev.callPackage ./nix/pkgs/android-sdk { }; + androidPkgs = final.androidEnvCustom.pkgs; + androidShell = final.androidEnvCustom.shell; + }) + ]; + } + ); + in rec { + packages = forAllSystems (system: let + zerokitPkg = zerokit.packages.${system}.default; + buildTarget = pkgsFor.${system}.callPackage ./nix/default.nix rec { + inherit stableSystems zerokitPkg; + src = self; + }; + build = targets: buildTarget.override { inherit targets; }; + in rec { + libwaku-android-amd64 = build ["libwaku-android-amd64"]; + libwaku-android-arm64 = build ["libwaku-android-arm64"]; + libwaku-android-arm = build ["libwaku-android-arm"]; + libwaku-android-x86 = build ["libwaku-android-x86"]; + default = libwaku-android-amd64; + }); + + devShells = forAllSystems (system: { + default = pkgsFor.${system}.callPackage ./nix/shell.nix { }; + }); + }; +} diff --git a/nix/README.md b/nix/README.md new file mode 100644 index 000000000..e928b7938 --- /dev/null +++ b/nix/README.md @@ -0,0 +1,35 @@ +# Usage + +## Shell + +A development shell can be started using: +```sh +nix develop +``` + +## Building + +To build a Codex you can use: +```sh +nix build '.?submodules=1#default' +``` +The `?submodules=1` part should eventually not be necessary. +For more details see: +https://github.com/NixOS/nix/issues/4423 + +It can be also done without even cloning the repo: +```sh +nix build 'git+https://github.com/waku-org/nwaku?submodules=1#' +``` + +## Running + +```sh +nix run 'git+https://github.com/waku-org/nwaku?submodules=1#'' +``` + +## Testing + +```sh +nix flake check ".?submodules=1#" +``` diff --git a/nix/atlas.nix b/nix/atlas.nix new file mode 100644 index 000000000..43336e07a --- /dev/null +++ b/nix/atlas.nix @@ -0,0 +1,12 @@ +{ pkgs ? import { } }: + +let + tools = pkgs.callPackage ./tools.nix {}; + sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim; +in pkgs.fetchFromGitHub { + owner = "nim-lang"; + repo = "atlas"; + rev = tools.findKeyValue "^ +AtlasStableCommit = \"([a-f0-9]+)\"$" sourceFile; + # WARNING: Requires manual updates when Nim compiler version changes. + hash = "sha256-G1TZdgbRPSgxXZ3VsBP2+XFCLHXVb3an65MuQx67o/k="; +} \ No newline at end of file diff --git a/nix/checksums.nix b/nix/checksums.nix new file mode 100644 index 000000000..d79345d24 --- /dev/null +++ b/nix/checksums.nix @@ -0,0 +1,12 @@ +{ pkgs ? import { } }: + +let + tools = pkgs.callPackage ./tools.nix {}; + sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim; +in pkgs.fetchFromGitHub { + owner = "nim-lang"; + repo = "checksums"; + rev = tools.findKeyValue "^ +ChecksumsStableCommit = \"([a-f0-9]+)\"$" sourceFile; + # WARNING: Requires manual updates when Nim compiler version changes. + hash = "sha256-Bm5iJoT2kAvcTexiLMFBa9oU5gf7d4rWjo3OiN7obWQ="; +} diff --git a/nix/csources.nix b/nix/csources.nix new file mode 100644 index 000000000..5aa90fd6f --- /dev/null +++ b/nix/csources.nix @@ -0,0 +1,12 @@ +{ pkgs ? import { } }: + +let + tools = pkgs.callPackage ./tools.nix {}; + sourceFile = ../vendor/nimbus-build-system/vendor/Nim/config/build_config.txt; +in pkgs.fetchFromGitHub { + owner = "nim-lang"; + repo = "csources_v2"; + rev = tools.findKeyValue "^nim_csourcesHash=([a-f0-9]+)$" sourceFile; + # WARNING: Requires manual updates when Nim compiler version changes. + hash = "sha256-UCLtoxOcGYjBdvHx7A47x6FjLMi6VZqpSs65MN7fpBs="; +} \ No newline at end of file diff --git a/nix/default.nix b/nix/default.nix new file mode 100644 index 000000000..6ba901297 --- /dev/null +++ b/nix/default.nix @@ -0,0 +1,103 @@ +{ + config ? {}, + pkgs ? import ./pkgs.nix { inherit config; }, + src ? ../., + targets ? ["libwaku-android-amd64"], + verbosity ? 2, + useSystemNim ? false, + stableSystems ? [ + "x86_64-linux" "aarch64-linux" + ], + zerokitPkg ? ( + builtins.getFlake "github:vacp2p/zerokit?ref=add-nix-flake-and-derivation" + ).packages.${builtins.currentSystem}.default +}: + +assert pkgs.lib.assertMsg ((src.submodules or true) == true) + "Unable to build without submodules. Append '?submodules=1#' to the URI."; + +let + inherit (pkgs) stdenv lib writeScriptBin callPackage; + + revision = lib.substring 0 8 (src.rev or "dirty"); + +in stdenv.mkDerivation rec { + + pname = "nwaku"; + + version = "1.0.0-${revision}"; + + inherit src; + + buildInputs = with pkgs; [ + openssl + gmp + ]; + + # Dependencies that should only exist in the build environment. + nativeBuildInputs = let + # Fix for Nim compiler calling 'git rev-parse' and 'lsb_release'. + fakeGit = writeScriptBin "git" "echo ${version}"; + # Fix for the zerokit package that is built with cargo/rustup/cross. + fakeCargo = writeScriptBin "cargo" "echo ${version}"; + # Fix for the zerokit package that is built with cargo/rustup/cross. + fakeRustup = writeScriptBin "rustup" "echo ${version}"; + # Fix for the zerokit package that is built with cargo/rustup/cross. + fakeCross = writeScriptBin "cross" "echo ${version}"; + in + with pkgs; [ + cmake + which + lsb-release + zerokitPkg + fakeGit + fakeCargo + fakeRustup + fakeCross + ]; + + # Environment variables required for Android builds + ANDROID_SDK_ROOT="${pkgs.androidPkgs.sdk}"; + ANDROID_NDK_HOME="${pkgs.androidPkgs.ndk}"; + NIMFLAGS = "-d:disableMarchNative -d:git_revision_override=${revision}"; + XDG_CACHE_HOME = "/tmp"; + + makeFlags = targets ++ [ + "V=${toString verbosity}" + ]; + + configurePhase = '' + patchShebangs . vendor/nimbus-build-system > /dev/null + make nimbus-build-system-paths + ''; + + preBuild = '' + pushd vendor/nimbus-build-system/vendor/Nim + mkdir dist + cp -r ${callPackage ./nimble.nix {}} dist/nimble + chmod 777 -R dist/nimble + mkdir -p dist/nimble/dist + cp -r ${callPackage ./checksums.nix {}} dist/checksums # need both + cp -r ${callPackage ./checksums.nix {}} dist/nimble/dist/checksums + cp -r ${callPackage ./atlas.nix {}} dist/atlas + chmod 777 -R dist/atlas + mkdir dist/atlas/dist + cp -r ${callPackage ./sat.nix {}} dist/nimble/dist/sat + cp -r ${callPackage ./sat.nix {}} dist/atlas/dist/sat + cp -r ${callPackage ./csources.nix {}} csources_v2 + chmod 777 -R dist/nimble csources_v2 + popd + ''; + + installPhase = '' + mkdir -p $out/build/android + cp -r ./build/android/* $out/build/android/ + ''; + + meta = with pkgs.lib; { + description = "NWaku derivation to build libwaku for mobile targets using Android NDK and Rust."; + homepage = "https://github.com/status-im/nwaku"; + license = licenses.mit; + platforms = stableSystems; + }; +} diff --git a/nix/nimble.nix b/nix/nimble.nix new file mode 100644 index 000000000..5bd7b0f32 --- /dev/null +++ b/nix/nimble.nix @@ -0,0 +1,12 @@ +{ pkgs ? import { } }: + +let + tools = pkgs.callPackage ./tools.nix {}; + sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim; +in pkgs.fetchFromGitHub { + owner = "nim-lang"; + repo = "nimble"; + rev = tools.findKeyValue "^ +NimbleStableCommit = \"([a-f0-9]+)\".+" sourceFile; + # WARNING: Requires manual updates when Nim compiler version changes. + hash = "sha256-MVHf19UbOWk8Zba2scj06PxdYYOJA6OXrVyDQ9Ku6Us="; +} \ No newline at end of file diff --git a/nix/overlay.nix b/nix/overlay.nix new file mode 100644 index 000000000..b872f44c2 --- /dev/null +++ b/nix/overlay.nix @@ -0,0 +1,55 @@ +# Override some packages and utilities in 'pkgs' +# and make them available globally via callPackage. +# +# For more details see: +# - https://nixos.wiki/wiki/Overlays +# - https://nixos.org/nixos/nix-pills/callpackage-design-pattern.html + +self: super: + +let + inherit (super) stdenv stdenvNoCC callPackage; +in { + # Fix for MacOS + mkShell = super.mkShell.override { stdenv = stdenvNoCC; }; + + lib = (super.lib or { }) // (import ./lib { + inherit (super) lib; + }); + + # Checks fail on darwin. + git-lfs = super.git-lfs.overrideAttrs (old: { + doCheck = false; + }); + + # Package version adjustments + nodejs = super.nodejs_20; + ruby = super.ruby_3_1; + yarn = super.yarn.override { nodejs = super.nodejs_20; }; + openjdk = super.openjdk17_headless; + go = super.go_1_21; + clang = super.clang_15; + buildGoPackage = super.buildGo121Package; + buildGoModule = super.buildGo121Module; + gomobile = (super.gomobile.overrideAttrs (old: { + patches = [ + (self.fetchurl { # https://github.com/golang/mobile/pull/84 + url = "https://github.com/golang/mobile/commit/f20e966e05b8f7e06bed500fa0da81cf6ebca307.patch"; + sha256 = "sha256-TZ/Yhe8gMRQUZFAs9G5/cf2b9QGtTHRSObBFD5Pbh7Y="; + }) + (self.fetchurl { # https://github.com/golang/go/issues/58426 + url = "https://github.com/golang/mobile/commit/406ed3a7b8e44dc32844953647b49696d8847d51.patch"; + sha256 = "sha256-dqbYukHkQEw8npOkKykOAzMC3ot/Y4DEuh7fE+ptlr8="; + }) + ]; + })).override { + # FIXME: No Android SDK packages for aarch64-darwin. + withAndroidPkgs = stdenv.system != "aarch64-darwin"; + androidPkgs = self.androidEnvCustom.compose; + }; + + # Android environment + androidEnvCustom = callPackage ./pkgs/android-sdk { }; + androidPkgs = self.androidEnvCustom.pkgs; + androidShell = self.androidEnvCustom.shell; +} diff --git a/nix/pkgs.nix b/nix/pkgs.nix new file mode 100644 index 000000000..a08fa188a --- /dev/null +++ b/nix/pkgs.nix @@ -0,0 +1,34 @@ +# This file controls the pinned version of nixpkgs we use for our Nix environment +# as well as which versions of package we use, including their overrides. +{ + config ? { }, + stableSystems ? [ + "x86_64-linux" "aarch64-linux" + ], +}: + +let + # For testing local version of nixpkgs + #nixpkgsSrc = (import { }).lib.cleanSource "/home/jakubgs/work/nixpkgs"; + + # We follow release 24-05 of nixpkgs + # https://github.com/NixOS/nixpkgs/releases/tag/24.05 + nixpkgsSrc = builtins.fetchTarball { + url = "https://github.com/NixOS/nixpkgs/archive/df27247e6f3e636c119e2610bf12d38b5e98cc79.tar.gz"; + sha256 = "sha256:0bbvimk7xb7akrx106mmsiwf9nzxnssisqmqffla03zz51d0kz2n"; + }; + + # Status specific configuration defaults + defaultConfig = { + android_sdk.accept_license = true; + allowUnfree = true; + }; + + # Override some packages and utilities + pkgsOverlay = import ./overlay.nix; + + (import nixpkgsSrc) { + config = defaultConfig // config; + system = stableSystems; + overlays = [ pkgsOverlay ]; + } diff --git a/nix/pkgs/android-sdk/compose.nix b/nix/pkgs/android-sdk/compose.nix new file mode 100644 index 000000000..c73aaee43 --- /dev/null +++ b/nix/pkgs/android-sdk/compose.nix @@ -0,0 +1,26 @@ +# +# This Nix expression centralizes the configuration +# for the Android development environment. +# + +{ androidenv, lib, stdenv }: + +assert lib.assertMsg (stdenv.system != "aarch64-darwin") + "aarch64-darwin not supported for Android SDK. Use: NIXPKGS_SYSTEM_OVERRIDE=x86_64-darwin"; + +# The "android-sdk-license" license is accepted +# by setting android_sdk.accept_license = true. +androidenv.composeAndroidPackages { + cmdLineToolsVersion = "9.0"; + toolsVersion = "26.1.1"; + platformToolsVersion = "33.0.3"; + buildToolsVersions = [ "34.0.0" ]; + platformVersions = [ "34" ]; + cmakeVersions = [ "3.22.1" ]; + ndkVersion = "25.2.9519653"; + includeNDK = true; + includeExtras = [ + "extras;android;m2repository" + "extras;google;m2repository" + ]; +} diff --git a/nix/pkgs/android-sdk/default.nix b/nix/pkgs/android-sdk/default.nix new file mode 100644 index 000000000..f3f795251 --- /dev/null +++ b/nix/pkgs/android-sdk/default.nix @@ -0,0 +1,14 @@ +# +# This Nix expression centralizes the configuration +# for the Android development environment. +# + +{ callPackage }: + +let + compose = callPackage ./compose.nix { }; + pkgs = callPackage ./pkgs.nix { inherit compose; }; + shell = callPackage ./shell.nix { androidPkgs = pkgs; }; +in { + inherit compose pkgs shell; +} diff --git a/nix/pkgs/android-sdk/pkgs.nix b/nix/pkgs/android-sdk/pkgs.nix new file mode 100644 index 000000000..645987b3a --- /dev/null +++ b/nix/pkgs/android-sdk/pkgs.nix @@ -0,0 +1,17 @@ +{ stdenv, compose }: + +# +# This derivation simply symlinks some stuff to get +# shorter paths as libexec/android-sdk is quite the mouthful. +# With this you can just do `androidPkgs.sdk` and `androidPkgs.ndk`. +# +stdenv.mkDerivation { + name = "${compose.androidsdk.name}-mod"; + phases = [ "symlinkPhase" ]; + outputs = [ "out" "sdk" "ndk" ]; + symlinkPhase = '' + ln -s ${compose.androidsdk} $out + ln -s ${compose.androidsdk}/libexec/android-sdk $sdk + ln -s ${compose.androidsdk}/libexec/android-sdk/ndk-bundle $ndk + ''; +} diff --git a/nix/pkgs/android-sdk/shell.nix b/nix/pkgs/android-sdk/shell.nix new file mode 100644 index 000000000..364c27884 --- /dev/null +++ b/nix/pkgs/android-sdk/shell.nix @@ -0,0 +1,18 @@ +{ mkShell, openjdk, androidPkgs }: + +mkShell { + name = "android-sdk-shell"; + buildInputs = [ openjdk ]; + + shellHook = '' + export ANDROID_HOME="${androidPkgs.sdk}" + export ANDROID_NDK_ROOT="${androidPkgs.ndk}" + export ANDROID_SDK_ROOT="$ANDROID_HOME" + + export PATH="$ANDROID_NDK_ROOT:$PATH" + export PATH="$ANDROID_SDK_ROOT/tools:$PATH" + export PATH="$ANDROID_SDK_ROOT/tools/bin:$PATH" + export PATH="$(echo $ANDROID_SDK_ROOT/cmdline-tools/*/bin):$PATH" + export PATH="$ANDROID_SDK_ROOT/platform-tools:$PATH" + ''; +} diff --git a/nix/sat.nix b/nix/sat.nix new file mode 100644 index 000000000..31f264468 --- /dev/null +++ b/nix/sat.nix @@ -0,0 +1,12 @@ +{ pkgs ? import { } }: + +let + tools = pkgs.callPackage ./tools.nix {}; + sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim; +in pkgs.fetchFromGitHub { + owner = "nim-lang"; + repo = "sat"; + rev = tools.findKeyValue "^ +SatStableCommit = \"([a-f0-9]+)\"$" sourceFile; + # WARNING: Requires manual updates when Nim compiler version changes. + hash = "sha256-JFrrSV+mehG0gP7NiQ8hYthL0cjh44HNbXfuxQNhq7c="; +} \ No newline at end of file diff --git a/nix/shell.nix b/nix/shell.nix new file mode 100644 index 000000000..dd160ff35 --- /dev/null +++ b/nix/shell.nix @@ -0,0 +1,28 @@ +{ + pkgs ? import ./pkgs.nix {}, +}: + +pkgs.mkShell { + buildInputs = with pkgs; [ + git + cmake + openssl + which + rustup + docker + cargo + ] ++ lib.optionals stdenv.isDarwin [ + libiconv + darwin.apple_sdk.frameworks.Security + ]; + + LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [ + pkgs.pcre + ]; + + shellHook = '' + export ANDROID_SDK_ROOT="${pkgs.androidPkgs.sdk}" + export ANDROID_NDK_HOME="${pkgs.androidPkgs.ndk}" + rustup default stable + ''; +} diff --git a/nix/tools.nix b/nix/tools.nix new file mode 100644 index 000000000..108d38606 --- /dev/null +++ b/nix/tools.nix @@ -0,0 +1,15 @@ +{ pkgs ? import { } }: + +let + + inherit (pkgs.lib) fileContents last splitString flatten remove; + inherit (builtins) map match; +in { + findKeyValue = regex: sourceFile: + let + linesFrom = file: splitString "\n" (fileContents file); + matching = regex: lines: map (line: match regex line) lines; + extractMatch = matches: last (flatten (remove null matches)); + in + extractMatch (matching regex (linesFrom sourceFile)); +} diff --git a/shell.nix b/shell.nix deleted file mode 100644 index ae2426a78..000000000 --- a/shell.nix +++ /dev/null @@ -1,22 +0,0 @@ -{ pkgs ? import (builtins.fetchTarball { - url = "https://github.com/NixOS/nixpkgs/archive/dbf1d73cd1a17276196afeee169b4cf7834b7a96.tar.gz"; - sha256 = "sha256:1k5nvn2yzw370cqsfh62lncsgydq2qkbjrx34cprzf0k6b93v7ch"; -}) {} }: - -pkgs.mkShell { - name = "nim-waku-build-shell"; - - # Versions dependent on nixpkgs commit. Update manually. - buildInputs = with pkgs; [ - git # 2.37.3 - which # 2.21 - rustc # 1.63.0 - ] ++ lib.optionals stdenv.isDarwin [ - libiconv - darwin.apple_sdk.frameworks.Security - ]; - - LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [ - pkgs.pcre - ]; -}