diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..0ea28afaa --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,7 @@ +# Changes to Nix and CI require approval from DevOps team +Makefile @logos-messaging/devops +/ci/ @logos-messaging/devops +/.github/ @logos-messaging/devops +/nix/ @logos-messaging/devops +/flake.nix @logos-messaging/devops +/flake.lock @logos-messaging/devops diff --git a/.github/workflows/ci-nix.yml b/.github/workflows/ci-nix.yml index 8fc7ac985..3e4c53f97 100644 --- a/.github/workflows/ci-nix.yml +++ b/.github/workflows/ci-nix.yml @@ -19,6 +19,7 @@ jobs: - libwaku - libwaku-android-arm64 - wakucanary + - liblogosdelivery exclude: # Android SDK limitation @@ -39,10 +40,10 @@ jobs: with: submodules: recursive - - name: 'Run Nix build for {{ matrix.nixpkg }}' + - name: 'Run Nix build for ${{ matrix.nixpkg }}' shell: bash run: nix build -L '.?submodules=1#${{ matrix.nixpkg }}' - name: 'Show result contents' shell: bash - run: find result -type f + run: find result/ -type f diff --git a/flake.lock b/flake.lock index b927e8807..43dbd5ab9 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,26 @@ { "nodes": { + "nimbusBuildSystem": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1771846495, + "narHash": "sha256-wwgf8dScFtpMJ2knaY8T2VQmilC/9RC1TCvU5m/2RSY=", + "rev": "77b2c37f6bb50aac4796a1153b9fdee25d71a26e", + "revCount": 239, + "submodules": true, + "type": "git", + "url": "file:./vendor/nimbus-build-system" + }, + "original": { + "submodules": true, + "type": "git", + "url": "file:./vendor/nimbus-build-system" + } + }, "nixpkgs": { "locked": { "lastModified": 1757590060, @@ -18,6 +39,7 @@ }, "root": { "inputs": { + "nimbusBuildSystem": "nimbusBuildSystem", "nixpkgs": "nixpkgs", "zerokit": "zerokit" } @@ -51,18 +73,16 @@ "rust-overlay": "rust-overlay" }, "locked": { - "lastModified": 1762211504, - "narHash": "sha256-SbDoBElFYJ4cYebltxlO2lYnz6qOaDAVY6aNJ5bqHDE=", - "ref": "refs/heads/master", - "rev": "3160d9504d07791f2fc9b610948a6cf9a58ed488", - "revCount": 342, + "lastModified": 1766055337, + "narHash": "sha256-v75agzTje4y0C4/aoe7fKWTT7PB55drSoAMA6Q4lcv4=", + "rev": "2071346174e3fea2a5db97549da0ccd30ba6bc4f", + "revCount": 351, "type": "git", - "url": "https://github.com/vacp2p/zerokit" + "url": "file:./vendor/zerokit" }, "original": { - "rev": "3160d9504d07791f2fc9b610948a6cf9a58ed488", "type": "git", - "url": "https://github.com/vacp2p/zerokit" + "url": "file:./vendor/zerokit" } } }, diff --git a/flake.nix b/flake.nix index ee24c8f13..f7d6be8e0 100644 --- a/flake.nix +++ b/flake.nix @@ -12,12 +12,16 @@ nixpkgs.url = "github:NixOS/nixpkgs/0ef228213045d2cdb5a169a95d63ded38670b293"; # WARNING: Remember to update commit and use 'nix flake update' to update flake.lock. zerokit = { - url = "git+https://github.com/vacp2p/zerokit?rev=3160d9504d07791f2fc9b610948a6cf9a58ed488"; + url = "git+file:./vendor/zerokit"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + nimbusBuildSystem = { + url = "git+file:./vendor/nimbus-build-system?submodules=1"; inputs.nixpkgs.follows = "nixpkgs"; }; }; - outputs = { self, nixpkgs, zerokit }: + outputs = { self, nixpkgs, zerokit, nimbusBuildSystem }: let stableSystems = [ "x86_64-linux" "aarch64-linux" @@ -47,42 +51,30 @@ in rec { packages = forAllSystems (system: let - pkgs = pkgsFor.${system}; - in rec { - libwaku-android-arm64 = pkgs.callPackage ./nix/default.nix { - inherit stableSystems; + nim = nimbusBuildSystem.packages.${system}.nim; + buildTargets = pkgsFor.${system}.callPackage ./nix/default.nix { + inherit stableSystems nim; src = self; + zerokitRln = zerokit.packages.${system}.rln; + }; + in rec { + libwaku-android-arm64 = buildTargets.override { targets = ["libwaku-android-arm64"]; abidir = "arm64-v8a"; zerokitRln = zerokit.packages.${system}.rln-android-arm64; }; - libwaku = pkgs.callPackage ./nix/default.nix { - inherit stableSystems; - src = self; - targets = ["libwaku"]; - zerokitRln = zerokit.packages.${system}.rln; - }; - - wakucanary = pkgs.callPackage ./nix/default.nix { - inherit stableSystems; - src = self; - targets = ["wakucanary"]; - zerokitRln = zerokit.packages.${system}.rln; - }; - - liblogosdelivery = pkgs.callPackage ./nix/default.nix { - inherit stableSystems; - src = self; - targets = ["liblogosdelivery"]; - zerokitRln = zerokit.packages.${system}.rln; - }; + libwaku = buildTargets.override { targets = ["libwaku"]; }; + wakucanary = buildTargets.override { targets = ["wakucanary"]; }; + liblogosdelivery = buildTargets.override { targets = ["liblogosdelivery"]; }; default = libwaku; }); devShells = forAllSystems (system: { - default = pkgsFor.${system}.callPackage ./nix/shell.nix {}; + default = pkgsFor.${system}.callPackage ./nix/shell.nix { + inherit (nimbusBuildSystem.packages.${system}) nim; + }; }); }; } diff --git a/nix/checksums.nix b/nix/checksums.nix deleted file mode 100644 index c9c9f3d45..000000000 --- a/nix/checksums.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ pkgs ? import { } }: - -let - tools = pkgs.callPackage ./tools.nix {}; - sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim; -in pkgs.fetchFromGitHub { - owner = "nim-lang"; - repo = "checksums"; - rev = tools.findKeyValue "^ +ChecksumsStableCommit = \"([a-f0-9]+)\".*$" sourceFile; - # WARNING: Requires manual updates when Nim compiler version changes. - hash = "sha256-JZhWqn4SrAgNw/HLzBK0rrj3WzvJ3Tv1nuDMn83KoYY="; -} diff --git a/nix/csources.nix b/nix/csources.nix deleted file mode 100644 index 5aa90fd6f..000000000 --- a/nix/csources.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ pkgs ? import { } }: - -let - tools = pkgs.callPackage ./tools.nix {}; - sourceFile = ../vendor/nimbus-build-system/vendor/Nim/config/build_config.txt; -in pkgs.fetchFromGitHub { - owner = "nim-lang"; - repo = "csources_v2"; - rev = tools.findKeyValue "^nim_csourcesHash=([a-f0-9]+)$" sourceFile; - # WARNING: Requires manual updates when Nim compiler version changes. - hash = "sha256-UCLtoxOcGYjBdvHx7A47x6FjLMi6VZqpSs65MN7fpBs="; -} \ No newline at end of file diff --git a/nix/default.nix b/nix/default.nix index 7df58df60..0c1e8f45a 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -1,13 +1,14 @@ { pkgs, src ? ../., - targets ? ["libwaku-android-arm64"], + # Nimbus-build-system package. + nim ? null, + # Options: 0,1,2 verbosity ? 1, - useSystemNim ? true, - quickAndDirty ? true, - stableSystems ? [ - "x86_64-linux" "aarch64-linux" - ], + # Make targets + targets ? ["libwaku-android-arm64"], + # These are the only platforms tested in CI and considered stable. + stableSystems ? ["x86_64-linux" "aarch64-linux"], abidir ? null, zerokitRln, }: @@ -17,16 +18,17 @@ assert pkgs.lib.assertMsg ((src.submodules or true) == true) let inherit (pkgs) stdenv lib writeScriptBin callPackage; + inherit (lib) any match substring optionals optionalString; - androidManifest = ""; + # Check if build is for android platform. + containsAndroid = s: (match ".*android.*" s) != null; + isAndroidBuild = any containsAndroid targets; tools = pkgs.callPackage ./tools.nix {}; version = tools.findKeyValue "^version = \"([a-f0-9.-]+)\"$" ../waku.nimble; revision = lib.substring 0 8 (src.rev or src.dirtyRev or "00000000"); - copyLibwaku = lib.elem "libwaku" targets; - copyLiblogosdelivery = lib.elem "liblogosdelivery" targets; - copyWakunode2 = lib.elem "wakunode2" targets; - hasKnownInstallTarget = copyLibwaku || copyLiblogosdelivery || copyWakunode2; + + androidManifest = ""; in stdenv.mkDerivation { pname = "logos-messaging-nim"; @@ -34,7 +36,15 @@ in stdenv.mkDerivation { inherit src; - # Runtime dependencies + env = { + # Disable CPU optimizations that make binary not portable. + NIMFLAGS = "-d:disableMarchNative"; + # Optional variables required for Android builds + ANDROID_SDK_ROOT = optionalString isAndroidBuild pkgs.androidPkgs.sdk; + ANDROID_NDK_HOME = optionalString isAndroidBuild pkgs.androidPkgs.ndk; + }; + + # Dependencies that should exist in the runtime environment. buildInputs = with pkgs; [ openssl gmp zip ]; @@ -44,100 +54,37 @@ in stdenv.mkDerivation { # Fix for Nim compiler calling 'git rev-parse' and 'lsb_release'. fakeGit = writeScriptBin "git" "echo ${version}"; in with pkgs; [ - cmake which zerokitRln nim-unwrapped-2_2 fakeGit - ] ++ lib.optionals stdenv.isDarwin [ + nim cmake which zerokitRln fakeGit + ] ++ optionals stdenv.isDarwin [ pkgs.darwin.cctools gcc # Necessary for libbacktrace ]; - # Environment variables required for Android builds - ANDROID_SDK_ROOT = "${pkgs.androidPkgs.sdk}"; - ANDROID_NDK_HOME = "${pkgs.androidPkgs.ndk}"; - NIMFLAGS = "-d:disableMarchNative -d:git_revision_override=${revision}"; - XDG_CACHE_HOME = "/tmp"; - makeFlags = targets ++ [ "V=${toString verbosity}" - "QUICK_AND_DIRTY_COMPILER=${if quickAndDirty then "1" else "0"}" - "QUICK_AND_DIRTY_NIMBLE=${if quickAndDirty then "1" else "0"}" - "USE_SYSTEM_NIM=${if useSystemNim then "1" else "0"}" "LIBRLN_FILE=${zerokitRln}/lib/librln.${if abidir != null then "so" else "a"}" "POSTGRES=1" + # Built from nimbus-build-system via flake. + "USE_SYSTEM_NIM=1" ]; configurePhase = '' + # Avoid /tmp write errors. + export XDG_CACHE_HOME=$TMPDIR/cache patchShebangs . vendor/nimbus-build-system > /dev/null - - # build_nim.sh guards "rm -rf dist/checksums" with NIX_BUILD_TOP != "/build", - # but on macOS the nix sandbox uses /private/tmp/... so the check fails and - # dist/checksums (provided via preBuild) gets deleted. Fix the check to skip - # the removal whenever NIX_BUILD_TOP is set (i.e. any nix build). - substituteInPlace vendor/nimbus-build-system/scripts/build_nim.sh \ - --replace 'if [[ "''${NIX_BUILD_TOP}" != "/build" ]]; then' \ - 'if [[ -z "''${NIX_BUILD_TOP}" ]]; then' - make nimbus-build-system-paths make nimbus-build-system-nimble-dir ''; - # For the Nim v2.2.4 built with NBS we added sat and zippy - preBuild = lib.optionalString (!useSystemNim) '' - pushd vendor/nimbus-build-system/vendor/Nim - mkdir dist - mkdir -p dist/nimble/vendor/sat - mkdir -p dist/nimble/vendor/checksums - mkdir -p dist/nimble/vendor/zippy - - cp -r ${callPackage ./nimble.nix {}}/. dist/nimble - cp -r ${callPackage ./checksums.nix {}}/. dist/checksums - cp -r ${callPackage ./csources.nix {}}/. csources_v2 - cp -r ${callPackage ./sat.nix {}}/. dist/nimble/vendor/sat - cp -r ${callPackage ./checksums.nix {}}/. dist/nimble/vendor/checksums - cp -r ${callPackage ./zippy.nix {}}/. dist/nimble/vendor/zippy - chmod 777 -R dist/nimble csources_v2 - popd - ''; - - installPhase = if abidir != null then '' + installPhase = if isAndroidBuild then '' mkdir -p $out/jni cp -r ./build/android/${abidir}/* $out/jni/ echo '${androidManifest}' > $out/jni/AndroidManifest.xml cd $out && zip -r libwaku.aar * '' else '' - mkdir -p $out/bin $out/include - - # Copy artifacts from build directory (created by Make during buildPhase) - # Note: build/ is in the source tree, not result/ (which is a post-build symlink) - if [ -d build ]; then - ${lib.optionalString copyLibwaku '' - cp build/libwaku.{so,dylib,dll,a,lib} $out/bin/ 2>/dev/null || true - ''} - - ${lib.optionalString copyLiblogosdelivery '' - cp build/liblogosdelivery.{so,dylib,dll,a,lib} $out/bin/ 2>/dev/null || true - ''} - - ${lib.optionalString copyWakunode2 '' - cp build/wakunode2 $out/bin/ 2>/dev/null || true - ''} - - ${lib.optionalString (!hasKnownInstallTarget) '' - cp build/lib*.{so,dylib,dll,a,lib} $out/bin/ 2>/dev/null || true - ''} - fi - - # Copy header files - ${lib.optionalString copyLibwaku '' - cp library/libwaku.h $out/include/ 2>/dev/null || true - ''} - - ${lib.optionalString copyLiblogosdelivery '' - cp liblogosdelivery/liblogosdelivery.h $out/include/ 2>/dev/null || true - ''} - - ${lib.optionalString (!hasKnownInstallTarget) '' - cp library/libwaku.h $out/include/ 2>/dev/null || true - cp liblogosdelivery/liblogosdelivery.h $out/include/ 2>/dev/null || true - ''} + mkdir -p $out/bin $out/lib $out/include + cp build/waku* $out/bin/ || true + cp build/lib* $out/lib/ || true + cp lib*/lib*.h $out/include/ ''; meta = with pkgs.lib; { diff --git a/nix/nimble.nix b/nix/nimble.nix deleted file mode 100644 index 337ecd672..000000000 --- a/nix/nimble.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ pkgs ? import { } }: - -let - tools = pkgs.callPackage ./tools.nix {}; - sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim; -in pkgs.fetchFromGitHub { - owner = "nim-lang"; - repo = "nimble"; - rev = tools.findKeyValue "^ +NimbleStableCommit = \"([a-f0-9]+)\".*$" sourceFile; - # WARNING: Requires manual updates when Nim compiler version changes. - hash = "sha256-8iutVgNzDtttZ7V+7S11KfLEuwhKA9TsgS51mlUI08k="; -} diff --git a/nix/sat.nix b/nix/sat.nix deleted file mode 100644 index 92db58a2e..000000000 --- a/nix/sat.nix +++ /dev/null @@ -1,13 +0,0 @@ -{ pkgs ? import { } }: - -let - tools = pkgs.callPackage ./tools.nix {}; - sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim; -in pkgs.fetchFromGitHub { - owner = "nim-lang"; - repo = "sat"; - rev = tools.findKeyValue "^ +SatStableCommit = \"([a-f0-9]+)\".*$" sourceFile; - # WARNING: Requires manual updates when Nim compiler version changes. - # WARNING: Requires manual updates when Nim compiler version changes. - hash = "sha256-JFrrSV+mehG0gP7NiQ8hYthL0cjh44HNbXfuxQNhq7c="; -} diff --git a/nix/shell.nix b/nix/shell.nix index 3b83ac93d..1f57d0aa9 100644 --- a/nix/shell.nix +++ b/nix/shell.nix @@ -1,19 +1,22 @@ -{ pkgs }: +{ + pkgs ? import { }, + nim ? null, +}: pkgs.mkShell { inputsFrom = [ pkgs.androidShell - ] ++ pkgs.lib.optionals pkgs.stdenv.isDarwin [ - pkgs.libiconv - pkgs.darwin.apple_sdk.frameworks.Security ]; buildInputs = with pkgs; [ git + nim cargo rustup rustc cmake - nim-unwrapped-2_2 + ] ++ pkgs.lib.optionals pkgs.stdenv.isDarwin [ + pkgs.libiconv + pkgs.darwin.apple_sdk.frameworks.Security ]; } diff --git a/nix/zippy.nix b/nix/zippy.nix deleted file mode 100644 index ec59dfc07..000000000 --- a/nix/zippy.nix +++ /dev/null @@ -1,9 +0,0 @@ -{ pkgs }: - -pkgs.fetchFromGitHub { - owner = "guzba"; - repo = "zippy"; - rev = "a99f6a7d8a8e3e0213b3cad0daf0ea974bf58e3f"; - # WARNING: Requires manual updates when Nim compiler version changes. - hash = "sha256-e2ma2Oyp0dlNx8pJsdZl5o5KnaoAX87tqfY0RLG3DZs="; -} \ No newline at end of file diff --git a/vendor/nimbus-build-system b/vendor/nimbus-build-system index e6c2c9da3..77b2c37f6 160000 --- a/vendor/nimbus-build-system +++ b/vendor/nimbus-build-system @@ -1 +1 @@ -Subproject commit e6c2c9da39c2d368d9cf420ac22692e99715d22c +Subproject commit 77b2c37f6bb50aac4796a1153b9fdee25d71a26e diff --git a/vendor/zerokit b/vendor/zerokit index a4bb3feb5..207134617 160000 --- a/vendor/zerokit +++ b/vendor/zerokit @@ -1 +1 @@ -Subproject commit a4bb3feb5054e6fd24827adf204493e6e173437b +Subproject commit 2071346174e3fea2a5db97549da0ccd30ba6bc4f