From ee2e77ac6c0d9657e4da2932fceaa97a9127c21f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Soko=C5=82owski?= Date: Thu, 29 Jan 2026 18:55:46 +0100 Subject: [PATCH] ci(nix): use Nix Flake from NBS repo to provide Nim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This way we can avoid rebuilding Nim every time we build packages. Compilation of logos-storage-nim with Nim: X minutes Compilation of logos-storage-nim without Nim: Y minutes Notably this upgrades from Nim 2.2.4 to 2.2.6. Depends on: https://github.com/status-im/nimbus-build-system/pull/112 Signed-off-by: Jakub SokoĊ‚owski --- .github/workflows/ci-nix.yml | 42 +++++++++++ flake.lock | 31 +++++++-- flake.nix | 130 +++++++++++++++++++---------------- nix/checksums.nix | 12 ---- nix/csources.nix | 12 ---- nix/default.nix | 72 +++++++------------ nix/nimble.nix | 13 ---- vendor/nimbus-build-system | 2 +- 8 files changed, 164 insertions(+), 150 deletions(-) create mode 100644 .github/workflows/ci-nix.yml delete mode 100644 nix/checksums.nix delete mode 100644 nix/csources.nix delete mode 100644 nix/nimble.nix diff --git a/.github/workflows/ci-nix.yml b/.github/workflows/ci-nix.yml new file mode 100644 index 00000000..a1547862 --- /dev/null +++ b/.github/workflows/ci-nix.yml @@ -0,0 +1,42 @@ +name: ci / nix +permissions: + contents: read + pull-requests: read + checks: write +on: + pull_request: + branches: [master] + +jobs: + build: + strategy: + fail-fast: false + matrix: + system: + - aarch64-darwin + - x86_64-linux + nixpkg: + - logos-storage-nim + - libstorage + + include: + - system: aarch64-darwin + runs_on: [self-hosted, macOS, ARM64] + + - system: x86_64-linux + runs_on: [self-hosted, Linux, X64] + + name: '${{ matrix.system }} / ${{ matrix.nixpkg }}' + runs-on: ${{ matrix.runs_on }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: 'Run Nix build for ${{ matrix.nixpkg }}' + shell: bash + run: nix build -L '.?submodules=1#${{ matrix.nixpkg }}' + + - name: 'Show result contents' + shell: bash + run: find result -type f diff --git a/flake.lock b/flake.lock index fe08f566..cc4a5959 100644 --- a/flake.lock +++ b/flake.lock @@ -1,23 +1,46 @@ { "nodes": { + "nimbusBuildSystem": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1769685810, + "narHash": "sha256-mANqcQcRXb08ZR8WF6SoyVrL/nXUePtMZfpNx25BlLI=", + "ref": "refs/heads/master", + "rev": "0e7a764edae92b224326b3700f062702489ce2b4", + "revCount": 238, + "submodules": true, + "type": "git", + "url": "file:./vendor/nimbus-build-system" + }, + "original": { + "submodules": true, + "type": "git", + "url": "file:./vendor/nimbus-build-system" + } + }, "nixpkgs": { "locked": { - "lastModified": 1751274312, - "narHash": "sha256-/bVBlRpECLVzjV19t5KMdMFWSwKLtb5RyXdjz3LJT+g=", + "lastModified": 1757590060, + "narHash": "sha256-EWwwdKLMZALkgHFyKW7rmyhxECO74+N+ZO5xTDnY/5c=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "50ab793786d9de88ee30ec4e4c24fb4236fc2674", + "rev": "0ef228213045d2cdb5a169a95d63ded38670b293", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-24.11", "repo": "nixpkgs", + "rev": "0ef228213045d2cdb5a169a95d63ded38670b293", "type": "github" } }, "root": { "inputs": { + "nimbusBuildSystem": "nimbusBuildSystem", "nixpkgs": "nixpkgs" } } diff --git a/flake.nix b/flake.nix index a73d08c5..9a43bd93 100644 --- a/flake.nix +++ b/flake.nix @@ -2,70 +2,78 @@ description = "Logos Storage build flake"; inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11"; + # We are pinning the commit because ultimately we want to use same commit across different projects. + # A commit from nixpkgs 24.11 release : https://github.com/NixOS/nixpkgs/tree/release-24.11 + nixpkgs.url = "github:NixOS/nixpkgs/0ef228213045d2cdb5a169a95d63ded38670b293"; + nimbusBuildSystem = { + url = "git+file:./vendor/nimbus-build-system?submodules=1"; + inputs.nixpkgs.follows = "nixpkgs"; + }; }; - outputs = { self, nixpkgs }: - let - stableSystems = [ - "x86_64-linux" "aarch64-linux" - "x86_64-darwin" "aarch64-darwin" - ]; - forAllSystems = f: nixpkgs.lib.genAttrs stableSystems (system: f system); - pkgsFor = forAllSystems (system: import nixpkgs { inherit system; }); - in rec { - packages = forAllSystems (system: let - buildTarget = pkgsFor.${system}.callPackage ./nix/default.nix rec { - inherit stableSystems; - src = self; - }; - build = targets: buildTarget.override { inherit targets; }; - in rec { - logos-storage-nim = build ["all"]; - libstorage = build ["libstorage"]; - default = logos-storage-nim; - }); - - nixosModules.logos-storage-nim = { config, lib, pkgs, ... }: import ./nix/service.nix { - inherit config lib pkgs self; + outputs = { self, nixpkgs, nimbusBuildSystem }: let + stableSystems = [ + "x86_64-linux" "aarch64-linux" + "x86_64-darwin" "aarch64-darwin" + ]; + forAllSystems = f: nixpkgs.lib.genAttrs stableSystems (system: f system); + pkgsFor = forAllSystems (system: import nixpkgs { inherit system; }); + in rec { + packages = forAllSystems (system: let + nim = nimbusBuildSystem.packages.${system}.nim; + buildTarget = pkgsFor.${system}.callPackage ./nix/default.nix rec { + inherit stableSystems nim; + src = self; }; + build = targets: buildTarget.override { inherit targets; }; + in rec { + logos-storage-nim = build ["all"]; + libstorage = build ["libstorage"]; - devShells = forAllSystems (system: let - pkgs = pkgsFor.${system}; - in { - default = pkgs.mkShell { - inputsFrom = [ - packages.${system}.logos-storage-nim - packages.${system}.libstorage - ]; - # Not using buildInputs to override fakeGit and fakeCargo. - nativeBuildInputs = with pkgs; [ git cargo nodejs_18 ]; - }; - }); + default = logos-storage-nim; + }); - checks = forAllSystems (system: let - pkgs = pkgsFor.${system}; - in { - logos-storage-nim-test = pkgs.nixosTest { - name = "logos-storage-nim-test"; - nodes = { - server = { config, pkgs, ... }: { - imports = [ self.nixosModules.logos-storage-nim ]; - services.logos-storage-nim.enable = true; - services.logos-storage-nim.settings = { - data-dir = "/var/lib/logos-storage-nim-test"; - }; - systemd.services.logos-storage-nim.serviceConfig.StateDirectory = "logos-storage-nim-test"; - }; - }; - testScript = '' - print("Starting test: logos-storage-nim-test") - machine.start() - machine.wait_for_unit("logos-storage-nim.service") - machine.succeed("test -d /var/lib/logos-storage-nim-test") - machine.wait_until_succeeds("journalctl -u logos-storage-nim.service | grep 'Started Storage node'", 10) - ''; - }; - }); + nixosModules.logos-storage-nim = { config, lib, pkgs, ... }: import ./nix/service.nix { + inherit config lib pkgs self; }; -} \ No newline at end of file + + devShells = forAllSystems (system: let + pkgs = pkgsFor.${system}; + nim = nimbusBuildSystem.packages.${system}.nim; + in { + default = pkgs.mkShell { + inputsFrom = [ + packages.${system}.logos-storage-nim + packages.${system}.libstorage + ]; + # Not using buildInputs to override fakeGit. + nativeBuildInputs = with pkgs; [ nim git cargo nodejs_20 ]; + }; + }); + + checks = forAllSystems (system: let + pkgs = pkgsFor.${system}; + in { + logos-storage-nim-test = pkgs.nixosTest { + name = "logos-storage-nim-test"; + nodes = { + server = { config, pkgs, ... }: { + imports = [ self.nixosModules.logos-storage-nim ]; + services.logos-storage-nim.enable = true; + services.logos-storage-nim.settings = { + data-dir = "/var/lib/logos-storage-nim-test"; + }; + systemd.services.logos-storage-nim.serviceConfig.StateDirectory = "logos-storage-nim-test"; + }; + }; + testScript = '' + print("Starting test: logos-storage-nim-test") + machine.start() + machine.wait_for_unit("logos-storage-nim.service") + machine.succeed("test -d /var/lib/logos-storage-nim-test") + machine.wait_until_succeeds("journalctl -u logos-storage-nim.service | grep 'Started Storage node'", 10) + ''; + }; + }); + }; +} diff --git a/nix/checksums.nix b/nix/checksums.nix deleted file mode 100644 index c9c9f3d4..00000000 --- a/nix/checksums.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ pkgs ? import { } }: - -let - tools = pkgs.callPackage ./tools.nix {}; - sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim; -in pkgs.fetchFromGitHub { - owner = "nim-lang"; - repo = "checksums"; - rev = tools.findKeyValue "^ +ChecksumsStableCommit = \"([a-f0-9]+)\".*$" sourceFile; - # WARNING: Requires manual updates when Nim compiler version changes. - hash = "sha256-JZhWqn4SrAgNw/HLzBK0rrj3WzvJ3Tv1nuDMn83KoYY="; -} diff --git a/nix/csources.nix b/nix/csources.nix deleted file mode 100644 index 33cd9d0f..00000000 --- a/nix/csources.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ pkgs ? import { } }: - -let - tools = pkgs.callPackage ./tools.nix {}; - sourceFile = ../vendor/nimbus-build-system/vendor/Nim/config/build_config.txt; -in pkgs.fetchFromGitHub { - owner = "nim-lang"; - repo = "csources_v2"; - rev = tools.findKeyValue "^nim_csourcesHash=([a-f0-9]+)$" sourceFile; - # WARNING: Requires manual updates when Nim compiler version changes. - hash = "sha256-UCLtoxOcGYjBdvHx7A47x6FjLMi6VZqpSs65MN7fpBs="; -} diff --git a/nix/default.nix b/nix/default.nix index 3ec55f7d..80172775 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -1,17 +1,17 @@ { pkgs ? import { }, src ? ../., - targets ? ["all"], + # Nimbus-build-system package. + nim ? null, # Options: 0,1,2 verbosity ? 1, - commit ? builtins.substring 0 7 (src.rev or "dirty"), + # Make targets + targets ? ["all"], # These are the only platforms tested in CI and considered stable. stableSystems ? [ "x86_64-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ], - # Perform 2-stage bootstrap instead of 3-stage to save time. - quickAndDirty ? true, }: assert pkgs.lib.assertMsg ((src.submodules or true) == true) @@ -20,54 +20,42 @@ assert pkgs.lib.assertMsg ((src.submodules or true) == true) let inherit (pkgs) lib writeScriptBin callPackage; - revision = lib.substring 0 8 (src.rev or "dirty"); tools = callPackage ./tools.nix {}; + version = tools.findKeyValue "version = \"([0-9]+\.[0-9]+\.[0-9]+)\"" ../codex.nimble; + revision = lib.substring 0 8 (src.rev or src.dirtyRev or "00000000"); + # Pin GCC/CLang versions stdenv = if pkgs.stdenv.isLinux then pkgs.gcc13Stdenv else pkgs.clang16Stdenv; -in stdenv.mkDerivation rec { - pname = "storage"; - - version = "${tools.findKeyValue "version = \"([0-9]+\.[0-9]+\.[0-9]+)\"" ../codex.nimble}-${revision}"; +in stdenv.mkDerivation { + pname = "logos-storage-nim"; + version = "${version}-${revision}"; inherit src; - # Dependencies that should exist in the runtime environment. - buildInputs = with pkgs; [ - openssl - gmp + # Disable CPU optimizations that make binary not portable. + env = { + NIMFLAGS = "-d:disableMarchNative"; + }; + + makeFlags = targets ++ [ + "V=${toString verbosity}" + # Built from nimbus-build-system via flake. + "USE_SYSTEM_NIM=1" ]; + # Dependencies that should exist in the runtime environment. + buildInputs = with pkgs; [ openssl gmp ]; + # Dependencies that should only exist in the build environment. nativeBuildInputs = let # Fix for Nim compiler calling 'git rev-parse' and 'lsb_release'. fakeGit = writeScriptBin "git" "echo ${version}"; - in with pkgs; [ - cmake - which - fakeGit - ] ++ lib.optionals stdenv.isLinux [ - lsb-release - ] ++ lib.optionals stdenv.isDarwin [ - darwin.cctools - ]; - - # Disable CPU optimizations that make binary not portable. - NIMFLAGS = "-d:disableMarchNative -d:git_revision_override=${revision}"; - - makeFlags = targets ++ [ - "V=${toString verbosity}" - "QUICK_AND_DIRTY_COMPILER=${if quickAndDirty then "1" else "0"}" - "QUICK_AND_DIRTY_NIMBLE=${if quickAndDirty then "1" else "0"}" - ]; - - # FIXME: Remove once permanent fix is applied to NBS: - patchPhase = '' - substituteInPlace vendor/nimbus-build-system/scripts/build_nim.sh \ - --replace-fail '"''${NIX_BUILD_TOP}" != "/build"' '-z $${NIX_BUILD_TOP}' - ''; + in with pkgs; [ nim cmake which fakeGit ] + ++ lib.optionals stdenv.isLinux [ lsb-release ] + ++ lib.optionals stdenv.isDarwin [ darwin.cctools ]; configurePhase = '' # Avoid Nim cache permission errors. @@ -76,16 +64,6 @@ in stdenv.mkDerivation rec { make nimbus-build-system-paths ''; - preBuild = '' - pushd vendor/nimbus-build-system/vendor/Nim - mkdir dist - cp -r ${callPackage ./nimble.nix {}} dist/nimble - cp -r ${callPackage ./checksums.nix {}} dist/checksums - cp -r ${callPackage ./csources.nix {}} csources_v2 - chmod 777 -R dist/nimble csources_v2 - popd - ''; - installPhase = '' if [ -f build/storage ]; then mkdir -p $out/bin diff --git a/nix/nimble.nix b/nix/nimble.nix deleted file mode 100644 index 6eb4c8a0..00000000 --- a/nix/nimble.nix +++ /dev/null @@ -1,13 +0,0 @@ -{ pkgs ? import { } }: - -let - tools = pkgs.callPackage ./tools.nix {}; - sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim; -in pkgs.fetchFromGitHub { - owner = "nim-lang"; - repo = "nimble"; - fetchSubmodules = true; - rev = tools.findKeyValue "^ +NimbleStableCommit = \"([a-f0-9]+)\".+" sourceFile; - # WARNING: Requires manual updates when Nim compiler version changes. - hash = "sha256-wgzFhModFkwB8st8F5vSkua7dITGGC2cjoDvgkRVZMs="; -} diff --git a/vendor/nimbus-build-system b/vendor/nimbus-build-system index 0be0663e..0e7a764e 160000 --- a/vendor/nimbus-build-system +++ b/vendor/nimbus-build-system @@ -1 +1 @@ -Subproject commit 0be0663e1af76e869837226a4ef3e586fcc737d3 +Subproject commit 0e7a764edae92b224326b3700f062702489ce2b4