mirror of
https://github.com/logos-storage/logos-storage-nim.git
synced 2026-03-02 10:23:07 +00:00
ci(nix): use Nix Flake from NBS repo to provide Nim
This way we can avoid rebuilding Nim every time we build packages. Compilation of logos-storage-nim with Nim: X minutes Compilation of logos-storage-nim without Nim: Y minutes Notably this upgrades from Nim 2.2.4 to 2.2.6. Depends on: https://github.com/status-im/nimbus-build-system/pull/112 Signed-off-by: Jakub Sokołowski <jakub@status.im>
This commit is contained in:
parent
e375223500
commit
ee2e77ac6c
42
.github/workflows/ci-nix.yml
vendored
Normal file
42
.github/workflows/ci-nix.yml
vendored
Normal file
@ -0,0 +1,42 @@
|
||||
name: ci / nix
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
checks: write
|
||||
on:
|
||||
pull_request:
|
||||
branches: [master]
|
||||
|
||||
jobs:
|
||||
build:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
system:
|
||||
- aarch64-darwin
|
||||
- x86_64-linux
|
||||
nixpkg:
|
||||
- logos-storage-nim
|
||||
- libstorage
|
||||
|
||||
include:
|
||||
- system: aarch64-darwin
|
||||
runs_on: [self-hosted, macOS, ARM64]
|
||||
|
||||
- system: x86_64-linux
|
||||
runs_on: [self-hosted, Linux, X64]
|
||||
|
||||
name: '${{ matrix.system }} / ${{ matrix.nixpkg }}'
|
||||
runs-on: ${{ matrix.runs_on }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: 'Run Nix build for ${{ matrix.nixpkg }}'
|
||||
shell: bash
|
||||
run: nix build -L '.?submodules=1#${{ matrix.nixpkg }}'
|
||||
|
||||
- name: 'Show result contents'
|
||||
shell: bash
|
||||
run: find result -type f
|
||||
31
flake.lock
generated
31
flake.lock
generated
@ -1,23 +1,46 @@
|
||||
{
|
||||
"nodes": {
|
||||
"nimbusBuildSystem": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1769685810,
|
||||
"narHash": "sha256-mANqcQcRXb08ZR8WF6SoyVrL/nXUePtMZfpNx25BlLI=",
|
||||
"ref": "refs/heads/master",
|
||||
"rev": "0e7a764edae92b224326b3700f062702489ce2b4",
|
||||
"revCount": 238,
|
||||
"submodules": true,
|
||||
"type": "git",
|
||||
"url": "file:./vendor/nimbus-build-system"
|
||||
},
|
||||
"original": {
|
||||
"submodules": true,
|
||||
"type": "git",
|
||||
"url": "file:./vendor/nimbus-build-system"
|
||||
}
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1751274312,
|
||||
"narHash": "sha256-/bVBlRpECLVzjV19t5KMdMFWSwKLtb5RyXdjz3LJT+g=",
|
||||
"lastModified": 1757590060,
|
||||
"narHash": "sha256-EWwwdKLMZALkgHFyKW7rmyhxECO74+N+ZO5xTDnY/5c=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "50ab793786d9de88ee30ec4e4c24fb4236fc2674",
|
||||
"rev": "0ef228213045d2cdb5a169a95d63ded38670b293",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixos-24.11",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "0ef228213045d2cdb5a169a95d63ded38670b293",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"nimbusBuildSystem": "nimbusBuildSystem",
|
||||
"nixpkgs": "nixpkgs"
|
||||
}
|
||||
}
|
||||
|
||||
130
flake.nix
130
flake.nix
@ -2,70 +2,78 @@
|
||||
description = "Logos Storage build flake";
|
||||
|
||||
inputs = {
|
||||
nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11";
|
||||
# We are pinning the commit because ultimately we want to use same commit across different projects.
|
||||
# A commit from nixpkgs 24.11 release : https://github.com/NixOS/nixpkgs/tree/release-24.11
|
||||
nixpkgs.url = "github:NixOS/nixpkgs/0ef228213045d2cdb5a169a95d63ded38670b293";
|
||||
nimbusBuildSystem = {
|
||||
url = "git+file:./vendor/nimbus-build-system?submodules=1";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
};
|
||||
|
||||
outputs = { self, nixpkgs }:
|
||||
let
|
||||
stableSystems = [
|
||||
"x86_64-linux" "aarch64-linux"
|
||||
"x86_64-darwin" "aarch64-darwin"
|
||||
];
|
||||
forAllSystems = f: nixpkgs.lib.genAttrs stableSystems (system: f system);
|
||||
pkgsFor = forAllSystems (system: import nixpkgs { inherit system; });
|
||||
in rec {
|
||||
packages = forAllSystems (system: let
|
||||
buildTarget = pkgsFor.${system}.callPackage ./nix/default.nix rec {
|
||||
inherit stableSystems;
|
||||
src = self;
|
||||
};
|
||||
build = targets: buildTarget.override { inherit targets; };
|
||||
in rec {
|
||||
logos-storage-nim = build ["all"];
|
||||
libstorage = build ["libstorage"];
|
||||
default = logos-storage-nim;
|
||||
});
|
||||
|
||||
nixosModules.logos-storage-nim = { config, lib, pkgs, ... }: import ./nix/service.nix {
|
||||
inherit config lib pkgs self;
|
||||
outputs = { self, nixpkgs, nimbusBuildSystem }: let
|
||||
stableSystems = [
|
||||
"x86_64-linux" "aarch64-linux"
|
||||
"x86_64-darwin" "aarch64-darwin"
|
||||
];
|
||||
forAllSystems = f: nixpkgs.lib.genAttrs stableSystems (system: f system);
|
||||
pkgsFor = forAllSystems (system: import nixpkgs { inherit system; });
|
||||
in rec {
|
||||
packages = forAllSystems (system: let
|
||||
nim = nimbusBuildSystem.packages.${system}.nim;
|
||||
buildTarget = pkgsFor.${system}.callPackage ./nix/default.nix rec {
|
||||
inherit stableSystems nim;
|
||||
src = self;
|
||||
};
|
||||
build = targets: buildTarget.override { inherit targets; };
|
||||
in rec {
|
||||
logos-storage-nim = build ["all"];
|
||||
libstorage = build ["libstorage"];
|
||||
|
||||
devShells = forAllSystems (system: let
|
||||
pkgs = pkgsFor.${system};
|
||||
in {
|
||||
default = pkgs.mkShell {
|
||||
inputsFrom = [
|
||||
packages.${system}.logos-storage-nim
|
||||
packages.${system}.libstorage
|
||||
];
|
||||
# Not using buildInputs to override fakeGit and fakeCargo.
|
||||
nativeBuildInputs = with pkgs; [ git cargo nodejs_18 ];
|
||||
};
|
||||
});
|
||||
default = logos-storage-nim;
|
||||
});
|
||||
|
||||
checks = forAllSystems (system: let
|
||||
pkgs = pkgsFor.${system};
|
||||
in {
|
||||
logos-storage-nim-test = pkgs.nixosTest {
|
||||
name = "logos-storage-nim-test";
|
||||
nodes = {
|
||||
server = { config, pkgs, ... }: {
|
||||
imports = [ self.nixosModules.logos-storage-nim ];
|
||||
services.logos-storage-nim.enable = true;
|
||||
services.logos-storage-nim.settings = {
|
||||
data-dir = "/var/lib/logos-storage-nim-test";
|
||||
};
|
||||
systemd.services.logos-storage-nim.serviceConfig.StateDirectory = "logos-storage-nim-test";
|
||||
};
|
||||
};
|
||||
testScript = ''
|
||||
print("Starting test: logos-storage-nim-test")
|
||||
machine.start()
|
||||
machine.wait_for_unit("logos-storage-nim.service")
|
||||
machine.succeed("test -d /var/lib/logos-storage-nim-test")
|
||||
machine.wait_until_succeeds("journalctl -u logos-storage-nim.service | grep 'Started Storage node'", 10)
|
||||
'';
|
||||
};
|
||||
});
|
||||
nixosModules.logos-storage-nim = { config, lib, pkgs, ... }: import ./nix/service.nix {
|
||||
inherit config lib pkgs self;
|
||||
};
|
||||
}
|
||||
|
||||
devShells = forAllSystems (system: let
|
||||
pkgs = pkgsFor.${system};
|
||||
nim = nimbusBuildSystem.packages.${system}.nim;
|
||||
in {
|
||||
default = pkgs.mkShell {
|
||||
inputsFrom = [
|
||||
packages.${system}.logos-storage-nim
|
||||
packages.${system}.libstorage
|
||||
];
|
||||
# Not using buildInputs to override fakeGit.
|
||||
nativeBuildInputs = with pkgs; [ nim git cargo nodejs_20 ];
|
||||
};
|
||||
});
|
||||
|
||||
checks = forAllSystems (system: let
|
||||
pkgs = pkgsFor.${system};
|
||||
in {
|
||||
logos-storage-nim-test = pkgs.nixosTest {
|
||||
name = "logos-storage-nim-test";
|
||||
nodes = {
|
||||
server = { config, pkgs, ... }: {
|
||||
imports = [ self.nixosModules.logos-storage-nim ];
|
||||
services.logos-storage-nim.enable = true;
|
||||
services.logos-storage-nim.settings = {
|
||||
data-dir = "/var/lib/logos-storage-nim-test";
|
||||
};
|
||||
systemd.services.logos-storage-nim.serviceConfig.StateDirectory = "logos-storage-nim-test";
|
||||
};
|
||||
};
|
||||
testScript = ''
|
||||
print("Starting test: logos-storage-nim-test")
|
||||
machine.start()
|
||||
machine.wait_for_unit("logos-storage-nim.service")
|
||||
machine.succeed("test -d /var/lib/logos-storage-nim-test")
|
||||
machine.wait_until_succeeds("journalctl -u logos-storage-nim.service | grep 'Started Storage node'", 10)
|
||||
'';
|
||||
};
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
@ -1,12 +0,0 @@
|
||||
{ pkgs ? import <nixpkgs> { } }:
|
||||
|
||||
let
|
||||
tools = pkgs.callPackage ./tools.nix {};
|
||||
sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim;
|
||||
in pkgs.fetchFromGitHub {
|
||||
owner = "nim-lang";
|
||||
repo = "checksums";
|
||||
rev = tools.findKeyValue "^ +ChecksumsStableCommit = \"([a-f0-9]+)\".*$" sourceFile;
|
||||
# WARNING: Requires manual updates when Nim compiler version changes.
|
||||
hash = "sha256-JZhWqn4SrAgNw/HLzBK0rrj3WzvJ3Tv1nuDMn83KoYY=";
|
||||
}
|
||||
@ -1,12 +0,0 @@
|
||||
{ pkgs ? import <nixpkgs> { } }:
|
||||
|
||||
let
|
||||
tools = pkgs.callPackage ./tools.nix {};
|
||||
sourceFile = ../vendor/nimbus-build-system/vendor/Nim/config/build_config.txt;
|
||||
in pkgs.fetchFromGitHub {
|
||||
owner = "nim-lang";
|
||||
repo = "csources_v2";
|
||||
rev = tools.findKeyValue "^nim_csourcesHash=([a-f0-9]+)$" sourceFile;
|
||||
# WARNING: Requires manual updates when Nim compiler version changes.
|
||||
hash = "sha256-UCLtoxOcGYjBdvHx7A47x6FjLMi6VZqpSs65MN7fpBs=";
|
||||
}
|
||||
@ -1,17 +1,17 @@
|
||||
{
|
||||
pkgs ? import <nixpkgs> { },
|
||||
src ? ../.,
|
||||
targets ? ["all"],
|
||||
# Nimbus-build-system package.
|
||||
nim ? null,
|
||||
# Options: 0,1,2
|
||||
verbosity ? 1,
|
||||
commit ? builtins.substring 0 7 (src.rev or "dirty"),
|
||||
# Make targets
|
||||
targets ? ["all"],
|
||||
# These are the only platforms tested in CI and considered stable.
|
||||
stableSystems ? [
|
||||
"x86_64-linux" "aarch64-linux"
|
||||
"x86_64-darwin" "aarch64-darwin"
|
||||
],
|
||||
# Perform 2-stage bootstrap instead of 3-stage to save time.
|
||||
quickAndDirty ? true,
|
||||
}:
|
||||
|
||||
assert pkgs.lib.assertMsg ((src.submodules or true) == true)
|
||||
@ -20,54 +20,42 @@ assert pkgs.lib.assertMsg ((src.submodules or true) == true)
|
||||
let
|
||||
inherit (pkgs) lib writeScriptBin callPackage;
|
||||
|
||||
revision = lib.substring 0 8 (src.rev or "dirty");
|
||||
|
||||
tools = callPackage ./tools.nix {};
|
||||
|
||||
version = tools.findKeyValue "version = \"([0-9]+\.[0-9]+\.[0-9]+)\"" ../codex.nimble;
|
||||
revision = lib.substring 0 8 (src.rev or src.dirtyRev or "00000000");
|
||||
|
||||
# Pin GCC/CLang versions
|
||||
stdenv = if pkgs.stdenv.isLinux then pkgs.gcc13Stdenv else pkgs.clang16Stdenv;
|
||||
|
||||
in stdenv.mkDerivation rec {
|
||||
pname = "storage";
|
||||
|
||||
version = "${tools.findKeyValue "version = \"([0-9]+\.[0-9]+\.[0-9]+)\"" ../codex.nimble}-${revision}";
|
||||
in stdenv.mkDerivation {
|
||||
pname = "logos-storage-nim";
|
||||
version = "${version}-${revision}";
|
||||
|
||||
inherit src;
|
||||
|
||||
# Dependencies that should exist in the runtime environment.
|
||||
buildInputs = with pkgs; [
|
||||
openssl
|
||||
gmp
|
||||
# Disable CPU optimizations that make binary not portable.
|
||||
env = {
|
||||
NIMFLAGS = "-d:disableMarchNative";
|
||||
};
|
||||
|
||||
makeFlags = targets ++ [
|
||||
"V=${toString verbosity}"
|
||||
# Built from nimbus-build-system via flake.
|
||||
"USE_SYSTEM_NIM=1"
|
||||
];
|
||||
|
||||
# Dependencies that should exist in the runtime environment.
|
||||
buildInputs = with pkgs; [ openssl gmp ];
|
||||
|
||||
# Dependencies that should only exist in the build environment.
|
||||
nativeBuildInputs = let
|
||||
# Fix for Nim compiler calling 'git rev-parse' and 'lsb_release'.
|
||||
fakeGit = writeScriptBin "git" "echo ${version}";
|
||||
in with pkgs; [
|
||||
cmake
|
||||
which
|
||||
fakeGit
|
||||
] ++ lib.optionals stdenv.isLinux [
|
||||
lsb-release
|
||||
] ++ lib.optionals stdenv.isDarwin [
|
||||
darwin.cctools
|
||||
];
|
||||
|
||||
# Disable CPU optimizations that make binary not portable.
|
||||
NIMFLAGS = "-d:disableMarchNative -d:git_revision_override=${revision}";
|
||||
|
||||
makeFlags = targets ++ [
|
||||
"V=${toString verbosity}"
|
||||
"QUICK_AND_DIRTY_COMPILER=${if quickAndDirty then "1" else "0"}"
|
||||
"QUICK_AND_DIRTY_NIMBLE=${if quickAndDirty then "1" else "0"}"
|
||||
];
|
||||
|
||||
# FIXME: Remove once permanent fix is applied to NBS:
|
||||
patchPhase = ''
|
||||
substituteInPlace vendor/nimbus-build-system/scripts/build_nim.sh \
|
||||
--replace-fail '"''${NIX_BUILD_TOP}" != "/build"' '-z $${NIX_BUILD_TOP}'
|
||||
'';
|
||||
in with pkgs; [ nim cmake which fakeGit ]
|
||||
++ lib.optionals stdenv.isLinux [ lsb-release ]
|
||||
++ lib.optionals stdenv.isDarwin [ darwin.cctools ];
|
||||
|
||||
configurePhase = ''
|
||||
# Avoid Nim cache permission errors.
|
||||
@ -76,16 +64,6 @@ in stdenv.mkDerivation rec {
|
||||
make nimbus-build-system-paths
|
||||
'';
|
||||
|
||||
preBuild = ''
|
||||
pushd vendor/nimbus-build-system/vendor/Nim
|
||||
mkdir dist
|
||||
cp -r ${callPackage ./nimble.nix {}} dist/nimble
|
||||
cp -r ${callPackage ./checksums.nix {}} dist/checksums
|
||||
cp -r ${callPackage ./csources.nix {}} csources_v2
|
||||
chmod 777 -R dist/nimble csources_v2
|
||||
popd
|
||||
'';
|
||||
|
||||
installPhase = ''
|
||||
if [ -f build/storage ]; then
|
||||
mkdir -p $out/bin
|
||||
|
||||
@ -1,13 +0,0 @@
|
||||
{ pkgs ? import <nixpkgs> { } }:
|
||||
|
||||
let
|
||||
tools = pkgs.callPackage ./tools.nix {};
|
||||
sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim;
|
||||
in pkgs.fetchFromGitHub {
|
||||
owner = "nim-lang";
|
||||
repo = "nimble";
|
||||
fetchSubmodules = true;
|
||||
rev = tools.findKeyValue "^ +NimbleStableCommit = \"([a-f0-9]+)\".+" sourceFile;
|
||||
# WARNING: Requires manual updates when Nim compiler version changes.
|
||||
hash = "sha256-wgzFhModFkwB8st8F5vSkua7dITGGC2cjoDvgkRVZMs=";
|
||||
}
|
||||
2
vendor/nimbus-build-system
vendored
2
vendor/nimbus-build-system
vendored
@ -1 +1 @@
|
||||
Subproject commit 0be0663e1af76e869837226a4ef3e586fcc737d3
|
||||
Subproject commit 0e7a764edae92b224326b3700f062702489ce2b4
|
||||
Loading…
x
Reference in New Issue
Block a user