add packages to Nix flake config, move to repo root

This way we can actually build and run a node using just:
```sh
nix run 'github:status-im/nimbus-eth2?submodules=1'
```
The `?submodules=1` part should eventually not be necessary.
For more details see:
https://github.com/NixOS/nix/issues/4423

Signed-off-by: Jakub Sokołowski <jakub@status.im>
This commit is contained in:
Jakub Sokołowski 2024-04-07 01:10:15 +03:00
parent c41fe698cf
commit 4a1a2c8196
No known key found for this signature in database
GPG Key ID: FE65CD384D5BF7B4
15 changed files with 327 additions and 91 deletions

View File

@ -244,7 +244,7 @@ jobs:
if: ${{ !cancelled() }} && github.event_name == 'pull_request' if: ${{ !cancelled() }} && github.event_name == 'pull_request'
run: | run: |
excluded_files="config.yaml" excluded_files="config.yaml"
excluded_extensions="ans|cfg|json|json\\.template|md|png|service|ssz|txt" excluded_extensions="ans|cfg|json|json\\.template|md|png|service|ssz|txt|lock|nix"
current_year=$(date +"%Y") current_year=$(date +"%Y")
problematic_files=() problematic_files=()

3
.gitignore vendored
View File

@ -1,5 +1,5 @@
# beacon_chain # beacon_chain
# Copyright (c) 2018-2023 Status Research & Development GmbH # Copyright (c) 2018-2024 Status Research & Development GmbH
# Licensed and distributed under either of # Licensed and distributed under either of
# * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT). # * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0). # * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
@ -65,6 +65,7 @@ geth-*.zip
# generated during Nim compilation # generated during Nim compilation
*.nim.generated.nim *.nim.generated.nim
result
/dist /dist
/benchmark_results /benchmark_results
/.update.timestamp /.update.timestamp

85
ci/Jenkinsfile.nix Normal file
View File

@ -0,0 +1,85 @@
#!/usr/bin/env groovy
/* beacon_chain
* Copyright (c) 2019-2024 Status Research & Development GmbH
* Licensed and distributed under either of
* * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
* * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
* at your option. This file may not be copied, modified, or distributed except according to those terms.
*/
library 'status-jenkins-lib@nix/flake-build'
pipeline {
/* This way we run the same Jenkinsfile on different platforms. */
agent { label params.AGENT_LABEL }
parameters {
string(
name: 'AGENT_LABEL',
description: 'Label for targetted CI slave host: linux/macos',
defaultValue: params.AGENT_LABEL ?: getAgentLabel(),
)
choice(
name: 'VERBOSITY',
description: 'Value for the V make flag to increase log verbosity',
choices: [0, 1, 2]
)
}
options {
timestamps()
ansiColor('xterm')
/* This also includes wait time in the queue. */
timeout(time: 1, unit: 'HOURS')
/* Limit builds retained. */
buildDiscarder(logRotator(
numToKeepStr: '5',
daysToKeepStr: '30',
))
/* Abort old builds for non-main branches. */
disableConcurrentBuilds(
abortPrevious: !isMainBranch()
)
}
stages {
stage('Beacon Node') {
steps { script {
nix.flake('beacon_node')
} }
}
stage('Version check') {
steps { script {
sh 'result/bin/nimbus_beacon_node --version'
} }
}
}
post {
always {
cleanWs(
disableDeferredWipeout: true,
deleteDirs: true
)
}
}
}
def isMainBranch() {
return ['stable', 'testing', 'unstable'].contains(env.BRANCH_NAME)
}
/* This allows us to use one Jenkinsfile and run
* jobs on different platforms based on job name. */
def getAgentLabel() {
if (params.AGENT_LABEL) { return params.AGENT_LABEL }
/* We extract the name of the job from currentThread because
* before an agent is picket env is not available. */
def tokens = Thread.currentThread().getName().split('/')
def labels = []
/* Check if the job path contains any of the valid labels. */
['linux', 'macos', 'x86_64', 'aarch64', 'arm64'].each {
if (tokens.contains(it)) { labels.add(it) }
}
return labels.join(' && ')
}

27
flake.lock Normal file
View File

@ -0,0 +1,27 @@
{
"nodes": {
"nixpkgs": {
"locked": {
"lastModified": 1712439257,
"narHash": "sha256-aSpiNepFOMk9932HOax0XwNxbA38GOUVOiXfUVPOrck=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "ff0dbd94265ac470dda06a657d5fe49de93b4599",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "master",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"nixpkgs": "nixpkgs"
}
}
},
"root": "root",
"version": 7
}

38
flake.nix Normal file
View File

@ -0,0 +1,38 @@
{
description = "nimbus-eth2";
inputs.nixpkgs.url = github:NixOS/nixpkgs/master;
outputs = { self, nixpkgs }:
let
stableSystems = [
"x86_64-linux" "aarch64-linux" "armv7a-linux"
"x86_64-darwin" "aarch64-darwin"
"x86_64-windows"
];
forEach = nixpkgs.lib.genAttrs;
forAllSystems = forEach stableSystems;
pkgsFor = forEach stableSystems (
system: import nixpkgs { inherit system; }
);
in rec {
packages = forAllSystems (system: let
buildTarget = pkgsFor.${system}.callPackage ./nix/default.nix {
inherit stableSystems; src = self;
};
build = targets: buildTarget.override { inherit targets; };
in rec {
beacon_node = build ["nimbus_beacon_node"];
signing_node = build ["nimbus_signing_node"];
validator_client = build ["nimbus_validator_client"];
ncli = build ["ncli"];
ncli_db = build ["ncli_db"];
default = beacon_node;
});
devShells = forAllSystems (system: {
default = pkgsFor.${system}.callPackage ./nix/shell.nix { };
});
};
}

View File

@ -1,2 +0,0 @@
# Local file unique for each user
.flake-profiles/

View File

@ -1,61 +0,0 @@
{
"nodes": {
"flake-utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1701680307,
"narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "4022d587cbbfd70fe950c1e2083a02621806a725",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1703992652,
"narHash": "sha256-C0o8AUyu8xYgJ36kOxJfXIroy9if/G6aJbNOpA5W0+M=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "32f63574c85fbc80e4ba1fbb932cde9619bad25e",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-23.11",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"flake-utils": "flake-utils",
"nixpkgs": "nixpkgs"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
}
},
"root": "root",
"version": 7
}

View File

@ -1,15 +0,0 @@
{
description = "nimbus-eth2";
inputs = {
nixpkgs.url = github:NixOS/nixpkgs/nixos-23.11;
flake-utils.url = github:numtide/flake-utils;
};
outputs = { self, nixpkgs, flake-utils }:
flake-utils.lib.simpleFlake {
inherit self nixpkgs;
name = "nimbus-eth2";
shell = ./shell.nix;
};
}

29
nix/README.md Normal file
View File

@ -0,0 +1,29 @@
# Usage
## Shell
A development shell can be started using:
```sh
nix develop
```
## Building
To build a beacon node you can use:
```sh
nix build '.?submodules=1#beacon_node'
```
The `?submodules=1` part should eventually not be necessary.
For more details see:
https://github.com/NixOS/nix/issues/4423
It can be also done without even cloning the repo:
```sh
nix build 'github:status-im/nimbus-eth2?submodules=1'
```
## Running
```sh
nix run 'github:status-im/nimbus-eth2?submodules=1'
```

12
nix/csources.nix Normal file
View File

@ -0,0 +1,12 @@
{ pkgs ? import <nixpkgs> { } }:
let
tools = pkgs.callPackage ./tools.nix {};
sourceFile = ../vendor/nimbus-build-system/vendor/Nim/config/build_config.txt;
in pkgs.fetchFromGitHub {
owner = "nim-lang";
repo = "csources_v1";
rev = tools.findKeyValue "^nim_csourcesHash=([a-f0-9]+)$" sourceFile;
# WARNING: Requires manual updates when Nim compiler version changes.
hash = "sha256-gwBFuR7lzO4zttR/6rgdjXMRxVhwKeLqDwpmOwMyU7A=";
}

91
nix/default.nix Normal file
View File

@ -0,0 +1,91 @@
{
pkgs ? import <nixpkgs> { },
# Source code of this repo.
src ? ../.,
# Options: nimbus_light_client, nimbus_validator_client, nimbus_signing_node, all
targets ? ["nimbus_beacon_node"],
# Options: 0,1,2
verbosity ? 0,
# Perform 2-stage bootstrap instead of 3-stage to save time.
quickAndDirty ? true,
# These are the only platforms tested in CI and considered stable.
stableSystems ? [
"x86_64-linux" "aarch64-linux" "armv7a-linux"
"x86_64-darwin" "aarch64-darwin"
"x86_64-windows"
],
}:
let
inherit (pkgs) stdenv lib writeScriptBin callPackage;
nimble = callPackage ./nimble.nix {};
csources = callPackage ./csources.nix {};
revision = lib.substring 0 8 (src.rev or "dirty");
in stdenv.mkDerivation rec {
pname = "nimbus-eth2";
version = "${callPackage ./version.nix {}}-${revision}";
inherit src;
# Fix for Nim compiler calling 'git rev-parse' and 'lsb_release'.
nativeBuildInputs = let
fakeGit = writeScriptBin "git" "echo ${version}";
fakeLsbRelease = writeScriptBin "lsb_release" "echo nix";
in
with pkgs; [ fakeGit fakeLsbRelease which cmake ]
++ lib.optionals stdenv.isDarwin [ pkgs.darwin.cctools ];
enableParallelBuilding = true;
# Disable CPU optmizations that make binary not portable.
NIMFLAGS = "-d:disableMarchNative -d:git_revision_override=${revision}";
# Avoid Nim cache permission errors.
XDG_CACHE_HOME = "/tmp";
makeFlags = targets ++ [
"V=${toString verbosity}"
# TODO: Compile Nim in a separate derivation to save time.
"QUICK_AND_DIRTY_COMPILER=${if quickAndDirty then "1" else "0"}"
"QUICK_AND_DIRTY_NIMBLE=${if quickAndDirty then "1" else "0"}"
];
# Generate the nimbus-build-system.paths file.
configurePhase = ''
patchShebangs scripts vendor/nimbus-build-system > /dev/null
make nimbus-build-system-paths
'';
# Avoid nimbus-build-system invoking `git clone` to build Nim.
preBuild = ''
pushd vendor/nimbus-build-system/vendor/Nim
mkdir dist
cp -r ${nimble} dist/nimble
cp -r ${csources} csources_v1
chmod 777 -R dist/nimble csources_v1
sed -i 's/isGitRepo(destDir)/false/' tools/deps.nim
popd
'';
installPhase = ''
mkdir -p $out/bin
rm -f build/generate_makefile
cp build/* $out/bin
'';
meta = with lib; {
homepage = "https://nimbus.guide/";
downloadPage = "https://github.com/status-im/nimbus-eth2/releases";
changelog = "https://github.com/status-im/nimbus-eth2/blob/stable/CHANGELOG.md";
description = "Nimbus is a lightweight client for the Ethereum consensus layer";
longDescription = ''
Nimbus is an extremely efficient consensus layer client implementation.
While it's optimised for embedded systems and resource-restricted devices --
including Raspberry Pis, its low resource usage also makes it an excellent choice
for any server or desktop (where it simply takes up fewer resources).
'';
license = with licenses; [asl20 mit];
mainProgram = "nimbus_beacon_node";
platforms = stableSystems;
};
}

12
nix/nimble.nix Normal file
View File

@ -0,0 +1,12 @@
{ pkgs ? import <nixpkgs> { } }:
let
tools = pkgs.callPackage ./tools.nix {};
sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim;
in pkgs.fetchFromGitHub {
owner = "nim-lang";
repo = "nimble";
rev = tools.findKeyValue "^ +NimbleStableCommit = \"([a-f0-9]+)\".+" sourceFile;
# WARNING: Requires manual updates when Nim compiler version changes.
hash = "sha256-qJcDKnc+9iUvYrZCMUbBbws+Qqa9vmWyCRsvOUEmq8U=";
}

View File

@ -1,11 +1,5 @@
# beacon_chain
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{ pkgs ? import <nixpkgs> {}}: { pkgs ? import <nixpkgs> {}}:
let let
mkdocs-packages = ps: with ps; [ mkdocs-packages = ps: with ps; [
mkdocs mkdocs
@ -14,11 +8,9 @@ let
pymdown-extensions pymdown-extensions
]; ];
mkdocs-python = pkgs.python3.withPackages mkdocs-packages; mkdocs-python = pkgs.python3.withPackages mkdocs-packages;
in in pkgs.mkShell {
with pkgs;
mkShell {
buildInputs = [ buildInputs = with pkgs; [
figlet figlet
git git
git-lfs git-lfs
@ -43,6 +35,7 @@ mkShell {
# For the purposes of compiling Nimbus, this behavior is not desired: # For the purposes of compiling Nimbus, this behavior is not desired:
export NIX_ENFORCE_NO_NATIVE=0 export NIX_ENFORCE_NO_NATIVE=0
export USE_SYSTEM_GETOPT=1 export USE_SYSTEM_GETOPT=1
export MAKEFLAGS="-j$NIX_BUILD_CORES"
figlet "Welcome to Nimbus-eth2" figlet "Welcome to Nimbus-eth2"
''; '';

15
nix/tools.nix Normal file
View File

@ -0,0 +1,15 @@
{ pkgs ? import <nixpkgs> { } }:
let
inherit (pkgs.lib) fileContents last splitString flatten remove;
inherit (builtins) map match;
in {
findKeyValue = regex: sourceFile:
let
linesFrom = sourceFile: splitString "\n" (fileContents sourceFile);
matching = regex: lines: map (line: match regex line) lines;
extractMatch = matches: last (flatten (remove null matches));
in
extractMatch (matching regex (linesFrom sourceFile));
}

11
nix/version.nix Normal file
View File

@ -0,0 +1,11 @@
{ pkgs ? import <nixpkgs> { } }:
let
tools = pkgs.callPackage ./tools.nix {};
source = ../beacon_chain/version.nim;
major = tools.findKeyValue " versionMajor\\* = ([0-9]+)$" source;
minor = tools.findKeyValue " versionMinor\\* = ([0-9]+)$" source;
build = tools.findKeyValue " versionBuild\\* = ([0-9]+)$" source;
in
"${major}.${minor}.${build}"