From 4a1a2c8196588ec0c28be0d9b351b219428c2b41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Soko=C5=82owski?= Date: Sun, 7 Apr 2024 01:10:15 +0300 Subject: [PATCH] add packages to Nix flake config, move to repo root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This way we can actually build and run a node using just: ```sh nix run 'github:status-im/nimbus-eth2?submodules=1' ``` The `?submodules=1` part should eventually not be necessary. For more details see: https://github.com/NixOS/nix/issues/4423 Signed-off-by: Jakub SokoĊ‚owski --- .github/workflows/ci.yml | 2 +- .gitignore | 3 +- ci/Jenkinsfile.nix | 85 +++++++++++++++++++++++++++++ flake.lock | 27 ++++++++++ flake.nix | 38 +++++++++++++ installer/nix/.gitignore | 2 - installer/nix/flake.lock | 61 --------------------- installer/nix/flake.nix | 15 ------ nix/README.md | 29 ++++++++++ nix/csources.nix | 12 +++++ nix/default.nix | 91 ++++++++++++++++++++++++++++++++ nix/nimble.nix | 12 +++++ {installer/nix => nix}/shell.nix | 15 ++---- nix/tools.nix | 15 ++++++ nix/version.nix | 11 ++++ 15 files changed, 327 insertions(+), 91 deletions(-) create mode 100644 ci/Jenkinsfile.nix create mode 100644 flake.lock create mode 100644 flake.nix delete mode 100644 installer/nix/.gitignore delete mode 100644 installer/nix/flake.lock delete mode 100644 installer/nix/flake.nix create mode 100644 nix/README.md create mode 100644 nix/csources.nix create mode 100644 nix/default.nix create mode 100644 nix/nimble.nix rename {installer/nix => nix}/shell.nix (66%) create mode 100644 nix/tools.nix create mode 100644 nix/version.nix diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c465e7a2..3b1b3a97f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -244,7 +244,7 @@ jobs: if: ${{ !cancelled() }} && github.event_name == 'pull_request' run: | excluded_files="config.yaml" - excluded_extensions="ans|cfg|json|json\\.template|md|png|service|ssz|txt" + excluded_extensions="ans|cfg|json|json\\.template|md|png|service|ssz|txt|lock|nix" current_year=$(date +"%Y") problematic_files=() diff --git a/.gitignore b/.gitignore index 6e2ee700b..8c112aa1d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ # beacon_chain -# Copyright (c) 2018-2023 Status Research & Development GmbH +# Copyright (c) 2018-2024 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0). @@ -65,6 +65,7 @@ geth-*.zip # generated during Nim compilation *.nim.generated.nim +result /dist /benchmark_results /.update.timestamp diff --git a/ci/Jenkinsfile.nix b/ci/Jenkinsfile.nix new file mode 100644 index 000000000..44607fcca --- /dev/null +++ b/ci/Jenkinsfile.nix @@ -0,0 +1,85 @@ +#!/usr/bin/env groovy +/* beacon_chain + * Copyright (c) 2019-2024 Status Research & Development GmbH + * Licensed and distributed under either of + * * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). + * * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). + * at your option. This file may not be copied, modified, or distributed except according to those terms. + */ +library 'status-jenkins-lib@nix/flake-build' + +pipeline { + /* This way we run the same Jenkinsfile on different platforms. */ + agent { label params.AGENT_LABEL } + + parameters { + string( + name: 'AGENT_LABEL', + description: 'Label for targetted CI slave host: linux/macos', + defaultValue: params.AGENT_LABEL ?: getAgentLabel(), + ) + choice( + name: 'VERBOSITY', + description: 'Value for the V make flag to increase log verbosity', + choices: [0, 1, 2] + ) + } + + options { + timestamps() + ansiColor('xterm') + /* This also includes wait time in the queue. */ + timeout(time: 1, unit: 'HOURS') + /* Limit builds retained. */ + buildDiscarder(logRotator( + numToKeepStr: '5', + daysToKeepStr: '30', + )) + /* Abort old builds for non-main branches. */ + disableConcurrentBuilds( + abortPrevious: !isMainBranch() + ) + } + + stages { + stage('Beacon Node') { + steps { script { + nix.flake('beacon_node') + } } + } + + stage('Version check') { + steps { script { + sh 'result/bin/nimbus_beacon_node --version' + } } + } + } + + post { + always { + cleanWs( + disableDeferredWipeout: true, + deleteDirs: true + ) + } + } +} + +def isMainBranch() { + return ['stable', 'testing', 'unstable'].contains(env.BRANCH_NAME) +} + +/* This allows us to use one Jenkinsfile and run + * jobs on different platforms based on job name. */ +def getAgentLabel() { + if (params.AGENT_LABEL) { return params.AGENT_LABEL } + /* We extract the name of the job from currentThread because + * before an agent is picket env is not available. */ + def tokens = Thread.currentThread().getName().split('/') + def labels = [] + /* Check if the job path contains any of the valid labels. */ + ['linux', 'macos', 'x86_64', 'aarch64', 'arm64'].each { + if (tokens.contains(it)) { labels.add(it) } + } + return labels.join(' && ') +} diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..0f60d43a3 --- /dev/null +++ b/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1712439257, + "narHash": "sha256-aSpiNepFOMk9932HOax0XwNxbA38GOUVOiXfUVPOrck=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ff0dbd94265ac470dda06a657d5fe49de93b4599", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "master", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..6391bdd00 --- /dev/null +++ b/flake.nix @@ -0,0 +1,38 @@ +{ + description = "nimbus-eth2"; + + inputs.nixpkgs.url = github:NixOS/nixpkgs/master; + + outputs = { self, nixpkgs }: + let + stableSystems = [ + "x86_64-linux" "aarch64-linux" "armv7a-linux" + "x86_64-darwin" "aarch64-darwin" + "x86_64-windows" + ]; + forEach = nixpkgs.lib.genAttrs; + forAllSystems = forEach stableSystems; + pkgsFor = forEach stableSystems ( + system: import nixpkgs { inherit system; } + ); + in rec { + packages = forAllSystems (system: let + buildTarget = pkgsFor.${system}.callPackage ./nix/default.nix { + inherit stableSystems; src = self; + }; + build = targets: buildTarget.override { inherit targets; }; + in rec { + beacon_node = build ["nimbus_beacon_node"]; + signing_node = build ["nimbus_signing_node"]; + validator_client = build ["nimbus_validator_client"]; + ncli = build ["ncli"]; + ncli_db = build ["ncli_db"]; + + default = beacon_node; + }); + + devShells = forAllSystems (system: { + default = pkgsFor.${system}.callPackage ./nix/shell.nix { }; + }); + }; +} diff --git a/installer/nix/.gitignore b/installer/nix/.gitignore deleted file mode 100644 index 0500e4799..000000000 --- a/installer/nix/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Local file unique for each user -.flake-profiles/ \ No newline at end of file diff --git a/installer/nix/flake.lock b/installer/nix/flake.lock deleted file mode 100644 index 262f0732f..000000000 --- a/installer/nix/flake.lock +++ /dev/null @@ -1,61 +0,0 @@ -{ - "nodes": { - "flake-utils": { - "inputs": { - "systems": "systems" - }, - "locked": { - "lastModified": 1701680307, - "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "4022d587cbbfd70fe950c1e2083a02621806a725", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "nixpkgs": { - "locked": { - "lastModified": 1703992652, - "narHash": "sha256-C0o8AUyu8xYgJ36kOxJfXIroy9if/G6aJbNOpA5W0+M=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "32f63574c85fbc80e4ba1fbb932cde9619bad25e", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-23.11", - "repo": "nixpkgs", - "type": "github" - } - }, - "root": { - "inputs": { - "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs" - } - }, - "systems": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/installer/nix/flake.nix b/installer/nix/flake.nix deleted file mode 100644 index d64520876..000000000 --- a/installer/nix/flake.nix +++ /dev/null @@ -1,15 +0,0 @@ -{ - description = "nimbus-eth2"; - - inputs = { - nixpkgs.url = github:NixOS/nixpkgs/nixos-23.11; - flake-utils.url = github:numtide/flake-utils; - }; - - outputs = { self, nixpkgs, flake-utils }: - flake-utils.lib.simpleFlake { - inherit self nixpkgs; - name = "nimbus-eth2"; - shell = ./shell.nix; - }; -} diff --git a/nix/README.md b/nix/README.md new file mode 100644 index 000000000..17825a941 --- /dev/null +++ b/nix/README.md @@ -0,0 +1,29 @@ +# Usage + +## Shell + +A development shell can be started using: +```sh +nix develop +``` + +## Building + +To build a beacon node you can use: +```sh +nix build '.?submodules=1#beacon_node' +``` +The `?submodules=1` part should eventually not be necessary. +For more details see: +https://github.com/NixOS/nix/issues/4423 + +It can be also done without even cloning the repo: +```sh +nix build 'github:status-im/nimbus-eth2?submodules=1' +``` + +## Running + +```sh +nix run 'github:status-im/nimbus-eth2?submodules=1' +``` diff --git a/nix/csources.nix b/nix/csources.nix new file mode 100644 index 000000000..ed217e5cf --- /dev/null +++ b/nix/csources.nix @@ -0,0 +1,12 @@ +{ pkgs ? import { } }: + +let + tools = pkgs.callPackage ./tools.nix {}; + sourceFile = ../vendor/nimbus-build-system/vendor/Nim/config/build_config.txt; +in pkgs.fetchFromGitHub { + owner = "nim-lang"; + repo = "csources_v1"; + rev = tools.findKeyValue "^nim_csourcesHash=([a-f0-9]+)$" sourceFile; + # WARNING: Requires manual updates when Nim compiler version changes. + hash = "sha256-gwBFuR7lzO4zttR/6rgdjXMRxVhwKeLqDwpmOwMyU7A="; +} diff --git a/nix/default.nix b/nix/default.nix new file mode 100644 index 000000000..a86470974 --- /dev/null +++ b/nix/default.nix @@ -0,0 +1,91 @@ +{ + pkgs ? import { }, + # Source code of this repo. + src ? ../., + # Options: nimbus_light_client, nimbus_validator_client, nimbus_signing_node, all + targets ? ["nimbus_beacon_node"], + # Options: 0,1,2 + verbosity ? 0, + # Perform 2-stage bootstrap instead of 3-stage to save time. + quickAndDirty ? true, + # These are the only platforms tested in CI and considered stable. + stableSystems ? [ + "x86_64-linux" "aarch64-linux" "armv7a-linux" + "x86_64-darwin" "aarch64-darwin" + "x86_64-windows" + ], +}: + +let + inherit (pkgs) stdenv lib writeScriptBin callPackage; + + nimble = callPackage ./nimble.nix {}; + csources = callPackage ./csources.nix {}; + revision = lib.substring 0 8 (src.rev or "dirty"); +in stdenv.mkDerivation rec { + pname = "nimbus-eth2"; + version = "${callPackage ./version.nix {}}-${revision}"; + + inherit src; + + # Fix for Nim compiler calling 'git rev-parse' and 'lsb_release'. + nativeBuildInputs = let + fakeGit = writeScriptBin "git" "echo ${version}"; + fakeLsbRelease = writeScriptBin "lsb_release" "echo nix"; + in + with pkgs; [ fakeGit fakeLsbRelease which cmake ] + ++ lib.optionals stdenv.isDarwin [ pkgs.darwin.cctools ]; + + enableParallelBuilding = true; + + # Disable CPU optmizations that make binary not portable. + NIMFLAGS = "-d:disableMarchNative -d:git_revision_override=${revision}"; + # Avoid Nim cache permission errors. + XDG_CACHE_HOME = "/tmp"; + + makeFlags = targets ++ [ + "V=${toString verbosity}" + # TODO: Compile Nim in a separate derivation to save time. + "QUICK_AND_DIRTY_COMPILER=${if quickAndDirty then "1" else "0"}" + "QUICK_AND_DIRTY_NIMBLE=${if quickAndDirty then "1" else "0"}" + ]; + + # Generate the nimbus-build-system.paths file. + configurePhase = '' + patchShebangs scripts vendor/nimbus-build-system > /dev/null + make nimbus-build-system-paths + ''; + + # Avoid nimbus-build-system invoking `git clone` to build Nim. + preBuild = '' + pushd vendor/nimbus-build-system/vendor/Nim + mkdir dist + cp -r ${nimble} dist/nimble + cp -r ${csources} csources_v1 + chmod 777 -R dist/nimble csources_v1 + sed -i 's/isGitRepo(destDir)/false/' tools/deps.nim + popd + ''; + + installPhase = '' + mkdir -p $out/bin + rm -f build/generate_makefile + cp build/* $out/bin + ''; + + meta = with lib; { + homepage = "https://nimbus.guide/"; + downloadPage = "https://github.com/status-im/nimbus-eth2/releases"; + changelog = "https://github.com/status-im/nimbus-eth2/blob/stable/CHANGELOG.md"; + description = "Nimbus is a lightweight client for the Ethereum consensus layer"; + longDescription = '' + Nimbus is an extremely efficient consensus layer client implementation. + While it's optimised for embedded systems and resource-restricted devices -- + including Raspberry Pis, its low resource usage also makes it an excellent choice + for any server or desktop (where it simply takes up fewer resources). + ''; + license = with licenses; [asl20 mit]; + mainProgram = "nimbus_beacon_node"; + platforms = stableSystems; + }; +} diff --git a/nix/nimble.nix b/nix/nimble.nix new file mode 100644 index 000000000..8eec4b414 --- /dev/null +++ b/nix/nimble.nix @@ -0,0 +1,12 @@ +{ pkgs ? import { } }: + +let + tools = pkgs.callPackage ./tools.nix {}; + sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim; +in pkgs.fetchFromGitHub { + owner = "nim-lang"; + repo = "nimble"; + rev = tools.findKeyValue "^ +NimbleStableCommit = \"([a-f0-9]+)\".+" sourceFile; + # WARNING: Requires manual updates when Nim compiler version changes. + hash = "sha256-qJcDKnc+9iUvYrZCMUbBbws+Qqa9vmWyCRsvOUEmq8U="; +} diff --git a/installer/nix/shell.nix b/nix/shell.nix similarity index 66% rename from installer/nix/shell.nix rename to nix/shell.nix index ea2499f81..5cacda064 100644 --- a/installer/nix/shell.nix +++ b/nix/shell.nix @@ -1,11 +1,5 @@ -# beacon_chain -# Copyright (c) 2023-2024 Status Research & Development GmbH -# Licensed and distributed under either of -# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). -# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). -# at your option. This file may not be copied, modified, or distributed except according to those terms. - { pkgs ? import {}}: + let mkdocs-packages = ps: with ps; [ mkdocs @@ -14,11 +8,9 @@ let pymdown-extensions ]; mkdocs-python = pkgs.python3.withPackages mkdocs-packages; -in -with pkgs; -mkShell { +in pkgs.mkShell { - buildInputs = [ + buildInputs = with pkgs; [ figlet git git-lfs @@ -43,6 +35,7 @@ mkShell { # For the purposes of compiling Nimbus, this behavior is not desired: export NIX_ENFORCE_NO_NATIVE=0 export USE_SYSTEM_GETOPT=1 + export MAKEFLAGS="-j$NIX_BUILD_CORES" figlet "Welcome to Nimbus-eth2" ''; diff --git a/nix/tools.nix b/nix/tools.nix new file mode 100644 index 000000000..1a9736862 --- /dev/null +++ b/nix/tools.nix @@ -0,0 +1,15 @@ +{ pkgs ? import { } }: + +let + + inherit (pkgs.lib) fileContents last splitString flatten remove; + inherit (builtins) map match; +in { + findKeyValue = regex: sourceFile: + let + linesFrom = sourceFile: splitString "\n" (fileContents sourceFile); + matching = regex: lines: map (line: match regex line) lines; + extractMatch = matches: last (flatten (remove null matches)); + in + extractMatch (matching regex (linesFrom sourceFile)); +} diff --git a/nix/version.nix b/nix/version.nix new file mode 100644 index 000000000..2c03f3d8a --- /dev/null +++ b/nix/version.nix @@ -0,0 +1,11 @@ +{ pkgs ? import { } }: + +let + tools = pkgs.callPackage ./tools.nix {}; + source = ../beacon_chain/version.nim; + + major = tools.findKeyValue " versionMajor\\* = ([0-9]+)$" source; + minor = tools.findKeyValue " versionMinor\\* = ([0-9]+)$" source; + build = tools.findKeyValue " versionBuild\\* = ([0-9]+)$" source; +in + "${major}.${minor}.${build}"