diff --git a/Makefile b/Makefile index f39a3394..8927d421 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,7 @@ # If NIM_COMMIT is set to "nimbusbuild", this will use the # version pinned by nimbus-build-system. PINNED_NIM_VERSION := v2.2.4 +export BUILD_SYSTEM_DIR := vendor/nimbus-build-system ifeq ($(NIM_COMMIT),) NIM_COMMIT := $(PINNED_NIM_VERSION) @@ -31,9 +32,6 @@ endif SHELL := bash # the shell used internally by Make -# used inside the included makefiles -BUILD_SYSTEM_DIR := vendor/nimbus-build-system - # -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics # -d:chronicles_colors:none - Necessary to disable colors in logs for Docker DOCKER_IMAGE_NIM_PARAMS ?= -d:chronicles_colors:none -d:insecure @@ -199,6 +197,11 @@ ifneq ($(USE_LIBBACKTRACE), 0) + $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT) endif +nimbus-build-system-nimble-dir: + NIMBLE_DIR="$(CURDIR)/$(NIMBLE_DIR)" \ + PWD_CMD="$(PWD)" \ + $(CURDIR)/scripts/generate_nimble_links.sh + ############ ## Format ## ############ diff --git a/flake.lock b/flake.lock index decca52e..8cb133a6 100644 --- a/flake.lock +++ b/flake.lock @@ -22,17 +22,17 @@ }, "nixpkgs": { "locked": { - "lastModified": 1736200483, - "narHash": "sha256-JO+lFN2HsCwSLMUWXHeOad6QUxOuwe9UOAF/iSl1J4I=", + "lastModified": 1740603184, + "narHash": "sha256-t+VaahjQAWyA+Ctn2idyo1yxRIYpaDxMgHkgCNiMJa4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "3f0a8ac25fb674611b98089ca3a5dd6480175751", + "rev": "f44bd8ca21e026135061a0a57dcf3d0775b67a49", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-24.11", "repo": "nixpkgs", + "rev": "f44bd8ca21e026135061a0a57dcf3d0775b67a49", "type": "github" } }, diff --git a/flake.nix b/flake.nix index 4302b3d2..96e6521f 100644 --- a/flake.nix +++ b/flake.nix @@ -1,8 +1,13 @@ { description = "Nim Codex build flake"; + nixConfig = { + extra-substituters = [ "https://nix-cache.status.im/" ]; + extra-trusted-public-keys = [ "nix-cache.status.im-1:x/93lOfLU+duPplwMSBR+OlY4+mo+dCN7n0mr4oPwgY=" ]; + }; + inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11"; + nixpkgs.url = "github:NixOS/nixpkgs?rev=f44bd8ca21e026135061a0a57dcf3d0775b67a49"; circom-compat = { url = "github:codex-storage/circom-compat-ffi"; inputs.nixpkgs.follows = "nixpkgs"; diff --git a/nix/atlas.nix b/nix/atlas.nix new file mode 100644 index 00000000..4ad6b25c --- /dev/null +++ b/nix/atlas.nix @@ -0,0 +1,12 @@ +{ pkgs ? import { } }: + +let + tools = pkgs.callPackage ./tools.nix {}; + sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim; +in pkgs.fetchFromGitHub { + owner = "nim-lang"; + repo = "atlas"; + rev = "26cecf4d0cc038d5422fc1aa737eec9c8803a82b"; + # WARNING: Requires manual updates when Nim compiler version changes. + hash = "sha256-k5/42XFjIMWYL1bxTKkHIOgjaEEqB68hOIpW3N/ub3E="; +} \ No newline at end of file diff --git a/nix/checksums.nix b/nix/checksums.nix index d79345d2..dc100f7d 100644 --- a/nix/checksums.nix +++ b/nix/checksums.nix @@ -6,7 +6,7 @@ let in pkgs.fetchFromGitHub { owner = "nim-lang"; repo = "checksums"; - rev = tools.findKeyValue "^ +ChecksumsStableCommit = \"([a-f0-9]+)\"$" sourceFile; + rev = "f8f6bd34bfa3fe12c64b919059ad856a96efcba0"; # WARNING: Requires manual updates when Nim compiler version changes. - hash = "sha256-Bm5iJoT2kAvcTexiLMFBa9oU5gf7d4rWjo3OiN7obWQ="; + hash = "sha256-JZhWqn4SrAgNw/HLzBK0rrj3WzvJ3Tv1nuDMn83KoYY="; } diff --git a/nix/csources.nix b/nix/csources.nix index 33cd9d0f..3be7fffa 100644 --- a/nix/csources.nix +++ b/nix/csources.nix @@ -6,7 +6,7 @@ let in pkgs.fetchFromGitHub { owner = "nim-lang"; repo = "csources_v2"; - rev = tools.findKeyValue "^nim_csourcesHash=([a-f0-9]+)$" sourceFile; + rev = "86742fb02c6606ab01a532a0085784effb2e753e"; # WARNING: Requires manual updates when Nim compiler version changes. hash = "sha256-UCLtoxOcGYjBdvHx7A47x6FjLMi6VZqpSs65MN7fpBs="; } diff --git a/nix/default.nix b/nix/default.nix index b5823f86..c4943fc3 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -3,7 +3,7 @@ src ? ../., targets ? ["all"], # Options: 0,1,2 - verbosity ? 1, + verbosity ? 2, commit ? builtins.substring 0 7 (src.rev or "dirty"), # These are the only platforms tested in CI and considered stable. stableSystems ? [ @@ -11,6 +11,7 @@ "x86_64-darwin" "aarch64-darwin" ], # Perform 2-stage bootstrap instead of 3-stage to save time. + useSystemNim ? false, quickAndDirty ? true, circomCompatPkg ? ( builtins.getFlake "github:codex-storage/circom-compat-ffi" @@ -32,7 +33,8 @@ in pkgs.gcc13Stdenv.mkDerivation rec { version = "${tools.findKeyValue "version = \"([0-9]+\.[0-9]+\.[0-9]+)\"" ../codex.nimble}-${revision}"; - inherit src; + # Improve caching of sources + src = builtins.path { path = ./..; name = "nim-codex"; }; # Dependencies that should exist in the runtime environment. buildInputs = with pkgs; [ @@ -52,6 +54,7 @@ in pkgs.gcc13Stdenv.mkDerivation rec { which lsb-release circomCompatPkg + nim fakeGit fakeCargo ]; @@ -65,19 +68,32 @@ in pkgs.gcc13Stdenv.mkDerivation rec { "V=${toString verbosity}" "QUICK_AND_DIRTY_COMPILER=${if quickAndDirty then "1" else "0"}" "QUICK_AND_DIRTY_NIMBLE=${if quickAndDirty then "1" else "0"}" + "USE_SYSTEM_NIM=${if useSystemNim then "1" else "0"}" ]; configurePhase = '' patchShebangs . vendor/nimbus-build-system > /dev/null make nimbus-build-system-paths + make nimbus-build-system-nimble-dir ''; preBuild = '' pushd vendor/nimbus-build-system/vendor/Nim mkdir dist cp -r ${callPackage ./nimble.nix {}} dist/nimble - cp -r ${callPackage ./checksums.nix {}} dist/checksums + chmod 777 -R dist/nimble + mkdir -p dist/nimble/dist + mkdir -p dist/atlas/dist + cp -r ${callPackage ./checksums.nix {}} dist/checksums # need both + cp -r ${callPackage ./checksums.nix {}} dist/nimble/dist/checksums + cp -r ${callPackage ./checksums.nix {}}/. dist/nimble/vendor/checksums + cp -r ${callPackage ./atlas.nix {}} dist/atlas + chmod 777 -R dist/atlas + cp -r ${callPackage ./sat.nix {}} dist/nimble/dist/sat + cp -r ${callPackage ./sat.nix {}}/. dist/nimble/vendor/sat + cp -r ${callPackage ./sat.nix {}}/. dist/atlas/dist/sat cp -r ${callPackage ./csources.nix {}} csources_v2 + cp -r ${callPackage ./zippy.nix {}}/. dist/nimble/vendor/zippy chmod 777 -R dist/nimble csources_v2 popd ''; diff --git a/nix/nimble.nix b/nix/nimble.nix index 39c5e0ff..26ff7102 100644 --- a/nix/nimble.nix +++ b/nix/nimble.nix @@ -6,8 +6,7 @@ let in pkgs.fetchFromGitHub { owner = "nim-lang"; repo = "nimble"; - fetchSubmodules = true; - rev = tools.findKeyValue "^ +NimbleStableCommit = \"([a-f0-9]+)\".+" sourceFile; + rev = "b1dc28450f028aead0b7cf5da8adf2267db65f89"; # WARNING: Requires manual updates when Nim compiler version changes. - hash = "sha256-Rz48sGUKZEAp+UySla+MlsOfsERekuGKw69Tm11fDz8="; -} + hash = "sha256-8iutVgNzDtttZ7V+7S11KfLEuwhKA9TsgS51mlUI08k="; +} \ No newline at end of file diff --git a/nix/sat.nix b/nix/sat.nix new file mode 100644 index 00000000..389c346a --- /dev/null +++ b/nix/sat.nix @@ -0,0 +1,9 @@ +{ pkgs }: + +pkgs.fetchFromGitHub { + owner = "nim-lang"; + repo = "sat"; + rev = "faf1617f44d7632ee9601ebc13887644925dcc01"; + # WARNING: Requires manual updates when Nim compiler version changes. + hash = "sha256-JFrrSV+mehG0gP7NiQ8hYthL0cjh44HNbXfuxQNhq7c="; +} \ No newline at end of file diff --git a/nix/zippy.nix b/nix/zippy.nix new file mode 100644 index 00000000..ec59dfc0 --- /dev/null +++ b/nix/zippy.nix @@ -0,0 +1,9 @@ +{ pkgs }: + +pkgs.fetchFromGitHub { + owner = "guzba"; + repo = "zippy"; + rev = "a99f6a7d8a8e3e0213b3cad0daf0ea974bf58e3f"; + # WARNING: Requires manual updates when Nim compiler version changes. + hash = "sha256-e2ma2Oyp0dlNx8pJsdZl5o5KnaoAX87tqfY0RLG3DZs="; +} \ No newline at end of file diff --git a/scripts/generate_nimble_links.sh b/scripts/generate_nimble_links.sh new file mode 100755 index 00000000..e01e6db4 --- /dev/null +++ b/scripts/generate_nimble_links.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# This script is used for building Nix derivation which doesn't allow Git commands. +# It implements similar logic as $(NIMBLE_DIR) target in nimbus-build-system Makefile. + +create_nimble_link_script_path="$(pwd)/${BUILD_SYSTEM_DIR}/scripts/create_nimble_link.sh" + +process_gitmodules() { + local gitmodules_file="$1" + local gitmodules_dir=$(dirname "$gitmodules_file") + + # Extract all submodule paths from the .gitmodules file + grep "path" $gitmodules_file | awk '{print $3}' | while read submodule_path; do + # Change pwd to the submodule dir and execute script + pushd "$gitmodules_dir/$submodule_path" > /dev/null + NIMBLE_DIR=$NIMBLE_DIR PWD_CMD=$PWD_CMD EXCLUDED_NIM_PACKAGES=$EXCLUDED_NIM_PACKAGES \ + "$create_nimble_link_script_path" "$submodule_path" + popd > /dev/null + done +} + +# Create the base directory if it doesn't exist +mkdir -p "${NIMBLE_DIR}/pkgs" + +# Find all .gitmodules files and process them +for gitmodules_file in $(find . -name '.gitmodules'); do + echo "Processing .gitmodules file: $gitmodules_file" + process_gitmodules "$gitmodules_file" +done