nix: try to make derivation to work with nim 2.2.4

Makefile

@@ -16,6 +16,7 @@
 # If NIM_COMMIT is set to "nimbusbuild", this will use the
 # version pinned by nimbus-build-system.
 PINNED_NIM_VERSION := v2.2.4
+export BUILD_SYSTEM_DIR := vendor/nimbus-build-system
 
 ifeq ($(NIM_COMMIT),)
 NIM_COMMIT := $(PINNED_NIM_VERSION)
@@ -31,9 +32,6 @@ endif
 
 SHELL := bash # the shell used internally by Make
 
-# used inside the included makefiles
-BUILD_SYSTEM_DIR := vendor/nimbus-build-system
-
 # -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics
 # -d:chronicles_colors:none - Necessary to disable colors in logs for Docker
 DOCKER_IMAGE_NIM_PARAMS ?= -d:chronicles_colors:none -d:insecure
@@ -199,6 +197,11 @@ ifneq ($(USE_LIBBACKTRACE), 0)
 	+ $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT)
 endif
 
+nimbus-build-system-nimble-dir:
+	NIMBLE_DIR="$(CURDIR)/$(NIMBLE_DIR)" \
+	PWD_CMD="$(PWD)" \
+	$(CURDIR)/scripts/generate_nimble_links.sh
+
 ############
 ## Format ##
 ############

flake.lock

@@ -22,17 +22,17 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1736200483,
+        "lastModified": 1740603184,
-        "narHash": "sha256-JO+lFN2HsCwSLMUWXHeOad6QUxOuwe9UOAF/iSl1J4I=",
+        "narHash": "sha256-t+VaahjQAWyA+Ctn2idyo1yxRIYpaDxMgHkgCNiMJa4=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "3f0a8ac25fb674611b98089ca3a5dd6480175751",
+        "rev": "f44bd8ca21e026135061a0a57dcf3d0775b67a49",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-24.11",
         "repo": "nixpkgs",
+        "rev": "f44bd8ca21e026135061a0a57dcf3d0775b67a49",
         "type": "github"
       }
     },

flake.nix

@@ -1,8 +1,13 @@
 {
   description = "Nim Codex build flake";
 
+  nixConfig = {
+    extra-substituters = [ "https://nix-cache.status.im/" ];
+    extra-trusted-public-keys = [ "nix-cache.status.im-1:x/93lOfLU+duPplwMSBR+OlY4+mo+dCN7n0mr4oPwgY=" ];
+  };
+
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11";
+    nixpkgs.url = "github:NixOS/nixpkgs?rev=f44bd8ca21e026135061a0a57dcf3d0775b67a49";
     circom-compat = {
       url = "github:codex-storage/circom-compat-ffi";
       inputs.nixpkgs.follows = "nixpkgs";

nix/atlas.nix

@@ -0,0 +1,12 @@
+{ pkgs ? import <nixpkgs> { } }:
+
+let
+  tools = pkgs.callPackage ./tools.nix {};
+  sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim;
+in pkgs.fetchFromGitHub {
+  owner = "nim-lang";
+  repo = "atlas";
+  rev = "26cecf4d0cc038d5422fc1aa737eec9c8803a82b";
+  # WARNING: Requires manual updates when Nim compiler version changes.
+  hash = "sha256-k5/42XFjIMWYL1bxTKkHIOgjaEEqB68hOIpW3N/ub3E=";
+}

nix/checksums.nix

@@ -6,7 +6,7 @@ let
 in pkgs.fetchFromGitHub {
   owner = "nim-lang";
   repo = "checksums";
-  rev = tools.findKeyValue "^ +ChecksumsStableCommit = \"([a-f0-9]+)\"$" sourceFile;
+  rev = "f8f6bd34bfa3fe12c64b919059ad856a96efcba0";
   # WARNING: Requires manual updates when Nim compiler version changes.
-  hash = "sha256-Bm5iJoT2kAvcTexiLMFBa9oU5gf7d4rWjo3OiN7obWQ=";
+  hash = "sha256-JZhWqn4SrAgNw/HLzBK0rrj3WzvJ3Tv1nuDMn83KoYY=";
 }

nix/csources.nix

@@ -6,7 +6,7 @@ let
 in pkgs.fetchFromGitHub {
   owner = "nim-lang";
   repo = "csources_v2";
-  rev = tools.findKeyValue "^nim_csourcesHash=([a-f0-9]+)$" sourceFile;
+  rev = "86742fb02c6606ab01a532a0085784effb2e753e";
   # WARNING: Requires manual updates when Nim compiler version changes.
   hash = "sha256-UCLtoxOcGYjBdvHx7A47x6FjLMi6VZqpSs65MN7fpBs=";
 }

nix/default.nix

@@ -3,7 +3,7 @@
   src ? ../.,
   targets ? ["all"],
   # Options: 0,1,2
-  verbosity ? 1,
+  verbosity ? 2,
   commit ? builtins.substring 0 7 (src.rev or "dirty"),
   # These are the only platforms tested in CI and considered stable.
   stableSystems ? [
@@ -11,6 +11,7 @@
     "x86_64-darwin" "aarch64-darwin"
   ],
   # Perform 2-stage bootstrap instead of 3-stage to save time.
+  useSystemNim ? false,
   quickAndDirty ? true,
   circomCompatPkg ? (
     builtins.getFlake "github:codex-storage/circom-compat-ffi"
@@ -32,7 +33,8 @@ in pkgs.gcc13Stdenv.mkDerivation rec {
 
   version = "${tools.findKeyValue "version = \"([0-9]+\.[0-9]+\.[0-9]+)\"" ../codex.nimble}-${revision}";
 
-  inherit src;
+  # Improve caching of sources
+  src = builtins.path { path = ./..; name = "nim-codex"; };
 
   # Dependencies that should exist in the runtime environment.
   buildInputs = with pkgs; [
@@ -52,6 +54,7 @@ in pkgs.gcc13Stdenv.mkDerivation rec {
       which
       lsb-release
       circomCompatPkg
+      nim
       fakeGit
       fakeCargo
   ];
@@ -65,19 +68,32 @@ in pkgs.gcc13Stdenv.mkDerivation rec {
     "V=${toString verbosity}"
     "QUICK_AND_DIRTY_COMPILER=${if quickAndDirty then "1" else "0"}"
     "QUICK_AND_DIRTY_NIMBLE=${if quickAndDirty then "1" else "0"}"
+    "USE_SYSTEM_NIM=${if useSystemNim then "1" else "0"}"
   ];
 
   configurePhase = ''
     patchShebangs . vendor/nimbus-build-system > /dev/null
     make nimbus-build-system-paths
+    make nimbus-build-system-nimble-dir
   '';
 
   preBuild = ''
     pushd vendor/nimbus-build-system/vendor/Nim
     mkdir dist
     cp -r ${callPackage ./nimble.nix {}}    dist/nimble
-    cp -r ${callPackage ./checksums.nix {}} dist/checksums
+    chmod 777 -R dist/nimble
+    mkdir -p dist/nimble/dist
+    mkdir -p dist/atlas/dist
+    cp -r ${callPackage ./checksums.nix {}} dist/checksums  # need both
+    cp -r ${callPackage ./checksums.nix {}} dist/nimble/dist/checksums
+    cp -r ${callPackage ./checksums.nix {}}/. dist/nimble/vendor/checksums
+    cp -r ${callPackage ./atlas.nix {}}     dist/atlas
+    chmod 777 -R dist/atlas
+    cp -r ${callPackage ./sat.nix {}}       dist/nimble/dist/sat
+    cp -r ${callPackage ./sat.nix {}}/.       dist/nimble/vendor/sat
+    cp -r ${callPackage ./sat.nix {}}/.       dist/atlas/dist/sat
     cp -r ${callPackage ./csources.nix {}}  csources_v2
+    cp -r ${callPackage ./zippy.nix {}}/.     dist/nimble/vendor/zippy
     chmod 777 -R dist/nimble csources_v2
     popd
   '';

nix/nimble.nix

@@ -6,8 +6,7 @@ let
 in pkgs.fetchFromGitHub {
   owner = "nim-lang";
   repo = "nimble";
-  fetchSubmodules = true;
+  rev = "b1dc28450f028aead0b7cf5da8adf2267db65f89";
-  rev = tools.findKeyValue "^ +NimbleStableCommit = \"([a-f0-9]+)\".+" sourceFile;
   # WARNING: Requires manual updates when Nim compiler version changes.
-  hash = "sha256-Rz48sGUKZEAp+UySla+MlsOfsERekuGKw69Tm11fDz8=";
+  hash = "sha256-8iutVgNzDtttZ7V+7S11KfLEuwhKA9TsgS51mlUI08k=";
-}
+}

nix/sat.nix

@@ -0,0 +1,9 @@
+{ pkgs }:
+
+pkgs.fetchFromGitHub {
+  owner = "nim-lang";
+  repo = "sat";
+  rev = "faf1617f44d7632ee9601ebc13887644925dcc01";
+  # WARNING: Requires manual updates when Nim compiler version changes.
+  hash = "sha256-JFrrSV+mehG0gP7NiQ8hYthL0cjh44HNbXfuxQNhq7c=";
+}

nix/zippy.nix

@@ -0,0 +1,9 @@
+{ pkgs }:
+
+pkgs.fetchFromGitHub {
+  owner = "guzba";
+  repo = "zippy";
+  rev = "a99f6a7d8a8e3e0213b3cad0daf0ea974bf58e3f";
+  # WARNING: Requires manual updates when Nim compiler version changes.
+  hash = "sha256-e2ma2Oyp0dlNx8pJsdZl5o5KnaoAX87tqfY0RLG3DZs=";
+}

scripts/generate_nimble_links.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# This script is used for building Nix derivation which doesn't allow Git commands.
+# It implements similar logic as $(NIMBLE_DIR) target in nimbus-build-system Makefile.
+
+create_nimble_link_script_path="$(pwd)/${BUILD_SYSTEM_DIR}/scripts/create_nimble_link.sh"
+
+process_gitmodules() {
+  local gitmodules_file="$1"
+  local gitmodules_dir=$(dirname "$gitmodules_file")
+  
+  # Extract all submodule paths from the .gitmodules file
+  grep "path" $gitmodules_file | awk '{print $3}' | while read submodule_path; do
+    # Change pwd to the submodule dir and execute script
+    pushd "$gitmodules_dir/$submodule_path" > /dev/null
+    NIMBLE_DIR=$NIMBLE_DIR PWD_CMD=$PWD_CMD EXCLUDED_NIM_PACKAGES=$EXCLUDED_NIM_PACKAGES \
+    "$create_nimble_link_script_path" "$submodule_path"
+    popd  > /dev/null
+  done
+}
+
+# Create the base directory if it doesn't exist
+mkdir -p "${NIMBLE_DIR}/pkgs"
+
+# Find all .gitmodules files and process them
+for gitmodules_file in $(find . -name '.gitmodules'); do
+  echo "Processing .gitmodules file: $gitmodules_file"
+  process_gitmodules "$gitmodules_file"
+done