From e8b41fafc1b1f6e9feffddeffec0e97f5c76212d Mon Sep 17 00:00:00 2001 From: Ivan FB Date: Fri, 10 Apr 2026 01:07:37 +0200 Subject: [PATCH] fix nix --- flake.nix | 14 ++- nix/default.nix | 75 +++++++++++++--- nix/deps.nix | 198 +++++++++++++++++++++++++++++++++--------- nix/shell.nix | 4 +- sds.nimble | 66 ++++++++------ tools/gen-nix-deps.sh | 80 +++++++++++++++++ 6 files changed, 350 insertions(+), 87 deletions(-) create mode 100755 tools/gen-nix-deps.sh diff --git a/flake.nix b/flake.nix index 6eaffdd..5b964fe 100644 --- a/flake.nix +++ b/flake.nix @@ -68,10 +68,16 @@ } ); - devShells = forAllSystems (system: { - default = pkgsFor.${system}.callPackage ./nix/shell.nix { - }; - }); + devShells = forAllSystems (system: + let pkgs = pkgsFor.${system}; in { + default = pkgs.mkShell { + nativeBuildInputs = with pkgs; [ + nim-2_2 + nimble + ]; + }; + } + ); }; } diff --git a/nix/default.nix b/nix/default.nix index 3710861..ec3326d 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -22,9 +22,60 @@ let revision = substring 0 8 (src.rev or src.dirtyRev or "00000000"); version = tools.findKeyValue "^version = \"([a-f0-9.-]+)\"$" ../sds.nimble; - nimbleDeps = callPackage ./deps.nix { - inherit src version revision; - }; + # Fetched dep sources, keyed by package name. + deps = import ./deps.nix { inherit pkgs; }; + + # nimble.lock metadata (version + checksums) for pkgs2 directory naming. + lockFile = builtins.fromJSON (builtins.readFile ../nimble.lock); + lockPkgs = lockFile.packages; + + # nimble.paths for the Nim compiler (read by config.nims). + # Paths must be double-quoted so that NimScript can parse the include correctly. + nimblePaths = pkgs.writeText "nimble.paths" ( + builtins.concatStringsSep "\n" ( + [ "--noNimblePath" ] ++ + builtins.concatMap (p: [ "--path:\"${p}\"" "--path:\"${p}/src\"" ]) + (builtins.attrValues deps) + ) + ); + + # Shell commands to populate pkgs2 with writable copies of only the Nim + # source files nimble needs for dependency resolution. Full source for + # compilation is provided via nimble.paths pointing to the Nix store. + # Using rsync (same file filter as the old fixed-output deps derivation). + # Each dir also gets a nimblemeta.json so nimble recognises it as installed + # and does not attempt to re-download the package. + pkgs2SetupCmds = lib.concatStringsSep "\n" ( + lib.mapAttrsToList (name: dep: + let + meta = lockPkgs.${name}; + dirName = "${name}-${meta.version}-${meta.checksums.sha1}"; + nimbleMeta = pkgs.writeText "${name}-nimblemeta.json" (builtins.toJSON { + version = 1; + metaData = { + url = meta.url; + downloadMethod = "git"; + vcsRevision = meta.vcsRevision; + files = []; + binaries = []; + specialVersions = [ meta.version ]; + }; + }); + in '' + mkdir -p "$NIMBLE_DIR/pkgs2/${dirName}" + rsync -a \ + --include='*/' \ + --include='*.nim' \ + --include='*.nims' \ + --include='*.nimble' \ + --include='*.json' \ + --exclude='*' \ + ${dep}/ "$NIMBLE_DIR/pkgs2/${dirName}/" + chmod -R u+w "$NIMBLE_DIR/pkgs2/${dirName}" + cp ${nimbleMeta} "$NIMBLE_DIR/pkgs2/${dirName}/nimblemeta.json" + '' + ) deps + ); in stdenv.mkDerivation { pname = "nim-sds"; @@ -38,12 +89,12 @@ in stdenv.mkDerivation { }; buildInputs = with pkgs; [ - openssl gmp zip nim git nimble + openssl gmp zip nim-2_2 git nimble ]; # Dependencies that should only exist in the build environment. nativeBuildInputs = with pkgs; [ - nim cmake which patchelf nimbleDeps + nim-2_2 nimble rsync cmake which patchelf ] ++ optionals stdenv.isLinux [ pkgs.lsb-release ]; @@ -52,14 +103,16 @@ in stdenv.mkDerivation { "V=${toString verbosity}" ]; - # Provide dependencies via Nimble deps derivation. configurePhase = '' export NIMBLE_DIR=$NIX_BUILD_TOP/nimbledeps - cp -r ${nimbleDeps}/nimbledeps $NIMBLE_DIR - cp ${nimbleDeps}/nimble.paths ./ - chmod 775 -R $NIMBLE_DIR - # Fix relative paths to absolute paths - sed -i "s|./nimbledeps|$NIMBLE_DIR|g" nimble.paths + mkdir -p $NIMBLE_DIR/pkgs2 + + # Populate pkgs2 with writable copies so nimble considers deps installed + # and does not attempt to download them (which fails in the Nix sandbox). + ${pkgs2SetupCmds} + + # Write nimble.paths so config.nims passes --path: flags to the Nim compiler. + cp ${nimblePaths} ./nimble.paths ''; installPhase = let diff --git a/nix/deps.nix b/nix/deps.nix index b50b607..062e9a9 100644 --- a/nix/deps.nix +++ b/nix/deps.nix @@ -1,55 +1,167 @@ -{ pkgs, stdenv, src, version, revision }: +# AUTOGENERATED from nimble.lock — do not edit manually. +# Regenerate with: ./tools/gen-nix-deps.sh nimble.lock nix/deps.nix +{ pkgs }: -stdenv.mkDerivation { - pname = "nim-sds-nimble-deps"; - version = "${version}-${revision}"; +{ + unittest2 = pkgs.fetchgit { + url = "https://github.com/status-im/nim-unittest2"; + rev = "26f2ef3ae0ec72a2a75bfe557e02e88f6a31c189"; + sha256 = "1n8n36kad50m97b64y7bzzknz9n7szffxhp0bqpk3g2v7zpda8sw"; + fetchSubmodules = true; + }; - inherit src; + bearssl = pkgs.fetchgit { + url = "https://github.com/status-im/nim-bearssl"; + rev = "11e798b62b8e6beabe958e048e9e24c7e0f9ee63"; + sha256 = "0qx36iiawrhmx9qjqcyfvz0134ph9dy8ryq3ch8d31gq6ir7aw84"; + fetchSubmodules = true; + }; - nativeBuildInputs = with pkgs; [ - jq rsync git nimble cacert moreutils - ]; + bearssl_pkey_decoder = pkgs.fetchgit { + url = "https://github.com/vacp2p/bearssl_pkey_decoder"; + rev = "21dd3710df9345ed2ad8bf8f882761e07863b8e0"; + sha256 = "0bl3f147zmkazbhdkr4cj1nipf9rqiw3g4hh1j424k9hpl55zdpg"; + fetchSubmodules = true; + }; - configurePhase = '' - export XDG_CACHE_HOME=$TMPDIR - export NIMBLE_DIR=$NIX_BUILD_TOP/nimbledir - export HOME=$TMPDIR - ''; + results = pkgs.fetchgit { + url = "https://github.com/arnetheduck/nim-results"; + rev = "df8113dda4c2d74d460a8fa98252b0b771bf1f27"; + sha256 = "1h7amas16sbhlr7zb7n3jb5434k98ji375vzw72k1fsc86vnmcr9"; + fetchSubmodules = true; + }; - buildPhase = '' - nimble --version - nimble --silent --localdeps setup - nimble --silent --localdeps install -y --depsOnly - ''; + stew = pkgs.fetchgit { + url = "https://github.com/status-im/nim-stew"; + rev = "b66168735d6f3841c5239c3169d3fe5fe98b1257"; + sha256 = "10n71vfa6klzd9dmal96jy0hiqk04gaj8wc9g91z6fclryf0yq92"; + fetchSubmodules = true; + }; - installPhase = '' - mkdir -p $out/nimbledeps + faststreams = pkgs.fetchgit { + url = "https://github.com/status-im/nim-faststreams"; + rev = "ce27581a3e881f782f482cb66dc5b07a02bd615e"; + sha256 = "0y6bw2scnmr8cxj4fg18w7f34l2bh9qwg5nhlgd84m9fpr5bqarn"; + fetchSubmodules = true; + }; - cp nimble.paths $out/nimble.paths + serialization = pkgs.fetchgit { + url = "https://github.com/status-im/nim-serialization"; + rev = "b0f2fa32960ea532a184394b0f27be37bd80248b"; + sha256 = "0wip1fjx7ka39ck1g1xvmyarzq1p5dlngpqil6zff8k8z5skiz27"; + fetchSubmodules = true; + }; - rsync -ra \ - --prune-empty-dirs \ - --include='*/' \ - --include='*.json' \ - --include='*.nim' \ - --include='*.nimble' \ - --exclude='*' \ - $NIMBLE_DIR/pkgs2 $out/nimbledeps - ''; + json_serialization = pkgs.fetchgit { + url = "https://github.com/status-im/nim-json-serialization"; + rev = "c343b0e243d9e17e2c40f3a8a24340f7c4a71d44"; + sha256 = "0i8sq51nqj8lshf6bfixaz9a7sq0ahsbvq3chkxdvv4khsqvam91"; + fetchSubmodules = true; + }; - fixupPhase = '' - # Replace build path with deterministic $out. - sed "s|$NIMBLE_DIR|./nimbledeps|g" $out/nimble.paths \ - | sort | sponge $out/nimble.paths + testutils = pkgs.fetchgit { + url = "https://github.com/status-im/nim-testutils"; + rev = "e4d37dc1652d5c63afb89907efb5a5e812261797"; + sha256 = "0nv0a9jm5b1rn3y52cxvyj8xz3jg235mp0xbirfp2cda0icgy1si"; + fetchSubmodules = true; + }; - # Nimble does not maintain order of files list. - for META_FILE in $(find $out -name nimblemeta.json); do - jq '.metaData.files |= sort' $META_FILE | sponge $META_FILE - done - ''; + chronicles = pkgs.fetchgit { + url = "https://github.com/status-im/nim-chronicles"; + rev = "27ec507429a4eb81edc20f28292ee8ec420be05b"; + sha256 = "1xx9fcfwgcaizq3s7i3s03mclz253r5j8va38l9ycl19fcbc96z9"; + fetchSubmodules = true; + }; + + httputils = pkgs.fetchgit { + url = "https://github.com/status-im/nim-http-utils"; + rev = "c53852d9e24205b6363bba517fa8ee7bde823691"; + sha256 = "1b332smfyp2yvhvfjrfqy4kvh9pc5w6hqh17f1yclz5z1j5xdpf1"; + fetchSubmodules = true; + }; + + chronos = pkgs.fetchgit { + url = "https://github.com/status-im/nim-chronos"; + rev = "0646c444fce7c7ed08ef6f2c9a7abfd172ffe655"; + sha256 = "1r499jl0lhnjq7hgddwgjl0gh3y1mprnqkhk0h6yh3cwgsmr5ym9"; + fetchSubmodules = true; + }; + + dnsclient = pkgs.fetchgit { + url = "https://github.com/ba0f3/dnsclient.nim"; + rev = "23214235d4784d24aceed99bbfe153379ea557c8"; + sha256 = "03mf3lw5c0m5nq9ppa49nylrl8ibkv2zzlc0wyhqg7w09kz6hks6"; + fetchSubmodules = true; + }; + + jwt = pkgs.fetchgit { + url = "https://github.com/vacp2p/nim-jwt.git"; + rev = "18f8378de52b241f321c1f9ea905456e89b95c6f"; + sha256 = "1986czmszdxj6g9yr7xn1fx8y2y9mwpb3f1bn9nc6973qawsdm0p"; + fetchSubmodules = true; + }; + + nimcrypto = pkgs.fetchgit { + url = "https://github.com/cheatfate/nimcrypto"; + rev = "b3dbc9c4d08e58c5b7bfad6dc7ef2ee52f2f4c08"; + sha256 = "1v4rz42lwcazs6isi3kmjylkisr84mh0kgmlqycx4i885dn3g0l4"; + fetchSubmodules = true; + }; + + metrics = pkgs.fetchgit { + url = "https://github.com/status-im/nim-metrics"; + rev = "11d0cddfb0e711aa2a8c75d1892ae24a64c299fc"; + sha256 = "1jrf2cf7v3iqjsk6grzvivxic1shhaxnvab6d35rxs2kcy6b5dv0"; + fetchSubmodules = true; + }; + + secp256k1 = pkgs.fetchgit { + url = "https://github.com/status-im/nim-secp256k1"; + rev = "d8f1288b7c72f00be5fc2c5ea72bf5cae1eafb15"; + sha256 = "1qjrmwbngb73f6r1fznvig53nyal7wj41d1cmqfksrmivk2sgrn2"; + fetchSubmodules = true; + }; + + zlib = pkgs.fetchgit { + url = "https://github.com/status-im/nim-zlib"; + rev = "e680f269fb01af2c34a2ba879ff281795a5258fe"; + sha256 = "1xw9f1gjsgqihdg7kdkbaq1wankgnx2vn9l3ihc6nqk2jzv5bvk5"; + fetchSubmodules = true; + }; + + websock = pkgs.fetchgit { + url = "https://github.com/status-im/nim-websock"; + rev = "35ae76f1559e835c80f9c1a3943bf995d3dd9eb5"; + sha256 = "1j6dklzb6b6bv2aiglbiyflja2vdpmyxfirv98f49y62mykq0yrw"; + fetchSubmodules = true; + }; + + lsquic = pkgs.fetchgit { + url = "https://github.com/vacp2p/nim-lsquic"; + rev = "4fb03ee7bfb39aecb3316889fdcb60bec3d0936f"; + sha256 = "0qdhcd4hyp185szc9sv3jvwdwc9zp3j0syy7glxv13k9bchfmkfg"; + fetchSubmodules = true; + }; + + libp2p = pkgs.fetchgit { + url = "https://github.com/vacp2p/nim-libp2p"; + rev = "ca48c3718246bb411ff0e354a70cb82d9a28de0d"; + sha256 = "07qfjjrq6w7bj9dbchvcrpla47jidngbrgmigbhl7fh3cfkdabc9"; + fetchSubmodules = true; + }; + + stint = pkgs.fetchgit { + url = "https://github.com/status-im/nim-stint"; + rev = "470b7892561b5179ab20bd389a69217d6213fe58"; + sha256 = "1isfwmbj98qfi5pm9acy0yyvq0vlz38nxp30xl43jx2mmaga2w22"; + fetchSubmodules = true; + }; + + taskpools = pkgs.fetchgit { + url = "https://github.com/status-im/nim-taskpools"; + rev = "9e8ccc754631ac55ac2fd495e167e74e86293edb"; + sha256 = "1y78l33vdjxmb9dkr455pbphxa73rgdsh8m9gpkf4d9b1wm1yivy"; + fetchSubmodules = true; + }; - # Make this a fixed-output derivation to allows internet access for Nimble. - outputHash = "sha256-OnirsXLj4HMVTbk+b4fcC+1K9MSMJyae6I7JO16WDno="; - outputHashAlgo = "sha256"; - outputHashMode = "recursive"; } diff --git a/nix/shell.nix b/nix/shell.nix index 53b0663..575d791 100644 --- a/nix/shell.nix +++ b/nix/shell.nix @@ -1,6 +1,5 @@ { pkgs ? import { }, - nim ? null, }: let @@ -12,7 +11,8 @@ in pkgs.mkShell { ]; buildInputs = with pkgs; [ - nim + nim-2_2 + nimble which git cmake diff --git a/sds.nimble b/sds.nimble index a7774dd..d188ad6 100644 --- a/sds.nimble +++ b/sds.nimble @@ -8,9 +8,9 @@ license = "MIT" srcDir = "sds" # Dependencies -requires "nim >= 2.2.6" -requires "chronos >= 4.2.0" -requires "libp2p >= 1.15.1" +requires "nim >= 2.2.4" +requires "chronos >= 4.0.4" +requires "libp2p >= 1.15.2" requires "chronicles" requires "stew" requires "stint" @@ -46,7 +46,8 @@ proc getMyCpu(): string = ## Returns a Nim-compatible CPU name (e.g. amd64, arm64) for the host. ## Respects the ARCH environment variable when set. let envArch = getEnv("ARCH") - if envArch != "": return envArch + if envArch != "": + return envArch when defined(arm64): return "arm64" elif defined(amd64): @@ -54,7 +55,13 @@ proc getMyCpu(): string = else: let (archFromUname, _) = gorgeEx("uname -m") let a = archFromUname.strip() - return if a == "x86_64": "amd64" elif a == "aarch64": "arm64" else: a + return + if a == "x86_64": + "amd64" + elif a == "aarch64": + "arm64" + else: + a # Tasks task test, "Run the test suite": @@ -84,10 +91,14 @@ task libsdsDynamicMac, "Generate bindings": let cpu = getMyCpu() let clangArch = if cpu == "amd64": "x86_64" else: cpu let sdkPath = staticExec("xcrun --show-sdk-path").strip() - let archFlags = "--cpu:" & cpu & " --passC:\"-arch " & clangArch & "\" --passL:\"-arch " & clangArch & "\" --passC:\"-isysroot " & sdkPath & "\" --passL:\"-isysroot " & sdkPath & "\"" + let archFlags = + "--cpu:" & cpu & " --passC:\"-arch " & clangArch & "\" --passL:\"-arch " & clangArch & + "\" --passC:\"-isysroot " & sdkPath & "\" --passL:\"-isysroot " & sdkPath & "\"" buildLibrary outLibNameAndExt, - name, "library/", - archFlags & " -d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE", + name, + "library/", + archFlags & + " -d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE", "dynamic" task libsdsStaticWindows, "Generate bindings": @@ -113,10 +124,14 @@ task libsdsStaticMac, "Generate bindings": let cpu = getMyCpu() let clangArch = if cpu == "amd64": "x86_64" else: cpu let sdkPath = staticExec("xcrun --show-sdk-path").strip() - let archFlags = "--cpu:" & cpu & " --passC:\"-arch " & clangArch & "\" --passL:\"-arch " & clangArch & "\" --passC:\"-isysroot " & sdkPath & "\" --passL:\"-isysroot " & sdkPath & "\"" + let archFlags = + "--cpu:" & cpu & " --passC:\"-arch " & clangArch & "\" --passL:\"-arch " & clangArch & + "\" --passC:\"-isysroot " & sdkPath & "\" --passL:\"-isysroot " & sdkPath & "\"" buildLibrary outLibNameAndExt, - name, "library/", - archFlags & " -d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE", + name, + "library/", + archFlags & + " -d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE", "static" # Build Mobile iOS @@ -140,23 +155,17 @@ proc buildMobileIOS(srcDir = ".", sdkPath = "") = # 1) Generate C sources from Nim (no linking) # Use unique symbol prefix to avoid conflicts with other Nim libraries - exec "nim c" & - " --nimcache:" & nimcacheDir & " --os:ios --cpu:" & cpu & - " --compileOnly:on" & - " --noMain --mm:refc" & - " --threads:on --opt:size --header" & - " --nimMainPrefix:libsds" & - " --cc:clang" & - " -d:useMalloc" & - " " & srcDir & "/libsds.nim" + exec "nim c" & " --nimcache:" & nimcacheDir & " --os:ios --cpu:" & cpu & + " --compileOnly:on" & " --noMain --mm:refc" & " --threads:on --opt:size --header" & + " --nimMainPrefix:libsds" & " --cc:clang" & " -d:useMalloc" & " " & srcDir & + "/libsds.nim" # 2) Compile all generated C files to object files with hidden visibility # This prevents symbol conflicts with other Nim libraries (e.g., libnim_status_client) let nimLibDir = getHomeDir() / ".choosenim/toolchains/nim-" & NimVersion & "/lib" - let clangFlags = "-arch " & clangArch & " -isysroot " & sdkPath & - " -I" & nimLibDir & - " -fembed-bitcode -miphoneos-version-min=16.2 -O2" & - " -fvisibility=hidden" + let clangFlags = + "-arch " & clangArch & " -isysroot " & sdkPath & " -I" & nimLibDir & + " -fembed-bitcode -miphoneos-version-min=16.2 -O2" & " -fvisibility=hidden" var objectFiles: seq[string] = @[] for cFile in listFiles(nimcacheDir): @@ -171,9 +180,11 @@ proc buildMobileIOS(srcDir = ".", sdkPath = "") = # 4) Use libtool to localize all non-public symbols # Keep only Sds* functions as global, hide everything else to prevent conflicts # with nim runtime symbols from libnim_status_client - let keepSymbols = "_Sds*:_libsdsNimMain:_libsdsDatInit*:_libsdsInit*:_NimMainModule__libsds*" + let keepSymbols = + "_Sds*:_libsdsNimMain:_libsdsDatInit*:_libsdsInit*:_NimMainModule__libsds*" exec "xcrun libtool -static -o " & aFile & " " & aFileTmp & - " -exported_symbols_list /dev/stdin <<< '" & keepSymbols & "' 2>/dev/null || cp " & aFileTmp & " " & aFile + " -exported_symbols_list /dev/stdin <<< '" & keepSymbols & "' 2>/dev/null || cp " & + aFileTmp & " " & aFile echo "✔ iOS library created: " & aFile @@ -197,7 +208,8 @@ proc buildMobileAndroid(srcDir = ".", extra_params = "") = exec "nim c" & " --out:" & outDir & "/libsds.so --threads:on --app:lib --opt:size --noMain --mm:refc --nimMainPrefix:libsds " & "-d:chronicles_sinks=textlines[dynamic] --header --passL:-L" & outdir & - " --passL:-llog --cpu:" & cpu & " --os:android -d:androidNDK -d:chronosEventEngine=epoll " & extra_params & " " & + " --passL:-llog --cpu:" & cpu & + " --os:android -d:androidNDK -d:chronosEventEngine=epoll " & extra_params & " " & srcDir & "/libsds.nim" task libsdsAndroid, "Build the mobile bindings for Android": diff --git a/tools/gen-nix-deps.sh b/tools/gen-nix-deps.sh new file mode 100755 index 0000000..c601ecd --- /dev/null +++ b/tools/gen-nix-deps.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Generates nix/deps.nix from nimble.lock using nix-prefetch-git. +# Usage: ./tools/gen-nix-deps.sh [nimble.lock] [nix/deps.nix] +set -euo pipefail + +usage() { + cat < + +Example: + $0 nimble.lock nix/deps.nix +EOF +} + +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage; exit 0 +fi + +if [[ $# -ne 2 ]]; then + usage; exit 1 +fi + +LOCKFILE="$1" +OUTFILE="$2" + +command -v jq >/dev/null || { echo "error: jq required"; exit 1; } +command -v nix-prefetch-git >/dev/null || { echo "error: nix-prefetch-git required"; exit 1; } + +if [[ ! -f "$LOCKFILE" ]]; then + echo "[!] $LOCKFILE not found" + echo "[*] Generating $LOCKFILE via 'nimble lock'" + nimble lock +fi + +echo "[*] Generating $OUTFILE from $LOCKFILE" +mkdir -p "$(dirname "$OUTFILE")" + +cat > "$OUTFILE" <<'EOF' +# AUTOGENERATED from nimble.lock — do not edit manually. +# Regenerate with: ./tools/gen-nix-deps.sh nimble.lock nix/deps.nix +{ pkgs }: + +{ +EOF + +jq -c ' + .packages + | to_entries[] + | select(.value.downloadMethod == "git") + | select(.key != "nim" and .key != "nimble") +' "$LOCKFILE" | while read -r entry; do + name=$(jq -r '.key' <<<"$entry") + url=$(jq -r '.value.url' <<<"$entry") + rev=$(jq -r '.value.vcsRevision' <<<"$entry") + + echo " [*] Prefetching $name @ $rev" + + sha=$(nix-prefetch-git \ + --url "$url" \ + --rev "$rev" \ + --fetch-submodules \ + | jq -r '.sha256') + + cat >> "$OUTFILE" <> "$OUTFILE" <<'EOF' +} +EOF + +echo "[✓] Wrote $OUTFILE"