diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f924d0f8b..84a7f0b8d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,9 @@ jobs: - 'nimble.lock' - 'waku.nimble' - 'Makefile' + - 'scripts/**' + - 'flake.nix' + - 'flake.lock' - 'library/**' - 'liblogosdelivery/**' v2: diff --git a/flake.lock b/flake.lock index 8d0db9269..411bf2430 100644 --- a/flake.lock +++ b/flake.lock @@ -19,8 +19,7 @@ "root": { "inputs": { "nixpkgs": "nixpkgs", - "rust-overlay": "rust-overlay", - "zerokit": "zerokit" + "rust-overlay": "rust-overlay" } }, "rust-overlay": { @@ -42,47 +41,6 @@ "repo": "rust-overlay", "type": "github" } - }, - "rust-overlay_2": { - "inputs": { - "nixpkgs": [ - "zerokit", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1771211437, - "narHash": "sha256-lcNK438i4DGtyA+bPXXyVLHVmJjYpVKmpux9WASa3ro=", - "owner": "oxalica", - "repo": "rust-overlay", - "rev": "c62195b3d6e1bb11e0c2fb2a494117d3b55d410f", - "type": "github" - }, - "original": { - "owner": "oxalica", - "repo": "rust-overlay", - "type": "github" - } - }, - "zerokit": { - "inputs": { - "nixpkgs": [ - "nixpkgs" - ], - "rust-overlay": "rust-overlay_2" - }, - "locked": { - "owner": "vacp2p", - "repo": "zerokit", - "rev": "5e64cb8822bee65eed6cf459f95ae72b80c6ba63", - "type": "github" - }, - "original": { - "owner": "vacp2p", - "repo": "zerokit", - "rev": "5e64cb8822bee65eed6cf459f95ae72b80c6ba63", - "type": "github" - } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 6c283780d..b32a53455 100644 --- a/flake.nix +++ b/flake.nix @@ -17,19 +17,9 @@ url = "github:oxalica/rust-overlay"; inputs.nixpkgs.follows = "nixpkgs"; }; - - # External flake input: Zerokit pinned to a specific commit. - # Update the rev here when a new zerokit version is needed. - zerokit = { - # Pinned to v2.0.2 (5e64cb8822bee65eed6cf459f95ae72b80c6ba63) to match - # the vendor/zerokit submodule. Keep these two in sync: the nix build - # links librln from this input, the Makefile build from the submodule. - url = "github:vacp2p/zerokit/5e64cb8822bee65eed6cf459f95ae72b80c6ba63"; - inputs.nixpkgs.follows = "nixpkgs"; - }; }; - outputs = { self, nixpkgs, rust-overlay, zerokit }: + outputs = { self, nixpkgs, rust-overlay }: let systems = [ "x86_64-linux" "aarch64-linux" @@ -69,19 +59,78 @@ inherit system; overlays = [ (import rust-overlay) nimbleOverlay ]; }; + + # Prebuilt zerokit librln, fetched from the upstream GitHub release + # rather than compiled from source. Compiling zerokit makes Nix download + # its many crate dependencies from crates.io in one parallel burst, which + # crates.io intermittently rejects with HTTP 403 (rate limiting from the + # self-hosted runners' shared IP), breaking the nix build. The release + # ships the exact `stateless` library this project links (see + # scripts/build_rln.sh), so we use it directly — no Rust toolchain and + # no crates.io access needed. + # + # Keep `rlnVersion` aligned with `LIBRLN_VERSION` in the Makefile and the + # vendor/zerokit submodule. Each hash is the sha256 of the release tarball + # for that platform; refresh all four when bumping the version. + rlnVersion = "v2.0.2"; + rlnAssets = { + "x86_64-linux" = { triple = "x86_64-unknown-linux-gnu"; hash = "sha256-qbrUdaetYKFhjzxUP/QcwD3JHWJ8qk/tCMK3yXceIAk="; }; + "aarch64-linux" = { triple = "aarch64-unknown-linux-gnu"; hash = "sha256-s4bWrmCcNTWHNyJwV73ilWNp58ZdAVG+TAgtWN1cTQs="; }; + "x86_64-darwin" = { triple = "x86_64-apple-darwin"; hash = "sha256-ZaHP5CApN66FYY7jxwOmGcF9kJR78Fng3k1qE2W08Mk="; }; + "aarch64-darwin" = { triple = "aarch64-apple-darwin"; hash = "sha256-f2YppkPsKFdN00j+IY8fpvsebWTIb9lW/V1/vOTiVKU="; }; + }; + + mkZerokitRln = system: pkgs: + let + asset = rlnAssets.${system} or + (throw "zerokit ${rlnVersion} has no prebuilt rln asset for system '${system}'"); + in pkgs.stdenv.mkDerivation { + pname = "librln"; + version = lib.removePrefix "v" rlnVersion; + + src = pkgs.fetchurl { + url = "https://github.com/vacp2p/zerokit/releases/download/" + + "${rlnVersion}/${asset.triple}-stateless-rln.tar.gz"; + hash = asset.hash; + }; + + # The tarball lays its files out under release/. + sourceRoot = "release"; + dontConfigure = true; + dontBuild = true; + + # The release .so was linked outside Nix, so it references system + # libraries (libgcc_s, libstdc++, glibc) by bare name. autoPatchelfHook + # points those at the Nix versions so the library loads correctly when + # used by the Nix build. It does nothing for the static .a, and the + # step is skipped on macOS (dylib paths are fixed in nix/default.nix). + nativeBuildInputs = + pkgs.lib.optionals pkgs.stdenv.isLinux [ pkgs.autoPatchelfHook ]; + buildInputs = + pkgs.lib.optionals pkgs.stdenv.isLinux [ pkgs.stdenv.cc.cc.lib ]; + + installPhase = '' + runHook preInstall + mkdir -p $out/lib + cp librln.a $out/lib/ 2>/dev/null || true + cp librln.so $out/lib/ 2>/dev/null || true + cp librln.dylib $out/lib/ 2>/dev/null || true + runHook postInstall + ''; + + meta = with pkgs.lib; { + description = "Prebuilt zerokit RLN library (stateless flavor)"; + homepage = "https://github.com/vacp2p/zerokit"; + license = with licenses; [ mit asl20 ]; + platforms = builtins.attrNames rlnAssets; + }; + }; in { packages = forAllSystems (system: let pkgs = pkgsFor system; - # HACK: Fix for stale cargoHash in 2.0.2 release. - zerokitRln = zerokit.packages.${system}.rln.overrideAttrs (old: { - cargoDeps = old.cargoDeps.overrideAttrs (oldCargoDeps: { - vendorStaging = oldCargoDeps.vendorStaging.overrideAttrs (_: { - outputHash = "sha256-PNwEdZLgGQPqQDrEK2hsQtSybVfBbD6xn4K47fPFJUU="; - }); - }); - }); + zerokitRln = mkZerokitRln system pkgs; liblogosdelivery = pkgs.callPackage ./nix/default.nix { inherit pkgs; @@ -94,14 +143,13 @@ inherit pkgs; src = ./.; targets = ["wakucanary"]; - zerokitRln = zerokit.packages.${system}.rln; + inherit zerokitRln; }; in { inherit liblogosdelivery wakucanary; - # Expose the cargoHash-corrected librln so downstream consumers + # Expose the prebuilt librln so downstream consumers # (e.g. logos-delivery-module) bundle the exact same librln this - # build links, instead of pulling zerokit's rln directly — whose - # committed cargoHash is stale for v2.0.2 (see zerokitRln above). + # build links against. rln = zerokitRln; default = liblogosdelivery; } diff --git a/scripts/build_rln.sh b/scripts/build_rln.sh index 35b5b8953..b028885e2 100755 --- a/scripts/build_rln.sh +++ b/scripts/build_rln.sh @@ -1,8 +1,15 @@ #!/usr/bin/env bash -# This script is used to build the rln library for the current platform. -# Previously downloaded prebuilt binaries, but due to compatibility issues -# we now always build from source. +# Provides the rln static library for the current platform. +# +# If zerokit publishes a prebuilt `stateless` release asset for this platform, +# download and use it: that is faster than compiling and avoids fetching +# zerokit's many crate dependencies from crates.io. The asset is selected by +# the Rust host target triple (the platform identifier reported by rustc, +# e.g. x86_64-unknown-linux-gnu or aarch64-apple-darwin). +# +# When no matching asset exists (e.g. Windows), build from the vendored +# zerokit submodule instead. set -e @@ -15,8 +22,26 @@ output_filename=$3 [[ -z "${rln_version}" ]] && { echo "No rln version specified"; exit 1; } [[ -z "${output_filename}" ]] && { echo "No output filename specified"; exit 1; } -echo "Building RLN library from source (version ${rln_version})..." +# --- Prefer the prebuilt release asset -------------------------------------- +# Host target triple, e.g. x86_64-unknown-linux-gnu / aarch64-apple-darwin. +host_triplet=$(rustc --version --verbose | awk '/host:/{print $2}') +tarball="${host_triplet}-stateless-rln.tar.gz" +url="https://github.com/vacp2p/zerokit/releases/download/${rln_version}/${tarball}" +echo "Looking for prebuilt RLN: ${url}" +if curl --silent --fail-with-body -L "${url}" -o "${tarball}"; then + echo "Downloaded prebuilt ${tarball}" + tar -xzf "${tarball}" + mv "release/librln.a" "${output_filename}" + rm -rf "${tarball}" release + echo "Using prebuilt ${output_filename}" + exit 0 +fi +# curl --fail-with-body writes the error body to the file on HTTP failure. +rm -f "${tarball}" +echo "No prebuilt asset for ${host_triplet} at ${rln_version}; building from source." + +# --- Fall back to building from the vendored submodule ---------------------- # Check if submodule version = version in Makefile cargo metadata --format-version=1 --no-deps --manifest-path "${build_dir}/rln/Cargo.toml" @@ -33,7 +58,6 @@ if [[ "v${submodule_version}" != "${rln_version}" ]]; then exit 1 fi -# Build rln from source. # `stateless` feature: logos-delivery does not maintain a local Merkle tree # (post-PR #3312); the contract is the source of truth and the path is fetched # via getMerkleProof(index). The stateless build compiles out tree code.