Fix bundle cross-build + cfgsync config generation

2026-01-02 13:23:13 +00:00 · 2025-12-15 17:08:48 +01:00 · 2025-12-15 17:08:48 +01:00 · 7998e86863
commit 7998e86863
parent bcbbb926c5
4 changed files with 135 additions and 61 deletions
--- a/scripts/build-bundle.sh
+++ b/scripts/build-bundle.sh
@ -3,12 +3,13 @@ set -euo pipefail

 # Build a nomos-binaries.tar.gz for the specified platform.
 #
-# Usage: scripts/build-bundle.sh [--platform host|linux] [--output PATH] [--rev REV | --path DIR] [--features LIST]
-#   --platform   Target platform for binaries (default: host)
-#   --output     Output path for the tarball (default: .tmp/nomos-binaries-<platform>-<version>.tar.gz)
-#   --rev        nomos-node git revision to build (overrides NOMOS_NODE_REV)
-#   --path       Use local nomos-node checkout at DIR (skip fetch/checkout)
-#   --features   Extra cargo features to enable (comma-separated); base always includes "testing"
+# Usage: scripts/build-bundle.sh [--platform host|linux] [--output PATH] [--rev REV | --path DIR] [--features LIST] [--docker-platform PLATFORM]
+#   --platform        Target platform for binaries (default: host)
+#   --output          Output path for the tarball (default: .tmp/nomos-binaries-<platform>-<version>.tar.gz)
+#   --rev             nomos-node git revision to build (overrides NOMOS_NODE_REV)
+#   --path            Use local nomos-node checkout at DIR (skip fetch/checkout)
+#   --features        Extra cargo features to enable (comma-separated); base always includes "testing"
+#   --docker-platform Docker platform for Linux bundle when running on non-Linux host (default: linux/amd64)

 # Always run under bash; bail out if someone invokes via sh.
 if [ -z "${BASH_VERSION:-}" ]; then
@ -16,29 +17,35 @@ if [ -z "${BASH_VERSION:-}" ]; then
 fi

 usage() {
-  cat <<'EOF'
+  cat <<'USAGE'
 Usage: scripts/build-bundle.sh [--platform host|linux] [--output PATH]

 Options:
-  --platform   Target platform for binaries (default: host)
-  --output     Output path for the tarball (default: .tmp/nomos-binaries-<platform>-<version>.tar.gz)
-  --rev        nomos-node git revision to build (overrides NOMOS_NODE_REV)
-  --path       Use local nomos-node checkout at DIR (skip fetch/checkout)
-  --features   Extra cargo features to enable (comma-separated); base always includes "testing"
+  --platform        Target platform for binaries (default: host)
+  --output          Output path for the tarball (default: .tmp/nomos-binaries-<platform>-<version>.tar.gz)
+  --rev             nomos-node git revision to build (overrides NOMOS_NODE_REV)
+  --path            Use local nomos-node checkout at DIR (skip fetch/checkout)
+  --features        Extra cargo features to enable (comma-separated); base always includes "testing"
+  --docker-platform Docker platform for Linux bundle when running on non-Linux host (default: linux/amd64)

 Notes:
  - For compose/k8s, use platform=linux. If running on macOS, this script will
    run inside a Linux Docker container to produce Linux binaries.
+  - On Apple silicon, Docker defaults to linux/arm64; for compose/k8s you likely
+    want linux/amd64 (the default here). Override with --docker-platform.
  - VERSION, NOMOS_NODE_REV, and optional NOMOS_NODE_PATH env vars are honored (defaults align with run-examples.sh).
-EOF
+USAGE
 }

 fail() { echo "$1" >&2; exit 1; }

 if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then
-  usage; exit 0
+  usage
+  exit 0
 fi

+NOMOS_EXTRA_FEATURES="${NOMOS_EXTRA_FEATURES:-}"
+
 ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 if [ -f "${ROOT_DIR}/versions.env" ]; then
  # shellcheck disable=SC1091
@ -54,6 +61,7 @@ PLATFORM="host"
 OUTPUT=""
 REV_OVERRIDE=""
 PATH_OVERRIDE=""
+DOCKER_PLATFORM="${NOMOS_BUNDLE_DOCKER_PLATFORM:-${NOMOS_BIN_PLATFORM:-linux/amd64}}"

 # To avoid confusing cache corruption errors inside the Dockerized Linux build,
 # always start from a clean cargo registry/git cache for the cross-build.
@ -61,16 +69,18 @@ rm -rf "${ROOT_DIR}/.tmp/cargo-linux/registry" "${ROOT_DIR}/.tmp/cargo-linux/git

 while [ "$#" -gt 0 ]; do
  case "$1" in
-    --platform|-p)
-      PLATFORM="${2:-}"; shift 2 ;;
-    --output|-o)
-      OUTPUT="${2:-}"; shift 2 ;;
-    --rev)
-      REV_OVERRIDE="${2:-}"; shift 2 ;;
-    --path)
-      PATH_OVERRIDE="${2:-}"; shift 2 ;;
-    --features)
-      NOMOS_EXTRA_FEATURES="${2:-}"; shift 2 ;;
+    --platform=*|-p=*) PLATFORM="${1#*=}"; shift ;;
+    --platform|-p) PLATFORM="${2:-}"; shift 2 ;;
+    --output=*|-o=*) OUTPUT="${1#*=}"; shift ;;
+    --output|-o) OUTPUT="${2:-}"; shift 2 ;;
+    --rev=*) REV_OVERRIDE="${1#*=}"; shift ;;
+    --rev) REV_OVERRIDE="${2:-}"; shift 2 ;;
+    --path=*) PATH_OVERRIDE="${1#*=}"; shift ;;
+    --path) PATH_OVERRIDE="${2:-}"; shift 2 ;;
+    --features=*) NOMOS_EXTRA_FEATURES="${1#*=}"; shift ;;
+    --features) NOMOS_EXTRA_FEATURES="${2:-}"; shift 2 ;;
+    --docker-platform=*) DOCKER_PLATFORM="${1#*=}"; shift ;;
+    --docker-platform) DOCKER_PLATFORM="${2:-}"; shift 2 ;;
    *) fail "Unknown argument: $1" ;;
  esac
 done
@ -99,10 +109,13 @@ fi
 echo "Bundle output: ${OUTPUT}"

 if [ "$PLATFORM" = "linux" ] && [ "$(uname -s)" != "Linux" ] && [ -z "${BUNDLE_IN_CONTAINER:-}" ]; then
-  # Re-run inside a Linux container to produce Linux binaries.
  if ! command -v docker >/dev/null 2>&1; then
    fail "Docker is required to build a Linux bundle from non-Linux host"
  fi
+  if [ -z "${DOCKER_PLATFORM}" ]; then
+    fail "--docker-platform must not be empty"
+  fi
+
  NODE_PATH_ENV="${NOMOS_NODE_PATH}"
  EXTRA_MOUNTS=()
  if [ -n "${NOMOS_NODE_PATH}" ]; then
@ -119,16 +132,24 @@ if [ "$PLATFORM" = "linux" ] && [ "$(uname -s)" != "Linux" ] && [ -z "${BUNDLE_I
        ;;
    esac
  fi
+
  echo "==> Building Linux bundle inside Docker"
-  # Map host OUTPUT path into container.
  container_output="/workspace${OUTPUT#"${ROOT_DIR}"}"
  mkdir -p "${ROOT_DIR}/.tmp/cargo-linux" "${ROOT_DIR}/.tmp/nomos-node-linux-target"
+
  FEATURES_ARGS=()
  if [ -n "${NOMOS_EXTRA_FEATURES:-}" ]; then
-    # Forward the outer --features flag into the inner Dockerized build
    FEATURES_ARGS+=(--features "${NOMOS_EXTRA_FEATURES}")
  fi
-  docker run --rm \
+
+  SRC_ARGS=()
+  if [ -n "${NODE_PATH_ENV}" ]; then
+    SRC_ARGS+=(--path "${NODE_PATH_ENV}")
+  else
+    SRC_ARGS+=(--rev "${NOMOS_NODE_REV}")
+  fi
+
+  docker run --rm --platform "${DOCKER_PLATFORM}" \
    -e VERSION="$VERSION" \
    -e NOMOS_NODE_REV="$NOMOS_NODE_REV" \
    -e NOMOS_NODE_PATH="$NODE_PATH_ENV" \
@ -145,32 +166,33 @@ if [ "$PLATFORM" = "linux" ] && [ "$(uname -s)" != "Linux" ] && [ -z "${BUNDLE_I
    "${EXTRA_MOUNTS[@]}" \
    -w /workspace \
    rust:1.80-bullseye \
-    bash -c "apt-get update && apt-get install -y clang llvm-dev libclang-dev pkg-config cmake libssl-dev rsync libgmp10 libgmp-dev libgomp1 nasm && ./scripts/build-bundle.sh --platform linux --path \"${NODE_PATH_ENV}\" --output \"${container_output}\" ${FEATURES_ARGS[*]}"
+    bash -c "apt-get update && apt-get install -y clang llvm-dev libclang-dev pkg-config cmake libssl-dev rsync libgmp10 libgmp-dev libgomp1 nasm && ./scripts/build-bundle.sh --platform linux --output \"${container_output}\" ${SRC_ARGS[*]} ${FEATURES_ARGS[*]}"
+
  exit 0
 fi

 echo "==> Preparing circuits (version ${VERSION})"
 if [ "$PLATFORM" = "host" ]; then
  CIRCUITS_DIR="${ROOT_DIR}/.tmp/nomos-circuits-host"
-  NODE_SRC_DEFAULT="${ROOT_DIR}/.tmp/nomos-node-host-src"
  NODE_TARGET="${ROOT_DIR}/.tmp/nomos-node-host-target"
 else
  CIRCUITS_DIR="${ROOT_DIR}/.tmp/nomos-circuits-linux"
-  NODE_SRC_DEFAULT="${ROOT_DIR}/.tmp/nomos-node-linux-src"
  NODE_TARGET="${ROOT_DIR}/.tmp/nomos-node-linux-target"
 fi
+
+NODE_SRC_DEFAULT="${ROOT_DIR}/.tmp/nomos-node-${PLATFORM}-src"
 NODE_SRC="${NOMOS_NODE_PATH:-${NODE_SRC_DEFAULT}}"
-DEFAULT_NODE_TARGET="${NODE_TARGET}"
 if [ -n "${NOMOS_NODE_PATH}" ]; then
-  # Avoid mixing stale cloned sources/targets when using a local checkout.
+  if [ ! -d "${NODE_SRC}" ]; then
+    fail "NOMOS_NODE_PATH does not exist: ${NODE_SRC}"
+  fi
  rm -rf "${NODE_SRC_DEFAULT}"
  if [ -d "${NODE_TARGET}" ]; then
-    # Target dir may be a mounted volume; avoid removing the mount point itself.
    find "${NODE_TARGET}" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
  fi
  NODE_TARGET="${NODE_TARGET}-local"
 fi
-echo "Using nomos-node source: ${NODE_SRC}"
+
 export NOMOS_CIRCUITS="${CIRCUITS_DIR}"
 mkdir -p "${ROOT_DIR}/.tmp" "${CIRCUITS_DIR}"
 if [ -f "${CIRCUITS_DIR}/${KZG_FILE:-kzgrs_test_params}" ]; then
@ -183,32 +205,27 @@ fi
 NODE_BIN="${NODE_TARGET}/debug/nomos-node"
 EXEC_BIN="${NODE_TARGET}/debug/nomos-executor"
 CLI_BIN="${NODE_TARGET}/debug/nomos-cli"
+
 FEATURES="testing"
 if [ -n "${NOMOS_EXTRA_FEATURES:-}" ]; then
  FEATURES="${FEATURES},${NOMOS_EXTRA_FEATURES}"
 fi

-echo "==> Building host binaries (platform=${PLATFORM})"
+echo "==> Building binaries (platform=${PLATFORM})"
 mkdir -p "${NODE_SRC}"
-if [ -n "${NOMOS_NODE_PATH}" ]; then
-  if [ ! -d "${NODE_SRC}" ]; then
-    fail "NOMOS_NODE_PATH does not exist: ${NODE_SRC}"
-  fi
-  echo "Using local nomos-node checkout at ${NODE_SRC} (no fetch/checkout)"
-else
-  if [ ! -d "${NODE_SRC}/.git" ]; then
-    git clone https://github.com/logos-co/nomos-node.git "${NODE_SRC}"
-  fi
-  (
-    cd "${NODE_SRC}"
+(
+  cd "${NODE_SRC}"
+  if [ -n "${NOMOS_NODE_PATH}" ]; then
+    echo "Using local nomos-node checkout at ${NODE_SRC} (no fetch/checkout)"
+  else
+    if [ ! -d "${NODE_SRC}/.git" ]; then
+      git clone https://github.com/logos-co/nomos-node.git "${NODE_SRC}"
+    fi
    git fetch --depth 1 origin "${NOMOS_NODE_REV}"
    git checkout "${NOMOS_NODE_REV}"
    git reset --hard
    git clean -fdx
-  )
-fi
-  (
-    cd "${NODE_SRC}"
+  fi
  RUSTFLAGS='--cfg feature="pol-dev-mode"' NOMOS_CIRCUITS="${CIRCUITS_DIR}" \
    cargo build --features "${FEATURES}" \
    -p nomos-node -p nomos-executor -p nomos-cli \
@ -224,6 +241,15 @@ mkdir -p "${bundle_dir}/artifacts"
 cp "${NODE_BIN}" "${bundle_dir}/artifacts/"
 cp "${EXEC_BIN}" "${bundle_dir}/artifacts/"
 cp "${CLI_BIN}" "${bundle_dir}/artifacts/"
+{
+  echo "nomos_node_path=${NOMOS_NODE_PATH:-}"
+  echo "nomos_node_rev=${NOMOS_NODE_REV:-}"
+  if [ -d "${NODE_SRC}/.git" ] && command -v git >/dev/null 2>&1; then
+    echo "nomos_node_git_head=$(git -C "${NODE_SRC}" rev-parse HEAD 2>/dev/null || true)"
+  fi
+  echo "platform=${PLATFORM}"
+  echo "features=${FEATURES}"
+} > "${bundle_dir}/artifacts/nomos-bundle-meta.env"

 mkdir -p "$(dirname "${OUTPUT}")"
 if tar --help 2>/dev/null | grep -q -- '--no-mac-metadata'; then
@ -234,10 +260,11 @@ else
  tar -czf "${OUTPUT}" -C "${bundle_dir}" artifacts
 fi
 echo "Bundle created at ${OUTPUT}"
+
 if [[ "${FEATURES}" == *profiling* ]]; then
-  cat <<'EOF'
+  cat <<'EOF_PROF'
 Profiling endpoints (enabled by --features profiling):
  CPU pprof (SVG):   curl "http://<node-host>:8722/debug/pprof/profile?seconds=15&format=svg" -o profile.svg
  CPU pprof (proto): go tool pprof -http=:8080 "http://<node-host>:8722/debug/pprof/profile?seconds=15&format=proto"
-EOF
+EOF_PROF
 fi
--- a/testing-framework/assets/stack/Dockerfile
+++ b/testing-framework/assets/stack/Dockerfile
@ -81,17 +81,59 @@ RUN mkdir -p /opt/circuits && \

 ENV NOMOS_CIRCUITS=/opt/circuits

-# Provide runtime binaries. Require prebuilt artifacts staged in testing-framework/assets/stack/bin.
+# Provide runtime binaries. Prefer prebuilt artifacts (when present) for speed;
+# otherwise build from source (or if prebuilt artifacts don't match the image
+# architecture).
 RUN set -eu; \
    mkdir -p /workspace/artifacts; \
-    if [ -f testing-framework/assets/stack/bin/nomos-node ] && [ -f testing-framework/assets/stack/bin/nomos-executor ]; then \
-      echo "Using prebuilt nomos binaries from testing-framework/assets/stack/bin (no in-image build)"; \
+    TARGET_ARCH="$(uname -m)"; \
+    expect_arch() { \
+      case "$1" in \
+        x86_64) echo "x86-64" ;; \
+        aarch64|arm64) echo "arm64" ;; \
+        *) echo "$1" ;; \
+      esac; \
+    }; \
+    have_prebuilt() { \
+      [ -f testing-framework/assets/stack/bin/nomos-node ] && \
+      [ -f testing-framework/assets/stack/bin/nomos-executor ] && \
+      [ -f testing-framework/assets/stack/bin/nomos-cli ]; \
+    }; \
+    bin_matches_arch() { \
+      BIN_INFO="$(file -b testing-framework/assets/stack/bin/nomos-node 2>/dev/null || true)"; \
+      case "$BIN_INFO" in \
+        *ELF*);; \
+        *) return 1 ;; \
+      esac; \
+      case "$TARGET_ARCH" in \
+        x86_64) PATTERN="x86-64|x86_64" ;; \
+        aarch64|arm64) PATTERN="arm64|aarch64" ;; \
+        *) PATTERN="$(expect_arch "$TARGET_ARCH")" ;; \
+      esac; \
+      [ -n "$BIN_INFO" ] && echo "$BIN_INFO" | grep -Eqi "$PATTERN"; \
+    }; \
+    if have_prebuilt && bin_matches_arch; then \
+      echo "Using prebuilt nomos binaries from testing-framework/assets/stack/bin"; \
      cp testing-framework/assets/stack/bin/nomos-node /workspace/artifacts/nomos-node; \
      cp testing-framework/assets/stack/bin/nomos-executor /workspace/artifacts/nomos-executor; \
      cp testing-framework/assets/stack/bin/nomos-cli /workspace/artifacts/nomos-cli; \
    else \
-      echo "ERROR: Prebuilt nomos binaries not found in testing-framework/assets/stack/bin; aborting build"; \
-      exit 1; \
+      if have_prebuilt; then \
+        echo "Prebuilt nomos binaries do not match target architecture (${TARGET_ARCH}); rebuilding from source"; \
+      fi; \
+      echo "Prebuilt nomos binaries missing or wrong architecture; building from source (rev ${NOMOS_NODE_REV})"; \
+      git clone https://github.com/logos-co/nomos-node.git /tmp/nomos-node && \
+      cd /tmp/nomos-node && \
+      git fetch --depth 1 origin "${NOMOS_NODE_REV}" && \
+      git checkout "${NOMOS_NODE_REV}" && \
+      git reset --hard && git clean -fdx && \
+      # Enable pol-dev-mode via cfg to let POL_PROOF_DEV_MODE short-circuit proofs in tests.
+      RUSTFLAGS='--cfg feature="pol-dev-mode"' NOMOS_CIRCUITS=/opt/circuits cargo build --features "testing" \
+        -p nomos-node -p nomos-executor -p nomos-cli; \
+      cp /tmp/nomos-node/target/debug/nomos-node /workspace/artifacts/nomos-node; \
+      cp /tmp/nomos-node/target/debug/nomos-executor /workspace/artifacts/nomos-executor; \
+      cp /tmp/nomos-node/target/debug/nomos-cli /workspace/artifacts/nomos-cli; \
+      rm -rf /tmp/nomos-node/target/debug/incremental; \
    fi

 # Strip local path patches so container builds use git sources.
--- a/testing-framework/core/src/nodes/common/config/injection.rs
+++ b/testing-framework/core/src/nodes/common/config/injection.rs
@ -37,7 +37,6 @@ pub fn inject_ibd_into_cryptarchia(yaml_value: &mut Value) {
    ensure_network_adapter(cryptarchia);
    ensure_sync_defaults(cryptarchia);
    ensure_ibd_bootstrap(cryptarchia);
-    normalize_ed25519_sigs(yaml_value);
 }

 fn cryptarchia_section(yaml_value: &mut Value) -> Option<&mut Mapping> {
--- a/testing-framework/tools/cfgsync/src/bin/cfgsync-client.rs
+++ b/testing-framework/tools/cfgsync/src/bin/cfgsync-client.rs
@ -15,7 +15,10 @@ use nomos_libp2p::PeerId;
 use nomos_node::Config as ValidatorConfig;
 use serde::{Serialize, de::DeserializeOwned};
 use subnetworks_assignations::{MembershipCreator, MembershipHandler, SubnetworkId};
-use testing_framework_core::constants::cfgsync_port as default_cfgsync_port;
+use testing_framework_core::{
+    constants::cfgsync_port as default_cfgsync_port,
+    nodes::common::config::injection::normalize_ed25519_sigs,
+};

 fn parse_ip(ip_str: &str) -> Ipv4Addr {
    ip_str.parse().unwrap_or_else(|_| {
@ -72,7 +75,10 @@ where
        apply_membership(&mut config, assignations);
    }

-    let yaml = serde_yaml::to_string(&config)
+    let mut yaml_value = serde_yaml::to_value(&config)
+        .map_err(|err| format!("Failed to serialize config to YAML value: {err}"))?;
+    normalize_ed25519_sigs(&mut yaml_value);
+    let yaml = serde_yaml::to_string(&yaml_value)
        .map_err(|err| format!("Failed to serialize config to YAML: {err}"))?;

    fs::write(config_file, yaml).map_err(|err| format!("Failed to write config to file: {err}"))?;