Merge origin/dev into ci-optimize-artifact-and-tool-reuse

.github/actions/run-in-ci-image/action.yml

@@ -0,0 +1,54 @@
+name: Run in CI image
+description: >
+  Runs a command inside the CI image on a plain runner, for jobs that need the
+  host's Docker daemon. `container:` cannot work for those: it puts the
+  workspace on a path the host daemon cannot resolve, and it forbids
+  `--network host`.
+
+inputs:
+  image:
+    description: CI image reference, from the ci-image workflow's output.
+    required: true
+  run:
+    description: Command to run inside the image.
+    required: true
+  env:
+    description: Newline-separated NAME=VALUE pairs to pass into the container.
+    required: false
+    default: ""
+
+runs:
+  using: "composite"
+  steps:
+    - name: Run in CI image
+      shell: bash
+      env:
+        INPUT_IMAGE: ${{ inputs.image }}
+        INPUT_RUN: ${{ inputs.run }}
+        INPUT_ENV: ${{ inputs.env }}
+      run: |
+        set -euo pipefail
+
+        env_args=()
+        while IFS= read -r pair; do
+          [[ -z "$pair" ]] && continue
+          env_args+=(--env "$pair")
+        done <<< "$INPUT_ENV"
+
+        # Cargo's registry is the only part of CARGO_HOME worth sharing with the
+        # host: mounting all of it would shadow the tools baked into the image.
+        mkdir -p "$HOME/.cargo/registry"
+
+        # Mounting the workspace on its own path is what makes this work. The
+        # daemon resolves compose bind mounts against the host, and the test
+        # binaries bake CARGO_MANIFEST_DIR in at compile time, so the path has
+        # to mean the same thing inside and outside the container.
+        docker run --rm \
+          --network host \
+          --volume /var/run/docker.sock:/var/run/docker.sock \
+          --volume "$PWD:$PWD" \
+          --volume "$HOME/.cargo/registry:/usr/local/cargo/registry" \
+          --workdir "$PWD" \
+          "${env_args[@]}" \
+          "$INPUT_IMAGE" \
+          bash -e -o pipefail -c "$INPUT_RUN"

.github/docker/ci.Dockerfile

@@ -0,0 +1,97 @@
+# Image behind the `container:` of the CI jobs. Everything the jobs used to
+# install per-run is baked in here instead, so a CI run no longer asks the risc0
+# install servers for anything.
+#
+# The image tag is a hash of this file plus rust-toolchain.toml, so editing
+# either rebuilds the image on the PR that changes it. See ci-image.yml.
+#
+# Keep the base tag in sync with rust-toolchain.toml.
+FROM rust:1.94.0-trixie
+
+# GitHub sets HOME=/github/home inside container jobs, which hides anything we
+# bake into the image's ~. rzup defaults to $HOME/.risc0, so pin it to a fixed
+# path; RUSTUP_HOME and CARGO_HOME already point at /usr/local from the base
+# image, and rzup honours all three.
+ENV RISC0_HOME=/usr/local/risc0
+ENV PATH=/usr/local/risc0/bin:$PATH
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    clang \
+    libclang-dev \
+    libssl-dev \
+    pkg-config \
+    curl \
+    git \
+    jq \
+    python3-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# The runner pre-creates the workspace as another uid, so git in a container job
+# calls it dubious. checkout's own fix is --global, which only holds while HOME
+# still points at the gitconfig it wrote; --system holds regardless.
+RUN git config --system --add safe.directory '*'
+
+# The base image already has this toolchain at the minimal profile, so the
+# `profile = "default"` in rust-toolchain.toml is a no-op here: rustup sees the
+# toolchain as installed and never applies it. Add what the jobs need by hand.
+COPY rust-toolchain.toml /tmp/toolchain/rust-toolchain.toml
+RUN cd /tmp/toolchain \
+    && rustup toolchain install \
+    && rustup component add clippy rustfmt \
+    && rm -rf /tmp/toolchain
+
+# For `cargo +nightly fmt` in the fmt-rs job.
+RUN rustup toolchain install nightly --profile minimal --component rustfmt
+
+# The bootstrap script only ever drops rzup in $HOME/.risc0/bin, so run it
+# against the build-time HOME and keep just the binary. rzup itself then honours
+# RISC0_HOME and installs each component under /usr/local.
+#
+# Versions pinned to the set currently validated in local dev (`rzup show`);
+# bare `rzup install` would float all four default components to latest. r0vm
+# and cargo-risczero track each other and the risc0-zkvm crate version.
+RUN curl -L https://risczero.com/install | bash \
+    && mv /root/.risc0/bin/rzup /usr/local/bin/rzup \
+    && rm -rf /root/.risc0 \
+    && rzup install rust 1.94.1 \
+    && rzup install cpp 2024.1.5 \
+    && rzup install r0vm 3.0.5 \
+    && rzup install cargo-risczero 3.0.5
+
+# Copying docker. Static Go binaries, so the alpine-built ones run fine here.
+COPY --from=docker:29.6.1-cli /usr/local/bin/docker /usr/local/bin/docker
+COPY --from=docker:29.6.1-cli /usr/local/libexec/docker/cli-plugins/docker-compose \
+     /usr/local/libexec/docker/cli-plugins/docker-compose
+COPY --from=docker:29.6.1-cli /usr/local/libexec/docker/cli-plugins/docker-buildx \
+     /usr/local/libexec/docker/cli-plugins/docker-buildx
+
+# Prebuilt binaries; compiling these from source would dominate the image build.
+# Versions pinned so a rebuild of the same image ships the same tools.
+ENV BINSTALL_VERSION=1.21.0
+RUN curl -L --proto '=https' --tlsv1.2 -sSf \
+      https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash \
+    && cargo binstall --no-confirm \
+        cargo-nextest@0.9.140 \
+        taplo-cli@0.10.0 \
+        cargo-machete@0.9.2 \
+        cargo-deny@0.20.2 \
+        just@1.56.0
+
+# Smoke-test every tool the jobs use. A bad install should fail here, not later
+# in some CI job that reuses this cached image. pyo3's `auto-initialize` links
+# libpython, and the base image ships python3 without it, so check the .so too.
+RUN cargo --version \
+    && cargo +nightly fmt --version \
+    && cargo clippy --version \
+    && ls /usr/lib/*/libpython3*.so \
+    && r0vm --version \
+    && cargo risczero --version \
+    && cargo nextest --version \
+    && taplo --version \
+    && cargo machete --version \
+    && cargo deny --version \
+    && just --version \
+    && docker --version \
+    && docker compose version \
+    && docker buildx version

.github/workflows/ci-image.yml

@@ -0,0 +1,75 @@
+name: CI image
+
+concurrency:
+  group: ci-image-${{ github.sha }}
+  cancel-in-progress: false
+
+# Resolves the CI image the calling workflow's jobs run in, building and pushing
+# it only when it isn't published yet.
+on:
+  workflow_call:
+    outputs:
+      image:
+        description: Image reference to pass to a job's `container:`.
+        value: ${{ jobs.ci-image.outputs.image }}
+
+jobs:
+  ci-image:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    outputs:
+      image: ${{ steps.image-ref.outputs.image }}
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.head_ref }}
+
+      # Tagging by content hash means a PR that touches the Dockerfile or the
+      # toolchain builds and tests against its own image, while every other PR
+      # reuses the one already in the registry.
+      - name: Compute image reference
+        id: image-ref
+        run: |
+          repo="$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')"
+          tag="$(cat .github/docker/ci.Dockerfile rust-toolchain.toml | sha256sum | cut -c1-16)"
+          echo "image=ghcr.io/${repo}/ci:${tag}" >> "$GITHUB_OUTPUT"
+          echo "CI image: ghcr.io/${repo}/ci:${tag}"
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Check whether the image is already published
+        id: probe
+        run: |
+          if docker manifest inspect "${{ steps.image-ref.outputs.image }}" > /dev/null 2>&1; then
+            echo "exists=true" >> "$GITHUB_OUTPUT"
+            echo "Image already published, skipping build."
+          else
+            echo "exists=false" >> "$GITHUB_OUTPUT"
+            echo "Image not published yet, building it."
+          fi
+
+      - name: Set up Docker Buildx
+        if: steps.probe.outputs.exists == 'false'
+        uses: docker/setup-buildx-action@v3
+
+      # A pull_request from a fork gets a read-only GITHUB_TOKEN, so this step
+      # fails there. It only runs when the fork changed the image, which is the
+      # case that needs a maintainer's eyes anyway.
+      - name: Build and push CI image
+        if: steps.probe.outputs.exists == 'false'
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./.github/docker/ci.Dockerfile
+          push: true
+          tags: ${{ steps.image-ref.outputs.image }}
+          # Links the package to the repo, so it shows up there and inherits its
+          # access rules.
+          labels: org.opencontainers.image.source=https://github.com/${{ github.repository }}
+          cache-from: type=gha,scope=ci-image
+          cache-to: type=gha,mode=max,scope=ci-image

flake.nix

@@ -117,6 +117,7 @@
             done
             if [ -n "$tool" ]; then
               unset DEVELOPER_DIR SDKROOT
+              export xcrun_nocache=1
             fi
             exec /usr/bin/xcrun "$@"
           '';