diff --git a/.github/actions/run-in-ci-image/action.yml b/.github/actions/run-in-ci-image/action.yml new file mode 100644 index 00000000..3fafe38c --- /dev/null +++ b/.github/actions/run-in-ci-image/action.yml @@ -0,0 +1,54 @@ +name: Run in CI image +description: > + Runs a command inside the CI image on a plain runner, for jobs that need the + host's Docker daemon. `container:` cannot work for those: it puts the + workspace on a path the host daemon cannot resolve, and it forbids + `--network host`. + +inputs: + image: + description: CI image reference, from the ci-image workflow's output. + required: true + run: + description: Command to run inside the image. + required: true + env: + description: Newline-separated NAME=VALUE pairs to pass into the container. + required: false + default: "" + +runs: + using: "composite" + steps: + - name: Run in CI image + shell: bash + env: + INPUT_IMAGE: ${{ inputs.image }} + INPUT_RUN: ${{ inputs.run }} + INPUT_ENV: ${{ inputs.env }} + run: | + set -euo pipefail + + env_args=() + while IFS= read -r pair; do + [[ -z "$pair" ]] && continue + env_args+=(--env "$pair") + done <<< "$INPUT_ENV" + + # Cargo's registry is the only part of CARGO_HOME worth sharing with the + # host: mounting all of it would shadow the tools baked into the image. + mkdir -p "$HOME/.cargo/registry" + + # Mounting the workspace on its own path is what makes this work. The + # daemon resolves compose bind mounts against the host, and the test + # binaries bake CARGO_MANIFEST_DIR in at compile time, so the path has + # to mean the same thing inside and outside the container. + docker run --rm \ + --network host \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + --volume "$PWD:$PWD" \ + --volume "$HOME/.cargo/registry:/usr/local/cargo/registry" \ + --workdir "$PWD" \ + "${env_args[@]}" \ + "$INPUT_IMAGE" \ + bash -e -o pipefail -c "$INPUT_RUN" diff --git a/.github/docker/ci.Dockerfile b/.github/docker/ci.Dockerfile new file mode 100644 index 00000000..a79be15f --- /dev/null +++ b/.github/docker/ci.Dockerfile @@ -0,0 +1,97 @@ +# Image behind the `container:` of the CI jobs. Everything the jobs used to +# install per-run is baked in here instead, so a CI run no longer asks the risc0 +# install servers for anything. +# +# The image tag is a hash of this file plus rust-toolchain.toml, so editing +# either rebuilds the image on the PR that changes it. See ci-image.yml. +# +# Keep the base tag in sync with rust-toolchain.toml. +FROM rust:1.94.0-trixie + +# GitHub sets HOME=/github/home inside container jobs, which hides anything we +# bake into the image's ~. rzup defaults to $HOME/.risc0, so pin it to a fixed +# path; RUSTUP_HOME and CARGO_HOME already point at /usr/local from the base +# image, and rzup honours all three. +ENV RISC0_HOME=/usr/local/risc0 +ENV PATH=/usr/local/risc0/bin:$PATH + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + clang \ + libclang-dev \ + libssl-dev \ + pkg-config \ + curl \ + git \ + jq \ + python3-dev \ + && rm -rf /var/lib/apt/lists/* + +# The runner pre-creates the workspace as another uid, so git in a container job +# calls it dubious. checkout's own fix is --global, which only holds while HOME +# still points at the gitconfig it wrote; --system holds regardless. +RUN git config --system --add safe.directory '*' + +# The base image already has this toolchain at the minimal profile, so the +# `profile = "default"` in rust-toolchain.toml is a no-op here: rustup sees the +# toolchain as installed and never applies it. Add what the jobs need by hand. +COPY rust-toolchain.toml /tmp/toolchain/rust-toolchain.toml +RUN cd /tmp/toolchain \ + && rustup toolchain install \ + && rustup component add clippy rustfmt \ + && rm -rf /tmp/toolchain + +# For `cargo +nightly fmt` in the fmt-rs job. +RUN rustup toolchain install nightly --profile minimal --component rustfmt + +# The bootstrap script only ever drops rzup in $HOME/.risc0/bin, so run it +# against the build-time HOME and keep just the binary. rzup itself then honours +# RISC0_HOME and installs each component under /usr/local. +# +# Versions pinned to the set currently validated in local dev (`rzup show`); +# bare `rzup install` would float all four default components to latest. r0vm +# and cargo-risczero track each other and the risc0-zkvm crate version. +RUN curl -L https://risczero.com/install | bash \ + && mv /root/.risc0/bin/rzup /usr/local/bin/rzup \ + && rm -rf /root/.risc0 \ + && rzup install rust 1.94.1 \ + && rzup install cpp 2024.1.5 \ + && rzup install r0vm 3.0.5 \ + && rzup install cargo-risczero 3.0.5 + +# Copying docker. Static Go binaries, so the alpine-built ones run fine here. +COPY --from=docker:29.6.1-cli /usr/local/bin/docker /usr/local/bin/docker +COPY --from=docker:29.6.1-cli /usr/local/libexec/docker/cli-plugins/docker-compose \ + /usr/local/libexec/docker/cli-plugins/docker-compose +COPY --from=docker:29.6.1-cli /usr/local/libexec/docker/cli-plugins/docker-buildx \ + /usr/local/libexec/docker/cli-plugins/docker-buildx + +# Prebuilt binaries; compiling these from source would dominate the image build. +# Versions pinned so a rebuild of the same image ships the same tools. +ENV BINSTALL_VERSION=1.21.0 +RUN curl -L --proto '=https' --tlsv1.2 -sSf \ + https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash \ + && cargo binstall --no-confirm \ + cargo-nextest@0.9.140 \ + taplo-cli@0.10.0 \ + cargo-machete@0.9.2 \ + cargo-deny@0.20.2 \ + just@1.56.0 + +# Smoke-test every tool the jobs use. A bad install should fail here, not later +# in some CI job that reuses this cached image. pyo3's `auto-initialize` links +# libpython, and the base image ships python3 without it, so check the .so too. +RUN cargo --version \ + && cargo +nightly fmt --version \ + && cargo clippy --version \ + && ls /usr/lib/*/libpython3*.so \ + && r0vm --version \ + && cargo risczero --version \ + && cargo nextest --version \ + && taplo --version \ + && cargo machete --version \ + && cargo deny --version \ + && just --version \ + && docker --version \ + && docker compose version \ + && docker buildx version diff --git a/.github/workflows/ci-image.yml b/.github/workflows/ci-image.yml new file mode 100644 index 00000000..c93782de --- /dev/null +++ b/.github/workflows/ci-image.yml @@ -0,0 +1,75 @@ +name: CI image + +concurrency: + group: ci-image-${{ github.sha }} + cancel-in-progress: false + +# Resolves the CI image the calling workflow's jobs run in, building and pushing +# it only when it isn't published yet. +on: + workflow_call: + outputs: + image: + description: Image reference to pass to a job's `container:`. + value: ${{ jobs.ci-image.outputs.image }} + +jobs: + ci-image: + runs-on: ubuntu-latest + timeout-minutes: 60 + outputs: + image: ${{ steps.image-ref.outputs.image }} + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ github.event.pull_request.head.sha || github.head_ref }} + + # Tagging by content hash means a PR that touches the Dockerfile or the + # toolchain builds and tests against its own image, while every other PR + # reuses the one already in the registry. + - name: Compute image reference + id: image-ref + run: | + repo="$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')" + tag="$(cat .github/docker/ci.Dockerfile rust-toolchain.toml | sha256sum | cut -c1-16)" + echo "image=ghcr.io/${repo}/ci:${tag}" >> "$GITHUB_OUTPUT" + echo "CI image: ghcr.io/${repo}/ci:${tag}" + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Check whether the image is already published + id: probe + run: | + if docker manifest inspect "${{ steps.image-ref.outputs.image }}" > /dev/null 2>&1; then + echo "exists=true" >> "$GITHUB_OUTPUT" + echo "Image already published, skipping build." + else + echo "exists=false" >> "$GITHUB_OUTPUT" + echo "Image not published yet, building it." + fi + + - name: Set up Docker Buildx + if: steps.probe.outputs.exists == 'false' + uses: docker/setup-buildx-action@v3 + + # A pull_request from a fork gets a read-only GITHUB_TOKEN, so this step + # fails there. It only runs when the fork changed the image, which is the + # case that needs a maintainer's eyes anyway. + - name: Build and push CI image + if: steps.probe.outputs.exists == 'false' + uses: docker/build-push-action@v5 + with: + context: . + file: ./.github/docker/ci.Dockerfile + push: true + tags: ${{ steps.image-ref.outputs.image }} + # Links the package to the repo, so it shows up there and inherits its + # access rules. + labels: org.opencontainers.image.source=https://github.com/${{ github.repository }} + cache-from: type=gha,scope=ci-image + cache-to: type=gha,mode=max,scope=ci-image diff --git a/flake.nix b/flake.nix index 0aaad40c..d0058bcb 100644 --- a/flake.nix +++ b/flake.nix @@ -117,6 +117,7 @@ done if [ -n "$tool" ]; then unset DEVELOPER_DIR SDKROOT + export xcrun_nocache=1 fi exec /usr/bin/xcrun "$@" '';