From c5ebee4d847e0f50b152007ad41a72677c87f926 Mon Sep 17 00:00:00 2001 From: Ricardo Guilherme Schmidt <3esmit@gmail.com> Date: Wed, 15 Jul 2026 20:12:56 -0300 Subject: [PATCH] fix(ci): upload artifacts from action context --- .github/scripts/artifact-client/upload.mjs | 173 ++++++++++++++------- .github/workflows/ci.yml | 65 ++------ 2 files changed, 130 insertions(+), 108 deletions(-) diff --git a/.github/scripts/artifact-client/upload.mjs b/.github/scripts/artifact-client/upload.mjs index 40c86538..0484eb89 100644 --- a/.github/scripts/artifact-client/upload.mjs +++ b/.github/scripts/artifact-client/upload.mjs @@ -1,65 +1,132 @@ -import { appendFile, lstat, realpath } from "node:fs/promises"; -import { basename, dirname, resolve } from "node:path"; +import { + appendFile, + lstat, + readFile, + realpath, + rm, + statfs, + writeFile, +} from "node:fs/promises"; +import { basename, dirname, join, resolve } from "node:path"; import { DefaultArtifactClient } from "@actions/artifact"; -const [input, ...extra] = process.argv.slice(2); +const GIB = 1024 ** 3; +const MIB = 1024 ** 2; -if (!input || extra.length > 0) { - throw new Error("usage: node upload.mjs "); +function humanSize(size) { + if (size >= GIB) return `${(size / GIB).toFixed(2)} GiB`; + return `${(size / MIB).toFixed(2)} MiB`; } -const archive = await realpath(resolve(input)); -const artifactDirectory = await realpath( - resolve(process.env.INTEGRATION_ARTIFACT_DIR ?? ""), -); -if (dirname(archive) !== artifactDirectory) { - throw new Error(`integration artifact is outside staging: ${archive}`); -} +export async function buildAndUploadIntegrationTargets({ core, exec }) { + const artifactDirectory = await realpath( + resolve(process.env.INTEGRATION_ARTIFACT_DIR ?? ""), + ); + const manifest = resolve(process.env.INTEGRATION_ARTIFACT_MANIFEST ?? ""); + const targetsPath = resolve(process.env.INTEGRATION_TARGETS_FILE ?? ""); -const artifactName = basename(archive); -const artifactMatch = /^integration-test-([a-z0-9_]+)\.tar\.zst$/.exec( - artifactName, -); -if (!artifactMatch) { - throw new Error(`invalid integration artifact name: ${artifactName}`); -} + if (dirname(manifest) !== artifactDirectory) { + throw new Error(`integration manifest is outside staging: ${manifest}`); + } + if (dirname(targetsPath) !== artifactDirectory) { + throw new Error(`integration target list is outside staging: ${targetsPath}`); + } + await writeFile(manifest, ""); -const archiveStat = await lstat(archive); -if (!archiveStat.isFile()) { - throw new Error(`integration artifact is not a regular file: ${archive}`); -} + const targets = (await readFile(targetsPath, "utf8")) + .split("\n") + .filter(Boolean); + if (targets.length === 0 || targets.length > 64) { + throw new Error(`invalid integration target count: ${targets.length}`); + } -const artifact = new DefaultArtifactClient(); -const { artifacts } = await artifact.listArtifacts({ latest: true }); -if (artifacts.some(({ name }) => name === artifactName)) { - await artifact.deleteArtifact(artifactName); -} + const artifact = new DefaultArtifactClient(); + let totalArchiveSize = 0; -const { digest, id } = await artifact.uploadArtifact( - artifactName, - [archive], - dirname(archive), - { - retentionDays: 1, - skipArchive: true, - }, -); -if (!digest || !id) { - throw new Error(`artifact upload returned incomplete metadata: ${artifactName}`); -} + for (const target of targets) { + if (!/^[a-z0-9_]+$/.test(target)) { + throw new Error(`invalid integration target: ${target}`); + } -const manifest = resolve(process.env.INTEGRATION_ARTIFACT_MANIFEST ?? ""); -if (dirname(manifest) !== artifactDirectory) { - throw new Error(`integration manifest is outside staging: ${manifest}`); -} -await appendFile( - manifest, - `${JSON.stringify({ - archive: artifactName, - artifact_id: id, - target: artifactMatch[1], - })}\n`, -); + const artifactName = `integration-test-${target}.tar.zst`; + const archive = join(artifactDirectory, artifactName); -console.log(`Uploaded ${artifactName} (artifact ${id}, digest ${digest}).`); + try { + const exitCode = await exec.exec("cargo", [ + "nextest", + "archive", + "-p", + "integration_tests", + "--test", + target, + "-E", + `binary(=${target})`, + "--archive-file", + archive, + "--no-pager", + ]); + if (exitCode !== 0) { + throw new Error(`nextest archive failed for ${target}`); + } + + const archiveStat = await lstat(archive); + if (!archiveStat.isFile()) { + throw new Error(`integration artifact is not a file: ${archive}`); + } + if (archiveStat.size > 128 * MIB) { + throw new Error(`${target} archive exceeds the 128 MiB limit`); + } + + totalArchiveSize += archiveStat.size; + if (totalArchiveSize > 4 * GIB) { + throw new Error("integration archives exceed the 4 GiB total limit"); + } + + await appendFile( + process.env.GITHUB_STEP_SUMMARY, + `| \`${target}\` | ${humanSize(archiveStat.size)} |\n`, + ); + + const { artifacts } = await artifact.listArtifacts({ latest: true }); + if (artifacts.some(({ name }) => name === artifactName)) { + await artifact.deleteArtifact(artifactName); + } + + const { digest, id } = await artifact.uploadArtifact( + artifactName, + [archive], + artifactDirectory, + { + retentionDays: 1, + skipArchive: true, + }, + ); + if (!digest || !id) { + throw new Error(`upload returned incomplete metadata: ${artifactName}`); + } + + await appendFile( + manifest, + `${JSON.stringify({ + archive: artifactName, + artifact_id: id, + target, + })}\n`, + ); + core.info(`Uploaded ${artifactName} (artifact ${id}, digest ${digest}).`); + } finally { + await rm(archive, { force: true }); + } + + const disk = await statfs(artifactDirectory); + if (disk.bavail * disk.bsize < 6 * GIB) { + throw new Error("less than 6 GiB remains while building archives"); + } + } + + await appendFile( + process.env.GITHUB_STEP_SUMMARY, + `\nBuilt ${targets.length} integration targets exactly once.\n`, + ); +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6766e64..b60de542 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -284,65 +284,20 @@ jobs: --no-fund - name: Build and upload integration test targets + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: + ARTIFACT_CLIENT_SCRIPT: ${{ github.workspace }}/.github/scripts/artifact-client/upload.mjs INTEGRATION_ARTIFACT_DIR: ${{ runner.temp }}/integration-test-targets INTEGRATION_ARTIFACT_MANIFEST: ${{ runner.temp }}/integration-test-targets/uploaded.jsonl + INTEGRATION_TARGETS_FILE: ${{ runner.temp }}/integration-test-targets/discovered.targets RISC0_DEV_MODE: "1" - run: | - target_dir="$RUNNER_TEMP/integration-test-targets" - : > "$INTEGRATION_ARTIFACT_MANIFEST" - archive="" - trap '[[ -z "$archive" ]] || rm -f "$archive"' EXIT - - target_count=0 - total_archive_bytes=0 - while IFS= read -r target; do - if [[ ! "$target" =~ ^[a-z0-9_]+$ ]]; then - echo "Invalid integration test target: $target" >&2 - exit 2 - fi - - archive="$target_dir/integration-test-$target.tar.zst" - cargo nextest archive \ - -p integration_tests \ - --test "$target" \ - -E "binary(=$target)" \ - --archive-file "$archive" \ - --no-pager - - archive_bytes="$(stat --format=%s "$archive")" - max_archive_bytes=$((128 * 1024 * 1024)) - if (( archive_bytes > max_archive_bytes )); then - echo "Target $target archive is larger than the 128 MiB limit." >&2 - exit 1 - fi - total_archive_bytes=$((total_archive_bytes + archive_bytes)) - max_total_archive_bytes=$((4 * 1024 * 1024 * 1024)) - if (( total_archive_bytes > max_total_archive_bytes )); then - echo "Integration target archives exceed the 4 GiB total limit." >&2 - exit 1 - fi - - archive_size="$(du -h "$archive" | cut -f1)" - echo "| \`$target\` | $archive_size |" >> "$GITHUB_STEP_SUMMARY" - - node .github/scripts/artifact-client/upload.mjs "$archive" - rm -f "$archive" - archive="" - target_count=$((target_count + 1)) - - available_kib="$(df --output=avail -k "$RUNNER_TEMP" | tail -n 1 | tr -d ' ')" - minimum_free_kib=$((6 * 1024 * 1024)) - if (( available_kib < minimum_free_kib )); then - echo "Less than 6 GiB remains while building integration archives." >&2 - df -h "$RUNNER_TEMP" >&2 - exit 1 - fi - done < "$target_dir/discovered.targets" - - echo >> "$GITHUB_STEP_SUMMARY" - echo "Built $target_count integration targets exactly once." \ - >> "$GITHUB_STEP_SUMMARY" + with: + script: | + const { pathToFileURL } = await import("node:url") + const artifactClient = await import( + pathToFileURL(process.env.ARTIFACT_CLIENT_SCRIPT) + ) + await artifactClient.buildAndUploadIntegrationTargets({ core, exec }) - name: Publish integration test matrix id: publish-matrix