mirror of
https://github.com/logos-blockchain/lssa.git
synced 2026-07-17 05:09:54 +00:00
fix(ci): upload artifacts from action context
This commit is contained in:
Ricardo Guilherme Schmidt
parent
35d6907aaa
commit
c5ebee4d84
173
.github/scripts/artifact-client/upload.mjs
vendored
173
.github/scripts/artifact-client/upload.mjs
vendored
@ -1,65 +1,132 @@
|
||||
import { appendFile, lstat, realpath } from "node:fs/promises";
|
||||
import { basename, dirname, resolve } from "node:path";
|
||||
import {
|
||||
appendFile,
|
||||
lstat,
|
||||
readFile,
|
||||
realpath,
|
||||
rm,
|
||||
statfs,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { basename, dirname, join, resolve } from "node:path";
|
||||
|
||||
import { DefaultArtifactClient } from "@actions/artifact";
|
||||
|
||||
const [input, ...extra] = process.argv.slice(2);
|
||||
const GIB = 1024 ** 3;
|
||||
const MIB = 1024 ** 2;
|
||||
|
||||
if (!input || extra.length > 0) {
|
||||
throw new Error("usage: node upload.mjs <integration-test archive>");
|
||||
function humanSize(size) {
|
||||
if (size >= GIB) return `${(size / GIB).toFixed(2)} GiB`;
|
||||
return `${(size / MIB).toFixed(2)} MiB`;
|
||||
}
|
||||
|
||||
const archive = await realpath(resolve(input));
|
||||
const artifactDirectory = await realpath(
|
||||
resolve(process.env.INTEGRATION_ARTIFACT_DIR ?? ""),
|
||||
);
|
||||
if (dirname(archive) !== artifactDirectory) {
|
||||
throw new Error(`integration artifact is outside staging: ${archive}`);
|
||||
}
|
||||
export async function buildAndUploadIntegrationTargets({ core, exec }) {
|
||||
const artifactDirectory = await realpath(
|
||||
resolve(process.env.INTEGRATION_ARTIFACT_DIR ?? ""),
|
||||
);
|
||||
const manifest = resolve(process.env.INTEGRATION_ARTIFACT_MANIFEST ?? "");
|
||||
const targetsPath = resolve(process.env.INTEGRATION_TARGETS_FILE ?? "");
|
||||
|
||||
const artifactName = basename(archive);
|
||||
const artifactMatch = /^integration-test-([a-z0-9_]+)\.tar\.zst$/.exec(
|
||||
artifactName,
|
||||
);
|
||||
if (!artifactMatch) {
|
||||
throw new Error(`invalid integration artifact name: ${artifactName}`);
|
||||
}
|
||||
if (dirname(manifest) !== artifactDirectory) {
|
||||
throw new Error(`integration manifest is outside staging: ${manifest}`);
|
||||
}
|
||||
if (dirname(targetsPath) !== artifactDirectory) {
|
||||
throw new Error(`integration target list is outside staging: ${targetsPath}`);
|
||||
}
|
||||
await writeFile(manifest, "");
|
||||
|
||||
const archiveStat = await lstat(archive);
|
||||
if (!archiveStat.isFile()) {
|
||||
throw new Error(`integration artifact is not a regular file: ${archive}`);
|
||||
}
|
||||
const targets = (await readFile(targetsPath, "utf8"))
|
||||
.split("\n")
|
||||
.filter(Boolean);
|
||||
if (targets.length === 0 || targets.length > 64) {
|
||||
throw new Error(`invalid integration target count: ${targets.length}`);
|
||||
}
|
||||
|
||||
const artifact = new DefaultArtifactClient();
|
||||
const { artifacts } = await artifact.listArtifacts({ latest: true });
|
||||
if (artifacts.some(({ name }) => name === artifactName)) {
|
||||
await artifact.deleteArtifact(artifactName);
|
||||
}
|
||||
const artifact = new DefaultArtifactClient();
|
||||
let totalArchiveSize = 0;
|
||||
|
||||
const { digest, id } = await artifact.uploadArtifact(
|
||||
artifactName,
|
||||
[archive],
|
||||
dirname(archive),
|
||||
{
|
||||
retentionDays: 1,
|
||||
skipArchive: true,
|
||||
},
|
||||
);
|
||||
if (!digest || !id) {
|
||||
throw new Error(`artifact upload returned incomplete metadata: ${artifactName}`);
|
||||
}
|
||||
for (const target of targets) {
|
||||
if (!/^[a-z0-9_]+$/.test(target)) {
|
||||
throw new Error(`invalid integration target: ${target}`);
|
||||
}
|
||||
|
||||
const manifest = resolve(process.env.INTEGRATION_ARTIFACT_MANIFEST ?? "");
|
||||
if (dirname(manifest) !== artifactDirectory) {
|
||||
throw new Error(`integration manifest is outside staging: ${manifest}`);
|
||||
}
|
||||
await appendFile(
|
||||
manifest,
|
||||
`${JSON.stringify({
|
||||
archive: artifactName,
|
||||
artifact_id: id,
|
||||
target: artifactMatch[1],
|
||||
})}\n`,
|
||||
);
|
||||
const artifactName = `integration-test-${target}.tar.zst`;
|
||||
const archive = join(artifactDirectory, artifactName);
|
||||
|
||||
console.log(`Uploaded ${artifactName} (artifact ${id}, digest ${digest}).`);
|
||||
try {
|
||||
const exitCode = await exec.exec("cargo", [
|
||||
"nextest",
|
||||
"archive",
|
||||
"-p",
|
||||
"integration_tests",
|
||||
"--test",
|
||||
target,
|
||||
"-E",
|
||||
`binary(=${target})`,
|
||||
"--archive-file",
|
||||
archive,
|
||||
"--no-pager",
|
||||
]);
|
||||
if (exitCode !== 0) {
|
||||
throw new Error(`nextest archive failed for ${target}`);
|
||||
}
|
||||
|
||||
const archiveStat = await lstat(archive);
|
||||
if (!archiveStat.isFile()) {
|
||||
throw new Error(`integration artifact is not a file: ${archive}`);
|
||||
}
|
||||
if (archiveStat.size > 128 * MIB) {
|
||||
throw new Error(`${target} archive exceeds the 128 MiB limit`);
|
||||
}
|
||||
|
||||
totalArchiveSize += archiveStat.size;
|
||||
if (totalArchiveSize > 4 * GIB) {
|
||||
throw new Error("integration archives exceed the 4 GiB total limit");
|
||||
}
|
||||
|
||||
await appendFile(
|
||||
process.env.GITHUB_STEP_SUMMARY,
|
||||
`| \`${target}\` | ${humanSize(archiveStat.size)} |\n`,
|
||||
);
|
||||
|
||||
const { artifacts } = await artifact.listArtifacts({ latest: true });
|
||||
if (artifacts.some(({ name }) => name === artifactName)) {
|
||||
await artifact.deleteArtifact(artifactName);
|
||||
}
|
||||
|
||||
const { digest, id } = await artifact.uploadArtifact(
|
||||
artifactName,
|
||||
[archive],
|
||||
artifactDirectory,
|
||||
{
|
||||
retentionDays: 1,
|
||||
skipArchive: true,
|
||||
},
|
||||
);
|
||||
if (!digest || !id) {
|
||||
throw new Error(`upload returned incomplete metadata: ${artifactName}`);
|
||||
}
|
||||
|
||||
await appendFile(
|
||||
manifest,
|
||||
`${JSON.stringify({
|
||||
archive: artifactName,
|
||||
artifact_id: id,
|
||||
target,
|
||||
})}\n`,
|
||||
);
|
||||
core.info(`Uploaded ${artifactName} (artifact ${id}, digest ${digest}).`);
|
||||
} finally {
|
||||
await rm(archive, { force: true });
|
||||
}
|
||||
|
||||
const disk = await statfs(artifactDirectory);
|
||||
if (disk.bavail * disk.bsize < 6 * GIB) {
|
||||
throw new Error("less than 6 GiB remains while building archives");
|
||||
}
|
||||
}
|
||||
|
||||
await appendFile(
|
||||
process.env.GITHUB_STEP_SUMMARY,
|
||||
`\nBuilt ${targets.length} integration targets exactly once.\n`,
|
||||
);
|
||||
}
|
||||
|
||||
65
.github/workflows/ci.yml
vendored
65
.github/workflows/ci.yml
vendored
@ -284,65 +284,20 @@ jobs:
|
||||
--no-fund
|
||||
|
||||
- name: Build and upload integration test targets
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
ARTIFACT_CLIENT_SCRIPT: ${{ github.workspace }}/.github/scripts/artifact-client/upload.mjs
|
||||
INTEGRATION_ARTIFACT_DIR: ${{ runner.temp }}/integration-test-targets
|
||||
INTEGRATION_ARTIFACT_MANIFEST: ${{ runner.temp }}/integration-test-targets/uploaded.jsonl
|
||||
INTEGRATION_TARGETS_FILE: ${{ runner.temp }}/integration-test-targets/discovered.targets
|
||||
RISC0_DEV_MODE: "1"
|
||||
run: |
|
||||
target_dir="$RUNNER_TEMP/integration-test-targets"
|
||||
: > "$INTEGRATION_ARTIFACT_MANIFEST"
|
||||
archive=""
|
||||
trap '[[ -z "$archive" ]] || rm -f "$archive"' EXIT
|
||||
|
||||
target_count=0
|
||||
total_archive_bytes=0
|
||||
while IFS= read -r target; do
|
||||
if [[ ! "$target" =~ ^[a-z0-9_]+$ ]]; then
|
||||
echo "Invalid integration test target: $target" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
archive="$target_dir/integration-test-$target.tar.zst"
|
||||
cargo nextest archive \
|
||||
-p integration_tests \
|
||||
--test "$target" \
|
||||
-E "binary(=$target)" \
|
||||
--archive-file "$archive" \
|
||||
--no-pager
|
||||
|
||||
archive_bytes="$(stat --format=%s "$archive")"
|
||||
max_archive_bytes=$((128 * 1024 * 1024))
|
||||
if (( archive_bytes > max_archive_bytes )); then
|
||||
echo "Target $target archive is larger than the 128 MiB limit." >&2
|
||||
exit 1
|
||||
fi
|
||||
total_archive_bytes=$((total_archive_bytes + archive_bytes))
|
||||
max_total_archive_bytes=$((4 * 1024 * 1024 * 1024))
|
||||
if (( total_archive_bytes > max_total_archive_bytes )); then
|
||||
echo "Integration target archives exceed the 4 GiB total limit." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
archive_size="$(du -h "$archive" | cut -f1)"
|
||||
echo "| \`$target\` | $archive_size |" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
node .github/scripts/artifact-client/upload.mjs "$archive"
|
||||
rm -f "$archive"
|
||||
archive=""
|
||||
target_count=$((target_count + 1))
|
||||
|
||||
available_kib="$(df --output=avail -k "$RUNNER_TEMP" | tail -n 1 | tr -d ' ')"
|
||||
minimum_free_kib=$((6 * 1024 * 1024))
|
||||
if (( available_kib < minimum_free_kib )); then
|
||||
echo "Less than 6 GiB remains while building integration archives." >&2
|
||||
df -h "$RUNNER_TEMP" >&2
|
||||
exit 1
|
||||
fi
|
||||
done < "$target_dir/discovered.targets"
|
||||
|
||||
echo >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "Built $target_count integration targets exactly once." \
|
||||
>> "$GITHUB_STEP_SUMMARY"
|
||||
with:
|
||||
script: |
|
||||
const { pathToFileURL } = await import("node:url")
|
||||
const artifactClient = await import(
|
||||
pathToFileURL(process.env.ARTIFACT_CLIENT_SCRIPT)
|
||||
)
|
||||
await artifactClient.buildAndUploadIntegrationTargets({ core, exec })
|
||||
|
||||
- name: Publish integration test matrix
|
||||
id: publish-matrix
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user