ci: Add package verifications to build workflow (#13294)

Co-authored-by: cskh <hui.kang@hashicorp.com>
This commit is contained in:
Evan Culver 2022-06-06 14:42:11 -07:00 committed by GitHub
parent 321e236891
commit 596432fe38
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 524 additions and 83 deletions

View File

@ -1,23 +0,0 @@
#!/bin/bash
set -euo pipefail
function main {
local image_name="${1:-${IMAGE_NAME:-}}"
local test_version
if [[ -z "${image_name}" ]]; then
echo "ERROR: IMAGE_NAME environment var or argument is required"
exit 1
fi
test_version="$(docker run "${image_name}" version | head -n1 | awk '{print $2}')"
if [ "${test_version}" != "v${version}" ]; then
echo "Test FAILED"
echo "Got: ${test_version}, Want: ${version}"
exit 1
fi
echo "Test PASSED"
}
main "$@"

240
.github/scripts/verify_artifact.sh vendored Executable file
View File

@ -0,0 +1,240 @@
#!/bin/bash
set -euo pipefail
# verify_artifact.sh is the top-level script that implements the logic to decide
# which individual verification script to invoke. It decides which verification
# script to use based on artifact name it is given. By putting the logic in here,
# it keeps the workflow file simpler and easier to manage. It also doubles as a means
# to run verifications locally when necessary.
# set this so we can locate and execute the individual verification scripts.
SCRIPT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
function usage {
echo "verify_artifact.sh <artifact_path> <expect_version>"
}
# Arguments:
# $1 - artifact path (eg. /artifacts/consul-1.13.0~dev-1.i386.rpm)
# $2 - expected version to match against (eg. v1.13.0-dev)
function main {
local artifact_path="${1:-}"
local expect_version="${2:-}"
if [[ -z "${artifact_path}" ]]; then
echo "ERROR: artifact path argument is required"
usage
exit 1
fi
if [[ -z "${expect_version}" ]]; then
echo "ERROR: expected version argument is required"
usage
exit 1
fi
if [[ ! -e "${artifact_path}" ]]; then
echo "ERROR: ${artifact_path} does not exist"
usage
exit 1
fi
# match against the various artifact names:
# deb packages: consul_${version}-1_${arch}.deb
# rpm packages: consul-${version}-1.${arch}.rpm
# zip packages: consul_${version}_${os}_${arch}.zip
case "${artifact_path}" in
*.rpm) verify_rpm "${artifact_path}" "${expect_version}";;
*.deb) verify_deb "${artifact_path}" "${expect_version}";;
*.zip) verify_zip "${artifact_path}" "${expect_version}";;
*)
echo "${artifact_path} did not match known patterns"
exit 1
;;
esac
}
# Arguments:
# $1 - path to rpm (eg. consul-1.13.0~dev-1.aarch64.rpm)
# $2 - expected version to match against (eg. v1.13.0-dev)
function verify_rpm {
local artifact_path="${1:-}"
local expect_version="${2:-}"
local docker_image
local docker_platform
case "${artifact_path}" in
*.i386.rpm)
docker_platform="linux/386"
docker_image="i386/centos:7"
;;
*.x86_64.rpm)
docker_platform="linux/amd64"
docker_image="amd64/centos:7"
;;
*.arm.rpm)
docker_platform="linux/arm/v7"
docker_image="arm32v7/fedora:36"
;;
*.aarch64.rpm)
docker_platform="linux/arm64"
docker_image="arm64v8/fedora:36"
;;
*)
echo "${artifact_path} did not match known patterns for rpms"
exit 1
;;
esac
echo "executing RPM verification in Docker with these parameters:"
echo "PLATFORM=${docker_platform}"
echo "IMAGE=${docker_image}"
docker run \
--platform=${docker_platform} \
-v $(pwd):/workdir \
-v ${SCRIPT_DIR}:/scripts \
-w /workdir \
${docker_image} \
/scripts/verify_rpm.sh \
"/workdir/${artifact_path}" \
"${expect_version}"
}
# Arguments:
# $1 - path to deb (eg. consul_1.13.0~dev-1_arm64.deb)
# $2 - expected version to match against (eg. v1.13.0-dev)
function verify_deb {
local artifact_path="${1:-}"
local expect_version="${2:-}"
local docker_image
local docker_platform
case "${artifact_path}" in
*_i386.deb)
docker_platform="linux/386"
docker_image="i386/debian:bullseye"
;;
*_amd64.deb)
docker_platform="linux/amd64"
docker_image="amd64/debian:bullseye"
;;
*_arm.deb)
docker_platform="linux/arm/v7"
docker_image="arm32v7/debian:bullseye"
;;
*_arm64.deb)
docker_platform="linux/arm64"
docker_image="arm64v8/debian:bullseye"
;;
*)
echo "${artifact_path} did not match known patterns for debs"
exit 1
;;
esac
echo "executing DEB verification in Docker with these parameters:"
echo "PLATFORM=${docker_platform}"
echo "IMAGE=${docker_image}"
docker run \
--platform=${docker_platform} \
-v $(pwd):/workdir \
-v ${SCRIPT_DIR}:/scripts \
-w /workdir \
${docker_image} \
/scripts/verify_deb.sh \
"/workdir/${artifact_path}" \
"${expect_version}"
}
# Arguments:
# $1 - path to zip (eg. consul_1.13.0-dev_linux_amd64.zip)
# $2 - expected version to match against (eg. v1.13.0-dev)
function verify_zip {
local artifact_path="${1:-}"
local expect_version="${2:-}"
local machine_os=$(uname -s)
local machine_arch=$(uname -m)
unzip "${artifact_path}"
if [[ ! -e ./consul ]]; then
echo "ERROR: ${artifact_path} did not contain a consul binary"
exit 1
fi
case "${artifact_path}" in
*_darwin_amd64.zip)
if [[ "${machine_os}" = 'Darwin' ]]; then
# run the darwin binary if the host is Darwin.
${SCRIPT_DIR}/verify_bin.sh ./consul ${expect_version}
else
echo "cannot run darwin binary on a non-darwin host (${machine_os})"
fi
;;
*_linux_386.zip | *_linux_amd64.zip)
if [[ "${machine_os}" = 'Linux' && "${machine_arch}" = "x86_64" ]]; then
# run the binary directly on the host when it's x86_64 Linux
${SCRIPT_DIR}/verify_bin.sh ./consul ${expect_version}
else
# otherwise, use Docker/QEMU
docker run \
--platform=linux/amd64 \
-v $(pwd):/workdir \
-v ${SCRIPT_DIR}:/scripts \
-w /workdir \
amd64/debian \
/scripts/verify_bin.sh \
./consul \
"${expect_version}"
fi
;;
*_linux_arm.zip)
if [[ "${machine_os}" = 'Linux' && "${machine_arch}" = arm* ]]; then
# run the binary directly on the host when it's x86_64 Linux
${SCRIPT_DIR}/verify_bin.sh ./consul ${expect_version}
else
# otherwise, use Docker/QEMU
docker run \
--platform=linux/arm/v7 \
-v $(pwd):/workdir \
-v ${SCRIPT_DIR}:/scripts \
-w /workdir \
arm32v7/debian \
/scripts/verify_bin.sh \
./consul \
"${expect_version}"
fi
;;
*_linux_arm64.zip)
if [[ "${machine_os}" = 'Linux' && "${machine_arch}" = arm* ]]; then
# run the binary directly on the host when it's x86_64 Linux
${SCRIPT_DIR}/verify_bin.sh ./consul ${expect_version}
else
# otherwise, use Docker/QEMU
docker run \
--platform=linux/arm64 \
-v $(pwd):/workdir \
-v ${SCRIPT_DIR}:/scripts \
-w /workdir \
arm64v8/debian \
/scripts/verify_bin.sh \
./consul \
"${expect_version}"
fi
;;
*)
echo "${artifact_path} did not match known patterns for zips"
exit 1
;;
esac
}
main "$@"

44
.github/scripts/verify_bin.sh vendored Executable file
View File

@ -0,0 +1,44 @@
#!/bin/bash
set -euo pipefail
# verify_bin.sh validates the file at the path given and then runs `./consul version` and inspects its output. If its
# output doesn't match the version given, the script will exit 1 and report why it failed.
# This is meant to be run as part of the build workflow to verify the built .zip meets some basic criteria for validity.
function usage {
echo "./verify_bin.sh <path_to_bin> <expect_version>"
}
function main {
local bin_path="${1:-}"
local expect_version="${2:-}"
local got_version
if [[ -z "${bin_path}" ]]; then
echo "ERROR: path to binary argument is required"
usage
exit 1
fi
if [[ -z "${expect_version}" ]]; then
echo "ERROR: expected version argument is required"
usage
exit 1
fi
if [[ ! -e "${bin_path}" ]]; then
echo "ERROR: package at ${bin_path} does not exist."
exit 1
fi
got_version="$( awk '{print $2}' <(head -n1 <(${bin_path} version)) )"
if [ "${got_version}" != "${expect_version}" ]; then
echo "Test FAILED"
echo "Got: ${got_version}, Want: ${expect_version}"
exit 1
fi
echo "Test PASSED"
}
main "$@"

57
.github/scripts/verify_deb.sh vendored Executable file
View File

@ -0,0 +1,57 @@
#!/bin/bash
set -euo pipefail
# verify_deb.sh tries to install the .deb package at the path given before running `consul version`
# to inspect its output. If its output doesn't match the version given, the script will exit 1 and
# report why it failed. This is meant to be run as part of the build workflow to verify the built
# .deb meets some basic criteria for validity.
# set this so we can locate and execute the verify_bin.sh script for verifying version output
SCRIPT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
function usage {
echo "./verify_deb.sh <path_to_deb> <expect_version>"
}
function main {
local deb_path="${1:-}"
local expect_version="${2:-}"
local got_version
if [[ -z "${deb_path}" ]]; then
echo "ERROR: package path argument is required"
usage
exit 1
fi
if [[ -z "${expect_version}" ]]; then
echo "ERROR: expected version argument is required"
usage
exit 1
fi
# expand globs for path names, if this fails, the script will exit
deb_path=$(echo ${deb_path})
if [[ ! -e "${deb_path}" ]]; then
echo "ERROR: package at ${deb_path} does not exist."
usage
exit 1
fi
# we have to install the 'arm' architecture in order to install the 'arm'
# package, otherwise we will git a 'package architecture does not match system' error
if [[ ${deb_path} = *_arm.deb ]]; then
dpkg --add-architecture arm
fi
apt -y update
apt -y install openssl
dpkg -i ${deb_path}
# use the script that should be located next to this one for verifying the output
exec "${SCRIPT_DIR}/verify_bin.sh" $(which consul) "${expect_version}"
}
main "$@"

44
.github/scripts/verify_docker.sh vendored Executable file
View File

@ -0,0 +1,44 @@
#!/bin/bash
set -euo pipefail
# verify_docker.sh invokes the given Docker image to run `consul version` and inspect its output.
# If its output doesn't match the version given, the script will exit 1 and report why it failed.
# This is meant to be run as part of the build workflow to verify the built image meets some basic
# criteria for validity.
#
# Because this is meant to be run as the `smoke_test` for the docker-build workflow, the script expects
# the image name parameter to be provided by the `IMAGE_NAME` environment variable, rather than a
# positional argument.
function usage {
echo "IMAGE_NAME=<image uri> ./verify_docker.sh <expect_version>"
}
function main {
local image_name="${IMAGE_NAME:-}"
local expect_version="${1:-}"
local got_version
if [[ -z "${image_name}" ]]; then
echo "ERROR: IMAGE_NAME is not set"
usage
exit 1
fi
if [[ -z "${expect_version}" ]]; then
echo "ERROR: expected version argument is required"
usage
exit 1
fi
got_version="$( awk '{print $2}' <(head -n1 <(docker run "${image_name}" version)) )"
if [ "${got_version}" != "${expect_version}" ]; then
echo "Test FAILED"
echo "Got: ${got_version}, Want: ${expect_version}"
exit 1
fi
echo "Test PASSED"
}
main "$@"

52
.github/scripts/verify_rpm.sh vendored Executable file
View File

@ -0,0 +1,52 @@
#!/bin/bash
set -euo pipefail
# verify_rpm.sh tries to install the .rpm package at the path given before running `consul version`
# to inspect its output. If its output doesn't match the version given, the script will exit 1 and
# report why it failed. This is meant to be run as part of the build workflow to verify the built
# .rpm meets some basic criteria for validity.
# set this so we can locate and execute the verify_bin.sh script for verifying version output
SCRIPT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
function usage {
echo "./verify_rpm.sh <path_to_rpm> <expect_version>"
}
function main {
local rpm_path="${1:-}"
local expect_version="${2:-}"
local got_version
if [[ -z "${rpm_path}" ]]; then
echo "ERROR: package path argument is required"
usage
exit 1
fi
if [[ -z "${expect_version}" ]]; then
echo "ERROR: expected version argument is required"
usage
exit 1
fi
# expand globs for path names, if this fails, the script will exit
rpm_path=$(echo ${rpm_path})
if [[ ! -e "${rpm_path}" ]]; then
echo "ERROR: package at ${rpm_path} does not exist."
usage
exit 1
fi
yum -y clean all
yum -y update
yum -y install which openssl
rpm --ignorearch -i ${rpm_path}
# use the script that should be located next to this one for verifying the output
exec "${SCRIPT_DIR}/verify_bin.sh" $(which consul) "${expect_version}"
}
main "$@"

View File

@ -218,22 +218,13 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
arch: ["arm", "arm64", "386", "amd64"]
arch: ["386", "amd64", "arm", "arm64"]
env:
repo: ${{github.event.repository.name}}
version: ${{needs.get-product-version.outputs.product-version}}
steps:
- uses: actions/checkout@v2
- name: Download artifact for arm verification
uses: actions/download-artifact@v2
if: ${{ matrix.arch == 'arm64' || matrix.arch == 'arm' }}
with:
name: ${{ env.PKG_NAME }}_${{env.version}}_linux_${{ matrix.arch }}.zip
- name: Unzip for ${{ matrix.arch }} linux zip
if: ${{ matrix.arch == 'arm64' || matrix.arch == 'arm' }}
run: |
unzip ${{ env.PKG_NAME }}_${{ env.version }}_linux_${{ matrix.arch }}.zip
- name: Docker Build (Action)
uses: hashicorp/actions-docker-build@v1
with:
@ -246,17 +237,7 @@ jobs:
dev_tags: |
docker.io/hashicorppreview/${{ env.repo }}:${{ env.version }}
docker.io/hashicorppreview/${{ env.repo }}:${{ env.version }}-${{ github.sha }}
smoke_test: .github/scripts/docker_smoke.sh
- name: Run verification in qemu docker for ${{ matrix.arch }} linux zip
if: ${{ matrix.arch == 'arm64' }}
run: |
docker run --platform=linux/arm64 -v $(pwd):/workdir -w /workdir \
arm64v8/debian ./consul version
- name: Run verification in qemu docker for ${{ matrix.arch }} linux zip
if: ${{ matrix.arch == 'arm' }}
run: |
docker run --platform=linux/arm/v7 -v $(pwd):/workdir -w /workdir \
arm32v7/debian ./consul version
smoke_test: .github/scripts/verify_docker.sh v${{ env.version }}
build-docker-redhat:
name: Docker Build UBI Image for RedHat
@ -276,29 +257,39 @@ jobs:
target: ubi
arch: amd64
redhat_tag: scan.connect.redhat.com/ospid-612d01d49f14588c41ebf67c/${{env.repo}}:${{env.version}}-ubi
smoke_test: .github/scripts/docker_smoke.sh
verify:
smoke_test: .github/scripts/verify_docker.sh v${{ env.version }}
verify-linux:
needs:
- get-product-version
- build
runs-on: ubuntu-latest
strategy:
matrix:
arch: ["386", "amd64"]
arch: ["386", "amd64", "arm", "arm64"]
fail-fast: true
env:
version: ${{needs.get-product-version.outputs.product-version}}
name: Verify ${{ matrix.arch }} linux zip
version: ${{ needs.get-product-version.outputs.product-version }}
zip_name: consul_${{ needs.get-product-version.outputs.product-version }}_linux_${{ matrix.arch }}.zip
name: Verify ${{ matrix.arch }} linux binary
steps:
- uses: actions/checkout@v2
- name: Download artifact
uses: actions/download-artifact@v2
- name: Download ${{ matrix.arch }} zip
uses: actions/download-artifact@v3
with:
name: ${{ env.PKG_NAME }}_${{ env.version }}_linux_${{ matrix.arch }}.zip
- name: Run verification for ${{ matrix.arch }} linux zip
run: |
unzip ${{ env.PKG_NAME }}_${{env.version}}_linux_${{ matrix.arch }}.zip
./consul version
name: ${{ env.zip_name }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
if: ${{ matrix.arch == 'arm' || matrix.arch == 'arm64' }}
with:
# this should be a comma-separated string as opposed to an array
platforms: arm,arm64
- name: Run verification for ${{ matrix.arch }} binary
run: .github/scripts/verify_artifact.sh ${{ env.zip_name }} v${{ env.version }}
verify-darwin:
needs:
@ -306,60 +297,96 @@ jobs:
- build-darwin
runs-on: macos-latest
strategy:
matrix:
arch: ["amd64", "arm64"]
fail-fast: true
env:
version: ${{needs.get-product-version.outputs.product-version}}
name: Verify ${{ matrix.arch }} darwin zip
zip_name: consul_${{ needs.get-product-version.outputs.product-version }}_darwin_amd64.zip
name: Verify amd64 darwin binary
steps:
- uses: actions/checkout@v2
- name: Download artifact
uses: actions/download-artifact@v2
- name: Download amd64 darwin zip
uses: actions/download-artifact@v3
with:
name: ${{ env.PKG_NAME }}_${{env.version}}_darwin_${{ matrix.arch }}.zip
- name: Run verification for ${{ matrix.arch }} linux zip
if: ${{ matrix.arch == 'amd64' }}
run: |
unzip ${{ env.PKG_NAME }}_${{env.version}}_darwin_${{ matrix.arch }}.zip
./consul version
name: ${{ env.zip_name }}
- name: Unzip amd64 darwin zip
run: unzip ${{ env.zip_name }}
- name: Run verification for amd64 darwin binary
run: .github/scripts/verify_bin.sh ./consul v${{ env.version }}
verify-linux-packages-deb:
needs: build
needs:
- build
- get-product-version
runs-on: ubuntu-latest
strategy:
matrix:
arch: [ "amd64", "arm", "arm64", "i386" ]
fail-fast: true
arch: ["i386", "amd64", "arm", "arm64"]
# fail-fast: true
env:
version: ${{ needs.get-product-version.outputs.product-version }}
name: Verify ${{ matrix.arch }} debian package
steps:
- uses: actions/checkout@v2
- name: Download ${{ matrix.arch }} debian package
- name: Set package version
run: |
echo "TODO: download ${{ matrix.arch }} debian package"
echo "pkg_version=$(echo ${{ needs.get-product-version.outputs.product-version }} | sed 's/\-/~/g')" >> $GITHUB_ENV
- name: Set package name
run: |
echo "pkg_name=consul_${{ env.pkg_version }}-1_${{ matrix.arch }}.deb" >> $GITHUB_ENV
- name: Download workflow artifacts
uses: actions/download-artifact@v3
with:
name: ${{ env.pkg_name }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
with:
platforms: all
- name: Verify ${{ matrix.arch }} debian package
run: |
echo "TODO: run verification for ${{ matrix.arch }} debian package"
run: ./.github/scripts/verify_artifact.sh ${{ env.pkg_name }} v${{ env.version }}
verify-linux-packages-rpm:
needs: build
needs:
- build
- get-product-version
runs-on: ubuntu-latest
strategy:
matrix:
arch: [ "aarch64", "arm", "i386", "x86_64" ]
fail-fast: true
arch: ["i386", "x86_64", "arm", "aarch64"]
# fail-fast: true
env:
version: ${{ needs.get-product-version.outputs.product-version }}
name: Verify ${{ matrix.arch }} rpm package
name: Verify ${{ matrix.arch }} rpm
steps:
- uses: actions/checkout@v2
- name: Download ${{ matrix.arch }} rpm package
- name: Set package version
run: |
echo "TODO: download ${{ matrix.arch }} rpm package"
echo "pkg_version=$(echo ${{ needs.get-product-version.outputs.product-version }} | sed 's/\-/~/g')" >> $GITHUB_ENV
- name: Verify ${{ matrix.arch }} rpm package
- name: Set package name
run: |
echo "TODO: run verification for ${{ matrix.arch }} rpm package"
echo "pkg_name=consul-${{ env.pkg_version }}-1.${{ matrix.arch }}.rpm" >> $GITHUB_ENV
- name: Download workflow artifacts
uses: actions/download-artifact@v3
with:
name: ${{ env.pkg_name }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
with:
platforms: all
- name: Verify ${{ matrix.arch }} rpm
run: ./.github/scripts/verify_artifact.sh ${{ env.pkg_name }} v${{ env.version }}