ci: Add package verifications to build workflow (#13294)

Co-authored-by: cskh <hui.kang@hashicorp.com>

.github/scripts/docker_smoke.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-
-function main {
-  local image_name="${1:-${IMAGE_NAME:-}}"
-  local test_version
-
-  if [[ -z "${image_name}" ]]; then
-    echo "ERROR: IMAGE_NAME environment var or argument is required"
-    exit 1
-  fi
-
-  test_version="$(docker run "${image_name}" version  | head -n1 | awk '{print $2}')"
-  if [ "${test_version}" != "v${version}" ]; then
-    echo "Test FAILED"
-    echo "Got: ${test_version}, Want: ${version}"
-    exit 1
-  fi
-  echo "Test PASSED"
-}
-
-main "$@"

.github/scripts/verify_artifact.sh

@@ -0,0 +1,240 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# verify_artifact.sh is the top-level script that implements the logic to decide
+# which individual verification script to invoke. It decides which verification
+# script to use based on artifact name it is given. By putting the logic in here,
+# it keeps the workflow file simpler and easier to manage. It also doubles as a means
+# to run verifications locally when necessary.
+
+# set this so we can locate and execute the individual verification scripts.
+SCRIPT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+function usage {
+  echo "verify_artifact.sh <artifact_path> <expect_version>"
+}
+
+# Arguments:
+#   $1 - artifact path (eg. /artifacts/consul-1.13.0~dev-1.i386.rpm)
+#   $2 - expected version to match against (eg. v1.13.0-dev)
+function main {
+  local artifact_path="${1:-}"
+  local expect_version="${2:-}"
+
+  if [[ -z "${artifact_path}" ]]; then
+    echo "ERROR: artifact path argument is required"
+    usage
+    exit 1
+  fi
+
+  if [[ -z "${expect_version}" ]]; then
+    echo "ERROR: expected version argument is required"
+    usage
+    exit 1
+  fi
+
+  if [[ ! -e "${artifact_path}" ]]; then
+    echo "ERROR: ${artifact_path} does not exist"
+    usage
+    exit 1
+  fi
+
+  # match against the various artifact names:
+  # deb packages: consul_${version}-1_${arch}.deb
+  # rpm packages: consul-${version}-1.${arch}.rpm
+  # zip packages: consul_${version}_${os}_${arch}.zip
+  case "${artifact_path}" in
+    *.rpm) verify_rpm "${artifact_path}" "${expect_version}";;
+    *.deb) verify_deb "${artifact_path}" "${expect_version}";;
+    *.zip) verify_zip "${artifact_path}" "${expect_version}";;
+    *)
+      echo "${artifact_path} did not match known patterns"
+      exit 1
+      ;;
+  esac
+}
+
+# Arguments:
+#   $1 - path to rpm (eg. consul-1.13.0~dev-1.aarch64.rpm)
+#   $2 - expected version to match against (eg. v1.13.0-dev)
+function verify_rpm {
+  local artifact_path="${1:-}"
+  local expect_version="${2:-}"
+  local docker_image
+  local docker_platform
+
+  case "${artifact_path}" in
+    *.i386.rpm)
+      docker_platform="linux/386"
+      docker_image="i386/centos:7"
+      ;;
+    *.x86_64.rpm)
+      docker_platform="linux/amd64"
+      docker_image="amd64/centos:7"
+      ;;
+    *.arm.rpm)
+      docker_platform="linux/arm/v7"
+      docker_image="arm32v7/fedora:36"
+      ;;
+    *.aarch64.rpm)
+      docker_platform="linux/arm64"
+      docker_image="arm64v8/fedora:36"
+      ;;
+    *)
+      echo "${artifact_path} did not match known patterns for rpms"
+      exit 1
+      ;;
+  esac
+
+  echo "executing RPM verification in Docker with these parameters:"
+  echo "PLATFORM=${docker_platform}"
+  echo "IMAGE=${docker_image}"
+
+  docker run \
+      --platform=${docker_platform} \
+      -v $(pwd):/workdir \
+      -v ${SCRIPT_DIR}:/scripts \
+      -w /workdir \
+    ${docker_image} \
+    /scripts/verify_rpm.sh \
+    "/workdir/${artifact_path}" \
+    "${expect_version}"
+}
+
+# Arguments:
+#   $1 - path to deb (eg. consul_1.13.0~dev-1_arm64.deb)
+#   $2 - expected version to match against (eg. v1.13.0-dev)
+function verify_deb {
+  local artifact_path="${1:-}"
+  local expect_version="${2:-}"
+  local docker_image
+  local docker_platform
+
+  case "${artifact_path}" in
+    *_i386.deb)
+      docker_platform="linux/386"
+      docker_image="i386/debian:bullseye"
+      ;;
+    *_amd64.deb)
+      docker_platform="linux/amd64"
+      docker_image="amd64/debian:bullseye"
+      ;;
+    *_arm.deb)
+      docker_platform="linux/arm/v7"
+      docker_image="arm32v7/debian:bullseye"
+      ;;
+    *_arm64.deb)
+      docker_platform="linux/arm64"
+      docker_image="arm64v8/debian:bullseye"
+      ;;
+    *)
+      echo "${artifact_path} did not match known patterns for debs"
+      exit 1
+      ;;
+  esac
+
+  echo "executing DEB verification in Docker with these parameters:"
+  echo "PLATFORM=${docker_platform}"
+  echo "IMAGE=${docker_image}"
+
+  docker run \
+      --platform=${docker_platform} \
+      -v $(pwd):/workdir \
+      -v ${SCRIPT_DIR}:/scripts \
+      -w /workdir \
+    ${docker_image} \
+    /scripts/verify_deb.sh \
+    "/workdir/${artifact_path}" \
+    "${expect_version}"
+}
+
+# Arguments:
+#   $1 - path to zip (eg. consul_1.13.0-dev_linux_amd64.zip)
+#   $2 - expected version to match against (eg. v1.13.0-dev)
+function verify_zip {
+  local artifact_path="${1:-}"
+  local expect_version="${2:-}"
+  local machine_os=$(uname -s)
+  local machine_arch=$(uname -m)
+
+  unzip "${artifact_path}"
+
+  if [[ ! -e ./consul ]]; then
+    echo "ERROR: ${artifact_path} did not contain a consul binary"
+    exit 1
+  fi
+
+  case "${artifact_path}" in
+
+    *_darwin_amd64.zip)
+      if [[ "${machine_os}" = 'Darwin' ]]; then
+        # run the darwin binary if the host is Darwin.
+        ${SCRIPT_DIR}/verify_bin.sh ./consul ${expect_version}
+      else
+        echo "cannot run darwin binary on a non-darwin host (${machine_os})"
+      fi
+      ;;
+
+    *_linux_386.zip | *_linux_amd64.zip)
+      if [[ "${machine_os}" = 'Linux' && "${machine_arch}" = "x86_64" ]]; then
+        # run the binary directly on the host when it's x86_64 Linux
+        ${SCRIPT_DIR}/verify_bin.sh ./consul ${expect_version}
+      else
+        # otherwise, use Docker/QEMU
+        docker run \
+          --platform=linux/amd64 \
+          -v $(pwd):/workdir \
+          -v ${SCRIPT_DIR}:/scripts \
+          -w /workdir  \
+        amd64/debian \
+        /scripts/verify_bin.sh \
+        ./consul \
+        "${expect_version}"
+      fi
+      ;;
+
+    *_linux_arm.zip)
+      if [[ "${machine_os}" = 'Linux' && "${machine_arch}" = arm* ]]; then
+        # run the binary directly on the host when it's x86_64 Linux
+        ${SCRIPT_DIR}/verify_bin.sh ./consul ${expect_version}
+      else
+        # otherwise, use Docker/QEMU
+        docker run \
+          --platform=linux/arm/v7 \
+          -v $(pwd):/workdir \
+          -v ${SCRIPT_DIR}:/scripts \
+          -w /workdir  \
+        arm32v7/debian \
+        /scripts/verify_bin.sh \
+        ./consul \
+        "${expect_version}"
+      fi
+      ;;
+
+    *_linux_arm64.zip)
+      if [[ "${machine_os}" = 'Linux' && "${machine_arch}" = arm* ]]; then
+        # run the binary directly on the host when it's x86_64 Linux
+        ${SCRIPT_DIR}/verify_bin.sh ./consul ${expect_version}
+      else
+        # otherwise, use Docker/QEMU
+        docker run \
+          --platform=linux/arm64 \
+          -v $(pwd):/workdir \
+          -v ${SCRIPT_DIR}:/scripts \
+          -w /workdir  \
+        arm64v8/debian \
+        /scripts/verify_bin.sh \
+        ./consul \
+        "${expect_version}"
+      fi
+      ;;
+
+    *)
+      echo "${artifact_path} did not match known patterns for zips"
+      exit 1
+      ;;
+  esac
+}
+
+main "$@"

.github/scripts/verify_bin.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# verify_bin.sh validates the file at the path given and then runs `./consul version` and inspects its output. If its
+# output doesn't match the version given, the script will exit 1 and report why it failed.
+# This is meant to be run as part of the build workflow to verify the built .zip meets some basic criteria for validity.
+
+function usage {
+  echo "./verify_bin.sh <path_to_bin> <expect_version>"
+}
+
+function main {
+  local bin_path="${1:-}"
+  local expect_version="${2:-}"
+  local got_version
+
+  if [[ -z "${bin_path}" ]]; then
+    echo "ERROR: path to binary argument is required"
+    usage
+    exit 1
+  fi
+
+  if [[ -z "${expect_version}" ]]; then
+    echo "ERROR: expected version argument is required"
+    usage
+    exit 1
+  fi
+
+  if [[ ! -e "${bin_path}" ]]; then
+    echo "ERROR: package at ${bin_path} does not exist."
+    exit 1
+  fi
+  
+  got_version="$( awk '{print $2}' <(head -n1 <(${bin_path} version)) )"
+  if [ "${got_version}" != "${expect_version}" ]; then
+    echo "Test FAILED"
+    echo "Got: ${got_version}, Want: ${expect_version}"
+    exit 1
+  fi
+  echo "Test PASSED"
+}
+
+main "$@"

.github/scripts/verify_deb.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# verify_deb.sh tries to install the .deb package at the path given before running `consul version`
+# to inspect its output. If its output doesn't match the version given, the script will exit 1 and
+# report why it failed. This is meant to be run as part of the build workflow to verify the built
+# .deb meets some basic criteria for validity.
+
+# set this so we can locate and execute the verify_bin.sh script for verifying version output
+SCRIPT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+function usage {
+  echo "./verify_deb.sh <path_to_deb> <expect_version>"
+}
+
+function main {
+  local deb_path="${1:-}"
+  local expect_version="${2:-}"
+  local got_version
+
+  if [[ -z "${deb_path}" ]]; then
+    echo "ERROR: package path argument is required"
+    usage
+    exit 1
+  fi
+
+  if [[ -z "${expect_version}" ]]; then
+    echo "ERROR: expected version argument is required"
+    usage
+    exit 1
+  fi
+
+  # expand globs for path names, if this fails, the script will exit
+  deb_path=$(echo ${deb_path})
+
+  if [[ ! -e "${deb_path}" ]]; then
+    echo "ERROR: package at ${deb_path} does not exist."
+    usage
+    exit 1
+  fi
+
+  # we have to install the 'arm' architecture in order to install the 'arm'
+  # package, otherwise we will git a 'package architecture does not match system' error
+  if [[ ${deb_path} = *_arm.deb ]]; then
+    dpkg --add-architecture arm
+  fi
+
+  apt -y update
+  apt -y install openssl
+  dpkg -i ${deb_path}
+
+  # use the script that should be located next to this one for verifying the output
+  exec "${SCRIPT_DIR}/verify_bin.sh" $(which consul) "${expect_version}"
+}
+
+main "$@"

.github/scripts/verify_docker.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# verify_docker.sh invokes the given Docker image to run `consul version` and inspect its output.
+# If its output doesn't match the version given, the script will exit 1 and report why it failed.
+# This is meant to be run as part of the build workflow to verify the built image meets some basic
+# criteria for validity.
+#
+# Because this is meant to be run as the `smoke_test` for the docker-build workflow, the script expects 
+# the image name parameter to be provided by the `IMAGE_NAME` environment variable, rather than a
+# positional argument. 
+
+function usage {
+  echo "IMAGE_NAME=<image uri> ./verify_docker.sh <expect_version>"
+}
+
+function main {
+  local image_name="${IMAGE_NAME:-}"
+  local expect_version="${1:-}"
+  local got_version
+
+  if [[ -z "${image_name}" ]]; then
+    echo "ERROR: IMAGE_NAME is not set"
+    usage
+    exit 1
+  fi
+
+  if [[ -z "${expect_version}" ]]; then
+    echo "ERROR: expected version argument is required"
+    usage
+    exit 1
+  fi
+
+  got_version="$( awk '{print $2}' <(head -n1 <(docker run "${image_name}" version)) )"
+  if [ "${got_version}" != "${expect_version}" ]; then
+    echo "Test FAILED"
+    echo "Got: ${got_version}, Want: ${expect_version}"
+    exit 1
+  fi
+  echo "Test PASSED"
+}
+
+main "$@"

.github/scripts/verify_rpm.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# verify_rpm.sh tries to install the .rpm package at the path given before running `consul version`
+# to inspect its output. If its output doesn't match the version given, the script will exit 1 and
+# report why it failed. This is meant to be run as part of the build workflow to verify the built
+# .rpm meets some basic criteria for validity.
+
+# set this so we can locate and execute the verify_bin.sh script for verifying version output
+SCRIPT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+function usage {
+  echo "./verify_rpm.sh <path_to_rpm> <expect_version>"
+}
+
+function main {
+  local rpm_path="${1:-}"
+  local expect_version="${2:-}"
+  local got_version
+
+  if [[ -z "${rpm_path}" ]]; then
+    echo "ERROR: package path argument is required"
+    usage
+    exit 1
+  fi
+
+  if [[ -z "${expect_version}" ]]; then
+    echo "ERROR: expected version argument is required"
+    usage
+    exit 1
+  fi
+
+  # expand globs for path names, if this fails, the script will exit
+  rpm_path=$(echo ${rpm_path})
+
+  if [[ ! -e "${rpm_path}" ]]; then
+    echo "ERROR: package at ${rpm_path} does not exist."
+    usage
+    exit 1
+  fi
+
+  yum -y clean all
+  yum -y update
+  yum -y install which openssl
+  rpm --ignorearch -i ${rpm_path}
+
+  # use the script that should be located next to this one for verifying the output
+  exec "${SCRIPT_DIR}/verify_bin.sh" $(which consul) "${expect_version}"
+}
+
+main "$@"

.github/workflows/build.yml

@@ -218,22 +218,13 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        arch: ["arm", "arm64", "386", "amd64"]
+        arch: ["386", "amd64", "arm", "arm64"]
     env:
       repo: ${{github.event.repository.name}}
       version: ${{needs.get-product-version.outputs.product-version}}
 
     steps:
       - uses: actions/checkout@v2
-      - name: Download artifact for arm verification
-        uses: actions/download-artifact@v2
-        if: ${{ matrix.arch == 'arm64' || matrix.arch == 'arm' }}
-        with:
-          name: ${{ env.PKG_NAME }}_${{env.version}}_linux_${{ matrix.arch }}.zip
-      - name: Unzip for ${{ matrix.arch }} linux zip
-        if: ${{ matrix.arch == 'arm64' || matrix.arch == 'arm' }}
-        run: |
-          unzip ${{ env.PKG_NAME }}_${{ env.version }}_linux_${{ matrix.arch }}.zip
       - name: Docker Build (Action)
         uses: hashicorp/actions-docker-build@v1
         with:
@@ -246,17 +237,7 @@ jobs:
           dev_tags: |
             docker.io/hashicorppreview/${{ env.repo }}:${{ env.version }}
             docker.io/hashicorppreview/${{ env.repo }}:${{ env.version }}-${{ github.sha }}
-          smoke_test: .github/scripts/docker_smoke.sh
+          smoke_test: .github/scripts/verify_docker.sh v${{ env.version }}
-      - name: Run verification in qemu docker for ${{ matrix.arch }} linux zip
-        if: ${{ matrix.arch == 'arm64' }}
-        run: |
-          docker run --platform=linux/arm64 -v $(pwd):/workdir -w /workdir  \
-            arm64v8/debian ./consul version
-      - name: Run verification in qemu docker for ${{ matrix.arch }} linux zip
-        if: ${{ matrix.arch == 'arm' }}
-        run: |
-          docker run --platform=linux/arm/v7 -v $(pwd):/workdir -w /workdir  \
-            arm32v7/debian ./consul version
 
   build-docker-redhat:
     name: Docker Build UBI Image for RedHat
@@ -276,29 +257,39 @@ jobs:
           target: ubi
           arch: amd64
           redhat_tag: scan.connect.redhat.com/ospid-612d01d49f14588c41ebf67c/${{env.repo}}:${{env.version}}-ubi
-          smoke_test: .github/scripts/docker_smoke.sh
+          smoke_test: .github/scripts/verify_docker.sh v${{ env.version }}
-  verify:
+
+  verify-linux:
     needs:
       - get-product-version
       - build
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        arch: ["386", "amd64"]
+        arch: ["386", "amd64", "arm", "arm64"]
       fail-fast: true
     env:
-      version: ${{needs.get-product-version.outputs.product-version}}
+      version: ${{ needs.get-product-version.outputs.product-version }}
-    name: Verify ${{ matrix.arch }} linux zip
+      zip_name: consul_${{ needs.get-product-version.outputs.product-version }}_linux_${{ matrix.arch }}.zip
+
+    name: Verify ${{ matrix.arch }} linux binary
     steps:
       - uses: actions/checkout@v2
-      - name: Download artifact
+
-        uses: actions/download-artifact@v2
+      - name: Download ${{ matrix.arch  }} zip
+        uses: actions/download-artifact@v3
         with:
-          name: ${{ env.PKG_NAME }}_${{ env.version }}_linux_${{ matrix.arch }}.zip
+          name: ${{ env.zip_name }}
-      - name: Run verification for ${{ matrix.arch }} linux zip
+
-        run: |
+      - name: Set up QEMU
-          unzip ${{ env.PKG_NAME }}_${{env.version}}_linux_${{ matrix.arch }}.zip
+        uses: docker/setup-qemu-action@v1
-          ./consul version
+        if: ${{ matrix.arch == 'arm' || matrix.arch == 'arm64' }}
+        with:
+          # this should be a comma-separated string as opposed to an array
+          platforms: arm,arm64
+
+      - name: Run verification for ${{ matrix.arch }} binary
+        run: .github/scripts/verify_artifact.sh ${{ env.zip_name }} v${{ env.version }}
 
   verify-darwin:
     needs:
@@ -306,60 +297,96 @@ jobs:
       - build-darwin
     runs-on: macos-latest
     strategy:
-      matrix:
-        arch: ["amd64", "arm64"]
       fail-fast: true
     env:
       version: ${{needs.get-product-version.outputs.product-version}}
-    name: Verify ${{ matrix.arch }} darwin zip
+      zip_name: consul_${{ needs.get-product-version.outputs.product-version }}_darwin_amd64.zip
+
+    name: Verify amd64 darwin binary
     steps:
       - uses: actions/checkout@v2
-      - name: Download artifact
+
-        uses: actions/download-artifact@v2
+      - name: Download amd64 darwin zip
+        uses: actions/download-artifact@v3
         with:
-          name: ${{ env.PKG_NAME }}_${{env.version}}_darwin_${{ matrix.arch }}.zip
+          name: ${{ env.zip_name }}
-      - name: Run verification for ${{ matrix.arch }} linux zip
+
-        if: ${{ matrix.arch == 'amd64' }}
+      - name: Unzip amd64 darwin zip
-        run: |
+        run: unzip ${{ env.zip_name }}
-          unzip ${{ env.PKG_NAME }}_${{env.version}}_darwin_${{ matrix.arch }}.zip
+
-          ./consul version
+      - name: Run verification for amd64 darwin binary
+        run: .github/scripts/verify_bin.sh ./consul v${{ env.version }}
 
   verify-linux-packages-deb:
-    needs: build
+    needs:
+      - build
+      - get-product-version
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        arch: [ "amd64", "arm", "arm64", "i386" ]
+        arch: ["i386", "amd64", "arm", "arm64"]
-      fail-fast: true
+      # fail-fast: true
+    env:
+      version: ${{ needs.get-product-version.outputs.product-version }}
 
     name: Verify ${{ matrix.arch }} debian package
     steps:
       - uses: actions/checkout@v2
-
+        
-      - name: Download ${{ matrix.arch }} debian package
+      - name: Set package version
         run: |
-          echo "TODO: download ${{ matrix.arch }} debian package"
+          echo "pkg_version=$(echo ${{ needs.get-product-version.outputs.product-version }} | sed 's/\-/~/g')" >> $GITHUB_ENV 
+
+      - name: Set package name
+        run: |
+          echo "pkg_name=consul_${{ env.pkg_version }}-1_${{ matrix.arch }}.deb" >> $GITHUB_ENV 
+
+      - name: Download workflow artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ env.pkg_name }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: all
 
       - name: Verify ${{ matrix.arch }} debian package
-        run: |
+        run: ./.github/scripts/verify_artifact.sh ${{ env.pkg_name }} v${{ env.version }}
-          echo "TODO: run verification for ${{ matrix.arch }} debian package"
 
   verify-linux-packages-rpm:
-    needs: build
+    needs:
+      - build
+      - get-product-version
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        arch: [ "aarch64", "arm", "i386", "x86_64" ]
+        arch: ["i386", "x86_64", "arm", "aarch64"]
-      fail-fast: true
+      # fail-fast: true
+    env:
+      version: ${{ needs.get-product-version.outputs.product-version }}
 
-    name: Verify ${{ matrix.arch }} rpm package
+    name: Verify ${{ matrix.arch }} rpm
     steps:
       - uses: actions/checkout@v2
 
-      - name: Download ${{ matrix.arch }} rpm package
+      - name: Set package version
         run: |
-          echo "TODO: download ${{ matrix.arch }} rpm package"
+          echo "pkg_version=$(echo ${{ needs.get-product-version.outputs.product-version }} | sed 's/\-/~/g')" >> $GITHUB_ENV 
 
-      - name: Verify ${{ matrix.arch }} rpm package
+      - name: Set package name
         run: |
-          echo "TODO: run verification for ${{ matrix.arch }} rpm package"
+          echo "pkg_name=consul-${{ env.pkg_version }}-1.${{ matrix.arch }}.rpm" >> $GITHUB_ENV 
+        
+      - name: Download workflow artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ env.pkg_name }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: all
+
+      - name: Verify ${{ matrix.arch }} rpm 
+        run: ./.github/scripts/verify_artifact.sh ${{ env.pkg_name }} v${{ env.version }}