consul/.github/workflows/go-tests.yml
Michael Zalimeni 1d9234a87a
ci: sanitize commit message for Slack failure alerts (#19876)
To ensure that shell code cannot be injected, capture the commit message
in an env var, then format it as needed.

Also fix several other issues with formatting and JSON escaping by
wrapping the entire message in a `toJSON` expression.
2023-12-08 16:04:45 -05:00

548 lines
21 KiB
YAML

name: go-tests
on:
pull_request:
branches-ignore:
- stable-website
- 'docs/**'
- 'ui/**'
- 'mktg-**' # Digital Team Terraform-generated branches' prefix
- 'backport/docs/**'
- 'backport/ui/**'
- 'backport/mktg-**'
push:
branches:
# Push events on the main branch
- main
- release/**
permissions:
contents: read
env:
TEST_RESULTS: /tmp/test-results
GOPRIVATE: github.com/hashicorp # Required for enterprise deps
SKIP_CHECK_BRANCH: ${{ github.head_ref || github.ref_name }}
# concurrency
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: true
jobs:
conditional-skip:
runs-on: ubuntu-latest
name: Get files changed and conditionally skip CI
outputs:
skip-ci: ${{ steps.read-files.outputs.skip-ci }}
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
with:
fetch-depth: 0
- name: Get changed files
id: read-files
run: ./.github/scripts/filter_changed_files_go_test.sh
setup:
needs: [conditional-skip]
name: Setup
if: needs.conditional-skip.outputs.skip-ci != 'true'
runs-on: ubuntu-latest
outputs:
compute-small: ${{ steps.setup-outputs.outputs.compute-small }}
compute-medium: ${{ steps.setup-outputs.outputs.compute-medium }}
compute-large: ${{ steps.setup-outputs.outputs.compute-large }}
compute-xl: ${{ steps.setup-outputs.outputs.compute-xl }}
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
- id: setup-outputs
name: Setup outputs
run: ./.github/scripts/get_runner_classes.sh
check-go-mod:
needs:
- setup
uses: ./.github/workflows/reusable-check-go-mod.yml
with:
runs-on: ${{ needs.setup.outputs.compute-small }}
repository-name: ${{ github.repository }}
secrets:
elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
check-generated-protobuf:
needs:
- setup
runs-on: ${{ fromJSON(needs.setup.outputs.compute-medium) }}
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
# NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.
- name: Setup Git
if: ${{ endsWith(github.repository, '-enterprise') }}
run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com"
- uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
with:
go-version-file: 'go.mod'
- run: make proto-tools
name: Install protobuf
- run: make proto-format
name: "Protobuf Format"
- run: make --always-make proto
- run: |
if ! git diff --exit-code; then
echo "Generated code was not updated correctly"
exit 1
fi
- run: make proto-lint
name: "Protobuf Lint"
check-codegen:
needs:
- setup
runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }}
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
# NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.
- name: Setup Git
if: ${{ endsWith(github.repository, '-enterprise') }}
run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com"
- uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
with:
go-version-file: 'go.mod'
- run: make --always-make codegen
- run: |
if ! git diff --exit-code; then
echo "Generated code was not updated correctly"
exit 1
fi
lint-enums:
needs:
- setup
runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }}
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
# NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.
- name: Setup Git
if: ${{ endsWith(github.repository, '-enterprise') }}
run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com"
- uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
with:
go-version-file: 'go.mod'
- run: go install github.com/reillywatson/enumcover/cmd/enumcover@master && enumcover ./...
lint-container-test-deps:
needs:
- setup
runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
# NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.
- name: Setup Git
run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com"
- uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
with:
go-version-file: 'go.mod'
- run: make lint-container-test-deps
lint-consul-retry:
needs:
- setup
runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
# NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.
- name: Setup Git
if: ${{ endsWith(github.repository, '-enterprise') }}
run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com"
- uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
with:
go-version-file: 'go.mod'
- run: make lint-consul-retry
lint:
needs:
- setup
uses: ./.github/workflows/reusable-lint.yml
with:
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
secrets:
elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
lint-32bit:
needs:
- setup
uses: ./.github/workflows/reusable-lint.yml
with:
go-arch: "386"
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
secrets:
elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
# create a development build
dev-build:
needs:
- setup
uses: ./.github/workflows/reusable-dev-build.yml
with:
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
secrets:
elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
# dev-build-s390x:
# if: ${{ endsWith(github.repository, '-enterprise') }}
# needs:
# - setup
# uses: ./.github/workflows/reusable-dev-build.yml
# with:
# uploaded-binary-name: 'consul-bin-s390x'
# runs-on: ${{ needs.setup.outputs.compute-large }}
# go-arch: "s390x"
# repository-name: ${{ github.repository }}
# secrets:
# elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
# dev-build-arm64:
# # only run on enterprise because GHA does not have arm64 runners in CE
# if: ${{ endsWith(github.repository, '-enterprise') }}
# needs:
# - setup
# uses: ./.github/workflows/reusable-dev-build.yml
# with:
# uploaded-binary-name: 'consul-bin-arm64'
# runs-on: ${{ needs.setup.outputs.compute-large }}
# go-arch: "arm64"
# repository-name: ${{ github.repository }}
# secrets:
# elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
# go-test-arm64:
# # only run on enterprise because GHA does not have arm64 runners in CE
# if: ${{ endsWith(github.repository, '-enterprise') }}
# needs:
# - setup
# - dev-build-arm64
# uses: ./.github/workflows/reusable-unit-split.yml
# with:
# directory: .
# uploaded-binary-name: 'consul-bin-arm64'
# runner-count: 12
# runs-on: "['self-hosted', 'ondemand', 'os=macos-arm', 'arm64']"
# go-test-flags: "${{ (github.ref_name != 'main' && !startsWith(github.ref_name, 'release/')) && '-short' || '' }}"
# repository-name: ${{ github.repository }}
# secrets:
# elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
# consul-license: ${{secrets.CONSUL_LICENSE}}
# datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
go-test-ce:
needs:
- setup
- dev-build
uses: ./.github/workflows/reusable-unit-split.yml
with:
directory: .
runner-count: 6
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
go-tags: ""
go-test-flags: "${{ (github.ref_name != 'main' && !startsWith(github.ref_name, 'release/')) && '-short' || '' }}"
permissions:
id-token: write # NOTE: this permission is explicitly required for Vault auth.
contents: read
secrets:
elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
consul-license: ${{secrets.CONSUL_LICENSE}}
datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
go-test-enterprise:
if: ${{ endsWith(github.repository, '-enterprise') }}
needs:
- setup
- dev-build
uses: ./.github/workflows/reusable-unit-split.yml
with:
directory: .
runner-count: 6
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consuldev' || '' }}"
go-test-flags: "${{ (github.ref_name != 'main' && !startsWith(github.ref_name, 'release/')) && '-short' || '' }}"
permissions:
id-token: write # NOTE: this permission is explicitly required for Vault auth.
contents: read
secrets:
elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
consul-license: ${{secrets.CONSUL_LICENSE}}
datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
go-test-race:
needs:
- setup
- dev-build
uses: ./.github/workflows/reusable-unit.yml
with:
directory: .
go-test-flags: "-race -gcflags=all=-d=checkptr=0"
package-names-command: "go list ./... | grep -E -v '^github.com/hashicorp/consul/agent(/consul|/local|/routine-leak-checker)?$' | grep -E -v '^github.com/hashicorp/consul(/command|/connect|/snapshot)'"
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consuldev' || '' }}"
permissions:
id-token: write # NOTE: this permission is explicitly required for Vault auth.
contents: read
secrets:
elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
consul-license: ${{secrets.CONSUL_LICENSE}}
datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
go-test-32bit:
needs:
- setup
- dev-build
uses: ./.github/workflows/reusable-unit.yml
with:
directory: .
go-arch: "386"
go-test-flags: "-short"
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consuldev' || '' }}"
permissions:
id-token: write # NOTE: this permission is explicitly required for Vault auth.
contents: read
secrets:
elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
consul-license: ${{secrets.CONSUL_LICENSE}}
datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
# go-test-s390x:
# if: ${{ endsWith(github.repository, '-enterprise') }}
# needs:
# - setup
# - dev-build-s390x
# uses: ./.github/workflows/reusable-unit.yml
# with:
# uploaded-binary-name: 'consul-bin-s390x'
# directory: .
# go-test-flags: -short"
# runs-on: ${{ needs.setup.outputs.compute-large }}
# repository-name: ${{ github.repository }}
# go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consuldev' || '' }}"
# permissions:
# id-token: write # NOTE: this permission is explicitly required for Vault auth.
# contents: read
# secrets:
# elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
# consul-license: ${{secrets.CONSUL_LICENSE}}
# datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
go-test-envoyextensions:
needs:
- setup
- dev-build
uses: ./.github/workflows/reusable-unit.yml
with:
directory: envoyextensions
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consuldev' || '' }}"
permissions:
id-token: write # NOTE: this permission is explicitly required for Vault auth.
contents: read
secrets:
elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
consul-license: ${{secrets.CONSUL_LICENSE}}
datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
go-test-troubleshoot:
needs:
- setup
- dev-build
uses: ./.github/workflows/reusable-unit.yml
with:
directory: troubleshoot
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consuldev' || '' }}"
permissions:
id-token: write # NOTE: this permission is explicitly required for Vault auth.
contents: read
secrets:
elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
consul-license: ${{secrets.CONSUL_LICENSE}}
datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
go-test-api-1-19:
needs:
- setup
- dev-build
uses: ./.github/workflows/reusable-unit.yml
with:
directory: api
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consuldev' || '' }}"
go-version: "1.19"
permissions:
id-token: write # NOTE: this permission is explicitly required for Vault auth.
contents: read
secrets:
elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
consul-license: ${{secrets.CONSUL_LICENSE}}
datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
go-test-api-1-20:
needs:
- setup
- dev-build
uses: ./.github/workflows/reusable-unit.yml
with:
directory: api
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consuldev' || '' }}"
go-version: "1.20"
permissions:
id-token: write # NOTE: this permission is explicitly required for Vault auth.
contents: read
secrets:
elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
consul-license: ${{secrets.CONSUL_LICENSE}}
datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
go-test-sdk-1-19:
needs:
- setup
- dev-build
uses: ./.github/workflows/reusable-unit.yml
with:
directory: sdk
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consuldev' || '' }}"
go-version: "1.19"
permissions:
id-token: write # NOTE: this permission is explicitly required for Vault auth.
contents: read
secrets:
elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
consul-license: ${{secrets.CONSUL_LICENSE}}
datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
go-test-sdk-1-20:
needs:
- setup
- dev-build
uses: ./.github/workflows/reusable-unit.yml
with:
directory: sdk
runs-on: ${{ needs.setup.outputs.compute-large }}
repository-name: ${{ github.repository }}
go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consuldev' || '' }}"
go-version: "1.20"
permissions:
id-token: write # NOTE: this permission is explicitly required for Vault auth.
contents: read
secrets:
elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
consul-license: ${{secrets.CONSUL_LICENSE}}
datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
noop:
runs-on: ubuntu-latest
steps:
- run: "echo ok"
# This is job is required for branch protection as a required gihub check
# because GitHub actions show up as checks at the job level and not the
# workflow level. This is currently a feature request:
# https://github.com/orgs/community/discussions/12395
#
# This job must:
# - be placed after the fanout of a workflow so that everything fans back in
# to this job.
# - "need" any job that is part of the fan out / fan in
# - implement the if logic because we have conditional jobs
# (go-test-enteprise) that this job needs and this would potentially get
# skipped if a previous job got skipped. So we use the if clause to make
# sure it does not get skipped.
go-tests-success:
needs:
- conditional-skip
- setup
- check-codegen
- check-generated-protobuf
- check-go-mod
- lint-consul-retry
- lint-container-test-deps
- lint-enums
- lint
- lint-32bit
# - go-test-arm64
- go-test-enterprise
- go-test-ce
- go-test-race
- go-test-envoyextensions
- go-test-troubleshoot
- go-test-api-1-19
- go-test-api-1-20
- go-test-sdk-1-19
- go-test-sdk-1-20
- go-test-32bit
# - go-test-s390x
runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
if: always() && needs.conditional-skip.outputs.skip-ci != 'true'
steps:
- name: evaluate upstream job results
run: |
# exit 1 if failure or cancelled result for any upstream job
# this ensures that we fail the PR check regardless of cancellation, rather than skip-passing it
# see https://docs.github.com/en/actions/using-jobs/using-conditions-to-control-job-execution#overview
if printf '${{ toJSON(needs) }}' | grep -E -i '\"result\": \"(failure|cancelled)\"'; then
printf "Tests failed or workflow cancelled:\n\n${{ toJSON(needs) }}"
exit 1
fi
- name: Set failure Slack commit message summary
# failure() ensures this runs even if the test eval step exits 1
if: failure()
env:
# Capturing in an env var makes this safe against GHA shell injection via commit message.
# See https://securitylab.github.com/research/github-actions-untrusted-input/
COMMIT_MESSAGE_FULL: ${{ github.event.head_commit.message }}
run: |
# if failure (not cancelled), notify Slack
if printf '${{ toJSON(needs) }}' | grep -E -i '\"result\": \"(failure)\"'; then
printf "Tests failed, notifying Slack"
echo "FAILED_TESTS=true" >> $GITHUB_ENV
# 'echo ... | head -n 1' does not work reliably, so use bash-ism to get first line.
COMMIT_MESSAGE_SUMMARY=${COMMIT_MESSAGE_FULL%%$'\n'*}
# Send multi-line env var to GITHUB_ENV.
# github.event.head_commit.message and github.ref_name both rely on this event occurring on a push / merge
echo "SLACK_MESSAGE_RAW<<EOF" >> $GITHUB_ENV
echo "❌ ${{ github.workflow }} workflow failed:
- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- Branch: ${{ github.ref_name }}
- Message: ${COMMIT_MESSAGE_SUMMARY}
- Author: ${{ github.event.sender.login }}" >> $GITHUB_ENV
echo "EOF" >> $GITHUB_ENV
fi
- name: Notify Slack
# failure() ensures this runs even if the test eval step exits 1
# FAILED_TESTS must also be checked to avoid running this step on cancellation due to the summary check above
if: ${{ failure() && env.FAILED_TESTS == 'true' && (github.ref_name == 'main' || startsWith(github.ref_name, 'release/')) }}
id: slack
uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
with:
# Escape entire message string to ensure valid JSON. If invalid, the notification will fail silently in CI.
payload: |
{
"message": ${{ toJSON(env.SLACK_MESSAGE_RAW) }}
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.CONSUL_PROTECTED_BRANCH_TEST_SLACK_WEBHOOK }}