From 3b3aa1f26064eb00cfee21340b0df54f860ab1d5 Mon Sep 17 00:00:00 2001 From: Dan Bond Date: Wed, 12 Jul 2023 16:10:34 -0700 Subject: [PATCH] [NET-4103] ci: build s390x (#18067) * ci: build s390x * ci: test s390x * ci: dev build s390x * no GOOS * ent only * build: publish s390x * fix syntax error * fix syntax error again * fix syntax error again x2 * test branch * Move s390x conditionals to step level * remove test branch --------- Co-authored-by: emilymianeil --- .github/workflows/build-distros.yml | 48 +++++--- .github/workflows/build.yml | 59 ++++++++- .github/workflows/go-tests.yml | 145 ++++++++++++++--------- .github/workflows/reusable-dev-build.yml | 2 +- 4 files changed, 182 insertions(+), 72 deletions(-) diff --git a/.github/workflows/build-distros.yml b/.github/workflows/build-distros.yml index 8b88345d2e..10c5208933 100644 --- a/.github/workflows/build-distros.yml +++ b/.github/workflows/build-distros.yml @@ -2,7 +2,7 @@ # It is aimed at checking new commits don't introduce any breaking build changes. name: build-distros -on: +on: pull_request: push: branches: @@ -33,7 +33,7 @@ jobs: run: ./.github/scripts/get_runner_classes.sh check-go-mod: - needs: + needs: - setup uses: ./.github/workflows/reusable-check-go-mod.yml with: @@ -43,8 +43,8 @@ jobs: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} build-386: - needs: - - setup + needs: + - setup - check-go-mod env: XC_OS: "freebsd linux windows" @@ -56,7 +56,7 @@ jobs: - name: Setup Git if: ${{ endsWith(github.repository, '-enterprise') }} run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" - + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 with: go-version-file: 'go.mod' @@ -68,7 +68,7 @@ jobs: build-amd64: needs: - - setup + - setup - check-go-mod env: XC_OS: "darwin freebsd linux solaris windows" @@ -80,7 +80,7 @@ jobs: - name: Setup Git if: ${{ endsWith(github.repository, '-enterprise') }} run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" - + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 with: go-version-file: 'go.mod' @@ -92,7 +92,7 @@ jobs: build-arm: needs: - - setup + - setup - check-go-mod runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} env: @@ -105,7 +105,7 @@ jobs: - name: Setup Git if: ${{ endsWith(github.repository, '-enterprise') }} run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" - + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 with: @@ -117,6 +117,26 @@ jobs: - run: CC=arm-linux-gnueabihf-gcc GOARCH=arm GOARM=6 go build -tags "${{ env.GOTAGS }}" - run: CC=aarch64-linux-gnu-gcc GOARCH=arm64 go build -tags "${{ env.GOTAGS }}" + + build-s390x: + if: ${{ endsWith(github.repository, '-enterprise') }} + needs: + - setup + - check-go-mod + runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} + steps: + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" + + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version-file: 'go.mod' + - name: Build + run: GOOS=linux GOARCH=s390x CGO_ENABLED=0 go build -tags "${{ env.GOTAGS }}" + # This is job is required for branch protection as a required gihub check # because GitHub actions show up as checks at the job level and not the # workflow level. This is currently a feature request: @@ -126,18 +146,18 @@ jobs: # - be placed after the fanout of a workflow so that everything fans back in # to this job. # - "need" any job that is part of the fan out / fan in - # - implement the if logic because we have conditional jobs - # (go-test-enteprise) that this job needs and this would potentially get - # skipped if a previous job got skipped. So we use the if clause to make + # - implement the if logic because we have conditional jobs + # (go-test-enteprise) that this job needs and this would potentially get + # skipped if a previous job got skipped. So we use the if clause to make # sure it does not get skipped. - build-distros-success: - needs: + needs: - setup - check-go-mod - build-386 - build-amd64 - build-arm + - build-s390x runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} if: ${{ always() }} steps: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9186f12bfe..c788a1536c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -57,6 +57,7 @@ jobs: echo "Product Date: ${{ needs.set-product-version.outputs.product-date }}" echo "Prerelease Version: ${{ needs.set-product-version.outputs.pre-version }}" echo "Ldflags: ${{ needs.set-product-version.outputs.shared-ldflags }}" + generate-metadata-file: needs: set-product-version runs-on: ubuntu-latest @@ -173,6 +174,57 @@ jobs: name: ${{ env.DEB_PACKAGE }} path: out/${{ env.DEB_PACKAGE }} + build-s390x: + needs: set-product-version + if: ${{ endsWith(github.repository, '-enterprise') }} + runs-on: ubuntu-latest + strategy: + matrix: + include: + - {go: "1.20.4", goos: "linux", goarch: "s390x"} + fail-fast: true + + name: Go ${{ matrix.go }} ${{ matrix.goos }} ${{ matrix.goarch }} build + steps: + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + + - name: Setup with node and yarn + uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0 + with: + node-version: '14' + cache: 'yarn' + cache-dependency-path: 'ui/yarn.lock' + + - name: Build UI + run: | + CONSUL_VERSION=${{ needs.set-product-version.outputs.product-version }} + CONSUL_DATE=${{ needs.set-product-version.outputs.product-date }} + CONSUL_BINARY_TYPE=${CONSUL_BINARY_TYPE} + CONSUL_COPYRIGHT_YEAR=$(git show -s --format=%cd --date=format:%Y HEAD) + echo "consul_version is ${CONSUL_VERSION}" + echo "consul_date is ${CONSUL_DATE}" + echo "consul binary type is ${CONSUL_BINARY_TYPE}" + echo "consul copyright year is ${CONSUL_COPYRIGHT_YEAR}" + cd ui && make && cd .. + rm -rf agent/uiserver/dist + mv ui/packages/consul-ui/dist agent/uiserver/ + - name: Go Build + env: + PRODUCT_VERSION: ${{ needs.set-product-version.outputs.product-version }} + PRERELEASE_VERSION: ${{ needs.set-product-version.outputs.pre-version }} + CGO_ENABLED: "0" + GOLDFLAGS: "${{needs.set-product-version.outputs.shared-ldflags}}" + uses: hashicorp/actions-go-build@v0.1.7 + with: + product_name: ${{ env.PKG_NAME }} + product_version: ${{ needs.set-product-version.outputs.product-version }} + go_version: ${{ matrix.go }} + os: ${{ matrix.goos }} + arch: ${{ matrix.goarch }} + reproducible: report + instructions: |- + go build -ldflags="$GOLDFLAGS" -o "$BIN_PATH" -trimpath -buildvcs=false + build-darwin: needs: set-product-version runs-on: macos-latest @@ -186,7 +238,7 @@ jobs: name: Go ${{ matrix.go }} ${{ matrix.goos }} ${{ matrix.goarch }} build steps: - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - + - name: Setup with node and yarn uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0 with: @@ -319,7 +371,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - arch: ["386", "amd64", "arm", "arm64"] + arch: ["386", "amd64", "arm", "arm64", "s390x"] fail-fast: true env: version: ${{ needs.set-product-version.outputs.product-version }} @@ -328,8 +380,10 @@ jobs: name: Verify ${{ matrix.arch }} linux binary steps: - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + if: ${{ endsWith(github.repository, '-enterprise') || matrix.arch != 's390x' }} - name: Download ${{ matrix.arch }} zip + if: ${{ endsWith(github.repository, '-enterprise') || matrix.arch != 's390x' }} uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: name: ${{ env.zip_name }} @@ -342,6 +396,7 @@ jobs: platforms: arm,arm64 - name: Run verification for ${{ matrix.arch }} binary + if: ${{ endsWith(github.repository, '-enterprise') || matrix.arch != 's390x' }} run: .github/scripts/verify_artifact.sh ${{ env.zip_name }} v${{ env.version }} verify-darwin: diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 831271f6f8..9baf90c505 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -1,6 +1,6 @@ name: go-tests -on: +on: pull_request: branches-ignore: - stable-website @@ -15,7 +15,7 @@ on: # Push events on the main branch - main - release/** - + permissions: contents: read @@ -39,7 +39,7 @@ jobs: run: ./.github/scripts/get_runner_classes.sh check-go-mod: - needs: + needs: - setup uses: ./.github/workflows/reusable-check-go-mod.yml with: @@ -49,8 +49,8 @@ jobs: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} check-generated-protobuf: - needs: - - setup + needs: + - setup runs-on: ${{ fromJSON(needs.setup.outputs.compute-medium) }} steps: - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 @@ -75,10 +75,10 @@ jobs: name: "Protobuf Lint" - name: Notify Slack if: ${{ failure() }} - run: .github/scripts/notify_slack.sh + run: .github/scripts/notify_slack.sh check-generated-deep-copy: - needs: - - setup + needs: + - setup runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} steps: - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 @@ -100,8 +100,8 @@ jobs: run: .github/scripts/notify_slack.sh lint-enums: - needs: - - setup + needs: + - setup runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} steps: - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 @@ -112,7 +112,7 @@ jobs: - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 with: go-version-file: 'go.mod' - - run: go install github.com/reillywatson/enumcover/cmd/enumcover@master && enumcover ./... + - run: go install github.com/reillywatson/enumcover/cmd/enumcover@master && enumcover ./... - name: Notify Slack if: ${{ failure() }} run: .github/scripts/notify_slack.sh @@ -135,8 +135,8 @@ jobs: run: .github/scripts/notify_slack.sh lint-consul-retry: - needs: - - setup + needs: + - setup runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} steps: - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 @@ -153,8 +153,8 @@ jobs: run: .github/scripts/notify_slack.sh lint: - needs: - - setup + needs: + - setup uses: ./.github/workflows/reusable-lint.yml with: runs-on: ${{ needs.setup.outputs.compute-xl }} @@ -163,8 +163,8 @@ jobs: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} lint-32bit: - needs: - - setup + needs: + - setup uses: ./.github/workflows/reusable-lint.yml with: go-arch: "386" @@ -175,20 +175,33 @@ jobs: # create a development build dev-build: - needs: - - setup + needs: + - setup uses: ./.github/workflows/reusable-dev-build.yml - with: + with: runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + dev-build-s390x: + if: ${{ endsWith(github.repository, '-enterprise') }} + needs: + - setup + uses: ./.github/workflows/reusable-dev-build.yml + with: + uploaded-binary-name: 'consul-bin-s390x' + runs-on: ${{ needs.setup.outputs.compute-xl }} + go-arch: "s390x" + repository-name: ${{ github.repository }} + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + # dev-build-arm64: # # only run on enterprise because GHA does not have arm64 runners in OSS # if: ${{ endsWith(github.repository, '-enterprise') }} - # needs: - # - setup + # needs: + # - setup # uses: ./.github/workflows/reusable-dev-build.yml # with: # uploaded-binary-name: 'consul-bin-arm64' @@ -201,8 +214,8 @@ jobs: # go-test-arm64: # # only run on enterprise because GHA does not have arm64 runners in OSS # if: ${{ endsWith(github.repository, '-enterprise') }} - # needs: - # - setup + # needs: + # - setup # - dev-build-arm64 # uses: ./.github/workflows/reusable-unit-split.yml # with: @@ -218,8 +231,8 @@ jobs: # datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-oss: - needs: - - setup + needs: + - setup - dev-build uses: ./.github/workflows/reusable-unit-split.yml with: @@ -230,7 +243,7 @@ jobs: go-tags: "" permissions: id-token: write # NOTE: this permission is explicitly required for Vault auth. - contents: read + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} @@ -238,8 +251,8 @@ jobs: go-test-enterprise: if: ${{ endsWith(github.repository, '-enterprise') }} - needs: - - setup + needs: + - setup - dev-build uses: ./.github/workflows/reusable-unit-split.yml with: @@ -250,15 +263,15 @@ jobs: go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" permissions: id-token: write # NOTE: this permission is explicitly required for Vault auth. - contents: read + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-race: - needs: - - setup + needs: + - setup - dev-build uses: ./.github/workflows/reusable-unit.yml with: @@ -270,15 +283,15 @@ jobs: go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" permissions: id-token: write # NOTE: this permission is explicitly required for Vault auth. - contents: read + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-32bit: - needs: - - setup + needs: + - setup - dev-build uses: ./.github/workflows/reusable-unit.yml with: @@ -290,7 +303,28 @@ jobs: go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" permissions: id-token: write # NOTE: this permission is explicitly required for Vault auth. - contents: read + contents: read + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" + + go-test-s390x: + if: ${{ endsWith(github.repository, '-enterprise') }} + needs: + - setup + - dev-build-s390x + uses: ./.github/workflows/reusable-unit.yml + with: + uploaded-binary-name: 'consul-bin-s390x' + directory: . + go-test-flags: 'export GO_TEST_FLAGS="-short"' + runs-on: ${{ needs.setup.outputs.compute-xl }} + repository-name: ${{ github.repository }} + go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} @@ -308,7 +342,7 @@ jobs: go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" permissions: id-token: write # NOTE: this permission is explicitly required for Vault auth. - contents: read + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} @@ -326,15 +360,15 @@ jobs: go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" permissions: id-token: write # NOTE: this permission is explicitly required for Vault auth. - contents: read + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-api-1-19: - needs: - - setup + needs: + - setup - dev-build uses: ./.github/workflows/reusable-unit.yml with: @@ -344,15 +378,15 @@ jobs: go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" permissions: id-token: write # NOTE: this permission is explicitly required for Vault auth. - contents: read + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-api-1-20: - needs: - - setup + needs: + - setup - dev-build uses: ./.github/workflows/reusable-unit.yml with: @@ -362,15 +396,15 @@ jobs: go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" permissions: id-token: write # NOTE: this permission is explicitly required for Vault auth. - contents: read + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-sdk-1-19: - needs: - - setup + needs: + - setup - dev-build uses: ./.github/workflows/reusable-unit.yml with: @@ -380,15 +414,15 @@ jobs: go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" permissions: id-token: write # NOTE: this permission is explicitly required for Vault auth. - contents: read + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-sdk-1-20: - needs: - - setup + needs: + - setup - dev-build uses: ./.github/workflows/reusable-unit.yml with: @@ -398,7 +432,7 @@ jobs: go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" permissions: id-token: write # NOTE: this permission is explicitly required for Vault auth. - contents: read + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} @@ -418,13 +452,13 @@ jobs: # - be placed after the fanout of a workflow so that everything fans back in # to this job. # - "need" any job that is part of the fan out / fan in - # - implement the if logic because we have conditional jobs - # (go-test-enteprise) that this job needs and this would potentially get - # skipped if a previous job got skipped. So we use the if clause to make + # - implement the if logic because we have conditional jobs + # (go-test-enteprise) that this job needs and this would potentially get + # skipped if a previous job got skipped. So we use the if clause to make # sure it does not get skipped. go-tests-success: - needs: + needs: - setup - check-generated-deep-copy - check-generated-protobuf @@ -436,7 +470,7 @@ jobs: - lint-32bit # - go-test-arm64 - go-test-enterprise - - go-test-oss + - go-test-oss - go-test-race - go-test-envoyextensions - go-test-troubleshoot @@ -445,6 +479,7 @@ jobs: - go-test-sdk-1-19 - go-test-sdk-1-20 - go-test-32bit + - go-test-s390x runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} if: ${{ always() }} steps: diff --git a/.github/workflows/reusable-dev-build.yml b/.github/workflows/reusable-dev-build.yml index 3ca6613985..2db9670655 100644 --- a/.github/workflows/reusable-dev-build.yml +++ b/.github/workflows/reusable-dev-build.yml @@ -1,4 +1,4 @@ -name: reusable-dev-build +name: reusable-dev-build on: workflow_call: