From 1339c79f8dccaf3dfb8eb59b106a1807e388b497 Mon Sep 17 00:00:00 2001 From: cskh Date: Wed, 17 May 2023 14:57:12 -0400 Subject: [PATCH] consul-container test: no splitting and on single runner (#17394) --- .github/workflows/test-integrations.yml | 99 ++----------------- .../libs/topology/peering_topology.go | 16 ++- .../test/gateways/http_route_test.go | 2 - .../rotate_server_and_ca_then_fail_test.go | 9 +- .../test/snapshot/snapshot_restore_test.go | 4 +- .../upgrade/basic/fullstopupgrade_test.go | 2 - .../peering/peering_control_plane_mgw_test.go | 9 +- .../test/upgrade/peering/peering_http_test.go | 36 ++++++- 8 files changed, 73 insertions(+), 104 deletions(-) diff --git a/.github/workflows/test-integrations.yml b/.github/workflows/test-integrations.yml index e1d86f031f..555d534f1a 100644 --- a/.github/workflows/test-integrations.yml +++ b/.github/workflows/test-integrations.yml @@ -343,51 +343,14 @@ jobs: DD_ENV: ci run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml - generate-compatibility-job-matrices: - needs: [setup] - runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} - name: Generate Compatibility Job Matrices - outputs: - compatibility-matrix: ${{ steps.set-matrix.outputs.compatibility-matrix }} - steps: - - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 - - name: Generate Compatibility Job Matrix - id: set-matrix - env: - TOTAL_RUNNERS: 6 - JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]' - run: | - cd ./test/integration/consul-container - NUM_RUNNERS=$TOTAL_RUNNERS - NUM_DIRS=$(find ./test -mindepth 1 -maxdepth 2 -type d | wc -l) - - if [ "$NUM_DIRS" -lt "$NUM_RUNNERS" ]; then - echo "TOTAL_RUNNERS is larger than the number of tests/packages to split." - NUM_RUNNERS=$((NUM_DIRS-1)) - fi - # fix issue where test splitting calculation generates 1 more split than TOTAL_RUNNERS. - NUM_RUNNERS=$((NUM_RUNNERS-1)) - { - echo -n "compatibility-matrix=" - find ./test -maxdepth 2 -type d -print0 | xargs -0 -n 1 \ - | grep -v util | grep -v upgrade \ - | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \ - | jq --compact-output 'map(join(" "))' - } >> "$GITHUB_OUTPUT" - compatibility-integration-test: runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} needs: - setup - dev-build - - generate-compatibility-job-matrices permissions: id-token: write # NOTE: this permission is explicitly required for Vault auth. contents: read - strategy: - fail-fast: false - matrix: - test-cases: ${{ fromJSON(needs.generate-compatibility-job-matrices.outputs.compatibility-matrix) }} env: ENVOY_VERSION: "1.25.4" steps: @@ -396,8 +359,10 @@ jobs: with: go-version-file: 'go.mod' - run: go env - - # Build the consul:local image from the already built binary + - name: docker env + run: | + docker version + docker info - name: fetch binary uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: @@ -405,7 +370,7 @@ jobs: path: . - name: restore mode+x run: chmod +x consul - + # Build the consul:local image from the already built binary - name: Build consul:local image run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile . - name: Build consul-envoy:target-version image @@ -426,21 +391,19 @@ jobs: mkdir -p "/tmp/test-results" cd ./test/integration/consul-container docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version - echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests" - # shellcheck disable=SC2001 - sed 's, ,\n,g' <<< "${{ matrix.test-cases }}" go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ --raw-command \ - --format=short-verbose \ + --format=standard-verbose \ --debug \ --rerun-fails=3 \ -- \ go test \ - -p=4 \ + -p=6 \ + -parallel=4 \ -tags "${{ env.GOTAGS }}" \ -timeout=30m \ -json \ - ${{ matrix.test-cases }} \ + `go list ./... | grep -v upgrade` \ --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \ --target-version local \ --latest-image docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }} \ @@ -483,52 +446,16 @@ jobs: DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" DD_ENV: ci run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml - - generate-upgrade-job-matrices: - needs: [setup] - runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} - name: Generate Upgrade Job Matrices - outputs: - upgrade-matrix: ${{ steps.set-matrix.outputs.upgrade-matrix }} - steps: - - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 - - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 - with: - go-version-file: 'go.mod' - - name: Generate Updgrade Job Matrix - id: set-matrix - env: - TOTAL_RUNNERS: 5 - JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]' - run: | - cd ./test/integration/consul-container/test/upgrade - NUM_RUNNERS=$TOTAL_RUNNERS - NUM_DIRS=$(go test ./... -list=. -json | jq -r '.Output | select (. !=null) | select(. | startswith("Test")) | gsub("[\\n\\t]"; "")' | wc -l) - - if [ "$NUM_DIRS" -lt "$NUM_RUNNERS" ]; then - echo "TOTAL_RUNNERS is larger than the number of tests/packages to split." - NUM_RUNNERS=$((NUM_DIRS-1)) - fi - # fix issue where test splitting calculation generates 1 more split than TOTAL_RUNNERS. - NUM_RUNNERS=$((NUM_RUNNERS-1)) - { - echo -n "upgrade-matrix=" - go test ./... -list=. -json | jq -r '.Output | select (. !=null) | select(. | startswith("Test")) | gsub("[\\n\\t]"; "")' \ - | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \ - | jq --compact-output 'map(join("|"))' - } >> "$GITHUB_OUTPUT" upgrade-integration-test: - runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} + runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} needs: - setup - dev-build - - generate-upgrade-job-matrices strategy: fail-fast: false matrix: consul-version: [ "1.14", "1.15"] - test-cases: ${{ fromJSON(needs.generate-upgrade-job-matrices.outputs.upgrade-matrix) }} env: CONSUL_LATEST_VERSION: ${{ matrix.consul-version }} ENVOY_VERSION: "1.24.6" @@ -576,9 +503,6 @@ jobs: mkdir -p "${{ env.TEST_RESULTS_DIR }}" cd ./test/integration/consul-container/test/upgrade docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version - echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests" - # shellcheck disable=SC2001 - sed 's,|,\n,g' <<< "${{ matrix.test-cases }}" go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ --raw-command \ --format=short-verbose \ @@ -591,7 +515,6 @@ jobs: -tags "${{ env.GOTAGS }}" \ -timeout=30m \ -json ./... \ - -run "${{ matrix.test-cases }}" \ --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \ --target-version local \ --latest-image docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }} \ @@ -617,9 +540,7 @@ jobs: - vault-integration-test - generate-envoy-job-matrices - envoy-integration-test - - generate-compatibility-job-matrices - compatibility-integration-test - - generate-upgrade-job-matrices - upgrade-integration-test runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} if: ${{ always() }} diff --git a/test/integration/consul-container/libs/topology/peering_topology.go b/test/integration/consul-container/libs/topology/peering_topology.go index 87c2743f3d..c684764d48 100644 --- a/test/integration/consul-container/libs/topology/peering_topology.go +++ b/test/integration/consul-container/libs/topology/peering_topology.go @@ -32,6 +32,13 @@ type BuiltCluster struct { Gateway libservice.Service } +type PeeringClusterSize struct { + AcceptingNumServers int + AcceptingNumClients int + DialingNumServers int + DialingNumClients int +} + // BasicPeeringTwoClustersSetup sets up a scenario for testing peering, which consists of // // - an accepting cluster with 3 servers and 1 client agent. The client should be used to @@ -46,11 +53,12 @@ func BasicPeeringTwoClustersSetup( t *testing.T, consulImage string, consulVersion string, + pcs PeeringClusterSize, peeringThroughMeshgateway bool, ) (*BuiltCluster, *BuiltCluster) { acceptingCluster, acceptingCtx, acceptingClient := NewCluster(t, &ClusterConfig{ - NumServers: 3, - NumClients: 1, + NumServers: pcs.AcceptingNumServers, + NumClients: pcs.AcceptingNumClients, BuildOpts: &libcluster.BuildOptions{ Datacenter: "dc1", ConsulImageName: consulImage, @@ -61,8 +69,8 @@ func BasicPeeringTwoClustersSetup( }) dialingCluster, dialingCtx, dialingClient := NewCluster(t, &ClusterConfig{ - NumServers: 1, - NumClients: 1, + NumServers: pcs.DialingNumServers, + NumClients: pcs.DialingNumClients, BuildOpts: &libcluster.BuildOptions{ Datacenter: "dc2", ConsulImageName: consulImage, diff --git a/test/integration/consul-container/test/gateways/http_route_test.go b/test/integration/consul-container/test/gateways/http_route_test.go index c8236294b0..0a33ef2fae 100644 --- a/test/integration/consul-container/test/gateways/http_route_test.go +++ b/test/integration/consul-container/test/gateways/http_route_test.go @@ -502,8 +502,6 @@ func TestHTTPRouteParentRefChange(t *testing.T) { t.Skip("too slow for testing.Short") } - t.Parallel() - // infrastructure set up address := "localhost" diff --git a/test/integration/consul-container/test/peering/rotate_server_and_ca_then_fail_test.go b/test/integration/consul-container/test/peering/rotate_server_and_ca_then_fail_test.go index ccf5b206ce..86285f2c60 100644 --- a/test/integration/consul-container/test/peering/rotate_server_and_ca_then_fail_test.go +++ b/test/integration/consul-container/test/peering/rotate_server_and_ca_then_fail_test.go @@ -53,7 +53,14 @@ import ( func TestPeering_RotateServerAndCAThenFail_(t *testing.T) { t.Parallel() - accepting, dialing := libtopology.BasicPeeringTwoClustersSetup(t, utils.GetTargetImageName(), utils.TargetVersion, false) + accepting, dialing := libtopology.BasicPeeringTwoClustersSetup(t, utils.GetTargetImageName(), utils.TargetVersion, + libtopology.PeeringClusterSize{ + AcceptingNumServers: 3, + AcceptingNumClients: 1, + DialingNumServers: 1, + DialingNumClients: 1, + }, + false) var ( acceptingCluster = accepting.Cluster dialingCluster = dialing.Cluster diff --git a/test/integration/consul-container/test/snapshot/snapshot_restore_test.go b/test/integration/consul-container/test/snapshot/snapshot_restore_test.go index 47a03f4323..70fa0d2d41 100644 --- a/test/integration/consul-container/test/snapshot/snapshot_restore_test.go +++ b/test/integration/consul-container/test/snapshot/snapshot_restore_test.go @@ -82,7 +82,9 @@ func testSnapShotRestoreForLogStore(t *testing.T, logStore libcluster.LogStore) libcluster.WaitForMembers(t, client2, 3) // Restore the saved snapshot - require.NoError(t, client2.Snapshot().Restore(nil, snapshot)) + retry.RunWith(libcluster.LongFailer(), t, func(r *retry.R) { + require.NoError(r, client2.Snapshot().Restore(nil, snapshot)) + }) libcluster.WaitForLeader(t, cluster2, client2) diff --git a/test/integration/consul-container/test/upgrade/basic/fullstopupgrade_test.go b/test/integration/consul-container/test/upgrade/basic/fullstopupgrade_test.go index e05155c639..ff5c9f36a5 100644 --- a/test/integration/consul-container/test/upgrade/basic/fullstopupgrade_test.go +++ b/test/integration/consul-container/test/upgrade/basic/fullstopupgrade_test.go @@ -32,8 +32,6 @@ var ( // Test upgrade a cluster of latest version to the target version func TestStandardUpgradeToTarget_fromLatest(t *testing.T) { - t.Parallel() - tcs = append(tcs, testcase{ // Use the case of "1.12.3" ==> "1.13.0" to verify the test can diff --git a/test/integration/consul-container/test/upgrade/peering/peering_control_plane_mgw_test.go b/test/integration/consul-container/test/upgrade/peering/peering_control_plane_mgw_test.go index 4e9355657f..ef245c7c2b 100644 --- a/test/integration/consul-container/test/upgrade/peering/peering_control_plane_mgw_test.go +++ b/test/integration/consul-container/test/upgrade/peering/peering_control_plane_mgw_test.go @@ -27,7 +27,14 @@ import ( func TestPeering_ControlPlaneMGW(t *testing.T) { t.Parallel() - accepting, dialing := libtopology.BasicPeeringTwoClustersSetup(t, utils.GetLatestImageName(), utils.LatestVersion, true) + accepting, dialing := libtopology.BasicPeeringTwoClustersSetup(t, utils.GetLatestImageName(), utils.LatestVersion, + libtopology.PeeringClusterSize{ + AcceptingNumServers: 1, + AcceptingNumClients: 1, + DialingNumServers: 1, + DialingNumClients: 1, + }, + true) var ( acceptingCluster = accepting.Cluster dialingCluster = dialing.Cluster diff --git a/test/integration/consul-container/test/upgrade/peering/peering_http_test.go b/test/integration/consul-container/test/upgrade/peering/peering_http_test.go index 86feccc8fd..08471eb7d2 100644 --- a/test/integration/consul-container/test/upgrade/peering/peering_http_test.go +++ b/test/integration/consul-container/test/upgrade/peering/peering_http_test.go @@ -20,14 +20,28 @@ import ( func TestPeering_Basic(t *testing.T) { t.Parallel() - accepting, dialing := libtopology.BasicPeeringTwoClustersSetup(t, utils.GetLatestImageName(), utils.LatestVersion, false) + accepting, dialing := libtopology.BasicPeeringTwoClustersSetup(t, utils.GetLatestImageName(), utils.LatestVersion, + libtopology.PeeringClusterSize{ + AcceptingNumServers: 1, + AcceptingNumClients: 1, + DialingNumServers: 1, + DialingNumClients: 1, + }, + false) peeringUpgrade(t, accepting, dialing, utils.TargetVersion) peeringPostUpgradeValidation(t, dialing) } func TestPeering_HTTPRouter(t *testing.T) { t.Parallel() - accepting, dialing := libtopology.BasicPeeringTwoClustersSetup(t, utils.GetLatestImageName(), utils.LatestVersion, false) + accepting, dialing := libtopology.BasicPeeringTwoClustersSetup(t, utils.GetLatestImageName(), utils.LatestVersion, + libtopology.PeeringClusterSize{ + AcceptingNumServers: 1, + AcceptingNumClients: 1, + DialingNumServers: 1, + DialingNumClients: 1, + }, + false) acceptingCluster := accepting.Cluster // Create a second static-server at the client agent of accepting cluster and @@ -90,7 +104,14 @@ func TestPeering_HTTPRouter(t *testing.T) { func TestPeering_HTTPResolverAndFailover(t *testing.T) { t.Parallel() - accepting, dialing := libtopology.BasicPeeringTwoClustersSetup(t, utils.GetLatestImageName(), utils.LatestVersion, false) + accepting, dialing := libtopology.BasicPeeringTwoClustersSetup(t, utils.GetLatestImageName(), utils.LatestVersion, + libtopology.PeeringClusterSize{ + AcceptingNumServers: 1, + AcceptingNumClients: 1, + DialingNumServers: 1, + DialingNumClients: 1, + }, + false) dialingCluster := dialing.Cluster require.NoError(t, dialingCluster.ConfigEntryWrite(&api.ProxyConfigEntry{ @@ -183,7 +204,14 @@ func TestPeering_HTTPResolverAndFailover(t *testing.T) { func TestPeering_HTTPResolverAndSplitter(t *testing.T) { t.Parallel() - accepting, dialing := libtopology.BasicPeeringTwoClustersSetup(t, utils.GetLatestImageName(), utils.LatestVersion, false) + accepting, dialing := libtopology.BasicPeeringTwoClustersSetup(t, utils.GetLatestImageName(), utils.LatestVersion, + libtopology.PeeringClusterSize{ + AcceptingNumServers: 1, + AcceptingNumClients: 1, + DialingNumServers: 1, + DialingNumClients: 1, + }, + false) dialingCluster := dialing.Cluster require.NoError(t, dialingCluster.ConfigEntryWrite(&api.ProxyConfigEntry{