consul/.circleci/config.yml

---
version: 2

references:
  images:
    go: &GOLANG_IMAGE circleci/golang:1.13.7
    middleman: &MIDDLEMAN_IMAGE hashicorp/middleman-hashicorp:0.3.40
    ember: &EMBER_IMAGE circleci/node:8-browsers

  paths:
    test-results: &TEST_RESULTS_DIR /tmp/test-results

  cache:
    yarn: &YARN_CACHE_KEY consul-ui-v1-{{ checksum "ui-v2/yarn.lock" }}
    rubygem: &RUBYGEM_CACHE_KEY static-site-gems-v1-{{ checksum "Gemfile.lock" }}

  environment: &ENVIRONMENT
    TEST_RESULTS_DIR: *TEST_RESULTS_DIR
    GOTESTSUM_RELEASE: 0.3.3
    EMAIL: noreply@hashicorp.com
    GIT_AUTHOR_NAME: circleci-consul
    GIT_COMMITTER_NAME: circleci-consul
    S3_ARTIFACT_BUCKET: consul-dev-artifacts
    BASH_ENV: .circleci/bash_env.sh
    VAULT_BINARY_VERSION: 1.2.2

jobs:
  # lint consul tests
  lint-consul-retry:
    docker:
      - image: *GOLANG_IMAGE
    steps:
      - checkout
      - run: go get -u github.com/hashicorp/lint-consul-retry && lint-consul-retry

  # Runs go fmt and go vet
  go-fmt-and-vet:
    docker:
      - image: *GOLANG_IMAGE
    steps:
      - checkout
      - run:
          command: go mod download
      - run:
          name: check go fmt
          command: |
            files="$(go fmt ./... ; (cd api && go fmt ./... | sed 's@^@api/@') ; (cd sdk && go fmt ./... | sed 's@^@sdk/@'))"
            if [ -n "$files" ]; then
                echo "The following file(s) do not conform to go fmt:"
                echo "$files"
                exit 1
            fi
      - run:
          command: |
            go vet ./... && \
              (cd api && go vet ./...) && \
              (cd sdk && go vet ./...)
    environment:
      <<: *ENVIRONMENT

  # checks vendor directory is correct
  check-vendor:
    docker:
      - image: *GOLANG_IMAGE
    environment:
      <<: *ENVIRONMENT
    steps:
      - checkout
      - run:
          command: make update-vendor
      - run: |
          if ! git diff --exit-code; then
            echo "Git directory has vendor changes"
            exit 1
          fi

  go-test:
    docker:
      - image: *GOLANG_IMAGE
    parallelism: 4
    environment:
      <<: *ENVIRONMENT
      GOTAGS: "" # No tags for OSS but there are for enterprise
      # GOMAXPROCS defaults to number of cores on underlying hardware, set explicitly to avoid OOM issues https://support.circleci.com/hc/en-us/articles/360034684273-common-GoLang-memory-issues
      GOMAXPROCS: 2 # medium (default) boxes are 2 vCPUs, 4GB RAM https://circleci.com/docs/2.0/configuration-reference/#docker-executor
    steps:
      - checkout
      - attach_workspace:
          at: /go/bin
      - run: mkdir -p $TEST_RESULTS_DIR
      - run: sudo apt-get update && sudo apt-get install -y rsyslog
      - run: sudo service rsyslog start
      - run: |
          PACKAGE_NAMES=$(go list ./... | circleci tests split --split-by=timings --timings-type=classname)
          echo "Running $(echo $PACKAGE_NAMES | wc -w) packages"
          echo $PACKAGE_NAMES
          gotestsum --format=short-verbose --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- -tags="$GOTAGS" -p 2 -cover -coverprofile=cov_$CIRCLE_NODE_INDEX.part $PACKAGE_NAMES

      # save coverage report parts
      - persist_to_workspace:
          root: .
          paths:
            - cov_*.part

      - store_test_results:
          path: *TEST_RESULTS_DIR
      - store_artifacts:
          path: *TEST_RESULTS_DIR

  # split off a job for the API package since it is separate
  go-test-api:
    docker:
      - image: *GOLANG_IMAGE
    environment:
      <<: *ENVIRONMENT
      GOTAGS: "" # No tags for OSS but there are for enterprise
    steps:
      - checkout
      - attach_workspace:
          at: /go/bin
      - run: mkdir -p $TEST_RESULTS_DIR
      - run:
          working_directory: api
          command: |
            PACKAGE_NAMES=$(go list ./... | circleci tests split --split-by=timings --timings-type=classname)
            gotestsum --format=short-verbose --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- -tags="$GOTAGS" -cover -coverprofile=cov_api.part $PACKAGE_NAMES

      # save coverage report parts
      - persist_to_workspace:
          root: ./api
          paths:
            - cov_*.part

      - store_test_results:
          path: *TEST_RESULTS_DIR
      - store_artifacts:
          path: *TEST_RESULTS_DIR

  # split off a job for the SDK package since it is separate
  go-test-sdk:
    docker:
      - image: *GOLANG_IMAGE
    environment:
      <<: *ENVIRONMENT
      GOTAGS: "" # No tags for OSS but there are for enterprise
    steps:
      - checkout
      - attach_workspace:
          at: /go/bin
      - run: mkdir -p $TEST_RESULTS_DIR
      - run:
          working_directory: sdk
          command: |
            PACKAGE_NAMES=$(go list ./... | circleci tests split --split-by=timings --timings-type=classname)
            gotestsum --format=short-verbose --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- -tags=$GOTAGS -cover -coverprofile=cov_sdk.part $PACKAGE_NAMES

      # save coverage report parts
      - persist_to_workspace:
          root: ./sdk
          paths:
            - cov_*.part

      - store_test_results:
          path: *TEST_RESULTS_DIR
      - store_artifacts:
          path: *TEST_RESULTS_DIR

  # combine code coverage results from the parallel circleci executors
  coverage-merge:
    docker:
      - image: *GOLANG_IMAGE
    environment:
      <<: *ENVIRONMENT
    steps:
      - checkout
      - attach_workspace:
          at: .
      - run: mkdir -p $TEST_RESULTS_DIR
      - run:
          name: merge coverage reports
          command: |
            echo "mode: set" > coverage.out
            grep -h -v "mode: set" cov_*.part >> coverage.out
            go tool cover -html=coverage.out -o $TEST_RESULTS_DIR/coverage.html
      - run:
          name: codecov upload
          command: bash <(curl -s https://codecov.io/bash) -v -C $CIRCLE_SHA1 -f '!agent/bindata_assetfs.go'
      - store_artifacts:
          path: *TEST_RESULTS_DIR

  # build all distros
  build-distros: &build-distros
    docker:
      - image: *GOLANG_IMAGE
    environment: &build-env
      <<: *ENVIRONMENT
    steps:
      - checkout
      - run: ./build-support/scripts/build-local.sh

      # save dev build to CircleCI
      - store_artifacts:
          path: ./pkg/bin

  # build all 386 architecture supported OS binaries
  build-386:
    <<: *build-distros
    environment:
      <<: *build-env
      XC_OS: "darwin freebsd linux windows"
      XC_ARCH: "386"

  # build all amd64 architecture supported OS binaries
  build-amd64:
    <<: *build-distros
    environment:
      <<: *build-env
      XC_OS: "darwin freebsd linux solaris windows"
      XC_ARCH: "amd64"

  # build all arm/arm64 architecture supported OS binaries
  build-arm:
    docker:
      - image: *GOLANG_IMAGE
    environment:
      <<: *ENVIRONMENT
      CGO_ENABLED: 1
      GOOS: linux
    steps:
      - checkout
      - run: sudo apt-get update && sudo apt-get install -y gcc-arm-linux-gnueabi gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu
      - run:
          environment:
            GOARM: 5
            CC: arm-linux-gnueabi-gcc
            GOARCH: arm
          command: go build -o ./pkg/bin/linux_armel/consul  -ldflags="${GOLDFLAGS}"
      - run:
          environment:
            GOARM: 6
            CC: arm-linux-gnueabihf-gcc
            GOARCH: arm
          command: go build -o ./pkg/bin/linux_armhf/consul  -ldflags="${GOLDFLAGS}"
      - run:
          environment:
            CC: aarch64-linux-gnu-gcc
            GOARCH: arm64
          command: go build -o ./pkg/bin/linux_aarch64/consul  -ldflags="${GOLDFLAGS}"
      - store_artifacts:
          path: ./pkg/bin

  # create a development build
  dev-build:
    docker:
      - image: *GOLANG_IMAGE
    environment:
      <<: *ENVIRONMENT
    steps:
      - checkout
      - run:
          command: make dev

      # save dev build to pass to downstream jobs
      - persist_to_workspace:
          root: /go/bin
          paths:
            - consul

  # upload development build to s3
  dev-upload-s3:
    docker:
      - image: circleci/python:stretch
    environment:
      <<: *ENVIRONMENT
    steps:
      - run:
          name: Install awscli
          command: sudo pip install awscli
      # get consul binary
      - attach_workspace:
          at: bin/
      - run:
          name: package binary
          command: tar -czf consul.tar.gz -C bin/ .
      - run:
          name: Upload to s3
          command: |
            if [ -n "${S3_ARTIFACT_PATH}" ]; then
              aws s3 cp \
              --metadata "CIRCLECI=${CIRCLECI},CIRCLE_BUILD_URL=${CIRCLE_BUILD_URL},CIRCLE_BRANCH=${CIRCLE_BRANCH}" \
              "consul.tar.gz" "s3://${S3_ARTIFACT_BUCKET}/${S3_ARTIFACT_PATH}/${CIRCLE_SHA1}.tar.gz"
            else
              echo "CircleCI - S3_ARTIFACT_PATH was not set"
              exit 1
            fi

  # upload dev docker image
  dev-upload-docker:
    docker:
      - image: circleci/golang:latest # use a circleci image so the attach_workspace step works (has ca-certs installed)
    environment:
      <<: *ENVIRONMENT
    steps:
      - checkout
      # get consul binary
      - attach_workspace:
          at: bin/
      - setup_remote_docker
      - run: make ci.dev-docker

  # Nomad 0.8 builds on go0.10
  # Run integration tests on nomad/v0.8.7
  nomad-integration-0_8:
    docker:
      - image: circleci/golang:1.10
    environment:
      <<: *ENVIRONMENT
      NOMAD_WORKING_DIR: &NOMAD_WORKING_DIR /go/src/github.com/hashicorp/nomad
      NOMAD_VERSION: v0.8.7
    steps: &NOMAD_INTEGRATION_TEST_STEPS
      - run: git clone https://github.com/hashicorp/nomad.git --branch ${NOMAD_VERSION} ${NOMAD_WORKING_DIR}

      # get consul binary
      - attach_workspace:
          at: /go/bin

      # make test result directory
      - run: mkdir -p $TEST_RESULTS_DIR

      # make dev build of nomad
      - run:
          command: make pkg/linux_amd64/nomad
          working_directory: *NOMAD_WORKING_DIR

      # update gotestsum
      - run: curl -sSL "https://github.com/gotestyourself/gotestsum/releases/download/v${GOTESTSUM_RELEASE}/gotestsum_${GOTESTSUM_RELEASE}_linux_amd64.tar.gz" | sudo tar --overwrite -xz -C /usr/local/bin gotestsum

      # run integration tests
      - run:
          command: gotestsum --format=short-verbose --junitfile $TEST_RESULTS_DIR/results.xml -- ./command/agent/consul -run TestConsul
          working_directory: *NOMAD_WORKING_DIR

      # store test results for CircleCI
      - store_test_results:
          path: *TEST_RESULTS_DIR
      - store_artifacts:
          path: *TEST_RESULTS_DIR

  # run integration tests on nomad/master
  nomad-integration-master:
    docker:
      - image: *GOLANG_IMAGE
    environment:
      <<: *ENVIRONMENT
      NOMAD_WORKING_DIR: /go/src/github.com/hashicorp/nomad
      NOMAD_VERSION: master
    steps: *NOMAD_INTEGRATION_TEST_STEPS

  build-website:
    # setting the working_directory along with the checkout path allows us to not have
    # to cd into the website/ directory for commands
    working_directory: ~/project/website
    docker:
      - image: *MIDDLEMAN_IMAGE
    steps:
      - checkout:
          path: ~/project

      # restores gem cache
      - restore_cache:
          key: *RUBYGEM_CACHE_KEY

      - run:
          name: install gems
          command: bundle check || bundle install --path vendor/bundle --retry=3

      # saves gem cache if we have changed the Gemfile
      - save_cache:
          key: *RUBYGEM_CACHE_KEY
          paths:
            - ~/project/website/vendor/bundle

      # exclude guides directory since they moved to learn.hashicorp.com
      # keep index.html which points to learn
      - run:
          name: exclude guides
          command: find ./source/docs/guides -type f -not -name 'index.html.md' -delete

      - run:
          name: middleman build
          command: bundle exec middleman build

      - run:
          name: add missing tar binary
          command: apk update && apk add tar

      # saves website build directory
      - persist_to_workspace:
          root: .
          paths:
            - build

  deploy-website:
    # setting the working_directory along with the checkout path allows us to not have
    # to cd into the website/ directory for commands
    working_directory: ~/project/website
    docker:
      - image: *MIDDLEMAN_IMAGE
    steps:
      - checkout:
          path: ~/project

      - run:
          name: add missing tar binary
          command: apk update && apk add tar

      # attach website build directory
      - attach_workspace:
          at: ~/project/website

      # restores gem cache
      - restore_cache:
          key: *RUBYGEM_CACHE_KEY
      # rerun build with 'ENV=production' to add analytics
      - run:
          name: install gems
          command: bundle check || bundle install --path vendor/bundle --retry=3

      # exclude guides directory since they moved to learn.hashicorp.com
      # keep index.html which points to learn
      - run:
          name: exclude guides
          command: find ./source/docs/guides -type f -not -name 'index.html.md' -delete

      # rerun build with 'ENV=production' to add analytics
      - run:
          name: middleman build
          command: bundle exec middleman build

      - run:
          name: website deploy
          command: ./scripts/deploy.sh

  # Link check on a temporary netlify deployed site
  docs-link-checker:
    docker:
      - image: circleci/node:lts
    steps:
      - checkout
      # attach website build directory
      - attach_workspace:
          at: ~/project/website
      - run: ./website/scripts/link-check.sh

  # build frontend yarn cache
  frontend-cache:
    docker:
      - image: *EMBER_IMAGE
    steps:
      - checkout

      # cache yarn deps
      - restore_cache:
          key: *YARN_CACHE_KEY

      - run:
          name: install yarn packages
          command: cd ui-v2 && yarn install

      - save_cache:
          key: *YARN_CACHE_KEY
          paths:
            - ui-v2/node_modules

  # build ember so frontend tests run faster
  ember-build:
    docker:
      - image: *EMBER_IMAGE
    steps:
      - checkout
      - restore_cache:
          key: *YARN_CACHE_KEY
      - run: cd ui-v2 && make build-ci

      # saves the build to a workspace to be passed to a downstream job
      - persist_to_workspace:
          root: ui-v2
          paths:
            - dist

  # run ember frontend tests
  ember-test-oss:
    docker:
      - image: *EMBER_IMAGE
    environment:
      EMBER_TEST_REPORT: test-results/report-oss.xml #outputs test report for CircleCI test summary
    steps:
      - checkout
      - restore_cache:
          key: *YARN_CACHE_KEY
      - attach_workspace:
          at: ui-v2
      - run:
          working_directory: ui-v2
          command: make test-oss-ci
      - store_test_results:
          path: ui-v2/test-results
  # run ember frontend tests
  ember-test-ent:
    docker:
      - image: *EMBER_IMAGE
    environment:
      EMBER_TEST_REPORT: test-results/report-ent.xml #outputs test report for CircleCI test summary
    steps:
      - checkout
      - restore_cache:
          key: *YARN_CACHE_KEY
      - attach_workspace:
          at: ui-v2
      - run:
          working_directory: ui-v2
          command: make test-ci
      - store_test_results:
          path: ui-v2/test-results

  envoy-integration-test-1.10.0:
    docker:
      # We only really need bash and docker-compose which is installed on all
      # Circle images but pick Go since we have to pick one of them.
      - image: *GOLANG_IMAGE
    environment:
      ENVOY_VERSIONS: "1.10.0"
    steps: &ENVOY_INTEGRATION_TEST_STEPS
      - checkout
      # Get go binary from workspace
      - attach_workspace:
          at: .
      - setup_remote_docker
      # Build the consul-dev image from the already built binary
      - run: docker build -t consul-dev -f ./build-support/docker/Consul-Dev.dockerfile .
      - run:
          name: Envoy Integration Tests
          command: make test-envoy-integ SKIP_DOCKER_BUILD=1
          environment:
            # tput complains if this isn't set to something.
            TERM: ansi
      - store_artifacts:
          path: ./test/integration/connect/envoy/workdir/logs
          destination: container-logs

  envoy-integration-test-1.11.2:
    docker:
      - image: *GOLANG_IMAGE
    environment:
      ENVOY_VERSIONS: "1.11.2"
    steps: *ENVOY_INTEGRATION_TEST_STEPS

  envoy-integration-test-1.12.2:
    docker:
      - image: *GOLANG_IMAGE
    environment:
      ENVOY_VERSIONS: "1.12.2"
    steps: *ENVOY_INTEGRATION_TEST_STEPS

  envoy-integration-test-1.13.0:
    docker:
      - image: *GOLANG_IMAGE
    environment:
      ENVOY_VERSIONS: "1.13.0"
    steps: *ENVOY_INTEGRATION_TEST_STEPS

  # run tests on vault ca provider integration tests
  vault-ca-provider:
    docker:
      - image: *GOLANG_IMAGE
    environment:
      <<: *ENVIRONMENT
    steps:
      # Install vault
      - run: |
          wget -q -O /tmp/vault.zip https://releases.hashicorp.com/vault/${VAULT_BINARY_VERSION}/vault_${VAULT_BINARY_VERSION}_linux_amd64.zip
          sudo unzip -d /usr/local/bin /tmp/vault.zip
          rm -rf /tmp/vault*
      # Create directory to store test results
      - run: mkdir -p $TEST_RESULTS_DIR
      # Gather deps to run go tests
      - checkout
      # Run go tests
      - run: make test-vault-ca-provider
      - store_test_results:
          path: *TEST_RESULTS_DIR

  # only runs on master: checks latest commit to see if the PR associated has a backport/* or docs* label to cherry-pick
  cherry-picker:
    docker:
      - image: alpine:3.11
    steps:
      - run: apk add --no-cache --no-progress git bash curl ncurses jq
      - checkout
      - add_ssh_keys: # needs a key to push cherry-picked commits back to github
          fingerprints:
            - "c9:04:b7:85:bf:0e:ce:93:5f:b8:0e:68:8e:16:f3:71"
      - run: .circleci/scripts/cherry-picker.sh

workflows:
  version: 2
  go-tests:
    jobs:
      - check-vendor:
          filters:
            branches:
              ignore:
                - stable-website
                - /^docs\/.*/
                - /^ui\/.*/
      - lint-consul-retry:
          filters:
            branches:
              ignore:
                - stable-website
                - /^docs\/.*/
                - /^ui\/.*/
      - go-fmt-and-vet:
          requires:
            - check-vendor
      - dev-build:
          requires:
            - go-fmt-and-vet
      - go-test: &go-test
          requires:
            - dev-build
      - go-test-api: *go-test
      - go-test-sdk: *go-test
      - coverage-merge:
          requires:
            - go-test
            - go-test-api
            - go-test-sdk
  build-distros:
    jobs:
      - check-vendor:
          filters:
            branches:
              ignore:
                - stable-website
                - /^docs\/.*/
                - /^ui\/.*/
      - build-386: &require-check-vendor
          requires:
            - check-vendor
      - build-amd64: *require-check-vendor
      - build-arm: *require-check-vendor
  test-integrations:
    jobs:
      - dev-build:
          filters:
            branches:
              ignore:
                - stable-website
                - /^docs\/.*/
                - /^ui\/.*/
      - dev-upload-s3: &dev-upload
          requires:
            - dev-build
          filters:
            branches:
              ignore:
                - /^pull\/.*$/ # only push dev builds from non forks
      - dev-upload-docker:
          <<: *dev-upload
          context: consul-ci
      - nomad-integration-master:
          requires:
            - dev-build
      - nomad-integration-0_8:
          requires:
            - dev-build
      - envoy-integration-test-1.10.0:
          requires:
            - dev-build
      - envoy-integration-test-1.11.2:
          requires:
            - dev-build
      - envoy-integration-test-1.12.2:
          requires:
            - dev-build
      - envoy-integration-test-1.13.0:
          requires:
            - dev-build
      - vault-ca-provider:
          requires:
            - dev-build
  website:
    jobs:
      - build-website
      - docs-link-checker:
          requires:
            - build-website
          filters:
            branches:
              ignore:
                - /^pull\/.*$/ # only run link checker on non forks
      - deploy-website:
          requires:
            - docs-link-checker
          context: static-sites
          filters:
            branches:
              only: stable-website
  frontend:
    jobs:
      - frontend-cache:
          filters:
            branches:
              ignore:
                - stable-website
                - /^docs\/.*/
      - ember-build:
          requires:
            - frontend-cache
      - ember-test-oss:
          requires:
            - ember-build
      - ember-test-ent:
          requires:
            - ember-build
  cherry-pick:
    jobs:
      - cherry-picker:
          context: team-consul
          filters:
            branches:
              only:
                - master