consul/.circleci/config.yml

---
version: 2

references:
  images:
    go: &GOLANG_IMAGE circleci/golang:1.12.1
    middleman: &MIDDLEMAN_IMAGE hashicorp/middleman-hashicorp:0.3.35
    ember: &EMBER_IMAGE circleci/node:8-browsers

  paths:
    test-results: &TEST_RESULTS_DIR /tmp/test-results

  cache:
    yarn: &YARN_CACHE_KEY consul-ui-v1-{{ checksum "ui-v2/yarn.lock" }}
    rubygem: &RUBYGEM_CACHE_KEY static-site-gems-v1-{{ checksum "Gemfile.lock" }}

  environment: &ENVIRONMENT
    TEST_RESULTS_DIR: *TEST_RESULTS_DIR
    GOTESTSUM_RELEASE: 0.3.3
    EMAIL: noreply@hashicorp.com
    GIT_AUTHOR_NAME: circleci-consul
    GIT_COMMITTER_NAME: circleci-consul
    S3_ARTIFACT_BUCKET:  consul-dev-artifacts

jobs:
  # Runs go fmt and go vet
  go-fmt-and-vet:
    docker:
      - image: *GOLANG_IMAGE
    steps:
      - checkout
      - restore_cache:
          keys:
          - consul-modcache-v1-{{ checksum "go.mod" }}
      - run:
          command: go mod download
      - save_cache:
          key: consul-modcache-v1-{{ checksum "go.mod" }}
          paths:
          - /go/pkg/mod
      - run:
          name: check go fmt
          command: |
            files=$(go fmt ./...)
            if [ -n "$files" ]; then
              echo "The following file(s) do not conform to go fmt:"
              echo "$files"
              exit 1
            fi
      - run:
          command: go vet ./...
    environment:
      <<: *ENVIRONMENT

  go-test:
    docker:
      - image: *GOLANG_IMAGE
    parallelism: 4
    environment:
      <<: *ENVIRONMENT
      GOTAGS: '' # No tags for OSS but there are for enterprise
    steps:
      - checkout
      - restore_cache: # restore cache from dev-build job
          keys:
          - consul-modcache-v1-{{ checksum "go.mod" }}
      - attach_workspace:
          at: /go/bin
      - run: mkdir -p $TEST_RESULTS_DIR
      - run: sudo apt-get update && sudo apt-get install -y rsyslog
      - run: sudo service rsyslog start
      # Use CircleCI test splitting by classname. Since there are no classes in go,
      # we fake it by taking everything after github.com/hashicorp/consul/ and setting
      # it as the classname.

      # This loop writes go test results to <reportname>.xml per go package
      - run: |
          for pkg in $(go list ./... | grep -v github.com/hashicorp/consul/agent/proxyprocess |circleci tests split --split-by=timings --timings-type=classname | tr '\n' ' '); do
            reportname=$(echo $pkg | cut -d '/' -f3- | sed "s#/#_#g")
            gotestsum --format=short-verbose --junitfile $TEST_RESULTS_DIR/$reportname.xml -- -tags=$GOTAGS $pkg
          done

      - store_test_results:
          path: /tmp/test-results
      - store_artifacts:
          path: /tmp/test-results

  # split off a job for the API package since it is separate
  go-test-api:
    docker:
      - image: *GOLANG_IMAGE
    environment:
      <<: *ENVIRONMENT
      GOTAGS: '' # No tags for OSS but there are for enterprise
    steps:
      - checkout
      - restore_cache: # restore cache from dev-build job
          keys:
          - consul-modcache-v1-{{ checksum "go.mod" }}
      - attach_workspace:
          at: /go/bin
      - run: mkdir -p $TEST_RESULTS_DIR
      # Use CircleCI test splitting by classname. Since there are no classes in go,
      # we fake it by taking everything after github.com/hashicorp/consul/ and setting
      # it as the classname.

      # This loop writes go test results to <reportname>.xml per go package
      - run:
          working_directory: api
          command: |
            for pkg in $(go list ./... | circleci tests split --split-by=timings --timings-type=classname | tr '\n' ' '); do
              reportname=$(echo $pkg | cut -d '/' -f3- | sed "s#/#_#g")
              gotestsum --format=short-verbose --junitfile $TEST_RESULTS_DIR/$reportname.xml -- -tags=$GOTAGS $pkg
            done

      - store_test_results:
          path: /tmp/test-results
      - store_artifacts:
          path: /tmp/test-results

  # build all distros
  build-distros: &build-distros
    docker:
      - image: *GOLANG_IMAGE
    environment: &build-env
      GOXPARALLEL: 2 # CircleCI containers are 2 CPU x 4GB RAM
    steps:
      - checkout
      - run: make tools
      - run: ./build-support/scripts/build-local.sh

      # save dev build to CircleCI
      - store_artifacts:
          path: ./pkg/bin

  # build all 386 architecture supported OS binaries
  build-386:
    <<: *build-distros
    environment:
      <<: *build-env
      XC_OS: "darwin freebsd linux windows"
      XC_ARCH: "386"

  # build all amd64 architecture supported OS binaries
  build-amd64:
    <<: *build-distros
    environment:
      <<: *build-env
      XC_OS: "darwin freebsd linux solaris windows"
      XC_ARCH: "amd64"

  # build all arm/arm64 architecture supported OS binaries
  build-arm-arm64:
    <<: *build-distros
    environment:
      <<: *build-env
      XC_OS: linux
      XC_ARCH: "arm arm64"

  # create a development build
  dev-build:
    docker:
      - image: *GOLANG_IMAGE
    steps:
      - checkout
      - restore_cache:
          keys:
          - consul-modcache-v1-{{ checksum "go.mod" }}
      - run:
          command: make dev
      - save_cache:
          key: consul-modcache-v1-{{ checksum "go.mod" }}
          paths:
          - /go/pkg/mod

      # save dev build to pass to downstream jobs
      - persist_to_workspace:
          root: /go/bin
          paths:
            - consul

  # upload development build to s3
  dev-upload-s3:
    docker:
      - image: circleci/python:stretch
    environment:
      <<: *ENVIRONMENT
    steps:
      - run:
          name: Install awscli
          command: sudo pip install awscli
      # get consul binary
      - attach_workspace:
          at: bin/
      - run:
          name: package binary
          command: tar -czf consul.tar.gz -C bin/ .
      - run:
          name: Upload to s3
          command: |
            if [ -n "${S3_ARTIFACT_PATH}" ]; then
              aws s3 cp \
              --metadata "CIRCLECI=${CIRCLECI},CIRCLE_BUILD_URL=${CIRCLE_BUILD_URL},CIRCLE_BRANCH=${CIRCLE_BRANCH}" \
              "consul.tar.gz" "s3://${S3_ARTIFACT_BUCKET}/${S3_ARTIFACT_PATH}/${CIRCLE_SHA1}.tar.gz"
            else
              echo "CircleCI - S3_ARTIFACT_PATH was not set"
              exit 1
            fi

  # Nomad 0.8 builds on go0.10
  # Run integration tests on nomad/v0.8.7
  nomad-integration-0_8:
    docker:
      - image: circleci/golang:1.10
    environment:
      <<: *ENVIRONMENT
      NOMAD_WORKING_DIR: &NOMAD_WORKING_DIR /go/src/github.com/hashicorp/nomad
      NOMAD_VERSION: v0.8.7
    steps: &NOMAD_INTEGRATION_TEST_STEPS
      - run: git clone https://github.com/hashicorp/nomad.git --branch ${NOMAD_VERSION} ${NOMAD_WORKING_DIR}

      # get consul binary
      - attach_workspace:
          at: /go/bin

      # make test result directory
      - run: mkdir -p $TEST_RESULTS_DIR

      # make dev build of nomad
      - run:
          command: make dev
          working_directory: *NOMAD_WORKING_DIR

      # update gotestsum
      - run: curl -sSL "https://github.com/gotestyourself/gotestsum/releases/download/v${GOTESTSUM_RELEASE}/gotestsum_${GOTESTSUM_RELEASE}_linux_amd64.tar.gz" | sudo tar --overwrite -xz -C /usr/local/bin gotestsum

      # run integration tests
      - run:
          command: gotestsum --format=short-verbose --junitfile $TEST_RESULTS_DIR/results.xml -- ./command/agent/consul -run TestConsul
          working_directory: *NOMAD_WORKING_DIR

      # store test results for CircleCI
      - store_test_results:
          path: *TEST_RESULTS_DIR
      - store_artifacts:
          path: *TEST_RESULTS_DIR

  # run integration tests on nomad/master
  nomad-integration-master:
    docker:
      - image: *GOLANG_IMAGE
    environment:
      <<: *ENVIRONMENT
      NOMAD_WORKING_DIR: /go/src/github.com/hashicorp/nomad
      NOMAD_VERSION: master
    steps: *NOMAD_INTEGRATION_TEST_STEPS

  build-website:
    # setting the working_directory along with the checkout path allows us to not have
    # to cd into the website/ directory for commands
    working_directory: ~/project/website
    docker:
      - image: *MIDDLEMAN_IMAGE
    steps:
      - checkout:
          path: ~/project

      # restores gem cache
      - restore_cache:
          key: *RUBYGEM_CACHE_KEY

      - run:
          name: install gems
          command: bundle check || bundle install --path vendor/bundle --retry=3

      # saves gem cache if we have changed the Gemfile
      - save_cache:
          key: *RUBYGEM_CACHE_KEY
          paths:
            - ~/project/website/vendor/bundle

      # exclude guides directory since they moved to learn.hashicorp.com
      # keep index.html which points to learn
      - run:
          name: exclude guides
          command: find ./source/docs/guides -type f -not -name 'index.html.md' -delete

      - run:
          name: middleman build
          command: bundle exec middleman build

      # saves website build directory
      - persist_to_workspace:
          root: .
          paths:
            - build

  deploy-website:
    # setting the working_directory along with the checkout path allows us to not have
    # to cd into the website/ directory for commands
    working_directory: ~/project/website
    docker:
      - image: *MIDDLEMAN_IMAGE
    steps:
      - checkout:
          path: ~/project

      # attach website build directory
      - attach_workspace:
          at: ~/project/website

      # restores gem cache
      - restore_cache:
          key: *RUBYGEM_CACHE_KEY
      # rerun build with 'ENV=production' to add analytics
      - run:
          name: install gems
          command: bundle check || bundle install --path vendor/bundle --retry=3

      # exclude guides directory since they moved to learn.hashicorp.com
      # keep index.html which points to learn
      - run:
          name: exclude guides
          command: find ./source/docs/guides -type f -not -name 'index.html.md' -delete

      # rerun build with 'ENV=production' to add analytics
      - run:
          name: middleman build
          command: bundle exec middleman build

      - run:
          name: website deploy
          command: ./scripts/deploy.sh

  # Link check on a temporary netlify deployed site
  docs-link-checker:
    docker:
      - image: circleci/node:lts
    steps:
      - checkout
      # attach website build directory
      - attach_workspace:
          at: ~/project/website
      - run: ./website/scripts/link-check.sh

  # build frontend yarn cache
  frontend-cache:
    docker:
      - image: *EMBER_IMAGE
    steps:
      - checkout

      # cache yarn deps
      - restore_cache:
          key: *YARN_CACHE_KEY

      - run:
          name: install yarn packages
          command: cd ui-v2 && yarn install

      - save_cache:
          key: *YARN_CACHE_KEY
          paths:
            - ui-v2/node_modules

  # build ember so frontend tests run faster
  ember-build:
    docker:
      - image: *EMBER_IMAGE
    steps:
      - checkout
      - restore_cache:
          key: *YARN_CACHE_KEY
      - run: cd ui-v2 && yarn build-ci --output-path=dist

      # saves the build to a workspace to be passed to a downstream job
      - persist_to_workspace:
          root: ui-v2
          paths:
            - dist

  # run ember frontend tests
  ember-test:
    docker:
      - image: *EMBER_IMAGE
    environment:
      EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam
      EMBER_TEST_REPORT: test-results/report.xml #outputs test report for CircleCI test summary
    parallelism: 4
    steps:
      - checkout
      - restore_cache:
          key: *YARN_CACHE_KEY
      - attach_workspace:
          at: ui-v2
      - run:
          working_directory: ui-v2
          command: node_modules/ember-cli/bin/ember exam --split=$CIRCLE_NODE_TOTAL --partition=`expr $CIRCLE_NODE_INDEX + 1` --path dist --silent -r xunit
      - store_test_results:
          path: ui-v2/test-results

  # Envoy integration tests. Require docker dev binary to be built already
  envoy-integration-test-1.8.0:
    docker:
      # We only really need bash and docker-compose which is installed on all
      # Circle images but pick Go since we have to pick one of them.
      - image: *GOLANG_IMAGE
    environment:
      ENVOY_VERSIONS: "1.8.0"
    steps:
      &ENVOY_INTEGRATION_TEST_STEPS
      - checkout
      # Get go binary from workspace
      - attach_workspace:
          at: .
      - setup_remote_docker:
          docker_layer_caching: true
      # Build the consul-dev image from the already built binary
      - run: docker build -t consul-dev -f ./build-support/docker/Consul-Dev.dockerfile .
      - run:
          name: Envoy Integration Tests
          command: make test-envoy-integ SKIP_DOCKER_BUILD=1
          environment:
            # tput complains if this isn't set to something.
            TERM: ansi
      - store_artifacts:
          path: ./test/integration/connect/envoy/workdir/logs
          destination: container-logs

  envoy-integration-test-1.9.1:
    docker:
      - image: *GOLANG_IMAGE
    environment:
      ENVOY_VERSIONS: "1.9.1"
    steps: *ENVOY_INTEGRATION_TEST_STEPS

  envoy-integration-test-1.10.0:
    docker:
      - image: *GOLANG_IMAGE
    environment:
      ENVOY_VERSIONS: "1.10.0"
    steps: *ENVOY_INTEGRATION_TEST_STEPS

  # This job merges master into the 'current' branch (${CIRCLE_BRANCH}) specified
  # in the branch filtering of the workflow
  merge-master:
    docker:
      - image: *GOLANG_IMAGE
    steps:
      - add_ssh_keys:
          fingerprints:
          - c6:96:98:82:dc:04:6c:39:dd:ac:83:05:e3:15:1c:98
      - checkout
      - run:
          name: Merge Consul OSS master branch into current branch
          command: |
            set -eu -o pipefail

            # Configure Git
            git config --global user.email "hashicorp-ci@users.noreply.github.com"
            git config --global user.name "hashicorp-ci"

            # Fetch latest master
            git fetch origin

            # Create a merge branch to run tests on
            git_merge_branch="ci/master-merge-$(date +%Y%m%d%H%M%S)"
            git checkout -b "${git_merge_branch}"
            latest_oss_commit="$(git rev-parse origin/master)"
            # adding two merge messages will add them into different paragraphs which will achieve what is in
            # https://help.github.com/en/articles/about-status-checks#skipping-and-requesting-checks-for-individual-commits
            git merge -m "Merge Consul OSS branch 'master' at commit ${latest_oss_commit}" -m "skip-checks: true" "${latest_oss_commit}"
            git push origin "${git_merge_branch}"
            sleep 15  # Wait for merge branch to start CircleCI pipeline

            # Wait for OSS merge branch CircleCI pipeline to finish
            # return shallow results for better performance
            project_url="https://circleci.com/api/v1.1/project/github/hashicorp/consul/tree/${git_merge_branch}?shallow=true"
            echo "Waiting for master merge branch CI pipeline to finish..."
            builds=
            unfinished_builds="will be populated in the until loop below"
            min_waited=0
            until [[ -z "${unfinished_builds}" ]]; do
              builds="$(curl \
                --header 'Accept: application/json' \
                --show-error \
                --silent \
                "${project_url}")"
              unfinished_builds="$(echo "${builds}" \
                | jq --raw-output '.[] | select(.lifecycle!="finished") | .workflows.job_name')"
              sleep 60
              let "min_waited += 1"
              echo "Waited ${min_waited} min..."
            done

            # Fail this job if the merge branch CI pipeline failed
            # unsuccessful_builds will be a multiline string variable with a line format of "  $job_name: $build_url"
            unsuccessful_builds="$(echo "${builds}" \
              | jq --raw-output '.[] | select(.outcome!="success") | "\(.workflows.job_name): \(.build_url)"')"
            if [[ -n "${unsuccessful_builds}" ]]; then
              printf "master merge CI pipeline jobs failed:\n${unsuccessful_builds}\n"
              curl -X POST -H 'Content-type: application/json' \
              --data \
              "{ \
                \"attachments\": [ \
                  { \
                    \"fallback\": \"Nightly Master Merge Failed!\", \
                    \"text\": \"Nightly *master* merge into *release/1-6* failed!\n\nBuild Log: ${CIRCLE_BUILD_URL}\n\nAttempted merge from: https://github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/tree/${git_merge_branch}\n\nThere may be a merge conflict to resolve.\", \
                    \"footer\": \"${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}\", \
                    \"ts\": \"$(date +%s)\", \
                    \"color\": \"danger\" \
                  } \
                ] \
              }" ${CONSUL_SLACK_WEBHOOK_URL}
              exit 1
            fi

            current_ref=$(git rev-parse HEAD)
            # Set CLA check to pass
            echo "Setting CLA check"
            curl -u "${HASHICORP_CI_GITHUB_TOKEN}:" -X POST \
              --header "Accept: application/json" \
              --show-error \
              --silent \
              --fail \
              --output /dev/null \
              -d "{ \"state\": \"success\", \
                    \"target_url\": \"https://cla.hashicorp.com/hashicorp/consul\", \
                    \"description\": \"Contributor License Agreement is signed.\", \
                    \"context\": \"license/cla\"}" https://api.github.com/repos/hashicorp/consul/statuses/${current_ref}

            # CircleCI jobs passed, merging to release branch
            echo "master merge CI pipeline passed successfully so merging to ${CIRCLE_BRANCH}!"
            git checkout "${CIRCLE_BRANCH}"
            git merge "${git_merge_branch}"
            git push origin "${CIRCLE_BRANCH}"
            git push --delete origin "${git_merge_branch}"
            curl -X POST -H 'Content-type: application/json' \
              --data \
              "{ \
                \"attachments\": [ \
                  { \
                    \"fallback\": \"Nightly master merge success!\", \
                    \"text\": \"Nightly *master* merge into *release/1-6* succeeded!\", \
                    \"footer\": \"${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}\", \
                    \"ts\": \"$(date +%s)\", \
                    \"color\": \"good\" \
                  } \
                ] \
              }" ${CONSUL_SLACK_WEBHOOK_URL}

workflows:
  version: 2
  merge-master-to-release:
    jobs:
      - merge-master
    triggers:
      - schedule:
          cron: "0 2 * * *" # 2AM UTC <> 10PM EST <> 7PM PST should have no impact
          filters:
            branches:
              only:
                - release/1-6
  build-distros:
    jobs:
      - go-fmt-and-vet
      - build-386: &require-go-fmt-vet
          requires:
            - go-fmt-and-vet
      - build-amd64: *require-go-fmt-vet
      - build-arm-arm64: *require-go-fmt-vet
  test-integrations:
    jobs:
      - dev-build
      - go-test: &go-test
          requires:
            - dev-build
          filters:
            branches:
              only:
                - /^bug\/flaky-test-.*$/ # only run go tests on bug/flaky-test-* for now since we are fixing tests
      - go-test-api: *go-test
      - dev-upload-s3:
          requires:
            - dev-build
          filters:
            branches:
              ignore:
                - /^pull\/.*$/ # only push dev builds from non forks
      - nomad-integration-master:
          requires:
            - dev-build
      - nomad-integration-0_8:
          requires:
            - dev-build
      - envoy-integration-test-1.8.0:
          requires:
            - dev-build
      - envoy-integration-test-1.9.1:
          requires:
            - dev-build
      - envoy-integration-test-1.10.0:
          requires:
            - dev-build
  website:
    jobs:
      - build-website
      - docs-link-checker:
          requires:
            - build-website
          filters:
            branches:
              ignore:
                - /^pull\/.*$/ # only run link checker on non forks
      - deploy-website:
          requires:
            - docs-link-checker
          context: static-sites
          filters:
            branches:
              only: stable-website
  frontend:
    jobs:
      - frontend-cache
      - ember-build:
          requires:
            - frontend-cache
      - ember-test:
          requires:
            - ember-build