status-im
/
consul
mirror of https://github.com/status-im/consul.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity
consul/.circleci/config.yml

720 lines
22 KiB
YAML
Raw Blame History

---
version: 2
references:
images:
go: &GOLANG_IMAGE circleci/golang:1.12.8
middleman: &MIDDLEMAN_IMAGE hashicorp/middleman-hashicorp:0.3.35
ember: &EMBER_IMAGE circleci/node:8-browsers
paths:
test-results: &TEST_RESULTS_DIR /tmp/test-results
cache:
yarn: &YARN_CACHE_KEY consul-ui-v1-{{ checksum "ui-v2/yarn.lock" }}
rubygem: &RUBYGEM_CACHE_KEY static-site-gems-v1-{{ checksum "Gemfile.lock" }}
environment: &ENVIRONMENT
TEST_RESULTS_DIR: *TEST_RESULTS_DIR
GOTESTSUM_RELEASE: 0.3.3
EMAIL: noreply@hashicorp.com
GIT_AUTHOR_NAME: circleci-consul
GIT_COMMITTER_NAME: circleci-consul
S3_ARTIFACT_BUCKET: consul-dev-artifacts
BASH_ENV: .circleci/bash_env.sh
jobs:
# lint consul tests
lint-consul-retry:
docker:
- image: *GOLANG_IMAGE
steps:
- checkout
- run: go get -u github.com/hashicorp/lint-consul-retry && lint-consul-retry
# Runs go fmt and go vet
go-fmt-and-vet:
docker:
- image: *GOLANG_IMAGE
steps:
- checkout
- restore_cache:
keys:
- consul-modcache-v1-{{ checksum "go.mod" }}
- run:
command: go mod download
- save_cache:
key: consul-modcache-v1-{{ checksum "go.mod" }}
paths:
- /go/pkg/mod
- run:
name: check go fmt
command: |
files=$(go fmt ./...)
if [ -n "$files" ]; then
echo "The following file(s) do not conform to go fmt:"
echo "$files"
exit 1
fi
- run:
command: go vet ./...
environment:
<<: *ENVIRONMENT
# checks vendor directory is correct
check-vendor:
docker:
- image: *GOLANG_IMAGE
environment:
<<: *ENVIRONMENT
steps:
- checkout
- restore_cache:
keys:
- consul-modcache-v1-{{ checksum "go.mod" }}
- run:
command: make update-vendor
- save_cache:
key: consul-modcache-v1-{{ checksum "go.mod" }}
paths:
- /go/pkg/mod
- run: |
if ! git diff --exit-code; then
echo "Git directory has vendor changes"
exit 1
fi
go-test:
docker:
- image: *GOLANG_IMAGE
parallelism: 8
environment:
<<: *ENVIRONMENT
GOTAGS: "" # No tags for OSS but there are for enterprise
steps:
- checkout
- restore_cache: # restore cache from dev-build job
keys:
- consul-modcache-v1-{{ checksum "go.mod" }}
- attach_workspace:
at: /go/bin
- run: mkdir -p $TEST_RESULTS_DIR
- run: sudo apt-get update && sudo apt-get install -y rsyslog
- run: sudo service rsyslog start
- run: |
PACKAGE_NAMES=$(go list ./... | circleci tests split --split-by=timings --timings-type=classname)
gotestsum --format=short-verbose --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- -tags=$GOTAGS -p 3 $PACKAGE_NAMES
- store_test_results:
path: /tmp/test-results
- store_artifacts:
path: /tmp/test-results
# split off a job for the API package since it is separate
go-test-api:
docker:
- image: *GOLANG_IMAGE
environment:
<<: *ENVIRONMENT
GOTAGS: "" # No tags for OSS but there are for enterprise
steps:
- checkout
- restore_cache: # restore cache from dev-build job
keys:
- consul-modcache-v1-{{ checksum "go.mod" }}
- attach_workspace:
at: /go/bin
- run: mkdir -p $TEST_RESULTS_DIR
- run:
working_directory: api
command: |
PACKAGE_NAMES=$(go list ./... | circleci tests split --split-by=timings --timings-type=classname)
gotestsum --format=short-verbose --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- -tags=$GOTAGS $PACKAGE_NAMES
- store_test_results:
path: /tmp/test-results
- store_artifacts:
path: /tmp/test-results
# build all distros
build-distros: &build-distros
docker:
- image: *GOLANG_IMAGE
environment: &build-env
<<: *ENVIRONMENT
GOXPARALLEL: 2 # CircleCI containers are 2 CPU x 4GB RAM
resource_class: large
steps:
- checkout
- run: make tools
- run: ./build-support/scripts/build-local.sh
# save dev build to CircleCI
- store_artifacts:
path: ./pkg/bin
# build all 386 architecture supported OS binaries
build-386:
<<: *build-distros
environment:
<<: *build-env
XC_OS: "darwin freebsd linux windows"
XC_ARCH: "386"
# build all amd64 architecture supported OS binaries
build-amd64:
<<: *build-distros
environment:
<<: *build-env
XC_OS: "darwin freebsd linux solaris windows"
XC_ARCH: "amd64"
# build all arm/arm64 architecture supported OS binaries
build-arm:
docker:
- image: *GOLANG_IMAGE
environment:
<<: *ENVIRONMENT
CGO_ENABLED: 1
GOOS: linux
steps:
- checkout
- run: sudo apt-get update && sudo apt-get install -y gcc-arm-linux-gnueabi gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu
- run:
environment:
GOARM: 5
CC: arm-linux-gnueabi-gcc
GOARCH: arm
command: go build -o ./pkg/bin/linux_armel/consul -ldflags="${GOLDFLAGS}"
- run:
environment:
GOARM: 6
CC: arm-linux-gnueabihf-gcc
GOARCH: arm
command: go build -o ./pkg/bin/linux_armhf/consul -ldflags="${GOLDFLAGS}"
- run:
environment:
CC: aarch64-linux-gnu-gcc
GOARCH: arm64
command: go build -o ./pkg/bin/linux_aarch64/consul -ldflags="${GOLDFLAGS}"
- store_artifacts:
path: ./pkg/bin
# create a development build
dev-build:
docker:
- image: *GOLANG_IMAGE
environment:
<<: *ENVIRONMENT
steps:
- checkout
- restore_cache:
keys:
- consul-modcache-v1-{{ checksum "go.mod" }}
- run:
command: make dev
- save_cache:
key: consul-modcache-v1-{{ checksum "go.mod" }}
paths:
- /go/pkg/mod
# save dev build to pass to downstream jobs
- persist_to_workspace:
root: /go/bin
paths:
- consul
# upload development build to s3
dev-upload-s3:
docker:
- image: circleci/python:stretch
environment:
<<: *ENVIRONMENT
steps:
- run:
name: Install awscli
command: sudo pip install awscli
# get consul binary
- attach_workspace:
at: bin/
- run:
name: package binary
command: tar -czf consul.tar.gz -C bin/ .
- run:
name: Upload to s3
command: |
if [ -n "${S3_ARTIFACT_PATH}" ]; then
aws s3 cp \
--metadata "CIRCLECI=${CIRCLECI},CIRCLE_BUILD_URL=${CIRCLE_BUILD_URL},CIRCLE_BRANCH=${CIRCLE_BRANCH}" \
"consul.tar.gz" "s3://${S3_ARTIFACT_BUCKET}/${S3_ARTIFACT_PATH}/${CIRCLE_SHA1}.tar.gz"
else
echo "CircleCI - S3_ARTIFACT_PATH was not set"
exit 1
fi
# upload dev docker image
dev-upload-docker:
docker:
- image: circleci/golang:latest # use a circleci image so the attach_workspace step works (has ca-certs installed)
environment:
<<: *ENVIRONMENT
steps:
- checkout
# get consul binary
- attach_workspace:
at: bin/
- setup_remote_docker:
docker_layer_caching: true
- run: make ci.dev-docker
# Nomad 0.8 builds on go0.10
# Run integration tests on nomad/v0.8.7
nomad-integration-0_8:
docker:
- image: circleci/golang:1.10
environment:
<<: *ENVIRONMENT
NOMAD_WORKING_DIR: &NOMAD_WORKING_DIR /go/src/github.com/hashicorp/nomad
NOMAD_VERSION: v0.8.7
steps: &NOMAD_INTEGRATION_TEST_STEPS
- run: git clone https://github.com/hashicorp/nomad.git --branch ${NOMAD_VERSION} ${NOMAD_WORKING_DIR}
# get consul binary
- attach_workspace:
at: /go/bin
# make test result directory
- run: mkdir -p $TEST_RESULTS_DIR
# make dev build of nomad
- run:
command: make dev
working_directory: *NOMAD_WORKING_DIR
# update gotestsum
- run: curl -sSL "https://github.com/gotestyourself/gotestsum/releases/download/v${GOTESTSUM_RELEASE}/gotestsum_${GOTESTSUM_RELEASE}_linux_amd64.tar.gz" | sudo tar --overwrite -xz -C /usr/local/bin gotestsum
# run integration tests
- run:
command: gotestsum --format=short-verbose --junitfile $TEST_RESULTS_DIR/results.xml -- ./command/agent/consul -run TestConsul
working_directory: *NOMAD_WORKING_DIR
# store test results for CircleCI
- store_test_results:
path: *TEST_RESULTS_DIR
- store_artifacts:
path: *TEST_RESULTS_DIR
# run integration tests on nomad/master
nomad-integration-master:
docker:
- image: *GOLANG_IMAGE
environment:
<<: *ENVIRONMENT
NOMAD_WORKING_DIR: /go/src/github.com/hashicorp/nomad
NOMAD_VERSION: master
steps: *NOMAD_INTEGRATION_TEST_STEPS
build-website:
# setting the working_directory along with the checkout path allows us to not have
# to cd into the website/ directory for commands
working_directory: ~/project/website
docker:
- image: *MIDDLEMAN_IMAGE
steps:
- checkout:
path: ~/project
# restores gem cache
- restore_cache:
key: *RUBYGEM_CACHE_KEY
- run:
name: install gems
command: bundle check || bundle install --path vendor/bundle --retry=3
# saves gem cache if we have changed the Gemfile
- save_cache:
key: *RUBYGEM_CACHE_KEY
paths:
- ~/project/website/vendor/bundle
# exclude guides directory since they moved to learn.hashicorp.com
# keep index.html which points to learn
- run:
name: exclude guides
command: find ./source/docs/guides -type f -not -name 'index.html.md' -delete
- run:
name: middleman build
command: bundle exec middleman build
# saves website build directory
- persist_to_workspace:
root: .
paths:
- build
deploy-website:
# setting the working_directory along with the checkout path allows us to not have
# to cd into the website/ directory for commands
working_directory: ~/project/website
docker:
- image: *MIDDLEMAN_IMAGE
steps:
- checkout:
path: ~/project
# attach website build directory
- attach_workspace:
at: ~/project/website
# restores gem cache
- restore_cache:
key: *RUBYGEM_CACHE_KEY
# rerun build with 'ENV=production' to add analytics
- run:
name: install gems
command: bundle check || bundle install --path vendor/bundle --retry=3
# exclude guides directory since they moved to learn.hashicorp.com
# keep index.html which points to learn
- run:
name: exclude guides
command: find ./source/docs/guides -type f -not -name 'index.html.md' -delete
# rerun build with 'ENV=production' to add analytics
- run:
name: middleman build
command: bundle exec middleman build
- run:
name: website deploy
command: ./scripts/deploy.sh
# Link check on a temporary netlify deployed site
docs-link-checker:
docker:
- image: circleci/node:lts
steps:
- checkout
# attach website build directory
- attach_workspace:
at: ~/project/website
- run: ./website/scripts/link-check.sh
# build frontend yarn cache
frontend-cache:
docker:
- image: *EMBER_IMAGE
steps:
- checkout
# cache yarn deps
- restore_cache:
key: *YARN_CACHE_KEY
- run:
name: install yarn packages
command: cd ui-v2 && yarn install
- save_cache:
key: *YARN_CACHE_KEY
paths:
- ui-v2/node_modules
# build ember so frontend tests run faster
ember-build:
docker:
- image: *EMBER_IMAGE
steps:
- checkout
- restore_cache:
key: *YARN_CACHE_KEY
- run: cd ui-v2 && yarn build-ci --output-path=dist
# saves the build to a workspace to be passed to a downstream job
- persist_to_workspace:
root: ui-v2
paths:
- dist
# run ember frontend tests
ember-test:
docker:
- image: *EMBER_IMAGE
environment:
EMBER_TEST_PARALLEL: true #enables test parallelization with ember-exam
EMBER_TEST_REPORT: test-results/report.xml #outputs test report for CircleCI test summary
parallelism: 4
steps:
- checkout
- restore_cache:
key: *YARN_CACHE_KEY
- attach_workspace:
at: ui-v2
- run:
working_directory: ui-v2
command: node_modules/ember-cli/bin/ember exam --split=$CIRCLE_NODE_TOTAL --partition=`expr $CIRCLE_NODE_INDEX + 1` --path dist --silent -r xunit
- store_test_results:
path: ui-v2/test-results
# Envoy integration tests. Require docker dev binary to be built already
envoy-integration-test-1.8.0:
docker:
# We only really need bash and docker-compose which is installed on all
# Circle images but pick Go since we have to pick one of them.
- image: *GOLANG_IMAGE
environment:
ENVOY_VERSIONS: "1.8.0"
steps: &ENVOY_INTEGRATION_TEST_STEPS
- checkout
# Get go binary from workspace
- attach_workspace:
at: .
- setup_remote_docker:
docker_layer_caching: true
# Build the consul-dev image from the already built binary
- run: docker build -t consul-dev -f ./build-support/docker/Consul-Dev.dockerfile .
- run:
name: Envoy Integration Tests
command: make test-envoy-integ SKIP_DOCKER_BUILD=1
environment:
# tput complains if this isn't set to something.
TERM: ansi
- store_artifacts:
path: ./test/integration/connect/envoy/workdir/logs
destination: container-logs
envoy-integration-test-1.9.1:
docker:
- image: *GOLANG_IMAGE
environment:
ENVOY_VERSIONS: "1.9.1"
steps: *ENVOY_INTEGRATION_TEST_STEPS
envoy-integration-test-1.10.0:
docker:
- image: *GOLANG_IMAGE
environment:
ENVOY_VERSIONS: "1.10.0"
steps: *ENVOY_INTEGRATION_TEST_STEPS
envoy-integration-test-1.11.1:
docker:
- image: *GOLANG_IMAGE
environment:
ENVOY_VERSIONS: "1.11.1"
steps: *ENVOY_INTEGRATION_TEST_STEPS
# This job merges master into the 'current' branch (${CIRCLE_BRANCH}) specified
# in the branch filtering of the workflow
merge-master:
docker:
- image: *GOLANG_IMAGE
steps:
- add_ssh_keys:
fingerprints:
- c6:96:98:82:dc:04:6c:39:dd:ac:83:05:e3:15:1c:98
- checkout
- run:
name: Merge Consul OSS master branch into current branch
command: |
set -eu -o pipefail
# Configure Git
git config --global user.email "hashicorp-ci@users.noreply.github.com"
git config --global user.name "hashicorp-ci"
# Fetch latest master
git fetch origin
# Create a merge branch to run tests on
git_merge_branch="ci/master-merge-${CIRCLE_BRANCH}-$(date +%Y%m%d%H%M%S)"
git checkout -b "${git_merge_branch}"
latest_oss_commit="$(git rev-parse origin/master)"
if ! errors=$(git merge -m "Merge Consul OSS branch 'master' at commit ${latest_oss_commit}" "${latest_oss_commit}"); then
printf "oss/master merge into ${CIRCLE_BRANCH} failed because git was unable to auto-merge!\n${errors}"
curl -X POST -H 'Content-type: application/json' \
--data \
"{ \
\"attachments\": [ \
{ \
\"fallback\": \"master merge into ${CIRCLE_BRANCH} failed because git was unable to auto-merge!\", \
\"text\": \"Nightly *master* merge into *${CIRCLE_BRANCH}* failed!\n\nBuild Log: ${CIRCLE_BUILD_URL}\n\nGit was unable to auto-merge due to possible merge conflict.\n\n*Errors:*\n${errors}\", \
\"footer\": \"${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}\", \
\"ts\": \"$(date +%s)\", \
\"color\": \"danger\" \
} \
] \
}" ${CONSUL_SLACK_WEBHOOK_URL}
exit 1
fi
git push origin "${git_merge_branch}"
sleep 15 # Wait for merge branch to start CircleCI pipeline
# Wait for OSS merge branch CircleCI pipeline to finish
# return shallow results for better performance
project_url="https://circleci.com/api/v1.1/project/github/hashicorp/consul/tree/${git_merge_branch}?shallow=true"
echo "Waiting for master merge branch CI pipeline to finish..."
builds=
unfinished_builds="will be populated in the until loop below"
min_waited=0
until [[ -z "${unfinished_builds}" ]]; do
builds="$(curl \
--header 'Accept: application/json' \
--show-error \
--silent \
"${project_url}")"
unfinished_builds="$(echo "${builds}" \
| jq --raw-output '.[] | select(.lifecycle!="finished") | .workflows.job_name')"
sleep 60
let "min_waited += 1"
echo "Waited ${min_waited} min..."
done
# Fail this job if the merge branch CI pipeline failed
# unsuccessful_builds will be a multiline string variable with a line format of " $job_name: $build_url"
unsuccessful_builds="$(echo "${builds}" \
| jq --raw-output '.[] | select(.outcome!="success") | "\(.workflows.job_name): \(.build_url)"')"
if [[ -n "${unsuccessful_builds}" ]]; then
printf "master merge CI pipeline jobs failed:\n${unsuccessful_builds}\n"
curl -X POST -H 'Content-type: application/json' \
--data \
"{ \
\"attachments\": [ \
{ \
\"fallback\": \"Nightly Master Merge Failed!\", \
\"text\": \"Nightly *master* merge into *${CIRCLE_BRANCH}* failed!\n\nBuild Log: ${CIRCLE_BUILD_URL}\n\n*master* merge CI pipeline jobs failed:\n${unsuccessful_builds}\", \
\"footer\": \"${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}\", \
\"ts\": \"$(date +%s)\", \
\"color\": \"danger\" \
} \
] \
}" ${CONSUL_SLACK_WEBHOOK_URL}
git push --delete origin "${git_merge_branch}"
exit 1
fi
current_ref=$(git rev-parse HEAD)
# Set CLA check to pass
echo "Setting CLA check"
curl -u "${HASHICORP_CI_GITHUB_TOKEN}:" -X POST \
--header "Accept: application/json" \
--show-error \
--silent \
--fail \
--output /dev/null \
-d "{ \"state\": \"success\", \
\"target_url\": \"https://cla.hashicorp.com/hashicorp/consul\", \
\"description\": \"Contributor License Agreement is signed.\", \
\"context\": \"license/cla\"}" https://api.github.com/repos/hashicorp/consul/statuses/${current_ref}
# CircleCI jobs passed, merging to release branch
echo "master merge CI pipeline passed successfully so merging to ${CIRCLE_BRANCH}!"
git checkout "${CIRCLE_BRANCH}"
git merge "${git_merge_branch}"
git push origin "${CIRCLE_BRANCH}"
git push --delete origin "${git_merge_branch}"
curl -X POST -H 'Content-type: application/json' \
--data \
"{ \
\"attachments\": [ \
{ \
\"fallback\": \"Nightly master merge success!\", \
\"text\": \"Nightly *master* merge into *${CIRCLE_BRANCH}* succeeded!\", \
\"footer\": \"${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}\", \
\"ts\": \"$(date +%s)\", \
\"color\": \"good\" \
} \
] \
}" ${CONSUL_SLACK_WEBHOOK_URL}
workflows:
version: 2
merge-master-to-release:
jobs:
- merge-master
triggers:
- schedule:
cron: "0 2 * * *" # 2AM UTC <> 10PM EST <> 7PM PST should have no impact
filters:
branches:
only:
- /^release\/.*$/
go-tests:
jobs:
- check-vendor
- lint-consul-retry
- go-fmt-and-vet:
requires:
- check-vendor
- dev-build:
requires:
- go-fmt-and-vet
- go-test: &go-test
requires:
- dev-build
- go-test-api: *go-test
build-distros:
jobs:
- build-386
- build-amd64
- build-arm
test-integrations:
jobs:
- dev-build
- dev-upload-s3: &dev-upload
requires:
- dev-build
filters:
branches:
ignore:
- /^pull\/.*$/ # only push dev builds from non forks
- dev-upload-docker:
<<: *dev-upload
context: consul-ci
- nomad-integration-master:
requires:
- dev-build
- nomad-integration-0_8:
requires:
- dev-build
- envoy-integration-test-1.8.0:
requires:
- dev-build
- envoy-integration-test-1.9.1:
requires:
- dev-build
- envoy-integration-test-1.10.0:
requires:
- dev-build
website:
jobs:
- build-website
- docs-link-checker:
requires:
- build-website
filters:
branches:
ignore:
- /^pull\/.*$/ # only run link checker on non forks
- deploy-website:
requires:
- docs-link-checker
context: static-sites
filters:
branches:
only: stable-website
frontend:
jobs:
- frontend-cache
- ember-build:
requires:
- frontend-cache
- ember-test:
requires:
- ember-build
Reference in New Issue View Git Blame Copy Permalink