From f5a22f8490443cbe9a57b8d64dd73e7bd04a241c Mon Sep 17 00:00:00 2001 From: Matt Keeler Date: Fri, 8 Jun 2018 10:20:54 -0400 Subject: [PATCH] Initial progress on build system updates --- GNUmakefile | 51 +- build-support/docker/Build-Go.dockerfile | 16 + .../docker/Build-UI-Legacy.dockerfile | 16 + build-support/docker/Build-UI.dockerfile | 14 + build-support/docker/Makefile | 21 + build-support/scripts/build.sh | 40 ++ build-support/scripts/functions.sh | 464 ++++++++++++++++++ ui-v2/GNUmakefile | 9 +- ui-v2/config/environment.js | 14 + 9 files changed, 638 insertions(+), 7 deletions(-) create mode 100644 build-support/docker/Build-Go.dockerfile create mode 100644 build-support/docker/Build-UI-Legacy.dockerfile create mode 100644 build-support/docker/Build-UI.dockerfile create mode 100644 build-support/docker/Makefile create mode 100755 build-support/scripts/build.sh create mode 100644 build-support/scripts/functions.sh diff --git a/GNUmakefile b/GNUmakefile index bebe8bce5a..a02db40667 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -20,13 +20,27 @@ GOOS=$(shell go env GOOS) GOARCH=$(shell go env GOARCH) GOPATH=$(shell go env GOPATH) +ASSETFS_PATH?=agent/bindata_assetfs.go # Get the git commit -GIT_COMMIT=$(shell git rev-parse --short HEAD) -GIT_DIRTY=$(shell test -n "`git status --porcelain`" && echo "+CHANGES" || true) -GIT_DESCRIBE=$(shell git describe --tags --always) +GIT_COMMIT?=$(shell git rev-parse --short HEAD) +GIT_DIRTY?=$(shell test -n "`git status --porcelain`" && echo "+CHANGES" || true) +GIT_DESCRIBE?=$(shell git describe --tags --always) GIT_IMPORT=github.com/hashicorp/consul/version GOLDFLAGS=-X $(GIT_IMPORT).GitCommit=$(GIT_COMMIT)$(GIT_DIRTY) -X $(GIT_IMPORT).GitDescribe=$(GIT_DESCRIBE) +GO_BUILD_TAG?=consul-build-go +UI_BUILD_TAG?=consul-build-ui +UI_LEGACY_BUILD_TAG?=consul-build-ui-legacy +BUILD_CONTAINER_NAME?=consul-builder + +export GO_BUILD_TAG +export UI_BUILD_TAG +export UI_LEGACY_BUILD_TAG +export BUILD_CONTAINER_NAME +export GIT_COMMIT +export GIT_DIRTY +export GIT_DESCRIBE +export GOTAGS export GOLDFLAGS # all builds binaries for all targets @@ -121,10 +135,37 @@ ui: # also run as part of the release build script when it verifies that there are no # changes to the UI assets that aren't checked in. static-assets: - @go-bindata-assetfs -pkg agent -prefix pkg -o agent/bindata_assetfs.go ./pkg/web_ui/... + @go-bindata-assetfs -pkg agent -prefix pkg -o $(ASSETFS_PATH) ./pkg/web_ui/... $(MAKE) format tools: go get -u -v $(GOTOOLS) -.PHONY: all ci bin dev dist cov test cover format vet ui static-assets tools vendorfmt +docker-images: + @$(MAKE) -C build-support/docker images + +go-build-image: + @$(MAKE) -C build-support/docker go-build-image + +ui-build-image: + @$(MAKE) -C build-support/docker ui-build-image + +ui-legacy-build-image: + @$(MAKE) -C build-support/docker ui-legacy-build-image + +static-assets-docker: go-build-image + @$(SHELL) $(CURDIR)/build-support/scripts/build.sh assetfs + +go-docker: go-build-image + @$(SHELL) $(CURDIR)/build-support/scripts/build.sh consul + +ui-docker: ui-build-image + @$(SHELL) $(CURDIR)/build-support/scripts/build.sh ui + +ui-legacy-docker: ui-legacy-build-image + @$(SHELL) $(CURDIR)/build-support/scripts/build.sh ui-legacy + +release-docker: ui-docker ui-legacy-docker static-assets-docker go-docker + +.PHONY: all ci bin dev dist cov test cover format vet ui static-assets tools vendorfmt +.PHONY: docker-images go-build-iamge ui-build-image ui-legacy-build-image static-assets-docker go-docker ui-docker ui-legacy-docker release-docker diff --git a/build-support/docker/Build-Go.dockerfile b/build-support/docker/Build-Go.dockerfile new file mode 100644 index 0000000000..ea0aa25fbb --- /dev/null +++ b/build-support/docker/Build-Go.dockerfile @@ -0,0 +1,16 @@ +ARG GOLANG_VERSION=1.10.1 +FROM golang:${GOLANG_VERSION} + +ARG GOTOOLS="github.com/elazarl/go-bindata-assetfs/... \ + github.com/hashicorp/go-bindata/... \ + github.com/magiconair/vendorfmt/cmd/vendorfmt \ + github.com/mitchellh/gox \ + golang.org/x/tools/cmd/cover \ + golang.org/x/tools/cmd/stringer \ + github.com/axw/gocov/gocov \ + gopkg.in/matm/v1/gocov-html" + +RUN go get -u -v ${GOTOOLS} && mkdir -p ${GOPATH}/src/github.com/hashicorp/consul + +WORKDIR $GOPATH/src/github.com/hashicorp/consul + diff --git a/build-support/docker/Build-UI-Legacy.dockerfile b/build-support/docker/Build-UI-Legacy.dockerfile new file mode 100644 index 0000000000..7f3a9e6b88 --- /dev/null +++ b/build-support/docker/Build-UI-Legacy.dockerfile @@ -0,0 +1,16 @@ +FROM ubuntu:bionic + +RUN mkdir -p /consul-src/ui + +RUN apt-get update -y && \ + apt-get install --no-install-recommends -y -q \ + build-essential \ + git \ + ruby \ + ruby-dev \ + zip \ + zlib1g-dev && \ + gem install bundler + +WORKDIR /consul-src/ui +CMD make dist diff --git a/build-support/docker/Build-UI.dockerfile b/build-support/docker/Build-UI.dockerfile new file mode 100644 index 0000000000..9a4a0198b0 --- /dev/null +++ b/build-support/docker/Build-UI.dockerfile @@ -0,0 +1,14 @@ +ARG ALPINE_VERSION=3.7 +FROM alpine:${ALPINE_VERSION} + +ARG NODEJS_VERSION=8.9.3-r1 +ARG MAKE_VERSION=4.2.1-r0 +ARG YARN_VERSION=1.7.0 + +RUN apk update && \ + apk add nodejs=${NODEJS_VERSION} nodejs-npm=${NODEJS_VERSION} make=${MAKE_VERSION} && \ + npm install --global yarn@${YARN_VERSION} && \ + mkdir /consul-src + +WORKDIR /consul-src +CMD make init build diff --git a/build-support/docker/Makefile b/build-support/docker/Makefile new file mode 100644 index 0000000000..d9af9ed5a8 --- /dev/null +++ b/build-support/docker/Makefile @@ -0,0 +1,21 @@ +ifeq ($(FORCE_REBUILD),1) +NOCACHE=--no-cache +else +NOCACHE= +endif +GO_BUILD_TAG?=consul-build-go +UI_BUILD_TAG?=consul-build-ui +UI_LEGACY_BUILD_TAG?=consul-build-ui-legacy + +images: go-build-image ui-build-image ui-legacy-build-image + +go-build-image: + docker build $(NOCACHE) -t $(GO_BUILD_TAG) -f Build-Go.dockerfile . + +ui-build-image: + docker build $(NOCACHE) -t $(UI_BUILD_TAG) -f Build-UI.dockerfile . + +ui-legacy-build-image: + docker build $(NOCACHE) -t $(UI_LEGACY_BUILD_TAG) -f Build-UI-Legacy.dockerfile . + +.PHONY: images go-build-image ui-build-image ui-legacy-build-image diff --git a/build-support/scripts/build.sh b/build-support/scripts/build.sh new file mode 100755 index 0000000000..f1c387b226 --- /dev/null +++ b/build-support/scripts/build.sh @@ -0,0 +1,40 @@ +#!/bin/bash +pushd $(dirname ${BASH_SOURCE[0]}) > /dev/null +SCRIPT_DIR=$(pwd) +pushd ../.. > /dev/null +SOURCE_DIR=$(pwd) +popd > /dev/null +popd > /dev/null + +source "${SCRIPT_DIR}/functions.sh" + +function main { + case "$1" in + consul ) + build_consul "${SOURCE_DIR}" "${GO_BUILD_TAG}" + return $? + ;; + ui ) + build_ui "${SOURCE_DIR}" "${UI_BUILD_TAG}" + return $? + ;; + ui-legacy ) + build_ui_legacy "${SOURCE_DIR}" "${UI_LEGACY_BUILD_TAG}" + return $? + ;; + version ) + parse_version "${SOURCE_DIR}" + return $? + ;; + assetfs ) + build_assetfs "${SOURCE_DIR}" "${GO_BUILD_TAG}" + return $? + ;; + *) + echo "Unkown build: '$1' - possible values are 'consul', 'ui', 'ui-legacy', 'version' and 'assetfs'" 1>&2 + return 1 + esac +} + +main $@ +exit $? \ No newline at end of file diff --git a/build-support/scripts/functions.sh b/build-support/scripts/functions.sh new file mode 100644 index 0000000000..2b44d24013 --- /dev/null +++ b/build-support/scripts/functions.sh @@ -0,0 +1,464 @@ +# GPG Key ID to use for publically released builds +HASHICORP_GPG_KEY="348FFC4C" + +UI_BUILD_CONTAINER_DEFAULT="consul-build-ui" +UI_LEGACY_BUILD_CONTAINER_DEFAULT="consul-build-ui-legacy" + +function is_set { + # Arguments: + # $1 - string value to check its truthiness + # + # Return: + # 0 - is truthy (backwards I know but allows syntax like `if is_set ` to work) + # 1 - is not truthy + + local val=$(tr '[:upper:]' '[:lower:]' <<< "$1") + case $val in + 1 | t | true | y | yes) + return 0 + ;; + *) + return 1 + ;; + esac +} + +function have_gpg_key { + # Arguments: + # $1 - GPG Key id to check if we have installed + # + # Return: + # 0 - success (we can use this key for signing) + # * - failure (key cannot be used) + + gpg --list-secret-keys $1 >dev/null 2>&1 + return $? +} + +function parse_version { + # Arguments: + # $1 - Path to the top level Consul source + # $2 - boolean value for whether to omit the release version from the version string + # + # Return: + # 0 - success (will write the version to stdout) + # * - error (no version output) + # + # Notes: + # If the GIT_DESCRIBE environment variable is present then it is used as the version + # If the GIT_COMMIT environment variable is preset it will be added to the end of + # the version string. + + local vfile="${1}/version/version.go" + + # ensure the version file exists + if ! test -f "${vfile}" + then + echo "Error - File not found: ${vfile}" 1>&2 + return 1 + fi + + # Get the main version out of the source file + version=$(awk '$1 == "Version" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < ${vfile}) + + # override the version from source with the value of the GIT_DESCRIBE env var if present + if test -n "$GIT_DESCRIBE" + then + version=$GIT_DESCRIBE + fi + + if ! is_set $2 + then + # Get the release version out of the source file + release=$(awk '$1 == "VersionPrerelease" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < ${vfile}) + + # When no GIT_DESCRIBE env var is present and no release is in the source then we + # are definitely in dev mode + if test -z "$GIT_DESCRIBE" -a -z "$release" + then + release="dev" + fi + + # Add the release to the version + if test -n "$release" + then + version="${version}-${release}" + + # add the git commit to the version + if test -n "$GIT_COMMIT" + then + version="${version} (${GIT_COMMIT})" + fi + fi + fi + + # Output the version + echo "$version" | tr -d "'" + return 0 +} + +function get_version { + # Arguments: + # $1 - Path to the top level Consul source + # $2 - Whether the release version should be parsed from source (optional) + # + # Returns: + # 0 - success (the version is also echoed to stdout) + # 1 - error + # + # Notes: + # If a VERSION environment variable is present it will override any parsing of the version from the source + # In addition to processing the main version.go, version_*.go files will be processed if they have + # a Go build tag that matches the one in the GOTAGS environment variable. This tag processing is + # primitive though and will not match complex build tags in the files with negation etc. + + local vers="$VERSION" + if test -z "$vers" + then + # parse the OSS version from version.go + vers="$(parse_version ${1} ${2})" + + # try to determine the version if we have build tags + for tag in "$GOTAGS" + do + for file in $(ls ${1}/version/version_*.go | sort) + do + if grep -q "// +build $tag" $file + then + vers=$(awk -F\" '/Version =/ {print $2; exit}' < $file ) + fi + done + done + fi + + if test -z "$vers" + then + return 1 + else + echo $vers + return 0 + fi +} + +function tag_release { + # Arguments: + # $1 - Version string to use for tagging the release + # $2 - Alternative GPG key id used for signing the release commit (optional) + # + # Returns: + # 0 - success + # * - error + # + # Notes: + # If the RELEASE_UNSIGNED environment variable is set then no gpg signing will occur + + if ! test -d "$1" + then + echo "ERROR: '$1' is not a directory. tag_release must be called with the path to the top level source as the first argument'" 1>&2 + return 1 + fi + + if test -z "$2" + then + echo "ERROR: tag_release must be called with a version number as the second argument" 1>&2 + return 1 + fi + + # determine whether the gpg key to use is being overridden + local gpg_key=${HASHICORP_GPG_KEY} + if test -n "$3" + then + gpg_key=$3 + fi + + pushd "$1" > /dev/null + local ret=0 + + # perform an usngined release if requested (mainly for testing locally) + if is_set "$RELEASE_UNSIGNED" + then + ( + git commit --allow-empty -a -m "Release v${2}" && + git tag -a -m "Version ${2}" "v${2}" master + ) + ret=$? + # perform a signed release (official releases should do this) + elif have_gpg_key ${gpg_key} + then + ( + git commit --allow-empty -a --gpg-sign=${gpg_key} -m "Release v${2}" && + git tag -a -m "Version ${2}" -s -u ${gpg_key} "v${2}" master + ) + ret=$? + # unsigned release not requested and gpg key isn't useable + else + echo "ERROR: GPG key ${gpg_key} is not in the local keychain - to continue set RELEASE_UNSIGNED=1 in the env" + ret=1 + fi + popd > /dev/null + return $ret +} + +function build_ui { + # Arguments: + # $1 - Path to the top level Consul source + # $2 - The docker image to run the build within (optional) + # + # Returns: + # 0 - success + # * - error + # + # Notes: + # Use the GIT_COMMIT environment variable to pass off to the build + + if ! test -d "$1" + then + echo "ERROR: '$1' is not a directory. build_ui must be called with the path to the top level source as the first argument'" 1>&2 + return 1 + fi + + local image_name=${UI_BUILD_CONTAINER_DEFAULT} + if test -n "$2" + then + image_name="$2" + fi + + local sdir="$1" + local ui_dir="${1}/ui-v2" + + # parse the version + version=$(parse_version "${sdir}") + + # make sure we run within the ui dir + pushd ${ui_dir} > /dev/null + + echo "Creating the UI Build Container" + local container_id=$(docker create -it -e "CONSUL_GIT_SHA=${GIT_COMMIT}" -e "CONSUL_VERSION=${version}" ${image_name}) + local ret=$? + if test $ret -eq 0 + then + echo "Copying the source from '${ui_dir}' to /consul-src within the container" + ( + docker cp . ${container_id}:/consul-src && + echo "Running build in container" && docker start -i ${container_id} && + rm -rf ${1}/ui-v2/dist && + echo "Copying back artifacts" && docker cp ${container_id}:/consul-src/dist ${1}/ui-v2/dist + ) + ret=$? + docker rm ${container_id} > /dev/null + fi + + if test $ret -eq 0 + then + rm -rf ${1}/pkg/web_ui/v2 + cp -r ${1}/ui-v2/dist ${1}/pkg/web_ui/v2 + fi + popd > /dev/null + return $ret +} + +function build_ui_legacy { + # Arguments: + # $1 - Path to the top level Consul source + # $2 - The docker image to run the build within (optional) + # + # Returns: + # 0 - success + # * - error + + if ! test -d "$1" + then + echo "ERROR: '$1' is not a directory. build_ui_legacy must be called with the path to the top level source as the first argument'" 1>&2 + return 1 + fi + + local sdir="$1" + local ui_legacy_dir="${sdir}/ui" + + local image_name=${UI_LEGACY_BUILD_CONTAINER_DEFAULT} + if test -n "$2" + then + image_name="$2" + fi + + pushd ${ui_legacy_dir} > /dev/null + echo "Creating the Legacy UI Build Container" + rm -r ${sdir}/pkg/web_ui/v1 >/dev/null 2>&1 + mkdir -p ${sdir}/pkg/web_ui/v1 + local container_id=$(docker create -it ${image_name}) + local ret=$? + if test $ret -eq 0 + then + echo "Copying the source from '${ui_legacy_dir}' to /consul-src/ui within the container" + ( + docker cp . ${container_id}:/consul-src/ui && + echo "Running build in container" && + docker start -i ${container_id} && + echo "Copying back artifacts" && + docker cp ${container_id}:/consul-src/pkg/web_ui ${sdir}/pkg/web_ui/v1 + ) + ret=$? + docker rm ${container_id} > /dev/null + fi + popd > /dev/null + return $ret +} + +function build_assetfs { + # Arguments: + # $1 - Path to the top level Consul source + # $2 - The docker image to run the build within (optional) + # + # Returns: + # 0 - success + # * - error + # + # Note: + # The GIT_COMMIT, GIT_DIRTY and GIT_DESCRIBE environment variables will be used if present + + if ! test -d "$1" + then + echo "ERROR: '$1' is not a directory. build_assetfs must be called with the path to the top level source as the first argument'" 1>&2 + return 1 + fi + + local sdir="$1" + local image_name=${GO_BUILD_CONTAINER_DEFAULT} + if test -n "$2" + then + image_name="$2" + fi + + pushd ${sdir} > /dev/null + echo "Creating the Go Build Container" + local container_id=$(docker create -it -e GIT_COMMIT=${GIT_COMMIT} -e GIT_DIRTY=${GIT_DIRTY} -e GIT_DESCRIBE=${GIT_DESCRIBE} ${image_name} make static-assets ASSETFS_PATH=bindata_assetfs.go) + local ret=$? + if test $ret -eq 0 + then + echo "Copying the sources from '${sdir}/(pkg|GNUmakefile)' to /go/src/github.com/hashicorp/consul/pkg" + ( + tar -c pkg/web_ui GNUmakefile | docker cp - ${container_id}:/go/src/github.com/hashicorp/consul && + echo "Running build in container" && docker start -i ${container_id} && + echo "Copying back artifacts" && docker cp ${container_id}:/go/src/github.com/hashicorp/consul/bindata_assetfs.go ${sdir}/agent/bindata_assetfs.go + ) + ret=$? + docker rm ${container_id} > /dev/null + fi + popd >/dev/null + return $ret +} + +function build_consul { + # Arguments: + # $1 - Path to the top level Consul source + # $2 - The docker image to run the build within (optional) + # + # Returns: + # 0 - success + # * - error + # + # Note: + # The GOLDFLAGS and GOTAGS environment variables will be used if set + # If the CONSUL_DEV environment var is truthy only the local platform/architecture is built. + # If the XC_OS or the XC_ARCH environment vars are present then only those platforms/architectures + # will be built. Otherwise all supported platform/architectures are built + + if ! test -d "$1" + then + echo "ERROR: '$1' is not a directory. build_consul must be called with the path to the top level source as the first argument'" 1>&2 + return 1 + fi + + local sdir="$1" + local image_name=${GO_BUILD_CONTAINER_DEFAULT} + if test -n "$2" + then + image_name="$2" + fi + + pushd ${sdir} > /dev/null + echo "Creating the Go Build Container" + if is_set "${CONSUL_DEV}" + then + XC_OS=$(go_env GOOS) + XC_ARCH=$(go env GOARCH) + else + XC_OS=${XC_OS:-"solaris darwin freebsd linux windows"} + XC_ARCH=${XC_ARCH:-"386 amd64 arm arm64"} + fi + + local container_id=$(docker create -it ${image_name} gox -os="${XC_OS}" -arch="${XC_ARCH}" -osarch="!darwin/arm !darwin/arm64" -ldflags "${GOLDFLAGS}" -output "pkg/{{.OS}}_{{.Arch}}/consul" -tags="${GOTAGS}") + ret=$? + + if test $ret -eq 0 + then + echo "Copying the source from '${sdir}' to /go/src/github.com/hashicorp/consul/pkg" + ( + tar -c $(ls | grep -v "ui\|ui-v2\|website\|bin\|.git") | docker cp - ${container_id}:/go/src/github.com/hashicorp/consul && + echo "Running build in container" && + docker start -i ${container_id} && + echo "Copying back artifacts" && + docker cp ${container_id}:/go/src/github.com/hashicorp/consul/pkg/ pkg.new + ) + ret=$? + docker rm ${container_id} > /dev/null + + DEV_PLATFORM="./pkg.new/$(go env GOOS)_$(go env GOARCH)" + for F in $(find ${DEV_PLATFORM} -mindepth 1 -maxdepth 1 -type f) + do + cp ${F} bin/ + cp ${F} ${GOPATH}/bin + done + + cp -r pkg.new/* pkg/ + rm -r pkg.new + fi + popd > /dev/null + return $ret +} + +function package_release { + # Arguments: + # $1 - Path to the top level Consul source + # $2 - Version to use in the names of the zip files (optional) + # + # Returns: + # 0 - success + # * - error + + if ! test -d "$1" + then + echo "ERROR: '$1' is not a directory. package_release must be called with the path to the top level source as the first argument'" 1>&2 + return 1 + fi + + local vers="${2}" + if test -z "${vers}" + then + vers=$(get_version $1 false) + ret=$? + if test "$ret" -ne 0 + then + echo "ERROR: failed to determine the version." 1>&2 + return $ret + fi + fi + + local sdir="$1" + local ret=0 + for platform in $(find "${sdir}/pkg" -mindepth 1 -maxdepth 1 -type d) + do + local os_arch=$(basename $platform) + pushd "${platform}" > /dev/null + zip "${sdir}/pkg/dist/consul_${vers}_${os_arch}.zip" ./* + ret=$? + popd > /dev/null + + if test "$ret" -ne 0 + then + break + fi + done + + return $ret +} \ No newline at end of file diff --git a/ui-v2/GNUmakefile b/ui-v2/GNUmakefile index 9b5cb8ba9b..70d15d3841 100644 --- a/ui-v2/GNUmakefile +++ b/ui-v2/GNUmakefile @@ -3,8 +3,13 @@ ROOT:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) server: yarn run start -dist: +init: + yarn install + +build: yarn run build + +dist: build mv dist ../pkg/web_ui/v2 lint: @@ -12,4 +17,4 @@ lint: format: yarn run format:js -.PHONY: server dist lint format +.PHONY: server build dist lint format diff --git a/ui-v2/config/environment.js b/ui-v2/config/environment.js index d1ac0949bf..b72e7b00de 100644 --- a/ui-v2/config/environment.js +++ b/ui-v2/config/environment.js @@ -29,12 +29,19 @@ module.exports = function(environment) { }; ENV = Object.assign({}, ENV, { CONSUL_GIT_SHA: (function() { + if (process.env.CONSUL_GIT_SHA) { + return process.env.CONSUL_GIT_SHA + } + return require('child_process') .execSync('git rev-parse --short HEAD') .toString() .trim(); })(), CONSUL_VERSION: (function() { + if (process.env.CONSUL_VERSION) { + return process.env.CONSUL_VERSION + } // see /scripts/dist.sh:8 const version_go = `${path.dirname(path.dirname(__dirname))}/version/version.go`; const contents = fs.readFileSync(version_go).toString(); @@ -46,6 +53,13 @@ module.exports = function(environment) { .trim() .split('"')[1]; })(), + CONSUL_BINARY_TYPE: (function() { + if (process.env.CONSUL_BINARY_TYPE) { + return process.env.CONSUL_BINARY_TYPE + } + + return "oss" + }), CONSUL_DOCUMENTATION_URL: 'https://www.consul.io/docs', CONSUL_COPYRIGHT_URL: 'https://www.hashicorp.com', CONSUL_COPYRIGHT_YEAR: '2018',