Merge pull request #1736 from hashicorp/f-makefile-cleanup

Refactors dist into a Docker-based build with extra safety checks.

GNUmakefile

@@ -5,33 +5,29 @@ VETARGS?=-asmdecl -atomic -bool -buildtags -copylocks -methods \
          -nilfunc -printf -rangeloops -shift -structtags -unsafeptr
 VERSION?=$(shell awk -F\" '/^const Version/ { print $$2; exit }' version.go)
 
-all: format tools
+# all builds binaries for all targets
+all: tools
 	@mkdir -p bin/
-	@bash --norc -i ./scripts/build.sh
-
-# bin generates the releasable binaries
-bin: generate
 	@sh -c "'$(CURDIR)/scripts/build.sh'"
 
 # dev creates binaries for testing locally - these are put into ./bin and $GOPATH
-dev: generate
+dev: format
 	@CONSUL_DEV=1 sh -c "'$(CURDIR)/scripts/build.sh'"
 
-# dist creates the binaries for distibution
+# dist builds binaries for all platforms and packages them for distribution
-dist: bin
+dist:
 	@sh -c "'$(CURDIR)/scripts/dist.sh' $(VERSION)"
 
 cov:
 	gocov test ./... | gocov-html > /tmp/coverage.html
 	open /tmp/coverage.html
 
-test:
+test: format
 	@$(MAKE) vet
 	@./scripts/verify_no_uuid.sh
 	@./scripts/test.sh
 
 cover:
-	./scripts/verify_no_uuid.sh
 	go list ./... | xargs -n1 go test --cover
 
 format:
@@ -54,12 +50,7 @@ vet:
 		echo "and fix them if necessary before submitting the code for reviewal."; \
 	fi
 
-# generate runs `go generate` to build the dynamically generated source files
+# generates the static web ui that's compiled into the binary
-generate:
-	find . -type f -name '.DS_Store' -delete
-	go generate ./...
-
-# generates the static web ui
 static-assets:
 	@echo "--> Generating static assets"
 	@go-bindata-assetfs -pkg agent -prefix pkg ./pkg/web_ui/...
@@ -69,10 +60,4 @@ static-assets:
 tools:
 	go get -u -v $(GOTOOLS)
 
-web:
+.PHONY: all bin dev dist cov test cover format vet static-assets tools
-	./scripts/website_run.sh
-
-web-push:
-	./scripts/website_push.sh
-
-.PHONY: all bin dev dist cov test vet web web-push generate static-assets tools

Vagrantfile

@@ -1,72 +0,0 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
-VAGRANTFILE_API_VERSION = '2'
-
-@script = <<SCRIPT
-GOVERSION="1.6"
-SRCROOT="/opt/go"
-SRCPATH="/opt/gopath"
-
-# Get the ARCH
-ARCH=`uname -m | sed 's|i686|386|' | sed 's|x86_64|amd64|'`
-
-# Install Go
-sudo apt-get update
-sudo apt-get install -y build-essential git-core zip curl
-
-# Install Go
-cd /tmp
-curl -s -O https://storage.googleapis.com/golang/go${GOVERSION}.linux-${ARCH}.tar.gz
-tar -xvf go${GOVERSION}.linux-${ARCH}.tar.gz
-sudo mv go $SRCROOT
-sudo chmod 775 $SRCROOT
-sudo chown vagrant:vagrant $SRCROOT
-
-# Setup the GOPATH; even though the shared folder spec gives the consul
-# directory the right user/group, we need to set it properly on the
-# parent path to allow subsequent "go get" commands to work. We can't do
-# normal -R here because VMWare complains if we try to update the shared
-# folder permissions, so we just update the folders that matter.
-sudo mkdir -p $SRCPATH
-find /opt/gopath -type d -maxdepth 3 | xargs sudo chown vagrant:vagrant
-
-cat <<EOF >/tmp/gopath.sh
-export GOPATH="$SRCPATH"
-export GOROOT="$SRCROOT"
-export PATH="$SRCROOT/bin:$SRCPATH/bin:\$PATH"
-EOF
-sudo mv /tmp/gopath.sh /etc/profile.d/gopath.sh
-sudo chmod 0755 /etc/profile.d/gopath.sh
-source /etc/profile.d/gopath.sh
-
-# Install go tools
-go get golang.org/x/tools/cmd/cover
-SCRIPT
-
-Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
-  config.vm.provision 'shell', inline: @script, privileged: false
-  config.vm.synced_folder '.', '/opt/gopath/src/github.com/hashicorp/consul'
-
-  %w[vmware_fusion vmware_workstation].each do |p|
-    config.vm.provider p do |v|
-      v.vmx['memsize'] = '2048'
-      v.vmx['numvcpus'] = '2'
-      v.vmx['cpuid.coresPerSocket'] = '1'
-    end
-  end
-
-  config.vm.provider "virtualbox" do |v|
-    v.memory = 2048
-    v.cpus = 2
-  end
-
-  config.vm.define '64bit' do |n1|
-    n1.vm.box = 'hashicorp/precise64'
-  end
-
-  config.push.define "www", strategy: "local-exec" do |push|
-    push.script = "scripts/website_push.sh"
-  end
-end

scripts/build.sh

@@ -4,6 +4,7 @@
 set -e
 
 export GO15VENDOREXPERIMENT=1
+export CGO_ENABLED=0
 
 # Get the parent directory of where this script is.
 SOURCE="${BASH_SOURCE[0]}"

scripts/consul-builder/Dockerfile

@@ -0,0 +1,30 @@
+FROM ubuntu:trusty
+
+ENV GOVERSION 1.6
+
+RUN apt-get update -y && \
+    apt-get install --no-install-recommends -y -q \
+            build-essential \
+            ca-certificates \
+            curl \
+            git \
+            ruby \
+            ruby-dev \
+            zip && \
+    gem install bundler
+
+RUN mkdir /goroot && \
+    mkdir /gopath && \
+    curl https://storage.googleapis.com/golang/go${GOVERSION}.linux-amd64.tar.gz | \
+         tar xzf - -C /goroot --strip-components=1
+
+# We want to ensure that release builds never have any cgo dependencies so we
+# switch that off at the highest level.
+ENV CGO_ENABLED 0
+ENV GOPATH /gopath
+ENV GOROOT /goroot
+ENV PATH $GOROOT/bin:$GOPATH/bin:$PATH
+
+RUN mkdir -p $GOPATH/src/github.com/hashicorp/consul
+WORKDIR $GOPATH/src/github.com/hashicorp/consul
+CMD ./scripts/dist_build.sh

scripts/dist.sh

@@ -3,7 +3,7 @@ set -e
 
 export GO15VENDOREXPERIMENT=1
 
-# Get the version from the command line
+# Get the version from the command line.
 VERSION=$1
 if [ -z $VERSION ]; then
     echo "Please specify a version."
@@ -15,23 +15,23 @@ SOURCE="${BASH_SOURCE[0]}"
 while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
 DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
 
-# Change into that dir because we expect that
+# Change into that dir because we expect that.
 cd $DIR
 
-# Generate the tag
+# Generate the tag.
 if [ -z $NOTAG ]; then
   echo "==> Tagging..."
   git commit --allow-empty -a --gpg-sign=348FFC4C -m "Release v$VERSION"
   git tag -a -m "Version $VERSION" -s -u 348FFC4C "v${VERSION}" master
 fi
 
-# Generate the UI
+# Do a hermetic build inside a Docker container.
-if [ -z $NOUI ]; then
+if [ -z $NOBUILD ]; then
-  echo "==> Generating Consul UI..."
+    docker build -t hashicorp/consul-builder scripts/consul-builder/
-  make -f ui/Makefile dist
+    docker run --rm -v "$(pwd)":/gopath/src/github.com/hashicorp/consul hashicorp/consul-builder
 fi
 
-# Zip all the files
+# Zip all the files.
 rm -rf ./pkg/dist
 mkdir -p ./pkg/dist
 for FILENAME in $(find ./pkg -mindepth 1 -maxdepth 1 -type f); do
@@ -39,7 +39,7 @@ for FILENAME in $(find ./pkg -mindepth 1 -maxdepth 1 -type f); do
   cp ./pkg/${FILENAME} ./pkg/dist/consul_${VERSION}_${FILENAME}
 done
 
-# Make the checksums
+# Make the checksums.
 pushd ./pkg/dist
 shasum -a256 * > ./consul_${VERSION}_SHA256SUMS
 if [ -z $NOSIGN ]; then
@@ -48,10 +48,4 @@ if [ -z $NOSIGN ]; then
 fi
 popd
 
-# Upload
-if [ -z $NORELEASE ]; then
-  echo "==> Uploading binaries..."
-  hc-releases -upload=./pkg/dist
-fi
-
 exit 0

scripts/dist_build.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+set -e
+
+export GO15VENDOREXPERIMENT=1
+
+# Get the parent directory of where this script is.
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
+DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
+
+# Change into that dir because we expect that.
+cd $DIR
+
+# Make sure build tools are abailable.
+make -f GNUMakefile tools
+
+# Build the standalone version of the web assets for the sanity check.
+pushd ui
+bundle
+make dist
+popd
+
+# Fixup the timestamps to match what's checked in. This will allow us to cleanly
+# verify that the checked-in content is up to date without spurious diffs of the
+# file mod times.
+pushd pkg
+cat ../command/agent/bindata_assetfs.go | ../scripts/fixup_times.sh
+popd
+
+# Regenerate the built-in web assets. If there are any diffs after doing this
+# then we know something is up.
+make -f GNUMakefile static-assets
+if ! git diff --quiet command/agent/bindata_assetfs.go; then
+   echo "Checked-in web assets are out of date, build aborted"
+   exit 1
+fi
+
+# Now we are ready to do a clean build of everything. The "all" build will blow
+# away our pkg folder so we have to regenerate the ui once more. This is probably
+# for the best since we have meddled with the timestamps.
+rm -rf pkg
+make -f GNUMakefile all
+pushd ui
+make dist
+popd
+
+exit 0

scripts/fixup_times.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -e
+regex='bindataFileInfo.*name: \"(.+)\".*time.Unix.(.+),'
+while read line; do
+    if [[ $line =~ $regex ]]; then
+        file=${BASH_REMATCH[1]}
+        ts=${BASH_REMATCH[2]}
+        touch --date @$ts $file
+    fi
+done