Refactor a little to accomodate overriding packaging using directories

build-support/functions/10-util.sh

@@ -754,3 +754,52 @@ function commit_dev_mode {
    popd >/dev/null
    return ${ret}
 }
+
+function gpg_detach_sign {
+   # Arguments:
+   #   $1 - File to sign
+   #   $2 - Alternative GPG key to use for signing
+   #
+   # Returns:
+   #   0 - success
+   #   * - failure
+   
+   # determine whether the gpg key to use is being overridden
+   local gpg_key=${HASHICORP_GPG_KEY}
+   if test -n "$2"
+   then
+      gpg_key=$2
+   fi
+   
+   gpg --default-key "${gpg_key}" --detach-sig --yes -v  "$1"
+   return $?
+}
+
+function shasum_directory {
+   # Arguments:
+   #   $1 - Path to directory containing the files to shasum
+   #   $2 - File to output sha sums to
+   #
+   # Returns:
+   #   0 - success
+   #   * - failure
+   
+   if ! test -d "$1"
+   then
+      err "ERROR: '$1' is not a directory and shasum_release requires passing a directory as the first argument"
+      return 1
+   fi
+   
+   if test -z "$2"
+   then
+      err "ERROR: shasum_release requires a second argument to be the filename to output the shasums to but none was given"
+      return 1 
+   fi
+   
+   pushd $1 > /dev/null
+   shasum -a256 * > "$2"
+   ret=$?
+   popd >/dev/null
+   
+   return $ret
+}

build-support/functions/20-build.sh

@@ -180,7 +180,7 @@ function build_assetfs {
 function build_consul_post {
    # Arguments
    #   $1 - Path to the top level Consul source
-   #   $2 - build suffix (Optional)
+   #   $2 - Subdirectory under pkg/bin (Optional)
    #
    # Returns:
    #   0 - success
@@ -199,17 +199,25 @@ function build_consul_post {
    
    local sdir="$1"
    
+   local extra_dir_name="$2"
+   local extra_dir=""
+   
+   if test -n "${extra_dir_name}"
+   then
+      extra_dir="${extra_dir_name}/"
+   fi
+   
    pushd "${sdir}" > /dev/null
    
    # recreate the pkg dir
-   rm -r pkg/bin/*${2} 2> /dev/null
+   rm -r pkg/bin/${extra_dir}* 2> /dev/null
-   mkdir -p pkg/bin 2> /dev/null
+   mkdir -p pkg/bin/${extra_dir} 2> /dev/null
 
    # move all files in pkg.new into pkg
-   cp -r pkg.bin.new/* pkg/bin/
+   cp -r pkg.bin.new/${extra_dir}* pkg/bin/${extra_dir}
    rm -r pkg.bin.new
       
-   DEV_PLATFORM="./pkg/bin/$(go env GOOS)_$(go env GOARCH)${2}"
+   DEV_PLATFORM="./pkg/bin/${extra_dir}$(go env GOOS)_$(go env GOARCH)"
    for F in $(find ${DEV_PLATFORM} -mindepth 1 -maxdepth 1 -type f)
    do
       # recreate the bin dir
@@ -228,7 +236,7 @@ function build_consul_post {
 function build_consul {
    # Arguments:
    #   $1 - Path to the top level Consul source
-   #   $2 - build suffix (optional - must specify if needing to specify the docker image)
+   #   $2 - Subdirectory to put binaries in under pkg/bin (optional - must specify if needing to specify the docker image)
    #   $3 - The docker image to run the build within (optional)
    #
    # Returns:
@@ -248,7 +256,8 @@ function build_consul {
    fi
    
    local sdir="$1"
-   local build_suffix="$2"
+   local extra_dir_name="$2"
+   local extra_dir=""
    local image_name=${GO_BUILD_CONTAINER_DEFAULT}
    if test -n "$3"
    then
@@ -272,14 +281,19 @@ function build_consul {
    XC_OS=${XC_OS:-"solaris darwin freebsd linux windows"}
    XC_ARCH=${XC_ARCH:-"386 amd64 arm arm64"}
    
-   local container_id=$(docker create -it -e CGO_ENABLED=0 ${image_name} gox -os="${XC_OS}" -arch="${XC_ARCH}" -osarch="!darwin/arm !darwin/arm64" -ldflags "${GOLDFLAGS}" -output "pkg/bin/{{.OS}}_{{.Arch}}${build_suffix}/consul" -tags="${GOTAGS}")
+   if test -n "${extra_dir_name}"
+   then
+      extra_dir="${extra_dir_name}/"
+   fi
+
+   local container_id=$(docker create -it -e CGO_ENABLED=0 ${image_name} gox -os="${XC_OS}" -arch="${XC_ARCH}" -osarch="!darwin/arm !darwin/arm64" -ldflags "${GOLDFLAGS}" -output "pkg/bin/${extra_dir}{{.OS}}_{{.Arch}}/consul" -tags="${GOTAGS}")
    ret=$?
 
    if test $ret -eq 0
    then
-      status "Copying the source from '${sdir}' to /go/src/github.com/hashicorp/consul/pkg"
+      status "Copying the source from '${sdir}' to /go/src/github.com/hashicorp/consul"
       (
-         tar -c $(ls | grep -v "ui\|ui-v2\|website\|bin\|.git") | docker cp - ${container_id}:/go/src/github.com/hashicorp/consul &&
+         tar -c $(ls | grep -v "^(ui\|ui-v2\|website\|bin\|pkg\|.git)") | docker cp - ${container_id}:/go/src/github.com/hashicorp/consul &&
          status "Running build in container" &&
          docker start -i ${container_id} &&
          status "Copying back artifacts" &&
@@ -290,7 +304,7 @@ function build_consul {
       
       if test $ret -eq 0
       then
-         build_consul_post "${sdir}" "${build_suffix}"
+         build_consul_post "${sdir}" "${extra_dir_name}"
          ret=$?
       else
          rm -r pkg.bin.new 2> /dev/null
@@ -305,7 +319,7 @@ function build_consul_local {
    #   $1 - Path to the top level Consul source
    #   $2 - Space separated string of OSes to build. If empty will use env vars for determination.
    #   $3 - Space separated string of architectures to build. If empty will use env vars for determination.
-   #   $4 - build suffix (optional)
+   #   $4 - Subdirectory to put binaries in under pkg/bin (optional)
    #
    # Returns:
    #   0 - success
@@ -326,7 +340,13 @@ function build_consul_local {
    local sdir="$1"
    local build_os="$2"
    local build_arch="$3"
-   local build_suffix="$4"
+   local extra_dir_name="$4"
+   local extra_dir=""  
+   
+   if test -n "${extra_dir_name}"
+   then
+      extra_dir="${extra_dir_name}/"
+   fi 
    
    pushd ${sdir} > /dev/null
    if is_set "${CONSUL_DEV}"
@@ -361,7 +381,7 @@ function build_consul_local {
       -arch="${build_arch}" \
       -osarch="!darwin/arm !darwin/arm64" \
       -ldflags="${GOLDFLAGS}" \
-      -output "pkg.bin.new/{{.OS}}_{{.Arch}}${build_suffix}/consul" \
+      -output "pkg.bin.new/${extra_dir}{{.OS}}_{{.Arch}}${build_suffix}/consul" \
       -tags="${GOTAGS}" \
       .
 
@@ -372,7 +392,7 @@ function build_consul_local {
       return 1
    fi
 
-   build_consul_post "${sdir}" "${build_suffix}"
+   build_consul_post "${sdir}" "${extra_dir_name}"
    if test $? -ne 0
    then
       err "ERROR: Failed postprocessing Consul binaries"

build-support/functions/30-release.sh

@@ -63,10 +63,54 @@ function tag_release {
    return $ret
 }
 
-function package_release {
+function package_binaries {
+   # Arguments:
+   #   $1 - Path to the directory containing the built binaries
+   #   $2 - Destination path of the packaged binaries
+   #   $3 - Version
+   #
+   # Returns:
+   #   0 - success
+   #   * - error
+   
+   local sdir="$1"
+   local ddir="$2"
+   local vers="$3"
+   local ret=0   
+
+   
+   if ! test -d "${sdir}"
+   then
+      err "ERROR: '$1' is not a directory. package_binaries must be called with the path to the directory containing the binaries" 
+      return 1
+   fi
+   
+   rm -rf "${ddir}" > /dev/null 2>&1 
+   mkdir -p "${ddir}" >/dev/null 2>&1 
+   for platform in $(find "${sdir}" -mindepth 1 -maxdepth 1 -type d)
+   do
+      local os_arch=$(basename $platform)
+      local dest="${ddir}/${CONSUL_PKG_NAME}_${vers}_${os_arch}.zip"
+      status "Compressing ${os_arch} directory into ${dest}"
+      pushd "${platform}" > /dev/null
+      zip "${ddir}/${CONSUL_PKG_NAME}_${vers}_${os_arch}.zip" ./*
+      ret=$?
+      popd > /dev/null
+      
+      if test "$ret" -ne 0
+      then
+         break
+      fi
+   done
+   
+   return ${ret}
+}
+
+function package_release_one {
    # Arguments:
    #   $1 - Path to the top level Consul source
    #   $2 - Version to use in the names of the zip files (optional)
+   #   $3 - Subdirectory under pkg/dist to use (optional)
    #
    # Returns:
    #   0 - success
@@ -78,9 +122,16 @@ function package_release {
       return 1
    fi
 
-   local sdir="${1}"
+   local sdir="$1"
    local ret=0   
-   local vers="${2}"
+   local vers="$2"
+   local extra_dir_name="$3"
+   local extra_dir=""
+   
+   if test -n "${extra_dir_name}"
+   then
+      extra_dir="${extra_dir_name}/"
+   fi
 
    if test -z "${vers}"
    then
@@ -93,81 +144,91 @@ function package_release {
       fi
    fi
    
-   rm -rf "${sdir}/pkg/dist" > /dev/null 2>&1 
+   package_binaries "${sdir}/pkg/bin/${extra_dir}" "${sdir}/pkg/dist/${extra_dir}" "${vers}"
-   mkdir -p "${sdir}/pkg/dist" >/dev/null 2>&1 
+   return $?
-   for platform in $(find "${sdir}/pkg/bin" -mindepth 1 -maxdepth 1 -type d)
+}
-   do
-      local os_arch=$(basename $platform)
-      local dest="${sdir}/pkg/dist/${CONSUL_PKG_NAME}_${vers}_${os_arch}.zip"
-      status "Compressing ${os_arch} directory into ${dest}"
-      pushd "${platform}" > /dev/null
-      zip "${sdir}/pkg/dist/${CONSUL_PKG_NAME}_${vers}_${os_arch}.zip" ./*
-      ret=$?
-      popd > /dev/null
 
-      if test "$ret" -ne 0
+function package_release {
-      then
+   # Arguments:
-         break
+   #   $1 - Path to the top level Consul source
-      fi
+   #   $2 - Version to use in the names of the zip files (optional)
-   done
+   #
+   # Returns:
+   #   0 - success
+   #   * - error
    
-   return $ret
+   package_release_one "$1" "$2" ""
+   return $?
 }
 
 function shasum_release {
    # Arguments:
-   #   $1 - Path to directory containing the files to shasum
+   #   $1 - Path to the dist directory
-   #   $2 - File to output sha sums to
+   #   $2 - Version of the release
    #
    # Returns:
    #   0 - success
    #   * - failure
    
+   local sdir="$1"
+   local vers="$2"
+   
    if ! test -d "$1"
    then
-      err "ERROR: '$1' is not a directory and shasum_release requires passing a directory as the first argument"
+      err "ERROR: sign_release requires a path to the dist dir as the first argument"
       return 1
    fi
    
-   if test -z "$2"
+   if test -z "${vers}"
    then
-      err "ERROR: shasum_release requires a second argument to be the filename to output the shasums to but none was given"
+      err "ERROR: sign_release requires a version to be specified as the second argument"
       return 1
    fi
    
-   pushd $1 > /dev/null
+   local hfile="${CONSUL_PKG_NAME}_${vers}_SHA256SUMS"
-   shasum -a256 * > "$2"
-   ret=$?
-   popd >/dev/null
    
-   return $ret
+   shasum_directory "${sdir}" "${sdir}/${hfile}"
+   return $?
 }
 
 function sign_release {
    # Arguments:
-   #   $1 - File to sign
+   #   $1 - Path to distribution directory
+   #   $2 - Version
    #   $2 - Alternative GPG key to use for signing
    #
    # Returns:
    #   0 - success
    #   * - failure
    
-   # determine whether the gpg key to use is being overridden
+   local sdir="$1"
-   local gpg_key=${HASHICORP_GPG_KEY}
+   local vers="$2"
-   if test -n "$2"
+   
+   if ! test -d "${sdir}"
    then
-      gpg_key=$2
+      err "ERROR: sign_release requires a path to the dist dir as the first argument"
+      return 1
    fi
    
-   gpg --default-key "${gpg_key}" --detach-sig --yes -v  "$1"
+   if test -z "${vers}"
-   return $?
+   then
+      err "ERROR: sign_release requires a version to be specified as the second argument"
+      return 1
+   fi
+   
+   local hfile="${CONSUL_PKG_NAME}_${vers}_SHA256SUMS"
+   
+   status_stage "==> Signing ${hfile}"
+   gpg_detach_sign "${1}/${hfile}" "$2" || return 1
+   return 0
 }
 
-function check_release {
+function check_release_one {
    # Arguments:
    #   $1 - Path to the release files
    #   $2 - Version to expect
    #   $3 - boolean whether to expect the signature file
+   #   $4 - Release Name (optional)
    #
    # Returns:
    #   0 - success
@@ -177,6 +238,13 @@ function check_release {
    
    declare -a expected_files
    
+   declare log_extra=""
+   
+   if test -n "$4"
+   then
+      log_extra="for $4 "
+   fi
+   
    expected_files+=("${CONSUL_PKG_NAME}_${2}_SHA256SUMS")
    echo "check sig: $3"
    if is_set "$3"
@@ -199,7 +267,7 @@ function check_release {
    
    declare -a found_files
    
-   status_stage "==> Verifying release contents - ${2}"
+   status_stage "==> Verifying release contents ${log_extra}- ${2}"
    debug "Expecting Files:"
    for fname in "${expected_files[@]}"
    do
@@ -254,6 +322,21 @@ function check_release {
    return $ret
 }
 
+function check_release {
+   # Arguments:
+   #   $1 - Path to the release files
+   #   $2 - Version to expect
+   #   $3 - boolean whether to expect the signature file
+   #
+   # Returns:
+   #   0 - success
+   #   * - failure
+   
+   check_release_one "$1" "$2" "$3"
+   return ${ret}
+}
+   
+
 function build_consul_release {
    build_consul "$1" "" "$2"  
 }
@@ -420,7 +503,7 @@ function build_release {
    fi
    
    status_stage "==> Generating SHA 256 Hashes for Binaries"
-   shasum_release "${sdir}/pkg/dist" "${CONSUL_PKG_NAME}_${vers}_SHA256SUMS"
+   shasum_release "${sdir}/pkg/dist" "${vers}"
    if test $? -ne 0
    then
       err "ERROR: Failed to generate SHA 256 hashes for the release"
@@ -429,7 +512,7 @@ function build_release {
    
    if is_set "${do_sha256}"
    then
-      sign_release "${sdir}/pkg/dist/${CONSUL_PKG_NAME}_${vers}_SHA256SUMS" "${gpg_key}"
+      sign_release "${sdir}/pkg/dist" "${vers}" "${gpg_key}"
       if test $? -ne 0
       then
          err "ERROR: Failed to sign the SHA 256 hashes file"