beacon-node-builds: refactor to rebuild after failures

Signed-off-by: Jakub Sokołowski <jakub@status.im>

ansible/roles/beacon-node-builds/templates/Dockerfile.j2

@@ -8,7 +8,7 @@ RUN apt-get -qq update \
  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 {% for target in item.targets %}
-COPY repo/build/{{ target }} /usr/local/bin/
+COPY repo/build/{{ target }}_${COMMIT} /usr/local/bin/{{ target }}
 {% endfor %}
 
 STOPSIGNAL SIGINT

ansible/roles/beacon-node-builds/templates/build.sh.j2

@@ -1,61 +1,95 @@
 #!/usr/bin/env bash
-
 set -e
 
+function headIsDetached() {
+    [[ $(git rev-parse --abbrev-ref --symbolic-full-name HEAD) == "HEAD" ]];
+}
+
+function dockerImageExists() {
+    docker image inspect "${IMAGE}:${COMMIT}" 2>&1 1>/dev/null;
+}
+
+function binaryExists() {
+    ls -l build/{{ item.targets | first }}_${COMMIT} 2>&1 1>/dev/null
+}
+
+function fetchChanges() {
+    # We cannot use "git pull" in here, because history may be changed upstream
+    git fetch
+    git reset --hard "origin/${BRANCH}"
+}
+
+function buildBinaries() {
+    # Lower CPU and I/O priority so it doesn't affect the running beacon node
+    NICE="nice -n 19 ionice -c2 -n7"
+
+    ${NICE} make -j1 update
+    ${NICE} make -j1 {{ item.targets | join(" ") }} \
+        LOG_LEVEL="TRACE" NIMFLAGS="-d:testnet_servers_image -d:noSignalHandler"
+
+    # Rename binaries to match commit the were built from.
+{% for target in item.targets %}
+    mv "build/{{ target }}" "build/{{ target }}_${COMMIT}"
+{% endfor %}
+
+    # Delete copies that are older than 7 days
+    find build -mtime +7 -exec rm '{}' \+
+}
+
+# Adds binary into a simple Alpine image
+function buildDockerImage() {
+    docker build -t "${IMAGE}:${COMMIT}" \
+        --build-arg=COMMIT=${COMMIT} \
+        --label "commit=${COMMIT}" .
+}
+
+function pushImageTag() {
+    docker tag "${IMAGE}:${COMMIT}" "${IMAGE}:${TAG}"
+    docker push "${IMAGE}:${TAG}"
+}
+
+#-------------------------------------------------------------------------------
+
+BRANCH="{{ item.version }}"
+IMAGE="{{ beacon_node_builds_image_name }}"
+TAG="{{ item.name }}"
+
 if [[ "${USER}" != "{{ beacon_node_builds_user }}" ]]; then
     echo "Incorrect user: ${USER}" >&2
     echo "Expected: {{ beacon_node_builds_user }}" >&2
     exit 1
 fi
 
-IMAGE="{{ beacon_node_builds_image_name }}"
-TAG="{{ item.name }}"
-
 # Build the Beacon node binaries
-pushd repo
+pushd repo >/dev/null
-
-COMMIT_BEFORE=$(git rev-parse --short=8 HEAD)
-SYMBOLIC_NAME=$(git rev-parse --abbrev-ref --symbolic-full-name HEAD)
-
-function headIsDetached() { [[ "${SYMBOLIC_NAME}" == "HEAD" ]]; }
 
 # Detached HEAD means we're probably on a tag
 if headIsDetached; then
-    echo "Deatached HEAD, nothing to fetch."
+    echo " >>> Deatached HEAD, nothing to fetch."
 else
-    # We cannot use "git pull" in here, because history may be changed upstream
+    echo " >>> Fetching changes..."
-    git fetch
+    fetchChanges
-    git reset --hard origin/{{ item.version }}
 fi
 
-COMMIT_AFTER=$(git rev-parse --short=8 HEAD)
+COMMIT=$(git rev-parse --short=8 HEAD)
 
-if [[ "$1" == "--force" ]]; then
+if binaryExists && [[ "$1" != "--force" ]]; then
-    echo "Forcing rebuild!"
+    echo " >>> Binary already built"
-elif ! headIsDetached && [[ "${COMMIT_BEFORE}" == "${COMMIT_AFTER}" ]]; then
+else
-    echo "Nothing new to build."
+    echo " >>> Building binaries..."
-    exit
+    buildBinaries
 fi
 
-# Lower CPU and I/O priority so it doesn't affect the running beacon node
+popd >/dev/null
-NICE="nice -n 19 ionice -c2 -n7"
 
-${NICE} make -j1 update
+if dockerImageExists && [[ "$1" != "--force" ]]; then
-${NICE} make -j1 {{ item.targets | join(" ") }} \
+    echo " >>> Image already built: ${IMAGE}:${COMMIT}"
-    LOG_LEVEL="TRACE" NIMFLAGS="-d:testnet_servers_image -d:noSignalHandler"
+else
+    echo " >>> Building Docker image..."
+    buildDockerImage
+fi
 
-# Keep some copies of the resulting binaries, to be used for debugging in case of core dumps
+echo " >>> Pushing Docker image..."
-cp -a build/nimbus_beacon_node "build/nimbus_beacon_node_{{ item.name }}_$(date +%F_%H-%M-%S)"
+pushImageTag
-# Delete copies that are older than 7 days
-find build -name 'nimbus_beacon_node_*' -mtime +7 -exec rm '{}' \+
 
-popd
+echo " >>> SUCCESS - Pushed: ${IMAGE}:${TAG}"
-
-# Add binary into a simple Alpine image
-docker build -t "${IMAGE}:${COMMIT_AFTER}" \
-    --label "commit=${COMMIT_AFTER}" \
-    --build-arg=COMMIT=${COMMIT_AFTER} .
-docker tag "${IMAGE}:${COMMIT_AFTER}" "${IMAGE}:${TAG}"
-docker push "${IMAGE}:${TAG}"
-
-echo "SUCCESS - pushed: ${IMAGE}:${TAG}"