beacon-node-builds: automate building multiple branches

Signed-off-by: Jakub Sokołowski <jakub@status.im>

ansible/group_vars/nimbus-master.yml

@@ -1,4 +1,7 @@
 ---
+beacon_node_builds_docker_hub_user: '{{lookup("passwordstore", "cloud/DockerHub/user")}}'
+beacon_node_builds_docker_hub_token: '{{lookup("passwordstore", "cloud/DockerHub/token")}}'
+
 # Master host has more memory, but it should be for builds
 beacon_node_mem_limit: '{{ (ansible_memtotal_mb * 0.25) | int }}'
 beacon_node_mem_reserve: '{{ (ansible_memtotal_mb * 0.2) | int }}'

ansible/nimbus.yml

@@ -7,6 +7,11 @@
         that: 'ansible_version.full is version_compare("2.8", ">=")'
         msg: 'Your Ansible version is lower than 2.8. Upgrade it.'
 
+- name: Configure beacon node builds
+  hosts: nimbus-master
+  roles:
+    - { role: beacon-node-builds,     tags: beacon-node-builds }
+
 - name: Configure shared testnet3 using master
   hosts: nimbus.test[0:3]
   vars:

ansible/roles/beacon-node-builds/defaults/main.yml

@@ -0,0 +1,25 @@
+---
+beacon_node_builds_user: 'nimbus'
+beacon_node_builds_path: '/opt/beacon-node-builds'
+beacon_node_builds_image_name: 'statusteam/nimbus_beacon_node'
+
+# Required for pushing the images
+beacon_node_builds_docker_hub_user: ~
+beacon_node_builds_docker_hub_token: ~
+
+# Timers
+beacon_node_build_timer_timeout: 1200
+
+beacon_node_builds_branches:
+  - name: 'master'
+    branch: 'master'
+    frequency: '*-*-* 02:00:00'
+    batches: [ 'a', 'b' ]
+  - name: 'devel'
+    branch: 'devel'
+    frequency: '*-*-* 10:00:00'
+    batches: [ 'a', 'b' ]
+  - name: 'libp2p'
+    branch: 'nim-libp2p-auto-bump'
+    frequency: '*-*-* 18:00:00'
+    batches: [ 'a', 'b' ]

ansible/roles/beacon-node-builds/files/Dockerfile

@@ -0,0 +1,11 @@
+FROM alpine:3.12.1
+
+ARG COMMIT
+
+RUN apk add --no-cache libgcc pcre
+
+COPY repo/build/* /usr/local/bin/
+
+STOPSIGNAL SIGINT
+
+ENTRYPOINT ["/usr/local/bin/beacon_node"]

ansible/roles/beacon-node-builds/files/dockerignore

@@ -0,0 +1,2 @@
+repo
+!repo/build/*

ansible/roles/beacon-node-builds/tasks/build.yml

@@ -0,0 +1,55 @@
+---
+- name: '{{ item.name }} - Create builds folder'
+  file:
+    path: '{{ beacon_node_builds_path }}'
+    owner: '{{ beacon_node_builds_user }}'
+    group: 'adm'
+    mode: 0755
+    state: 'directory'
+
+- name: '{{ item.name }} - Clone branch'
+  git:
+    repo: 'https://github.com/status-im/nimbus-eth2'
+    version: '{{ item.branch }}'
+    dest: '{{ beacon_node_builds_path }}/{{ item.name }}/repo'
+    update: true
+    force: true
+  become_user: '{{ beacon_node_builds_user }}'
+
+- name: '{{ item.name }} - Create script'
+  template:
+    src: 'build.sh.j2'
+    dest: '{{ beacon_node_builds_path }}/{{ item.name }}/build.sh'
+    owner: '{{ beacon_node_builds_user }}'
+    group: 'adm'
+    mode: 0755
+
+- name: '{{ item.name }} - Create Dockerfile'
+  copy:
+    src: 'Dockerfile'
+    dest: '{{ beacon_node_builds_path }}/{{ item.name }}/Dockerfile'
+    owner: '{{ beacon_node_builds_user }}'
+    group: 'adm'
+    mode: 0644
+
+- name: '{{ item.name }} - Create .dockerignore'
+  copy:
+    src: 'dockerignore'
+    dest: '{{ beacon_node_builds_path }}/{{ item.name }}/.dockerignore'
+    owner: '{{ beacon_node_builds_user }}'
+    group: 'adm'
+    mode: 0644
+
+
+- name: Create timer for rebuilding image
+  include_role: name=systemd-timer
+  vars:
+    systemd_timer_name: 'beacon-node-build-{{ item.name }}'
+    systemd_timer_user: '{{ beacon_node_builds_user }}'
+    systemd_timer_description: 'Docker image build for Nimbus beacon node'
+    systemd_timer_requires_extra: 'docker.service'
+    systemd_timer_start_on_creation: false
+    systemd_timer_frequency: '{{ item.frequency }}'
+    systemd_timer_timeout_sec: '{{ beacon_node_build_timer_timeout }}'
+    systemd_timer_work_dir: '{{ beacon_node_builds_path }}/{{ item.name }}'
+    systemd_timer_script_path: '{{ beacon_node_builds_path }}/{{ item.name }}/build.sh'

ansible/roles/beacon-node-builds/tasks/install.yml

@@ -0,0 +1,6 @@
+---
+- name: Install build dependencies
+  apt:
+    name:
+      - build-essential
+      - libpcre3-dev

ansible/roles/beacon-node-builds/tasks/main.yml

@@ -0,0 +1,5 @@
+---
+- include_tasks: install.yml
+- include_tasks: user.yml
+- include_tasks: build.yml
+  with_items: '{{ beacon_node_builds_branches }}'

ansible/roles/beacon-node-builds/tasks/user.yml

@@ -0,0 +1,17 @@
+---
+- name: Create user for beacon node builds
+  user:
+    name: '{{ beacon_node_builds_user }}'
+    group: 'adm'
+    groups: ['docker']
+    shell: '/bin/zsh'
+
+- name: Set disable rebase as merge strategy
+  command: git config --global pull.rebase false
+  become_user: '{{ beacon_node_builds_user }}'
+
+- name: Configure access to Docker Hub
+  command: |
+    docker login \
+      --username {{ beacon_node_builds_docker_hub_user | mandatory }} \
+      --password {{ beacon_node_builds_docker_hub_token | mandatory }}

ansible/roles/beacon-node-builds/templates/build.sh.j2

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+set -e
+
+if [[ "${USER}" != "{{ beacon_node_builds_user }}" ]]; then
+    echo "Incorrect user: ${USER}" >&2
+    echo "Expected: {{ beacon_node_builds_user }}" >&2
+    exit 1
+fi
+
+IMAGE="{{ beacon_node_builds_image_name }}:{{ item.name }}"
+BATCHES=({{ item.batches | join(" ") }})
+
+# Build the Beacon node binaries
+{
+  cd repo
+  COMMIT_BEFORE=$(git rev-parse --short=8 HEAD)
+  git pull
+  COMMIT_AFTER=$(git rev-parse --short=8 HEAD)
+
+  if [[ "${COMMIT_BEFORE}" == "${COMMIT_AFTER}" ]]; then
+      echo "Nothing new to build."
+      exit
+  fi
+
+  make update
+  make \
+    LOG_LEVEL="TRACE" \
+    MAKEFLAGS="-j$(nproc)" \
+    NIMFLAGS="-d:insecure -d:testnet_servers_image" \
+    beacon_node signing_process
+}
+
+# Add binary into a simple Alpine image
+docker build -t "${IMAGE}" --build-arg=COMMIT=${COMMIT_AFTER} .
+docker push "${IMAGE}"