status-im
/
infra-nimbus
mirror of https://github.com/status-im/infra-nimbus.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

ansible: add roles.py script to manage roles 

https://github.com/status-im/infra-template/pull/5
Signed-off-by: Jakub Sokołowski <jakub@status.im>
This commit is contained in:
Jakub Sokołowski 2024-06-13 17:19:32 +02:00
parent f8c12213f8
commit eca6fc2e76
No known key found for this signature in database
GPG Key ID: FE65CD384D5BF7B4
13 changed files with 419 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
Makefile
View File

@ -16,16 +16,19 @@ PROVISIONER_ARCHIVE = $(PROVISIONER_NAME)-$(subst _,-,$(ARCH))_$(PROVISIONER_VER
PROVISIONER_URL = https://github.com/radekg/terraform-provisioner-ansible/releases/download/$(PROVISIONER_VERSION)/$(PROVISIONER_ARCHIVE)
PROVISIONER_PATH = $(TF_PLUGINS_DIR)/$(ARCH)/$(PROVISIONER_NAME)_$(PROVISIONER_VERSION)
all: requirements install-provisioner secrets init-terraform
all: roles-install install-provisioner secrets init-terraform
@echo "Success!"
requirements-install:
ansible-galaxy install --keep-scm-meta --ignore-errors --force -r ansible/requirements.yml
roles-install:
ansible/roles.py --install
requirements-check:
ansible/versioncheck.py
roles-check:
ansible/roles.py --check
requirements: requirements-install requirements-check
roles-update:
ansible/roles.py --update
roles: roles-install roles-check
$(PROVISIONER_PATH):
@mkdir -p $(TF_PLUGINS_DIR)/$(ARCH); \
@ -46,9 +49,3 @@ init-terraform:
cleanup:
rm -r $(TF_PLUGINS_DIR)/$(ARCHIVE)
ssh-config: export SSH_CONFIG_DIR ?= $(HOME)/.ssh/config.d
ssh-config: export SSH_CONFIG_FILE ?= infra-nimbus
ssh-config: export SSH_USERNAME ?= $(USER)
ssh-config:
scripts/create-ssh-config.sh

2
ansible/bootstrap.yml
View File

@ -16,7 +16,7 @@
run_once: true
gather_facts: false
tasks:
- local_action: command ./versioncheck.py
- local_action: command ./roles.py --check
changed_when: false
- name: Bootstrap Python support for Ansible

2
ansible/era.yml
View File

@ -6,7 +6,7 @@
run_once: true
gather_facts: false
tasks:
- local_action: command ./versioncheck.py
- local_action: command ./roles.py --check
changed_when: false
- name: Configure ERA files hosting

2
ansible/eth1.yml
View File

@ -6,7 +6,7 @@
run_once: true
gather_facts: false
tasks:
- local_action: command ./versioncheck.py
- local_action: command ./roles.py --check
changed_when: false
- name: Configure RocketPool & Eth1 nodes

2
ansible/fluffy.yml
View File

@ -6,7 +6,7 @@
run_once: true
gather_facts: false
tasks:
- local_action: command ./versioncheck.py
- local_action: command ./roles.py --check
changed_when: false
- name: Configure Nimbus Fluffy nodes

2
ansible/geth.yml
View File

@ -6,7 +6,7 @@
run_once: true
gather_facts: false
tasks:
- local_action: command ./versioncheck.py
- local_action: command ./roles.py --check
changed_when: false
- name: Configure geth nodes

2
ansible/holesky.yml
View File

@ -6,7 +6,7 @@
run_once: true
gather_facts: false
tasks:
- local_action: command ./versioncheck.py
- local_action: command ./roles.py --check
changed_when: false
- name: Deploy Holesky Linux Beacon Nodes

2
ansible/logs.yml
View File

@ -6,7 +6,7 @@
run_once: true
gather_facts: false
tasks:
- local_action: command ./versioncheck.py
- local_action: command ./roles.py --check
changed_when: false
- name: Configure ElasticSearch servers

2
ansible/mainnet.yml
View File

@ -6,7 +6,7 @@
run_once: true
gather_facts: false
tasks:
- local_action: command ./versioncheck.py
- local_action: command ./roles.py --check
changed_when: false
- name: Configure network mainnet bootnodes

30
ansible/requirements.yml
View File

@ -1,139 +1,109 @@
---
- name: infra-role-bootstrap-linux
src: git@github.com:status-im/infra-role-bootstrap-linux.git
scm: git
- name: infra-role-bootstrap-windows
src: git@github.com:status-im/infra-role-bootstrap-windows.git
scm: git
- name: infra-role-bootstrap-macos
src: git@github.com:status-im/infra-role-bootstrap-macos.git
scm: git
- name: infra-role-wireguard
src: git@github.com:status-im/infra-role-wireguard.git
scm: git
- name: infra-role-open-ports
src: git@github.com:status-im/infra-role-open-ports.git
scm: git
- name: infra-role-swap-file
src: git@github.com:status-im/infra-role-swap-file.git
scm: git
- name: infra-role-nginx
src: git@github.com:status-im/infra-role-nginx.git
version: 3043c998cbc92a634a71dc48363b3e2525696d26
scm: git
- name: infra-role-origin-certs
src: git@github.com:status-im/infra-role-origin-certs.git
version: f2b061da8976444d6576b755578f0e799cde4372
scm: git
- name: infra-role-oauth-proxy
src: git@github.com:status-im/infra-role-oauth-proxy.git
version: 9119f7af1feb809b28fcf68ce044795affe1aa76
scm: git
- name: infra-role-consul-service
src: git@github.com:status-im/infra-role-consul-service.git
version: d62993069bd045edade5d5fd90c098655089db85
scm: git
- name: infra-role-beacon-node-linux
src: git@github.com:status-im/infra-role-beacon-node-linux.git
version: c2ba324a5b442b6e46e2bdf4746320e8e8a24579
scm: git
- name: infra-role-beacon-node-windows
src: git@github.com:status-im/infra-role-beacon-node-windows.git
version: ebfece8f56ef18ba6dbb413778fbf7f56de06683
scm: git
- name: infra-role-beacon-node-macos
src: git@github.com:status-im/infra-role-beacon-node-macos.git
version: f1411a2f0dfc3f2b35cdcda618cd81766a22e4e7
scm: git
- name: infra-role-validator-client
src: git@github.com:status-im/infra-role-validator-client.git
version: aaf86e765ab8024bcc359e26d72f7767dc24ab3d
scm: git
- name: infra-role-nimbus-eth1
src: git@github.com:status-im/infra-role-nimbus-eth1.git
version: f4dfbb31eea9a1fa1fed575749723f9576844c84
scm: git
- name: infra-role-nimbus-fluffy
src: git@github.com:status-im/infra-role-nimbus-fluffy.git
version: 164cd9b1f8bf18cdcc3cce7be43e73743394adaf
scm: git
- name: infra-role-dist-validators
src: git@github.com:status-im/infra-role-dist-validators.git
version: 95a5de787aef58712121d2ac6c0bd97b6948baec
scm: git
- name: infra-role-rocketpool
src: git@github.com:status-im/infra-role-rocketpool.git
version: ab6834c7d291c9bf115a8e106afc83246bf803ae
scm: git
- name: infra-role-winsw
src: git@github.com:status-im/infra-role-winsw.git
version: 93c8a5b5ac091a128efe937c028b548751295297
scm: git
- name: infra-role-kibana
src: git@github.com:status-im/infra-role-kibana.git
version: ccc1c3a41d2ea2f125db9a3418e447958c72bfce
scm: git
- name: infra-role-elasticsearch
src: git@github.com:status-im/infra-role-elasticsearch.git
version: 1e86928b7848ab5520ead99a5f1432770af4fc22
scm: git
- name: infra-role-elasticsearch-lb
src: git@github.com:status-im/infra-role-elasticsearch-lb.git
version: 5ff5e5d526e38a7faf1ff7dfb809c6b6e5bdc17a
scm: git
- name: infra-role-systemd-timer
src: git@github.com:status-im/infra-role-systemd-timer.git
version: 9bbb72fe9df7379c5c63cfe9f144efa30ed828d9
scm: git
- name: infra-role-launchd-timer
src: git@github.com:status-im/infra-role-launchd-timer.git
version: 791174a9de57ffbbb4d60cdde05d74a66415f1bd
scm: git
- name: infra-role-geth
src: git@github.com:status-im/infra-role-geth.git
version: cd4fd6403549673e86047c10e2289cb6d34a4e20
scm: git
- name: infra-role-geth-exporter
src: git@github.com:status-im/infra-role-geth-exporter.git
version: c129dc06a407b153371c1467e5293b7ebaca04ab
scm: git
- name: infra-role-erigon
src: git@github.com:status-im/infra-role-erigon.git
version: 8568c7c0b8fefe85bad9a704d0e4d56f9c628631
scm: git
- name: infra-role-nethermind
src: git@github.com:status-im/infra-role-nethermind.git
version: c014302215821b0b7df07947a1259027b60a614e
scm: git
- name: infra-role-smart-metrics
src: git@github.com:status-im/infra-role-smart-metrics.git
version: 76aba3c459e6c5e8afc6979dd9532c8cea0fbeea
scm: git

401
ansible/roles.py Executable file
View File

@ -0,0 +1,401 @@
#!/usr/bin/env python3
import yaml
import logging
import ansible
import argparse
import subprocess
import functools
from enum import Enum
from os import path, getenv, symlink, readlink
from packaging.version import parse as version_parse
from concurrent import futures
HELP_DESCRIPTION='''
This tool managed Ansible roles as Git repositories.
It is both faster and simpler than Ansible Galaxy.
By default ~/.ansible/roles is symlinked to ~/work.
Override it using --roles-symlink or ROLES_SYMLINK.
Installation behavior:
- If no version is specified newest is pulled.
- If version is matching nothing is done.
- If repo is dirty or detached nothing is done.
- If version is newer user is notified.
'''
HELP_EXAMPLE='''Examples:
./roles.py --install
./roles.py --check
./roles.py --update
'''
SCRIPT_DIR = path.dirname(path.realpath(__file__))
# Where Ansible looks for installed roles.
ROLES_PATH = path.join(path.expanduser('~'), '.ansible/roles')
ROLES_SYMLINK = path.join(path.expanduser('~'), 'work')
ROLES_WORKERS = 20
REQUIREMENTS_PATH = path.join(SCRIPT_DIR, 'requirements.yml')
# Setup logging.
log_format = '[%(levelname)s] %(message)s'
logging.basicConfig(level=logging.INFO, format=log_format)
LOG = logging.getLogger(__name__)
# Colors
RST = '\033[0m'
NORMAL = lambda x: f'\033[00m{x}{RST}'
PURPLE = lambda x: f'\033[35m{x}{RST}'
YELLOW = lambda x: f'\033[33m{x}{RST}'
BLUE = lambda x: f'\033[34m{x}{RST}'
GREY = lambda x: f'\033[90m{x}{RST}'
RED = lambda x: f'\033[31m{x}{RST}'
ORANGE = lambda x: f'\033[91m{x}{RST}'
GREEN = lambda x: f'\033[32m{x}{RST}'
CYAN = lambda x: f'\033[36m{x}{RST}'
BOLD = lambda x: f'\033[1m{x}{RST}{RST}'
class State(Enum):
# Order is priority. Higher status trumps lower.
UNKNOWN = 0
EXISTS = 1
WRONG_VERSION = 2
NEWER_VERSION = 3
DIRTY = 4
DETACHED = 5
NO_VERSION = 6
CLONE_FAILURE = 7
MISSING = 8
CLONED = 9
UPDATED = 10
VALID = 11
SKIPPED = 12
def __str__(self):
match self:
case State.NEWER_VERSION: color = BOLD
case State.WRONG_VERSION: color = RED
case State.DIRTY: color = YELLOW
case State.DETACHED: color = YELLOW
case State.NO_VERSION: color = PURPLE
case State.CLONE_FAILURE: color = RED
case State.MISSING: color = RED
case State.CLONED: color = GREEN
case State.UPDATED: color = GREEN
case State.VALID: color = GREEN
case State.SKIPPED: color = GREY
case _: color = NORMAL
return color(self.name.replace('_', ' '))
# Allow calling max() to compare with previous state.
def __gt__(self, other):
if other is None:
return True
if self.__class__ is other.__class__:
return self.value > other.value
return NotImplemented
# Decorator to manage Role state based on function return value.
def update(success=None, failure=None):
def decorator(func):
@functools.wraps(func)
def wrapper_decorator(self, *args, **kwargs):
# Set state to failure one on exception.
try:
rval = func(self, *args, **kwargs)
except:
self.state = max(failure, self.state)
raise
# Set state based on truthiness of result, higher one wins.
if rval:
self.state = max(success, self.state)
else:
self.state = max(failure, self.state)
LOG.debug('[%-27s]: %s%s: state = %s',
self.name, func.__name__, args, self.state)
return rval
return wrapper_decorator
return decorator
class Role:
def __init__(self, name, src, required):
self.state = State.UNKNOWN
self.name = name
self.src = src
self.required = required
@classmethod
def from_requirement(cls, obj):
return cls(obj['name'], obj.get('src'), obj.get('version'))
def __repr__(self):
return 'Role(name=%s, src=%s, required=%s, state=%s)' % (
self.name, self.src, self.required, self.state,
)
def to_dict(self):
obj = {
'name': self.name,
'src': self.src,
}
if self.required:
obj['version'] = self.required
return obj
def _git(self, *args, cwd=None):
cmd = ['git'] + list(args)
LOG.debug('[%-27s]: COMMAND: %s', self.name, ' '.join(cmd))
rval = subprocess.run(
cmd,
capture_output=True,
cwd=cwd or self.path
)
LOG.debug('[%-27s]: RETURN: %d', self.name, rval.returncode)
if rval.stdout:
LOG.debug('[%-27s]: STDOUT: %s', self.name, rval.stdout.decode().strip())
if rval.stderr:
LOG.debug('[%-27s]: STDERR: %s', self.name, rval.stderr.decode().strip())
rval.check_returncode()
return str(rval.stdout.strip(), 'utf-8')
def _git_fail_is_false(self, *args, cwd=None):
try:
self._git(*args, cwd=cwd)
except:
return True
else:
return False
@property
def repo_parent_dir(self):
return self.path.removesuffix(self.name)
@property
def branch(self):
return self._git('rev-parse', '--abbrev-ref', 'HEAD')
@property
def current_commit(self):
if not self.exists():
return '........'
return self._git('rev-parse', 'HEAD')
@State.update(success=State.DIRTY)
def is_dirty(self):
return self._git_fail_is_false('diff-files', '--quiet')
@State.update(success=State.DETACHED)
def is_detached(self):
return self._git_fail_is_false('symbolic-ref', 'HEAD')
@State.update(success=State.NEWER_VERSION)
def is_ancestor(self):
if self.required is None or self.required == self.current_commit:
return False
return not self._git_fail_is_false(
'merge-base', self.required, '--is-ancestor', self.current_commit
)
@property
@State.update(failure=State.NO_VERSION)
def version(self):
return self.required
@version.setter
@State.update(success=State.UPDATED, failure=State.SKIPPED)
def version(self, version):
if self.required is not None:
self.required = version
return self.required
@State.update(success=State.VALID, failure=State.WRONG_VERSION)
def valid_version(self):
return self.required == self.current_commit
@State.update(success=State.VALID, failure=State.WRONG_VERSION)
def pull(self):
self._git('remote', 'update')
status = self._git('status', '--untracked-files=no')
if 'branch is behind' not in status:
return None
rval = self._git('pull')
return self.valid_version()
@State.update(success=State.CLONED, failure=State.CLONE_FAILURE)
def clone(self):
LOG.debug('Clogning: %s', self.src)
try:
self._git(
'clone',
self.src, self.name,
cwd=self.repo_parent_dir
)
except Exception as ex:
LOG.error('Clone failed: %s', ex.stderr.decode())
return False
return True
@property
def path(self):
return path.join(ROLES_PATH, self.name)
@State.update(success=State.EXISTS, failure=State.MISSING)
def exists(self):
return path.isdir(self.path)
def handle_role(role, check=False, update=False, install=False):
LOG.debug('[%-27s]: Processing role...', role.name)
if not role.exists():
if not check and not update:
role.clone()
return role
# Check if current version is newer.
if not update and role.is_ancestor():
return role
# Check if current version matches required.
if role.valid_version():
return role
# Verify if git repo is not dirty or has detached head.
if not update and (role.is_dirty() or role.is_detached()):
return role
# No need to fail if no version is set.
if check and not role.version:
return role
# Update config version or pull new changes.
if update:
role.version = role.current_commit
elif install:
# If version is not specified we just want the newest.
role.pull()
return role
# Special function to preserve order and separating newlines.
def roles_to_yaml(old_reqs, processed_roles):
# Get processed role when available, use original one when not.
return '\n'.join([
yaml.dump([processed_roles.get(role.name, role).to_dict()])
for role in old_reqs
])
def commit_or_any(commit):
return '*' if commit is None else commit[:8]
# Symlink only if folder or link doesn't exist.
def symlink_roles_dir(roles_symlink):
if path.islink(ROLES_PATH):
dest = readlink(ROLES_PATH)
if dest != roles_symlink:
LOG.error('Roles path is already a link to: %s', dest)
exit(1)
else:
return
elif path.isdir(ROLES_PATH):
LOG.error('Roles path is a directory, cannot symlink!')
exit(1)
symlink(roles_symlink, ROLES_PATH)
def parse_args():
parser = argparse.ArgumentParser(
epilog=HELP_EXAMPLE,
description=HELP_DESCRIPTION,
formatter_class=argparse.RawTextHelpFormatter,
)
parser.add_argument('-f', '--filter', default='',
help='Filter role repo names.')
parser.add_argument('-w', '--workers', default=getenv('ROLES_WORKERS', ROLES_WORKERS), type=int,
help='Max workers to run in parallel.')
parser.add_argument('-r', '--requirements', default=getenv('REQUIREMENTS_PATH', REQUIREMENTS_PATH),
help='Location of requirements.yml file.')
parser.add_argument('-s', '--roles-symlink', default=getenv('ROLES_SYMLINK', ROLES_SYMLINK),
help='Actual location of installed roles.')
parser.add_argument('-l', '--log-level', default='INFO',
help='Logging level.')
parser.add_argument('-d', '--fail-dirty', action='store_true',
help='Fail if repo is dirty.')
parser.add_argument('-a', '--fail-detached', action='store_true',
help='Fail if repo has detached head.')
group = parser.add_mutually_exclusive_group()
group.add_argument('-i', '--install', action='store_true',
help='Clone and update required roles.')
group.add_argument('-c', '--check', action='store_true',
help='Only check roles, no installing.')
group.add_argument('-u', '--update', action='store_true',
help='Update requirements with current commits.')
args = parser.parse_args()
assert args.install or args.check or args.update, \
parser.error('Pick one: --install, --check, --update')
return args
def main():
args = parse_args()
LOG.setLevel(args.log_level.upper())
# Verify Ansible version is 2.8 or newer.
if version_parse(ansible.__version__) < version_parse("2.8"):
LOG.error('Your Ansible version is lower than 2.8. Upgrade it.')
exit(1)
# Symlink ansible roles directory to work directory.
symlink_roles_dir(args.roles_symlink)
# Read Ansible requirements file.
with open(args.requirements, 'r') as f:
requirements = yaml.load(f, Loader=yaml.FullLoader)
requirements = [
Role.from_requirement(req) for req in requirements
]
# Check if each Ansible role is installed and has correct version.
with futures.ProcessPoolExecutor(max_workers=args.workers) as executor:
these_futures = [
executor.submit(handle_role, role, args.check, args.update, args.install)
for role in requirements
if args.filter in role.name
]
# Wait for all the workers to finishe and return their role.
processed_roles = {
r.name: r for r in
[r.result() for r in futures.as_completed(these_futures)]
}
# Use the same order as requirements.yml file.
for req in requirements:
if args.filter not in req.name:
continue
role = processed_roles[req.name]
print('%s%-44s --- %22s (Git: %s | Req: %s)' %
(RST, BOLD(role.name), role.state,
CYAN(role.current_commit[:8]),
commit_or_any(role.required)))
if args.update:
with open(args.requirements, 'w') as f:
f.write(roles_to_yaml(requirements, processed_roles))
fail_states = set([State.MISSING, State.WRONG_VERSION])
if args.fail_dirty:
fail_states.append(State.DIRTY)
if args.fail_detached:
fail_states.append(State.DETACHED)
if fail_states.intersection([r.state for r in processed_roles.values()]):
exit(1)
if __name__ == "__main__":
main()

2
ansible/sepolia.yml
View File

@ -6,7 +6,7 @@
run_once: true
gather_facts: false
tasks:
- local_action: command ./versioncheck.py
- local_action: command ./roles.py --check
changed_when: false
- name: Deploy Spolia Testnet nodes

71
ansible/versioncheck.py
View File

@ -1,71 +0,0 @@
#!/usr/bin/env python3
# WARNING: If importing this fails set PYTHONPATH.
import yaml
import ansible
import subprocess
from os import path, environ
from packaging import version
SCRIPT_DIR = path.dirname(path.realpath(__file__))
# Where Ansible looks for installed roles.
ANSIBLE_ROLES_PATH = path.join(environ['HOME'], '.ansible/roles')
class Role:
def __init__(self, name, version):
self.name = name
self.version = version
@property
def path(self):
return path.join(ANSIBLE_ROLES_PATH, self.name)
def exists(self):
return path.isdir(self.path)
def local_version(self):
cmd = subprocess.run(
['git', 'rev-parse', 'HEAD'],
capture_output=True,
cwd=self.path
)
cmd.check_returncode()
return str(cmd.stdout.strip(), 'utf-8')
# Verify Ansible version is 2.8 or newer.
if version.parse(ansible.__version__) < version.parse("2.8"):
print('Your Ansible version is lower than 2.8. Upgrade it.')
exit(1)
# Read Ansible requirements file.
with open(path.join(SCRIPT_DIR, 'requirements.yml'), 'r') as f:
requirements = yaml.load(f, Loader=yaml.FullLoader)
# Check if each Ansible role is installed and has correct version.
errors = 0
for req in requirements:
role = Role(req['name'], req.get('version'))
if not role.exists():
print('%25s - MISSING!' % role.name)
errors += 1
continue
# For now we allow not specifying versions for everyhing.
if role.version is None:
print('%25s - No version!' % role.name)
continue
local_version = role.local_version()
if role.version != local_version:
print('%25s - MISMATCH: %s != %s' %
(role.name, role.version[:8], local_version[:8]))
errors += 1
continue
print('%25s - VALID' % role.name)
# Any issue with any role should cause failure.
if errors > 0:
exit(1)