From 49df6d5996dded306907eb3ebb5cbfdc6e384a8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Soko=C5=82owski?= Date: Tue, 12 Mar 2024 10:53:13 +0100 Subject: [PATCH] roles.py: script to replace Ansible Galaxy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Usage: ``` usage: roles.py [-h] [-f FILTER] [-w WORKERS] [-r REQUIREMENTS] [-s ROLES_SYMLINK] [-l LOG_LEVEL] [-d] [-a] [-i | -c | -u] This tool managed Ansible roles as Git repositories. It is both faster and simpler than Ansible Galaxy. By default ~/.ansible/roles is symlinked to ~/work. Override it using --roles-symlink or ROLES_SYMLINK. Installation behavior: - If no version is specified newest is pulled. - If version is matching nothing is done. - If repo is dirty or detached nothing is done. - If version is newer user is notified. options: -h, --help show this help message and exit -f FILTER, --filter FILTER Filter role repo names. -w WORKERS, --workers WORKERS Max workers to run in parallel. -r REQUIREMENTS, --requirements REQUIREMENTS Location of requirements.yml file. -s ROLES_SYMLINK, --roles-symlink ROLES_SYMLINK Actual location of installed roles. -l LOG_LEVEL, --log-level LOG_LEVEL Logging level. -d, --fail-dirty Fail if repo is dirty. -a, --fail-detached Fail if repo has detached head. -i, --install Clone and update required roles. -c, --check Only check roles, no installing. -u, --update Update requirements with current commits. Examples: ./roles.py --install ./roles.py --check ./roles.py --update ``` Signed-off-by: Jakub SokoĊ‚owski --- Makefile | 15 +- ansible/requirements.yml | 7 - ansible/roles.py | 401 +++++++++++++++++++++++++++++++++++++++ ansible/versioncheck.py | 71 ------- 4 files changed, 410 insertions(+), 84 deletions(-) create mode 100755 ansible/roles.py delete mode 100755 ansible/versioncheck.py diff --git a/Makefile b/Makefile index e7ceec1..1246074 100644 --- a/Makefile +++ b/Makefile @@ -16,16 +16,19 @@ PROVISIONER_ARCHIVE = $(PROVISIONER_NAME)-$(subst _,-,$(ARCH))_$(PROVISIONER_VER PROVISIONER_URL = https://github.com/radekg/terraform-provisioner-ansible/releases/download/$(PROVISIONER_VERSION)/$(PROVISIONER_ARCHIVE) PROVISIONER_PATH = $(TF_PLUGINS_DIR)/$(ARCH)/$(PROVISIONER_NAME)_$(PROVISIONER_VERSION) -all: requirements install-provisioner secrets init-terraform +all: roles-install install-provisioner secrets init-terraform @echo "Success!" -requirements-install: - ansible-galaxy install --keep-scm-meta --ignore-errors --force -r ansible/requirements.yml +roles-install: + ansible/roles.py --install -requirements-check: - ansible/versioncheck.py +roles-check: + ansible/roles.py --check -requirements: requirements-install requirements-check +roles-update: + ansible/roles.py --update + +roles: roles-install roles-check $(PROVISIONER_PATH): @mkdir -p $(TF_PLUGINS_DIR)/$(ARCH); \ diff --git a/ansible/requirements.yml b/ansible/requirements.yml index 31e8b72..793d2d5 100644 --- a/ansible/requirements.yml +++ b/ansible/requirements.yml @@ -1,24 +1,17 @@ ---- - name: infra-role-bootstrap-linux src: git@github.com:status-im/infra-role-bootstrap-linux.git - scm: git - name: infra-role-wireguard src: git@github.com:status-im/infra-role-wireguard.git - scm: git - name: infra-role-open-ports src: git@github.com:status-im/infra-role-open-ports.git - scm: git - name: infra-role-swap-file src: git@github.com:status-im/infra-role-swap-file.git - scm: git - name: infra-role-consul-service src: git@github.com:status-im/infra-role-consul-service.git - scm: git - name: infra-role-systemd-timer src: git@github.com:status-im/infra-role-systemd-timer.git - scm: git diff --git a/ansible/roles.py b/ansible/roles.py new file mode 100755 index 0000000..f912227 --- /dev/null +++ b/ansible/roles.py @@ -0,0 +1,401 @@ +#!/usr/bin/env python3 +import yaml +import logging +import ansible +import argparse +import subprocess +import functools +from enum import Enum +from os import path, getenv, symlink, readlink +from packaging.version import parse as version_parse +from concurrent import futures + +HELP_DESCRIPTION=''' +This tool managed Ansible roles as Git repositories. +It is both faster and simpler than Ansible Galaxy. + +By default ~/.ansible/roles is symlinked to ~/work. +Override it using --roles-symlink or ROLES_SYMLINK. + +Installation behavior: +- If no version is specified newest is pulled. +- If version is matching nothing is done. +- If repo is dirty or detached nothing is done. +- If version is newer user is notified. +''' +HELP_EXAMPLE='''Examples: +./roles.py --install +./roles.py --check +./roles.py --update +''' + +SCRIPT_DIR = path.dirname(path.realpath(__file__)) +# Where Ansible looks for installed roles. +ROLES_PATH = path.join(path.expanduser('~'), '.ansible/roles') +ROLES_SYMLINK = path.join(path.expanduser('~'), 'work') +ROLES_WORKERS = 20 +REQUIREMENTS_PATH = path.join(SCRIPT_DIR, 'requirements.yml') + +# Setup logging. +log_format = '[%(levelname)s] %(message)s' +logging.basicConfig(level=logging.INFO, format=log_format) +LOG = logging.getLogger(__name__) + +# Colors +RST = '\033[0m' +NORMAL = lambda x: f'\033[00m{x}{RST}' +PURPLE = lambda x: f'\033[35m{x}{RST}' +YELLOW = lambda x: f'\033[33m{x}{RST}' +BLUE = lambda x: f'\033[34m{x}{RST}' +GREY = lambda x: f'\033[90m{x}{RST}' +RED = lambda x: f'\033[31m{x}{RST}' +ORANGE = lambda x: f'\033[91m{x}{RST}' +GREEN = lambda x: f'\033[32m{x}{RST}' +CYAN = lambda x: f'\033[36m{x}{RST}' +BOLD = lambda x: f'\033[1m{x}{RST}{RST}' + +class State(Enum): + # Order is priority. Higher status trumps lower. + UNKNOWN = 0 + EXISTS = 1 + WRONG_VERSION = 2 + NEWER_VERSION = 3 + DIRTY = 4 + DETACHED = 5 + NO_VERSION = 6 + CLONE_FAILURE = 7 + MISSING = 8 + CLONED = 9 + UPDATED = 10 + VALID = 11 + SKIPPED = 12 + + def __str__(self): + match self: + case State.NEWER_VERSION: color = BOLD + case State.WRONG_VERSION: color = RED + case State.DIRTY: color = YELLOW + case State.DETACHED: color = YELLOW + case State.NO_VERSION: color = PURPLE + case State.CLONE_FAILURE: color = RED + case State.MISSING: color = RED + case State.CLONED: color = GREEN + case State.UPDATED: color = GREEN + case State.VALID: color = GREEN + case State.SKIPPED: color = GREY + case _: color = NORMAL + return color(self.name.replace('_', ' ')) + + # Allow calling max() to compare with previous state. + def __gt__(self, other): + if other is None: + return True + if self.__class__ is other.__class__: + return self.value > other.value + return NotImplemented + + # Decorator to manage Role state based on function return value. + def update(success=None, failure=None): + def decorator(func): + @functools.wraps(func) + def wrapper_decorator(self, *args, **kwargs): + # Set state to failure one on exception. + try: + rval = func(self, *args, **kwargs) + except: + self.state = max(failure, self.state) + raise + # Set state based on truthiness of result, higher one wins. + if rval: + self.state = max(success, self.state) + else: + self.state = max(failure, self.state) + LOG.debug('[%-27s]: %s%s: state = %s', + self.name, func.__name__, args, self.state) + return rval + return wrapper_decorator + return decorator + +class Role: + + def __init__(self, name, src, required): + self.state = State.UNKNOWN + self.name = name + self.src = src + self.required = required + + @classmethod + def from_requirement(cls, obj): + return cls(obj['name'], obj.get('src'), obj.get('version')) + + def __repr__(self): + return 'Role(name=%s, src=%s, required=%s, state=%s)' % ( + self.name, self.src, self.required, self.state, + ) + + def to_dict(self): + obj = { + 'name': self.name, + 'src': self.src, + } + if self.required: + obj['version'] = self.required + return obj + + def _git(self, *args, cwd=None): + cmd = ['git'] + list(args) + LOG.debug('[%-27s]: COMMAND: %s', self.name, ' '.join(cmd)) + rval = subprocess.run( + cmd, + capture_output=True, + cwd=cwd or self.path + ) + LOG.debug('[%-27s]: RETURN: %d', self.name, rval.returncode) + if rval.stdout: + LOG.debug('[%-27s]: STDOUT: %s', self.name, rval.stdout.decode().strip()) + if rval.stderr: + LOG.debug('[%-27s]: STDERR: %s', self.name, rval.stderr.decode().strip()) + rval.check_returncode() + return str(rval.stdout.strip(), 'utf-8') + + def _git_fail_is_false(self, *args, cwd=None): + try: + self._git(*args, cwd=cwd) + except: + return True + else: + return False + + @property + def repo_parent_dir(self): + return self.path.removesuffix(self.name) + + @property + def branch(self): + return self._git('rev-parse', '--abbrev-ref', 'HEAD') + + @property + def current_commit(self): + if not self.exists(): + return '........' + return self._git('rev-parse', 'HEAD') + + @State.update(success=State.DIRTY) + def is_dirty(self): + return self._git_fail_is_false('diff-files', '--quiet') + + @State.update(success=State.DETACHED) + def is_detached(self): + return self._git_fail_is_false('symbolic-ref', 'HEAD') + + @State.update(success=State.NEWER_VERSION) + def is_ancestor(self): + if self.required is None or self.required == self.current_commit: + return False + return self._git_fail_is_false( + self.required, '--is-ancestor', self.current_commit + ) + + @property + @State.update(failure=State.NO_VERSION) + def version(self): + return self.required + + @version.setter + @State.update(success=State.UPDATED, failure=State.SKIPPED) + def version(self, version): + if self.required is not None: + self.required = version + return self.required + + @State.update(success=State.VALID, failure=State.WRONG_VERSION) + def valid_version(self): + return self.required == self.current_commit + + @State.update(success=State.VALID, failure=State.WRONG_VERSION) + def pull(self): + self._git('remote', 'update') + status = self._git('status', '--untracked-files=no') + if 'branch is behind' not in status: + return None + + rval = self._git('pull') + return self.valid_version() + + @State.update(success=State.CLONED, failure=State.CLONE_FAILURE) + def clone(self): + LOG.debug('Clogning: %s', self.src) + try: + self._git( + 'clone', + self.src, self.name, + cwd=self.repo_parent_dir + ) + except Exception as ex: + LOG.error('Clone failed: %s', ex.stderr.decode()) + return False + return True + + @property + def path(self): + return path.join(ROLES_PATH, self.name) + + @State.update(success=State.EXISTS, failure=State.MISSING) + def exists(self): + return path.isdir(self.path) + + +def handle_role(role, check=False, update=False, install=False): + LOG.debug('[%-27s]: Processing role...', role.name) + if not role.exists(): + if not check and not update: + role.clone() + return role + + # Check if current version is newer. + if role.is_ancestor(): + return role + + # Check if current version matches required. + if role.valid_version(): + return role + + # Verify if git repo is not dirty or has detached head. + if (role.is_dirty() or role.is_detached()) and not update: + return role + + # No need to fail if no version is set. + if not role.version and check: + return role + + # Update config version or pull new changes. + if update: + role.version = role.current_commit + elif install: + # If version is not specified we just want the newest. + role.pull() + return role + + +# Special function to preserve order and separating newlines. +def roles_to_yaml(old_reqs, processed_roles): + # Get processed role when available, use original one when not. + return '\n'.join([ + yaml.dump([processed_roles.get(role.name, role).to_dict()]) + for role in old_reqs + ]) + +def commit_or_any(commit): + return '*' if commit is None else commit[:8] + +# Symlink only if folder or link doesn't exist. +def symlink_roles_dir(roles_symlink): + if path.islink(ROLES_PATH): + dest = readlink(ROLES_PATH) + if dest != roles_symlink: + LOG.error('Roles path is already a link to: %s', dest) + exit(1) + else: + return + elif path.isdir(ROLES_PATH): + LOG.error('Roles path is a directory, cannot symlink!') + exit(1) + + symlink(roles_symlink, ROLES_PATH) + +def parse_args(): + parser = argparse.ArgumentParser( + epilog=HELP_EXAMPLE, + description=HELP_DESCRIPTION, + formatter_class=argparse.RawTextHelpFormatter, + ) + + parser.add_argument('-f', '--filter', default='', + help='Filter role repo names.') + parser.add_argument('-w', '--workers', default=getenv('ROLES_WORKERS', ROLES_WORKERS), type=int, + help='Max workers to run in parallel.') + parser.add_argument('-r', '--requirements', default=getenv('REQUIREMENTS_PATH', REQUIREMENTS_PATH), + help='Location of requirements.yml file.') + parser.add_argument('-s', '--roles-symlink', default=getenv('ROLES_SYMLINK', ROLES_SYMLINK), + help='Actual location of installed roles.') + parser.add_argument('-l', '--log-level', default='INFO', + help='Logging level.') + parser.add_argument('-d', '--fail-dirty', action='store_true', + help='Fail if repo is dirty.') + parser.add_argument('-a', '--fail-detached', action='store_true', + help='Fail if repo has detached head.') + + group = parser.add_mutually_exclusive_group() + group.add_argument('-i', '--install', action='store_true', + help='Clone and update required roles.') + group.add_argument('-c', '--check', action='store_true', + help='Only check roles, no installing.') + group.add_argument('-u', '--update', action='store_true', + help='Update requirements with current commits.') + + args = parser.parse_args() + + assert args.install or args.check or args.update, \ + parser.error('Pick one: --install, --check, --update') + + return args + + +def main(): + args = parse_args() + + LOG.setLevel(args.log_level.upper()) + + # Verify Ansible version is 2.8 or newer. + if version_parse(ansible.__version__) < version_parse("2.8"): + LOG.error('Your Ansible version is lower than 2.8. Upgrade it.') + exit(1) + + # Symlink ansible roles directory to work directory. + symlink_roles_dir(args.roles_symlink) + + # Read Ansible requirements file. + with open(args.requirements, 'r') as f: + requirements = yaml.load(f, Loader=yaml.FullLoader) + + requirements = [ + Role.from_requirement(req) for req in requirements + ] + + # Check if each Ansible role is installed and has correct version. + with futures.ProcessPoolExecutor(max_workers=args.workers) as executor: + these_futures = [ + executor.submit(handle_role, role, args.check, args.update, args.install) + for role in requirements + if args.filter in role.name + ] + # Wait for all the workers to finishe and return their role. + processed_roles = { + r.name: r for r in + [r.result() for r in futures.as_completed(these_futures)] + } + + # Use the same order as requirements.yml file. + for req in requirements: + if args.filter not in req.name: + continue + role = processed_roles[req.name] + print('%-40s --- %22s (Git: %s | Req: %s)' % + (BOLD(role.name), role.state, + CYAN(role.current_commit[:8]), + commit_or_any(role.required))) + + if args.update: + with open(args.requirements, 'w') as f: + f.write(roles_to_yaml(requirements, processed_roles)) + + fail_states = set([State.MISSING, State.WRONG_VERSION]) + if args.fail_dirty: + fail_states.append(State.DIRTY) + if args.fail_detached: + fail_states.append(State.DETACHED) + if fail_states.intersection([r.state for r in processed_roles.values()]): + exit(1) + +if __name__ == "__main__": + main() diff --git a/ansible/versioncheck.py b/ansible/versioncheck.py deleted file mode 100755 index d88a47e..0000000 --- a/ansible/versioncheck.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python3 -# WARNING: If importing this fails set PYTHONPATH. -import yaml -import ansible -import subprocess -from os import path, environ -from packaging import version - -SCRIPT_DIR = path.dirname(path.realpath(__file__)) -# Where Ansible looks for installed roles. -ANSIBLE_ROLES_PATH = path.join(environ['HOME'], '.ansible/roles') - - -class Role: - def __init__(self, name, version): - self.name = name - self.version = version - - @property - def path(self): - return path.join(ANSIBLE_ROLES_PATH, self.name) - - def exists(self): - return path.isdir(self.path) - - def local_version(self): - cmd = subprocess.run( - ['git', 'rev-parse', 'HEAD'], - capture_output=True, - cwd=self.path - ) - cmd.check_returncode() - return str(cmd.stdout.strip(), 'utf-8') - - -# Verify Ansible version is 2.8 or newer. -if version.parse(ansible.__version__) < version.parse("2.8"): - print('Your Ansible version is lower than 2.8. Upgrade it.') - exit(1) - -# Read Ansible requirements file. -with open(path.join(SCRIPT_DIR, 'requirements.yml'), 'r') as f: - requirements = yaml.load(f, Loader=yaml.FullLoader) - -# Check if each Ansible role is installed and has correct version. -errors = 0 -for req in requirements: - role = Role(req['name'], req.get('version')) - - if not role.exists(): - print('%25s - MISSING!' % role.name) - errors += 1 - continue - - # For now we allow not specifying versions for everyhing. - if role.version is None: - print('%25s - No version!' % role.name) - continue - - local_version = role.local_version() - if role.version != local_version: - print('%25s - MISMATCH: %s != %s' % - (role.name, role.version[:8], local_version[:8])) - errors += 1 - continue - - print('%25s - VALID' % role.name) - -# Any issue with any role should cause failure. -if errors > 0: - exit(1)