roles.py: script to replace Ansible Galaxy

Usage:
```
usage: roles.py [-h] [-f FILTER] [-w WORKERS] [-r REQUIREMENTS] [-s ROLES_SYMLINK] [-l LOG_LEVEL] [-d] [-a] [-i | -c | -u]

This tool managed Ansible roles as Git repositories.
It is both faster and simpler than Ansible Galaxy.

By default ~/.ansible/roles is symlinked to ~/work.
Override it using --roles-symlink or ROLES_SYMLINK.

Installation behavior:
- If no version is specified newest is pulled.
- If version is matching nothing is done.
- If repo is dirty or detached nothing is done.
- If version is newer user is notified.

options:
  -h, --help            show this help message and exit
  -f FILTER, --filter FILTER
                        Filter role repo names.
  -w WORKERS, --workers WORKERS
                        Max workers to run in parallel.
  -r REQUIREMENTS, --requirements REQUIREMENTS
                        Location of requirements.yml file.
  -s ROLES_SYMLINK, --roles-symlink ROLES_SYMLINK
                        Actual location of installed roles.
  -l LOG_LEVEL, --log-level LOG_LEVEL
                        Logging level.
  -d, --fail-dirty      Fail if repo is dirty.
  -a, --fail-detached   Fail if repo has detached head.
  -i, --install         Clone and update required roles.
  -c, --check           Only check roles, no installing.
  -u, --update          Update requirements with current commits.

Examples:
./roles.py --install
./roles.py --check
./roles.py --update
```

Signed-off-by: Jakub Sokołowski <jakub@status.im>
This commit is contained in:
Jakub Sokołowski 2024-03-12 10:53:13 +01:00
parent 23d081362e
commit 49df6d5996
No known key found for this signature in database
GPG Key ID: FE65CD384D5BF7B4
4 changed files with 410 additions and 84 deletions

View File

@ -16,16 +16,19 @@ PROVISIONER_ARCHIVE = $(PROVISIONER_NAME)-$(subst _,-,$(ARCH))_$(PROVISIONER_VER
PROVISIONER_URL = https://github.com/radekg/terraform-provisioner-ansible/releases/download/$(PROVISIONER_VERSION)/$(PROVISIONER_ARCHIVE) PROVISIONER_URL = https://github.com/radekg/terraform-provisioner-ansible/releases/download/$(PROVISIONER_VERSION)/$(PROVISIONER_ARCHIVE)
PROVISIONER_PATH = $(TF_PLUGINS_DIR)/$(ARCH)/$(PROVISIONER_NAME)_$(PROVISIONER_VERSION) PROVISIONER_PATH = $(TF_PLUGINS_DIR)/$(ARCH)/$(PROVISIONER_NAME)_$(PROVISIONER_VERSION)
all: requirements install-provisioner secrets init-terraform all: roles-install install-provisioner secrets init-terraform
@echo "Success!" @echo "Success!"
requirements-install: roles-install:
ansible-galaxy install --keep-scm-meta --ignore-errors --force -r ansible/requirements.yml ansible/roles.py --install
requirements-check: roles-check:
ansible/versioncheck.py ansible/roles.py --check
requirements: requirements-install requirements-check roles-update:
ansible/roles.py --update
roles: roles-install roles-check
$(PROVISIONER_PATH): $(PROVISIONER_PATH):
@mkdir -p $(TF_PLUGINS_DIR)/$(ARCH); \ @mkdir -p $(TF_PLUGINS_DIR)/$(ARCH); \

View File

@ -1,24 +1,17 @@
---
- name: infra-role-bootstrap-linux - name: infra-role-bootstrap-linux
src: git@github.com:status-im/infra-role-bootstrap-linux.git src: git@github.com:status-im/infra-role-bootstrap-linux.git
scm: git
- name: infra-role-wireguard - name: infra-role-wireguard
src: git@github.com:status-im/infra-role-wireguard.git src: git@github.com:status-im/infra-role-wireguard.git
scm: git
- name: infra-role-open-ports - name: infra-role-open-ports
src: git@github.com:status-im/infra-role-open-ports.git src: git@github.com:status-im/infra-role-open-ports.git
scm: git
- name: infra-role-swap-file - name: infra-role-swap-file
src: git@github.com:status-im/infra-role-swap-file.git src: git@github.com:status-im/infra-role-swap-file.git
scm: git
- name: infra-role-consul-service - name: infra-role-consul-service
src: git@github.com:status-im/infra-role-consul-service.git src: git@github.com:status-im/infra-role-consul-service.git
scm: git
- name: infra-role-systemd-timer - name: infra-role-systemd-timer
src: git@github.com:status-im/infra-role-systemd-timer.git src: git@github.com:status-im/infra-role-systemd-timer.git
scm: git

401
ansible/roles.py Executable file
View File

@ -0,0 +1,401 @@
#!/usr/bin/env python3
import yaml
import logging
import ansible
import argparse
import subprocess
import functools
from enum import Enum
from os import path, getenv, symlink, readlink
from packaging.version import parse as version_parse
from concurrent import futures
HELP_DESCRIPTION='''
This tool managed Ansible roles as Git repositories.
It is both faster and simpler than Ansible Galaxy.
By default ~/.ansible/roles is symlinked to ~/work.
Override it using --roles-symlink or ROLES_SYMLINK.
Installation behavior:
- If no version is specified newest is pulled.
- If version is matching nothing is done.
- If repo is dirty or detached nothing is done.
- If version is newer user is notified.
'''
HELP_EXAMPLE='''Examples:
./roles.py --install
./roles.py --check
./roles.py --update
'''
SCRIPT_DIR = path.dirname(path.realpath(__file__))
# Where Ansible looks for installed roles.
ROLES_PATH = path.join(path.expanduser('~'), '.ansible/roles')
ROLES_SYMLINK = path.join(path.expanduser('~'), 'work')
ROLES_WORKERS = 20
REQUIREMENTS_PATH = path.join(SCRIPT_DIR, 'requirements.yml')
# Setup logging.
log_format = '[%(levelname)s] %(message)s'
logging.basicConfig(level=logging.INFO, format=log_format)
LOG = logging.getLogger(__name__)
# Colors
RST = '\033[0m'
NORMAL = lambda x: f'\033[00m{x}{RST}'
PURPLE = lambda x: f'\033[35m{x}{RST}'
YELLOW = lambda x: f'\033[33m{x}{RST}'
BLUE = lambda x: f'\033[34m{x}{RST}'
GREY = lambda x: f'\033[90m{x}{RST}'
RED = lambda x: f'\033[31m{x}{RST}'
ORANGE = lambda x: f'\033[91m{x}{RST}'
GREEN = lambda x: f'\033[32m{x}{RST}'
CYAN = lambda x: f'\033[36m{x}{RST}'
BOLD = lambda x: f'\033[1m{x}{RST}{RST}'
class State(Enum):
# Order is priority. Higher status trumps lower.
UNKNOWN = 0
EXISTS = 1
WRONG_VERSION = 2
NEWER_VERSION = 3
DIRTY = 4
DETACHED = 5
NO_VERSION = 6
CLONE_FAILURE = 7
MISSING = 8
CLONED = 9
UPDATED = 10
VALID = 11
SKIPPED = 12
def __str__(self):
match self:
case State.NEWER_VERSION: color = BOLD
case State.WRONG_VERSION: color = RED
case State.DIRTY: color = YELLOW
case State.DETACHED: color = YELLOW
case State.NO_VERSION: color = PURPLE
case State.CLONE_FAILURE: color = RED
case State.MISSING: color = RED
case State.CLONED: color = GREEN
case State.UPDATED: color = GREEN
case State.VALID: color = GREEN
case State.SKIPPED: color = GREY
case _: color = NORMAL
return color(self.name.replace('_', ' '))
# Allow calling max() to compare with previous state.
def __gt__(self, other):
if other is None:
return True
if self.__class__ is other.__class__:
return self.value > other.value
return NotImplemented
# Decorator to manage Role state based on function return value.
def update(success=None, failure=None):
def decorator(func):
@functools.wraps(func)
def wrapper_decorator(self, *args, **kwargs):
# Set state to failure one on exception.
try:
rval = func(self, *args, **kwargs)
except:
self.state = max(failure, self.state)
raise
# Set state based on truthiness of result, higher one wins.
if rval:
self.state = max(success, self.state)
else:
self.state = max(failure, self.state)
LOG.debug('[%-27s]: %s%s: state = %s',
self.name, func.__name__, args, self.state)
return rval
return wrapper_decorator
return decorator
class Role:
def __init__(self, name, src, required):
self.state = State.UNKNOWN
self.name = name
self.src = src
self.required = required
@classmethod
def from_requirement(cls, obj):
return cls(obj['name'], obj.get('src'), obj.get('version'))
def __repr__(self):
return 'Role(name=%s, src=%s, required=%s, state=%s)' % (
self.name, self.src, self.required, self.state,
)
def to_dict(self):
obj = {
'name': self.name,
'src': self.src,
}
if self.required:
obj['version'] = self.required
return obj
def _git(self, *args, cwd=None):
cmd = ['git'] + list(args)
LOG.debug('[%-27s]: COMMAND: %s', self.name, ' '.join(cmd))
rval = subprocess.run(
cmd,
capture_output=True,
cwd=cwd or self.path
)
LOG.debug('[%-27s]: RETURN: %d', self.name, rval.returncode)
if rval.stdout:
LOG.debug('[%-27s]: STDOUT: %s', self.name, rval.stdout.decode().strip())
if rval.stderr:
LOG.debug('[%-27s]: STDERR: %s', self.name, rval.stderr.decode().strip())
rval.check_returncode()
return str(rval.stdout.strip(), 'utf-8')
def _git_fail_is_false(self, *args, cwd=None):
try:
self._git(*args, cwd=cwd)
except:
return True
else:
return False
@property
def repo_parent_dir(self):
return self.path.removesuffix(self.name)
@property
def branch(self):
return self._git('rev-parse', '--abbrev-ref', 'HEAD')
@property
def current_commit(self):
if not self.exists():
return '........'
return self._git('rev-parse', 'HEAD')
@State.update(success=State.DIRTY)
def is_dirty(self):
return self._git_fail_is_false('diff-files', '--quiet')
@State.update(success=State.DETACHED)
def is_detached(self):
return self._git_fail_is_false('symbolic-ref', 'HEAD')
@State.update(success=State.NEWER_VERSION)
def is_ancestor(self):
if self.required is None or self.required == self.current_commit:
return False
return self._git_fail_is_false(
self.required, '--is-ancestor', self.current_commit
)
@property
@State.update(failure=State.NO_VERSION)
def version(self):
return self.required
@version.setter
@State.update(success=State.UPDATED, failure=State.SKIPPED)
def version(self, version):
if self.required is not None:
self.required = version
return self.required
@State.update(success=State.VALID, failure=State.WRONG_VERSION)
def valid_version(self):
return self.required == self.current_commit
@State.update(success=State.VALID, failure=State.WRONG_VERSION)
def pull(self):
self._git('remote', 'update')
status = self._git('status', '--untracked-files=no')
if 'branch is behind' not in status:
return None
rval = self._git('pull')
return self.valid_version()
@State.update(success=State.CLONED, failure=State.CLONE_FAILURE)
def clone(self):
LOG.debug('Clogning: %s', self.src)
try:
self._git(
'clone',
self.src, self.name,
cwd=self.repo_parent_dir
)
except Exception as ex:
LOG.error('Clone failed: %s', ex.stderr.decode())
return False
return True
@property
def path(self):
return path.join(ROLES_PATH, self.name)
@State.update(success=State.EXISTS, failure=State.MISSING)
def exists(self):
return path.isdir(self.path)
def handle_role(role, check=False, update=False, install=False):
LOG.debug('[%-27s]: Processing role...', role.name)
if not role.exists():
if not check and not update:
role.clone()
return role
# Check if current version is newer.
if role.is_ancestor():
return role
# Check if current version matches required.
if role.valid_version():
return role
# Verify if git repo is not dirty or has detached head.
if (role.is_dirty() or role.is_detached()) and not update:
return role
# No need to fail if no version is set.
if not role.version and check:
return role
# Update config version or pull new changes.
if update:
role.version = role.current_commit
elif install:
# If version is not specified we just want the newest.
role.pull()
return role
# Special function to preserve order and separating newlines.
def roles_to_yaml(old_reqs, processed_roles):
# Get processed role when available, use original one when not.
return '\n'.join([
yaml.dump([processed_roles.get(role.name, role).to_dict()])
for role in old_reqs
])
def commit_or_any(commit):
return '*' if commit is None else commit[:8]
# Symlink only if folder or link doesn't exist.
def symlink_roles_dir(roles_symlink):
if path.islink(ROLES_PATH):
dest = readlink(ROLES_PATH)
if dest != roles_symlink:
LOG.error('Roles path is already a link to: %s', dest)
exit(1)
else:
return
elif path.isdir(ROLES_PATH):
LOG.error('Roles path is a directory, cannot symlink!')
exit(1)
symlink(roles_symlink, ROLES_PATH)
def parse_args():
parser = argparse.ArgumentParser(
epilog=HELP_EXAMPLE,
description=HELP_DESCRIPTION,
formatter_class=argparse.RawTextHelpFormatter,
)
parser.add_argument('-f', '--filter', default='',
help='Filter role repo names.')
parser.add_argument('-w', '--workers', default=getenv('ROLES_WORKERS', ROLES_WORKERS), type=int,
help='Max workers to run in parallel.')
parser.add_argument('-r', '--requirements', default=getenv('REQUIREMENTS_PATH', REQUIREMENTS_PATH),
help='Location of requirements.yml file.')
parser.add_argument('-s', '--roles-symlink', default=getenv('ROLES_SYMLINK', ROLES_SYMLINK),
help='Actual location of installed roles.')
parser.add_argument('-l', '--log-level', default='INFO',
help='Logging level.')
parser.add_argument('-d', '--fail-dirty', action='store_true',
help='Fail if repo is dirty.')
parser.add_argument('-a', '--fail-detached', action='store_true',
help='Fail if repo has detached head.')
group = parser.add_mutually_exclusive_group()
group.add_argument('-i', '--install', action='store_true',
help='Clone and update required roles.')
group.add_argument('-c', '--check', action='store_true',
help='Only check roles, no installing.')
group.add_argument('-u', '--update', action='store_true',
help='Update requirements with current commits.')
args = parser.parse_args()
assert args.install or args.check or args.update, \
parser.error('Pick one: --install, --check, --update')
return args
def main():
args = parse_args()
LOG.setLevel(args.log_level.upper())
# Verify Ansible version is 2.8 or newer.
if version_parse(ansible.__version__) < version_parse("2.8"):
LOG.error('Your Ansible version is lower than 2.8. Upgrade it.')
exit(1)
# Symlink ansible roles directory to work directory.
symlink_roles_dir(args.roles_symlink)
# Read Ansible requirements file.
with open(args.requirements, 'r') as f:
requirements = yaml.load(f, Loader=yaml.FullLoader)
requirements = [
Role.from_requirement(req) for req in requirements
]
# Check if each Ansible role is installed and has correct version.
with futures.ProcessPoolExecutor(max_workers=args.workers) as executor:
these_futures = [
executor.submit(handle_role, role, args.check, args.update, args.install)
for role in requirements
if args.filter in role.name
]
# Wait for all the workers to finishe and return their role.
processed_roles = {
r.name: r for r in
[r.result() for r in futures.as_completed(these_futures)]
}
# Use the same order as requirements.yml file.
for req in requirements:
if args.filter not in req.name:
continue
role = processed_roles[req.name]
print('%-40s --- %22s (Git: %s | Req: %s)' %
(BOLD(role.name), role.state,
CYAN(role.current_commit[:8]),
commit_or_any(role.required)))
if args.update:
with open(args.requirements, 'w') as f:
f.write(roles_to_yaml(requirements, processed_roles))
fail_states = set([State.MISSING, State.WRONG_VERSION])
if args.fail_dirty:
fail_states.append(State.DIRTY)
if args.fail_detached:
fail_states.append(State.DETACHED)
if fail_states.intersection([r.state for r in processed_roles.values()]):
exit(1)
if __name__ == "__main__":
main()

View File

@ -1,71 +0,0 @@
#!/usr/bin/env python3
# WARNING: If importing this fails set PYTHONPATH.
import yaml
import ansible
import subprocess
from os import path, environ
from packaging import version
SCRIPT_DIR = path.dirname(path.realpath(__file__))
# Where Ansible looks for installed roles.
ANSIBLE_ROLES_PATH = path.join(environ['HOME'], '.ansible/roles')
class Role:
def __init__(self, name, version):
self.name = name
self.version = version
@property
def path(self):
return path.join(ANSIBLE_ROLES_PATH, self.name)
def exists(self):
return path.isdir(self.path)
def local_version(self):
cmd = subprocess.run(
['git', 'rev-parse', 'HEAD'],
capture_output=True,
cwd=self.path
)
cmd.check_returncode()
return str(cmd.stdout.strip(), 'utf-8')
# Verify Ansible version is 2.8 or newer.
if version.parse(ansible.__version__) < version.parse("2.8"):
print('Your Ansible version is lower than 2.8. Upgrade it.')
exit(1)
# Read Ansible requirements file.
with open(path.join(SCRIPT_DIR, 'requirements.yml'), 'r') as f:
requirements = yaml.load(f, Loader=yaml.FullLoader)
# Check if each Ansible role is installed and has correct version.
errors = 0
for req in requirements:
role = Role(req['name'], req.get('version'))
if not role.exists():
print('%25s - MISSING!' % role.name)
errors += 1
continue
# For now we allow not specifying versions for everyhing.
if role.version is None:
print('%25s - No version!' % role.name)
continue
local_version = role.local_version()
if role.version != local_version:
print('%25s - MISMATCH: %s != %s' %
(role.name, role.version[:8], local_version[:8]))
errors += 1
continue
print('%25s - VALID' % role.name)
# Any issue with any role should cause failure.
if errors > 0:
exit(1)