roles.py: script to replace Ansible Galaxy
Usage:
```
usage: roles.py [-h] [-f FILTER] [-w WORKERS] [-r REQUIREMENTS] [-s ROLES_SYMLINK] [-l LOG_LEVEL] [-d] [-a] [-i | -c | -u]
This tool managed Ansible roles as Git repositories.
It is both faster and simpler than Ansible Galaxy.
By default ~/.ansible/roles is symlinked to ~/work.
Override it using --roles-symlink or ROLES_SYMLINK.
Installation behavior:
- If no version is specified newest is pulled.
- If version is matching nothing is done.
- If repo is dirty or detached nothing is done.
- If version is newer user is notified.
options:
-h, --help show this help message and exit
-f FILTER, --filter FILTER
Filter role repo names.
-w WORKERS, --workers WORKERS
Max workers to run in parallel.
-r REQUIREMENTS, --requirements REQUIREMENTS
Location of requirements.yml file.
-s ROLES_SYMLINK, --roles-symlink ROLES_SYMLINK
Actual location of installed roles.
-l LOG_LEVEL, --log-level LOG_LEVEL
Logging level.
-d, --fail-dirty Fail if repo is dirty.
-a, --fail-detached Fail if repo has detached head.
-i, --install Clone and update required roles.
-c, --check Only check roles, no installing.
-u, --update Update requirements with current commits.
Examples:
./roles.py --install
./roles.py --check
./roles.py --update
```
Signed-off-by: Jakub Sokołowski <jakub@status.im>
This commit is contained in:
parent
23d081362e
commit
49df6d5996
15
Makefile
15
Makefile
|
|
@ -16,16 +16,19 @@ PROVISIONER_ARCHIVE = $(PROVISIONER_NAME)-$(subst _,-,$(ARCH))_$(PROVISIONER_VER
|
|||
PROVISIONER_URL = https://github.com/radekg/terraform-provisioner-ansible/releases/download/$(PROVISIONER_VERSION)/$(PROVISIONER_ARCHIVE)
|
||||
PROVISIONER_PATH = $(TF_PLUGINS_DIR)/$(ARCH)/$(PROVISIONER_NAME)_$(PROVISIONER_VERSION)
|
||||
|
||||
all: requirements install-provisioner secrets init-terraform
|
||||
all: roles-install install-provisioner secrets init-terraform
|
||||
@echo "Success!"
|
||||
|
||||
requirements-install:
|
||||
ansible-galaxy install --keep-scm-meta --ignore-errors --force -r ansible/requirements.yml
|
||||
roles-install:
|
||||
ansible/roles.py --install
|
||||
|
||||
requirements-check:
|
||||
ansible/versioncheck.py
|
||||
roles-check:
|
||||
ansible/roles.py --check
|
||||
|
||||
requirements: requirements-install requirements-check
|
||||
roles-update:
|
||||
ansible/roles.py --update
|
||||
|
||||
roles: roles-install roles-check
|
||||
|
||||
$(PROVISIONER_PATH):
|
||||
@mkdir -p $(TF_PLUGINS_DIR)/$(ARCH); \
|
||||
|
|
|
|||
|
|
@ -1,24 +1,17 @@
|
|||
---
|
||||
- name: infra-role-bootstrap-linux
|
||||
src: git@github.com:status-im/infra-role-bootstrap-linux.git
|
||||
scm: git
|
||||
|
||||
- name: infra-role-wireguard
|
||||
src: git@github.com:status-im/infra-role-wireguard.git
|
||||
scm: git
|
||||
|
||||
- name: infra-role-open-ports
|
||||
src: git@github.com:status-im/infra-role-open-ports.git
|
||||
scm: git
|
||||
|
||||
- name: infra-role-swap-file
|
||||
src: git@github.com:status-im/infra-role-swap-file.git
|
||||
scm: git
|
||||
|
||||
- name: infra-role-consul-service
|
||||
src: git@github.com:status-im/infra-role-consul-service.git
|
||||
scm: git
|
||||
|
||||
- name: infra-role-systemd-timer
|
||||
src: git@github.com:status-im/infra-role-systemd-timer.git
|
||||
scm: git
|
||||
|
|
|
|||
|
|
@ -0,0 +1,401 @@
|
|||
#!/usr/bin/env python3
|
||||
import yaml
|
||||
import logging
|
||||
import ansible
|
||||
import argparse
|
||||
import subprocess
|
||||
import functools
|
||||
from enum import Enum
|
||||
from os import path, getenv, symlink, readlink
|
||||
from packaging.version import parse as version_parse
|
||||
from concurrent import futures
|
||||
|
||||
HELP_DESCRIPTION='''
|
||||
This tool managed Ansible roles as Git repositories.
|
||||
It is both faster and simpler than Ansible Galaxy.
|
||||
|
||||
By default ~/.ansible/roles is symlinked to ~/work.
|
||||
Override it using --roles-symlink or ROLES_SYMLINK.
|
||||
|
||||
Installation behavior:
|
||||
- If no version is specified newest is pulled.
|
||||
- If version is matching nothing is done.
|
||||
- If repo is dirty or detached nothing is done.
|
||||
- If version is newer user is notified.
|
||||
'''
|
||||
HELP_EXAMPLE='''Examples:
|
||||
./roles.py --install
|
||||
./roles.py --check
|
||||
./roles.py --update
|
||||
'''
|
||||
|
||||
SCRIPT_DIR = path.dirname(path.realpath(__file__))
|
||||
# Where Ansible looks for installed roles.
|
||||
ROLES_PATH = path.join(path.expanduser('~'), '.ansible/roles')
|
||||
ROLES_SYMLINK = path.join(path.expanduser('~'), 'work')
|
||||
ROLES_WORKERS = 20
|
||||
REQUIREMENTS_PATH = path.join(SCRIPT_DIR, 'requirements.yml')
|
||||
|
||||
# Setup logging.
|
||||
log_format = '[%(levelname)s] %(message)s'
|
||||
logging.basicConfig(level=logging.INFO, format=log_format)
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
# Colors
|
||||
RST = '\033[0m'
|
||||
NORMAL = lambda x: f'\033[00m{x}{RST}'
|
||||
PURPLE = lambda x: f'\033[35m{x}{RST}'
|
||||
YELLOW = lambda x: f'\033[33m{x}{RST}'
|
||||
BLUE = lambda x: f'\033[34m{x}{RST}'
|
||||
GREY = lambda x: f'\033[90m{x}{RST}'
|
||||
RED = lambda x: f'\033[31m{x}{RST}'
|
||||
ORANGE = lambda x: f'\033[91m{x}{RST}'
|
||||
GREEN = lambda x: f'\033[32m{x}{RST}'
|
||||
CYAN = lambda x: f'\033[36m{x}{RST}'
|
||||
BOLD = lambda x: f'\033[1m{x}{RST}{RST}'
|
||||
|
||||
class State(Enum):
|
||||
# Order is priority. Higher status trumps lower.
|
||||
UNKNOWN = 0
|
||||
EXISTS = 1
|
||||
WRONG_VERSION = 2
|
||||
NEWER_VERSION = 3
|
||||
DIRTY = 4
|
||||
DETACHED = 5
|
||||
NO_VERSION = 6
|
||||
CLONE_FAILURE = 7
|
||||
MISSING = 8
|
||||
CLONED = 9
|
||||
UPDATED = 10
|
||||
VALID = 11
|
||||
SKIPPED = 12
|
||||
|
||||
def __str__(self):
|
||||
match self:
|
||||
case State.NEWER_VERSION: color = BOLD
|
||||
case State.WRONG_VERSION: color = RED
|
||||
case State.DIRTY: color = YELLOW
|
||||
case State.DETACHED: color = YELLOW
|
||||
case State.NO_VERSION: color = PURPLE
|
||||
case State.CLONE_FAILURE: color = RED
|
||||
case State.MISSING: color = RED
|
||||
case State.CLONED: color = GREEN
|
||||
case State.UPDATED: color = GREEN
|
||||
case State.VALID: color = GREEN
|
||||
case State.SKIPPED: color = GREY
|
||||
case _: color = NORMAL
|
||||
return color(self.name.replace('_', ' '))
|
||||
|
||||
# Allow calling max() to compare with previous state.
|
||||
def __gt__(self, other):
|
||||
if other is None:
|
||||
return True
|
||||
if self.__class__ is other.__class__:
|
||||
return self.value > other.value
|
||||
return NotImplemented
|
||||
|
||||
# Decorator to manage Role state based on function return value.
|
||||
def update(success=None, failure=None):
|
||||
def decorator(func):
|
||||
@functools.wraps(func)
|
||||
def wrapper_decorator(self, *args, **kwargs):
|
||||
# Set state to failure one on exception.
|
||||
try:
|
||||
rval = func(self, *args, **kwargs)
|
||||
except:
|
||||
self.state = max(failure, self.state)
|
||||
raise
|
||||
# Set state based on truthiness of result, higher one wins.
|
||||
if rval:
|
||||
self.state = max(success, self.state)
|
||||
else:
|
||||
self.state = max(failure, self.state)
|
||||
LOG.debug('[%-27s]: %s%s: state = %s',
|
||||
self.name, func.__name__, args, self.state)
|
||||
return rval
|
||||
return wrapper_decorator
|
||||
return decorator
|
||||
|
||||
class Role:
|
||||
|
||||
def __init__(self, name, src, required):
|
||||
self.state = State.UNKNOWN
|
||||
self.name = name
|
||||
self.src = src
|
||||
self.required = required
|
||||
|
||||
@classmethod
|
||||
def from_requirement(cls, obj):
|
||||
return cls(obj['name'], obj.get('src'), obj.get('version'))
|
||||
|
||||
def __repr__(self):
|
||||
return 'Role(name=%s, src=%s, required=%s, state=%s)' % (
|
||||
self.name, self.src, self.required, self.state,
|
||||
)
|
||||
|
||||
def to_dict(self):
|
||||
obj = {
|
||||
'name': self.name,
|
||||
'src': self.src,
|
||||
}
|
||||
if self.required:
|
||||
obj['version'] = self.required
|
||||
return obj
|
||||
|
||||
def _git(self, *args, cwd=None):
|
||||
cmd = ['git'] + list(args)
|
||||
LOG.debug('[%-27s]: COMMAND: %s', self.name, ' '.join(cmd))
|
||||
rval = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
cwd=cwd or self.path
|
||||
)
|
||||
LOG.debug('[%-27s]: RETURN: %d', self.name, rval.returncode)
|
||||
if rval.stdout:
|
||||
LOG.debug('[%-27s]: STDOUT: %s', self.name, rval.stdout.decode().strip())
|
||||
if rval.stderr:
|
||||
LOG.debug('[%-27s]: STDERR: %s', self.name, rval.stderr.decode().strip())
|
||||
rval.check_returncode()
|
||||
return str(rval.stdout.strip(), 'utf-8')
|
||||
|
||||
def _git_fail_is_false(self, *args, cwd=None):
|
||||
try:
|
||||
self._git(*args, cwd=cwd)
|
||||
except:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
@property
|
||||
def repo_parent_dir(self):
|
||||
return self.path.removesuffix(self.name)
|
||||
|
||||
@property
|
||||
def branch(self):
|
||||
return self._git('rev-parse', '--abbrev-ref', 'HEAD')
|
||||
|
||||
@property
|
||||
def current_commit(self):
|
||||
if not self.exists():
|
||||
return '........'
|
||||
return self._git('rev-parse', 'HEAD')
|
||||
|
||||
@State.update(success=State.DIRTY)
|
||||
def is_dirty(self):
|
||||
return self._git_fail_is_false('diff-files', '--quiet')
|
||||
|
||||
@State.update(success=State.DETACHED)
|
||||
def is_detached(self):
|
||||
return self._git_fail_is_false('symbolic-ref', 'HEAD')
|
||||
|
||||
@State.update(success=State.NEWER_VERSION)
|
||||
def is_ancestor(self):
|
||||
if self.required is None or self.required == self.current_commit:
|
||||
return False
|
||||
return self._git_fail_is_false(
|
||||
self.required, '--is-ancestor', self.current_commit
|
||||
)
|
||||
|
||||
@property
|
||||
@State.update(failure=State.NO_VERSION)
|
||||
def version(self):
|
||||
return self.required
|
||||
|
||||
@version.setter
|
||||
@State.update(success=State.UPDATED, failure=State.SKIPPED)
|
||||
def version(self, version):
|
||||
if self.required is not None:
|
||||
self.required = version
|
||||
return self.required
|
||||
|
||||
@State.update(success=State.VALID, failure=State.WRONG_VERSION)
|
||||
def valid_version(self):
|
||||
return self.required == self.current_commit
|
||||
|
||||
@State.update(success=State.VALID, failure=State.WRONG_VERSION)
|
||||
def pull(self):
|
||||
self._git('remote', 'update')
|
||||
status = self._git('status', '--untracked-files=no')
|
||||
if 'branch is behind' not in status:
|
||||
return None
|
||||
|
||||
rval = self._git('pull')
|
||||
return self.valid_version()
|
||||
|
||||
@State.update(success=State.CLONED, failure=State.CLONE_FAILURE)
|
||||
def clone(self):
|
||||
LOG.debug('Clogning: %s', self.src)
|
||||
try:
|
||||
self._git(
|
||||
'clone',
|
||||
self.src, self.name,
|
||||
cwd=self.repo_parent_dir
|
||||
)
|
||||
except Exception as ex:
|
||||
LOG.error('Clone failed: %s', ex.stderr.decode())
|
||||
return False
|
||||
return True
|
||||
|
||||
@property
|
||||
def path(self):
|
||||
return path.join(ROLES_PATH, self.name)
|
||||
|
||||
@State.update(success=State.EXISTS, failure=State.MISSING)
|
||||
def exists(self):
|
||||
return path.isdir(self.path)
|
||||
|
||||
|
||||
def handle_role(role, check=False, update=False, install=False):
|
||||
LOG.debug('[%-27s]: Processing role...', role.name)
|
||||
if not role.exists():
|
||||
if not check and not update:
|
||||
role.clone()
|
||||
return role
|
||||
|
||||
# Check if current version is newer.
|
||||
if role.is_ancestor():
|
||||
return role
|
||||
|
||||
# Check if current version matches required.
|
||||
if role.valid_version():
|
||||
return role
|
||||
|
||||
# Verify if git repo is not dirty or has detached head.
|
||||
if (role.is_dirty() or role.is_detached()) and not update:
|
||||
return role
|
||||
|
||||
# No need to fail if no version is set.
|
||||
if not role.version and check:
|
||||
return role
|
||||
|
||||
# Update config version or pull new changes.
|
||||
if update:
|
||||
role.version = role.current_commit
|
||||
elif install:
|
||||
# If version is not specified we just want the newest.
|
||||
role.pull()
|
||||
return role
|
||||
|
||||
|
||||
# Special function to preserve order and separating newlines.
|
||||
def roles_to_yaml(old_reqs, processed_roles):
|
||||
# Get processed role when available, use original one when not.
|
||||
return '\n'.join([
|
||||
yaml.dump([processed_roles.get(role.name, role).to_dict()])
|
||||
for role in old_reqs
|
||||
])
|
||||
|
||||
def commit_or_any(commit):
|
||||
return '*' if commit is None else commit[:8]
|
||||
|
||||
# Symlink only if folder or link doesn't exist.
|
||||
def symlink_roles_dir(roles_symlink):
|
||||
if path.islink(ROLES_PATH):
|
||||
dest = readlink(ROLES_PATH)
|
||||
if dest != roles_symlink:
|
||||
LOG.error('Roles path is already a link to: %s', dest)
|
||||
exit(1)
|
||||
else:
|
||||
return
|
||||
elif path.isdir(ROLES_PATH):
|
||||
LOG.error('Roles path is a directory, cannot symlink!')
|
||||
exit(1)
|
||||
|
||||
symlink(roles_symlink, ROLES_PATH)
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
epilog=HELP_EXAMPLE,
|
||||
description=HELP_DESCRIPTION,
|
||||
formatter_class=argparse.RawTextHelpFormatter,
|
||||
)
|
||||
|
||||
parser.add_argument('-f', '--filter', default='',
|
||||
help='Filter role repo names.')
|
||||
parser.add_argument('-w', '--workers', default=getenv('ROLES_WORKERS', ROLES_WORKERS), type=int,
|
||||
help='Max workers to run in parallel.')
|
||||
parser.add_argument('-r', '--requirements', default=getenv('REQUIREMENTS_PATH', REQUIREMENTS_PATH),
|
||||
help='Location of requirements.yml file.')
|
||||
parser.add_argument('-s', '--roles-symlink', default=getenv('ROLES_SYMLINK', ROLES_SYMLINK),
|
||||
help='Actual location of installed roles.')
|
||||
parser.add_argument('-l', '--log-level', default='INFO',
|
||||
help='Logging level.')
|
||||
parser.add_argument('-d', '--fail-dirty', action='store_true',
|
||||
help='Fail if repo is dirty.')
|
||||
parser.add_argument('-a', '--fail-detached', action='store_true',
|
||||
help='Fail if repo has detached head.')
|
||||
|
||||
group = parser.add_mutually_exclusive_group()
|
||||
group.add_argument('-i', '--install', action='store_true',
|
||||
help='Clone and update required roles.')
|
||||
group.add_argument('-c', '--check', action='store_true',
|
||||
help='Only check roles, no installing.')
|
||||
group.add_argument('-u', '--update', action='store_true',
|
||||
help='Update requirements with current commits.')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
assert args.install or args.check or args.update, \
|
||||
parser.error('Pick one: --install, --check, --update')
|
||||
|
||||
return args
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
|
||||
LOG.setLevel(args.log_level.upper())
|
||||
|
||||
# Verify Ansible version is 2.8 or newer.
|
||||
if version_parse(ansible.__version__) < version_parse("2.8"):
|
||||
LOG.error('Your Ansible version is lower than 2.8. Upgrade it.')
|
||||
exit(1)
|
||||
|
||||
# Symlink ansible roles directory to work directory.
|
||||
symlink_roles_dir(args.roles_symlink)
|
||||
|
||||
# Read Ansible requirements file.
|
||||
with open(args.requirements, 'r') as f:
|
||||
requirements = yaml.load(f, Loader=yaml.FullLoader)
|
||||
|
||||
requirements = [
|
||||
Role.from_requirement(req) for req in requirements
|
||||
]
|
||||
|
||||
# Check if each Ansible role is installed and has correct version.
|
||||
with futures.ProcessPoolExecutor(max_workers=args.workers) as executor:
|
||||
these_futures = [
|
||||
executor.submit(handle_role, role, args.check, args.update, args.install)
|
||||
for role in requirements
|
||||
if args.filter in role.name
|
||||
]
|
||||
# Wait for all the workers to finishe and return their role.
|
||||
processed_roles = {
|
||||
r.name: r for r in
|
||||
[r.result() for r in futures.as_completed(these_futures)]
|
||||
}
|
||||
|
||||
# Use the same order as requirements.yml file.
|
||||
for req in requirements:
|
||||
if args.filter not in req.name:
|
||||
continue
|
||||
role = processed_roles[req.name]
|
||||
print('%-40s --- %22s (Git: %s | Req: %s)' %
|
||||
(BOLD(role.name), role.state,
|
||||
CYAN(role.current_commit[:8]),
|
||||
commit_or_any(role.required)))
|
||||
|
||||
if args.update:
|
||||
with open(args.requirements, 'w') as f:
|
||||
f.write(roles_to_yaml(requirements, processed_roles))
|
||||
|
||||
fail_states = set([State.MISSING, State.WRONG_VERSION])
|
||||
if args.fail_dirty:
|
||||
fail_states.append(State.DIRTY)
|
||||
if args.fail_detached:
|
||||
fail_states.append(State.DETACHED)
|
||||
if fail_states.intersection([r.state for r in processed_roles.values()]):
|
||||
exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,71 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# WARNING: If importing this fails set PYTHONPATH.
|
||||
import yaml
|
||||
import ansible
|
||||
import subprocess
|
||||
from os import path, environ
|
||||
from packaging import version
|
||||
|
||||
SCRIPT_DIR = path.dirname(path.realpath(__file__))
|
||||
# Where Ansible looks for installed roles.
|
||||
ANSIBLE_ROLES_PATH = path.join(environ['HOME'], '.ansible/roles')
|
||||
|
||||
|
||||
class Role:
|
||||
def __init__(self, name, version):
|
||||
self.name = name
|
||||
self.version = version
|
||||
|
||||
@property
|
||||
def path(self):
|
||||
return path.join(ANSIBLE_ROLES_PATH, self.name)
|
||||
|
||||
def exists(self):
|
||||
return path.isdir(self.path)
|
||||
|
||||
def local_version(self):
|
||||
cmd = subprocess.run(
|
||||
['git', 'rev-parse', 'HEAD'],
|
||||
capture_output=True,
|
||||
cwd=self.path
|
||||
)
|
||||
cmd.check_returncode()
|
||||
return str(cmd.stdout.strip(), 'utf-8')
|
||||
|
||||
|
||||
# Verify Ansible version is 2.8 or newer.
|
||||
if version.parse(ansible.__version__) < version.parse("2.8"):
|
||||
print('Your Ansible version is lower than 2.8. Upgrade it.')
|
||||
exit(1)
|
||||
|
||||
# Read Ansible requirements file.
|
||||
with open(path.join(SCRIPT_DIR, 'requirements.yml'), 'r') as f:
|
||||
requirements = yaml.load(f, Loader=yaml.FullLoader)
|
||||
|
||||
# Check if each Ansible role is installed and has correct version.
|
||||
errors = 0
|
||||
for req in requirements:
|
||||
role = Role(req['name'], req.get('version'))
|
||||
|
||||
if not role.exists():
|
||||
print('%25s - MISSING!' % role.name)
|
||||
errors += 1
|
||||
continue
|
||||
|
||||
# For now we allow not specifying versions for everyhing.
|
||||
if role.version is None:
|
||||
print('%25s - No version!' % role.name)
|
||||
continue
|
||||
|
||||
local_version = role.local_version()
|
||||
if role.version != local_version:
|
||||
print('%25s - MISMATCH: %s != %s' %
|
||||
(role.name, role.version[:8], local_version[:8]))
|
||||
errors += 1
|
||||
continue
|
||||
|
||||
print('%25s - VALID' % role.name)
|
||||
|
||||
# Any issue with any role should cause failure.
|
||||
if errors > 0:
|
||||
exit(1)
|
||||
Loading…
Reference in New Issue