From 0b50bcc867a0bd38ad7e69157d244bfed29e2c8c Mon Sep 17 00:00:00 2001 From: Mikhail Konyakhin Date: Mon, 25 Nov 2019 14:23:54 +0300 Subject: [PATCH 1/9] Fix systemd unit file for mongodb and add manage limits. --- defaults/main.yml | 2 ++ tasks/install.debian.yml | 4 ++-- files/mongodb.service => templates/mongodb.service.j2 | 7 ++++--- 3 files changed, 8 insertions(+), 5 deletions(-) rename files/mongodb.service => templates/mongodb.service.j2 (79%) diff --git a/defaults/main.yml b/defaults/main.yml index be3b175..b5d1bd5 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -16,6 +16,8 @@ mongodb_pymongo_pip_version: 3.7.1 mongodb_user_update_password: "on_create" # MongoDB user password update default policy mongodb_manage_service: true mongodb_manage_systemd_unit: true +mongodb_systemd_unit_limit_nofile: 64000 +mongodb_systemd_unit_limit_nproc: 64000 mongodb_disable_transparent_hugepages: false diff --git a/tasks/install.debian.yml b/tasks/install.debian.yml index 826aea9..2d78dd4 100644 --- a/tasks/install.debian.yml +++ b/tasks/install.debian.yml @@ -59,8 +59,8 @@ state: present - name: Add systemd configuration if present - copy: - src: mongodb.service + template: + src: mongodb.service.j2 dest: "/lib/systemd/system/{{mongodb_daemon_name}}.service" owner: root group: root diff --git a/files/mongodb.service b/templates/mongodb.service.j2 similarity index 79% rename from files/mongodb.service rename to templates/mongodb.service.j2 index ea90f2d..ce37667 100644 --- a/files/mongodb.service +++ b/templates/mongodb.service.j2 @@ -1,9 +1,10 @@ +# {{ ansible_managed }} [Unit] Description=An object/document-oriented database Documentation=man:mongod(1) [Service] -User=mongodb +User={{ mongodb_user }} ExecStart=/usr/bin/numactl --interleave=all /usr/bin/mongod --config /etc/mongod.conf # file size LimitFSIZE=infinity @@ -12,9 +13,9 @@ LimitCPU=infinity # virtual memory size LimitAS=infinity # open files -LimitNOFILE=64000 +LimitNOFILE={{ mongodb_systemd_unit_limit_nofile }} # processes/threads -LimitNPROC=64000 +LimitNPROC={{ mongodb_systemd_unit_limit_nproc }} # locked memory LimitMEMLOCK=infinity # total threads (user+kernel) From 56af8c7e9b244d3e12b7e25f755d20a9de5f1533 Mon Sep 17 00:00:00 2001 From: Mikhail Konyakhin Date: Sat, 21 Dec 2019 12:56:19 +0300 Subject: [PATCH 2/9] Remove not needed conditions from handlers. --- handlers/main.yml | 16 +--------------- tasks/configure.yml | 1 - tasks/disable_transparent_hugepages.yml | 3 +++ tasks/main.yml | 6 ++++++ tasks/mms-agent.yml | 11 ++++++++--- 5 files changed, 18 insertions(+), 19 deletions(-) diff --git a/handlers/main.yml b/handlers/main.yml index 3e3d32d..62251cf 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -3,9 +3,6 @@ - name: reload systemd systemd: daemon_reload: yes - when: - - ansible_service_mgr == "systemd" - - mongodb_manage_service | bool - name: run disable-transparent-hugepages systemd: @@ -13,13 +10,6 @@ enabled: true state: started daemon_reload: yes - when: ansible_service_mgr == "systemd" and mongodb_disable_transparent_hugepages | bool - -- name: mongodb reload - service: - name: "{{ mongodb_daemon_name }}" - state: reloaded - when: mongodb_manage_service | bool - name: mongodb restart service: @@ -38,11 +28,6 @@ name: sysfsutils state: restarted -- name: service started - service: - name: "{{ mongodb_daemon_name }}" - state: started - - name: wait when mongodb is started wait_for: host: "{{ item }}" @@ -56,3 +41,4 @@ port: "{{ mongodb_net_port }}" delay: 5 timeout: 120 + diff --git a/tasks/configure.yml b/tasks/configure.yml index f9a040b..ce06b8d 100644 --- a/tasks/configure.yml +++ b/tasks/configure.yml @@ -81,7 +81,6 @@ mode: 0644 notify: - mongodb restart - - service started - wait when mongodb is started - name: Flush all handlers at this point diff --git a/tasks/disable_transparent_hugepages.yml b/tasks/disable_transparent_hugepages.yml index 4f41d34..654674b 100644 --- a/tasks/disable_transparent_hugepages.yml +++ b/tasks/disable_transparent_hugepages.yml @@ -10,4 +10,7 @@ - "disable-transparent-hugepages.{{ ansible_distribution | lower }}.service.j2" - "disable-transparent-hugepages.{{ ansible_os_family | lower }}.service.j2" notify: "run disable-transparent-hugepages" + when: + - ansible_service_mgr == "systemd" + - mongodb_disable_transparent_hugepages | bool diff --git a/tasks/main.yml b/tasks/main.yml index e0a1d95..33d0e46 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -124,6 +124,12 @@ no_log: false tags: [mongodb] +- name: service started + service: + name: "{{ mongodb_daemon_name }}" + state: started + enabled: yes + - name: Include MMS Agent configuration include: mms-agent.yml when: mongodb_mms_api_key | length > 0 diff --git a/tasks/mms-agent.yml b/tasks/mms-agent.yml index 9bc646a..84aba16 100644 --- a/tasks/mms-agent.yml +++ b/tasks/mms-agent.yml @@ -19,9 +19,14 @@ mode: 0755 - name: Configure the MMS agent pt. 2 - template: src=monitoring-agent.config.j2 dest=/etc/mongodb-mms/monitoring-agent.config + template: + src: monitoring-agent.config.j2 + dest: /etc/mongodb-mms/monitoring-agent.config notify: mongodb-mms-monitoring-agent restart - name: Ensure that the MMS agent is started - service: name=mongodb-mms-monitoring-agent state=started enabled=yes - when: mongodb_manage_service + service: + name: mongodb-mms-monitoring-agent + state: started + enabled: yes + From 6a38960fe63560fd276920c72f1b942d183a2f6f Mon Sep 17 00:00:00 2001 From: Mikhail Konyakhin Date: Sat, 21 Dec 2019 15:53:21 +0300 Subject: [PATCH 3/9] Add mitogen and enable pipelining in tests. --- tests/scripts/before_install.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/scripts/before_install.sh b/tests/scripts/before_install.sh index eea35aa..48fc3c9 100644 --- a/tests/scripts/before_install.sh +++ b/tests/scripts/before_install.sh @@ -12,7 +12,14 @@ set -o errexit set -o pipefail # Latest Ansible install -pip install docker ansible +pip install docker ansible mitogen + +cat << EOF > ansible.cfg +[defaults] +pipelining = True +strategy = mitogen_linear +strategy_plugins = /home/travis/virtualenv/python2.7.15/lib/python2.7/site-packages/ansible_mitogen/plugins/strategy +EOF # Pull docker image or build it if [ -f tests/Dockerfile.${DISTRIBUTION}_${DIST_VERSION} ] From bb8ad9e55592a51814cd05bdb9601b3f2e301a32 Mon Sep 17 00:00:00 2001 From: Mikhail Konyakhin Date: Sat, 21 Dec 2019 15:54:25 +0300 Subject: [PATCH 4/9] Add ansible profiler of tasks for tests. --- tests/callback_plugins/profile_tasks.py | 74 +++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 tests/callback_plugins/profile_tasks.py diff --git a/tests/callback_plugins/profile_tasks.py b/tests/callback_plugins/profile_tasks.py new file mode 100644 index 0000000..e8afd89 --- /dev/null +++ b/tests/callback_plugins/profile_tasks.py @@ -0,0 +1,74 @@ +# profile_tasks.py: an Ansible plugin for timing tasks + +# Copyright (C) 2014 Jharrod LaFon +# https://github.com/jlafon/ansible-profile/ +# Included with permission + + +# The MIT License (MIT) +# +# Copyright (c) 2014 Jharrod LaFon +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +from ansible.plugins.callback import CallbackBase +import time + + +class CallbackModule(CallbackBase): + """ + A plugin for timing tasks + """ + def __init__(self): + self.stats = {} + self.current = None + + def playbook_on_task_start(self, name, is_conditional): + """ + Logs the start of each task + """ + if self.current is not None: + # Record the running time of the last executed task + self.stats[self.current] = time.time() - self.stats[self.current] + + # Record the start time of the current task + self.current = name + self.stats[self.current] = time.time() + + def playbook_on_stats(self, stats): + """ + Prints the timings + """ + # Record the timing of the very last task + if self.current is not None: + self.stats[self.current] = time.time() - self.stats[self.current] + + # Sort the tasks by their running time + results = sorted(self.stats.items(), + key=lambda value: value[1], reverse=True) + + # Just keep the top 25 + results = results[:25] + + # Print the timings + for name, elapsed in results: + print("{0:-<70}{1:->9}".format( + '{0} '.format(name), + ' {0:.02f}s'.format(elapsed))) From f8981d7c4552b49f9d2f54d0f5c3b3d0d836909d Mon Sep 17 00:00:00 2001 From: Mikhail Konyakhin Date: Sat, 21 Dec 2019 21:30:24 +0300 Subject: [PATCH 5/9] Update docker files and bind python version. --- .travis.yml | 2 +- tests/Dockerfile.amazonlinux_2-builded | 8 +++----- tests/Dockerfile.centos_6-builded | 4 +--- tests/Dockerfile.centos_7-builded | 4 +--- tests/Dockerfile.ubuntu_16_04-builded | 16 +++++++++++++++- tests/Dockerfile.ubuntu_18_04-builded | 13 +++++++++++++ 6 files changed, 34 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 775fcc8..3c2132d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ dist: trusty sudo: required language: python python: - - "2.7" + - "2.7.15" env: - > DISTRIBUTION=ubuntu diff --git a/tests/Dockerfile.amazonlinux_2-builded b/tests/Dockerfile.amazonlinux_2-builded index 8818337..eae97dd 100644 --- a/tests/Dockerfile.amazonlinux_2-builded +++ b/tests/Dockerfile.amazonlinux_2-builded @@ -1,14 +1,12 @@ FROM amazonlinux:2 # This is needed so that ansible managed to read "ansible_default_ipv4" -RUN yum install iproute -y +# This step is needed since standard CentOS docker image does not come with i +# This package seems to be required for Mongo 3.2 and downwards +RUN yum install iproute initscripts -y # we can has SSH EXPOSE 22 -# This step is needed since standard CentOS docker image does not come with i -# This package seems to be required for Mongo 3.2 and downwards -RUN yum install initscripts -y - # pepare for takeoff CMD ["/usr/sbin/init"] diff --git a/tests/Dockerfile.centos_6-builded b/tests/Dockerfile.centos_6-builded index 06fb3b3..9a472b5 100644 --- a/tests/Dockerfile.centos_6-builded +++ b/tests/Dockerfile.centos_6-builded @@ -1,10 +1,8 @@ FROM centos:6 # This is needed so that ansible managed to read "ansible_default_ipv4" -RUN yum install iproute -y - # This step is needed since standard CentOS docker image does not come with EPEL installed by default -RUN yum install epel-release -y +RUN yum install iproute epel-release -y # we can has SSH EXPOSE 22 diff --git a/tests/Dockerfile.centos_7-builded b/tests/Dockerfile.centos_7-builded index 55da0ca..f67c6c9 100644 --- a/tests/Dockerfile.centos_7-builded +++ b/tests/Dockerfile.centos_7-builded @@ -1,10 +1,8 @@ FROM centos:7 # This is needed so that ansible managed to read "ansible_default_ipv4" -RUN yum install iproute -y - # This step is needed since standard CentOS docker image does not come with EPEL installed by default -RUN yum install epel-release -y +RUN yum install iproute epel-release -y # we can has SSH EXPOSE 22 diff --git a/tests/Dockerfile.ubuntu_16_04-builded b/tests/Dockerfile.ubuntu_16_04-builded index 953e907..d61d9cd 100644 --- a/tests/Dockerfile.ubuntu_16_04-builded +++ b/tests/Dockerfile.ubuntu_16_04-builded @@ -2,6 +2,20 @@ FROM ubuntu:16.04 ARG DEBIAN_FRONTEND=noninteractive +RUN apt update && \ + apt install --yes python-pip curl jq && \ + pip install apt-select && \ + EXTERNAL_IP=`curl -s https://ipinfo.io/ip` && \ + echo "EXTERNAL_IP is ${EXTERNAL_IP}" && \ + DATA_IP=`curl -s https://json.geoiplookup.io/$EXTERNAL_IP` && \ + echo $DATA_IP && \ + COUNTRY_IP=`echo $DATA_IP | jq '.country_code' | tr -d '"'` && \ + apt-select -C $COUNTRY_IP -t 1 -m one-week-behind && \ + cat sources.list && \ + cp /etc/apt/sources.list /etc/apt/sources.list.backup && \ + mv -f sources.list /etc/apt/ + RUN apt update && \ apt install --yes python-minimal iproute2 && \ - rm /lib/systemd/system/getty@.service + rm -f /lib/systemd/system/getty@.service + diff --git a/tests/Dockerfile.ubuntu_18_04-builded b/tests/Dockerfile.ubuntu_18_04-builded index d2817bb..3c908e2 100644 --- a/tests/Dockerfile.ubuntu_18_04-builded +++ b/tests/Dockerfile.ubuntu_18_04-builded @@ -2,6 +2,19 @@ FROM ubuntu:18.04 ARG DEBIAN_FRONTEND=noninteractive +RUN apt update && \ + apt install --yes python-pip curl jq && \ + pip install apt-select && \ + EXTERNAL_IP=`curl -s https://ipinfo.io/ip` && \ + echo "EXTERNAL_IP is ${EXTERNAL_IP}" && \ + DATA_IP=`curl -s https://json.geoiplookup.io/$EXTERNAL_IP` && \ + echo $DATA_IP && \ + COUNTRY_IP=`echo $DATA_IP | jq '.country_code' | tr -d '"'` && \ + apt-select -C $COUNTRY_IP -t 1 -m one-week-behind && \ + cat sources.list && \ + cp /etc/apt/sources.list /etc/apt/sources.list.backup && \ + mv -f sources.list /etc/apt/ + RUN apt update && \ apt install --yes python-minimal systemd gnupg iproute2 From d21c149fdfd079115e3dfc96135da98d5a7c4ecc Mon Sep 17 00:00:00 2001 From: Mikhail Konyakhin Date: Thu, 26 Dec 2019 17:34:24 +0300 Subject: [PATCH 6/9] Update .gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a1d8754..48204ad 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ tests/*.retry \#*\# .\#* /.python-version +*.pyc From 1fa78ca6dabd3db2ec783a30069a2ab8e41ab3e6 Mon Sep 17 00:00:00 2001 From: Mikhail Konyakhin Date: Thu, 26 Dec 2019 17:34:49 +0300 Subject: [PATCH 7/9] Add docker container proxy for cache packages of yum, apt and pip repos. --- .travis.yml | 6 +-- tests/Dockerfile.amazonlinux_2-builded | 2 +- tests/Dockerfile.centos_6-builded | 2 +- tests/Dockerfile.centos_7-builded | 2 +- tests/Dockerfile.debian_8-builded | 2 +- tests/Dockerfile.debian_9-builded | 2 +- tests/Dockerfile.ubuntu-upstart_14_04-builded | 7 +++ tests/Dockerfile.ubuntu_16_04-builded | 17 +------ tests/Dockerfile.ubuntu_18_04-builded | 15 +----- tests/files/etc/apt/apt.conf.d/proxy.conf | 3 ++ tests/files/etc/environment | 2 + tests/files/etc/squid/squid.conf | 34 ++++++++++++++ tests/site.yml | 47 +++++++++++++++++++ 13 files changed, 104 insertions(+), 37 deletions(-) create mode 100644 tests/Dockerfile.ubuntu-upstart_14_04-builded create mode 100644 tests/files/etc/apt/apt.conf.d/proxy.conf create mode 100644 tests/files/etc/environment create mode 100644 tests/files/etc/squid/squid.conf diff --git a/.travis.yml b/.travis.yml index 3c2132d..ba31490 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,17 +46,17 @@ env: DOCKER_PRIVILEGED=true - > DISTRIBUTION=ubuntu-upstart - DIST_VERSION=14.04 + DIST_VERSION=14_04-builded MONGODB_VERSION=4.0 DOCKER_PRIVILEGED=true - > DISTRIBUTION=ubuntu-upstart - DIST_VERSION=14.04 + DIST_VERSION=14_04-builded MONGODB_VERSION=3.6 DOCKER_PRIVILEGED=true - > DISTRIBUTION=ubuntu-upstart - DIST_VERSION=14.04 + DIST_VERSION=14_04-builded MONGODB_VERSION=3.4 DOCKER_PRIVILEGED=true - > diff --git a/tests/Dockerfile.amazonlinux_2-builded b/tests/Dockerfile.amazonlinux_2-builded index eae97dd..8c25c8e 100644 --- a/tests/Dockerfile.amazonlinux_2-builded +++ b/tests/Dockerfile.amazonlinux_2-builded @@ -3,7 +3,7 @@ FROM amazonlinux:2 # This is needed so that ansible managed to read "ansible_default_ipv4" # This step is needed since standard CentOS docker image does not come with i # This package seems to be required for Mongo 3.2 and downwards -RUN yum install iproute initscripts -y +RUN yum install iproute initscripts python-pip python-devel -y # we can has SSH EXPOSE 22 diff --git a/tests/Dockerfile.centos_6-builded b/tests/Dockerfile.centos_6-builded index 9a472b5..2a65657 100644 --- a/tests/Dockerfile.centos_6-builded +++ b/tests/Dockerfile.centos_6-builded @@ -2,7 +2,7 @@ FROM centos:6 # This is needed so that ansible managed to read "ansible_default_ipv4" # This step is needed since standard CentOS docker image does not come with EPEL installed by default -RUN yum install iproute epel-release -y +RUN yum install iproute epel-release python-pip python-devel -y # we can has SSH EXPOSE 22 diff --git a/tests/Dockerfile.centos_7-builded b/tests/Dockerfile.centos_7-builded index f67c6c9..8ab589a 100644 --- a/tests/Dockerfile.centos_7-builded +++ b/tests/Dockerfile.centos_7-builded @@ -2,7 +2,7 @@ FROM centos:7 # This is needed so that ansible managed to read "ansible_default_ipv4" # This step is needed since standard CentOS docker image does not come with EPEL installed by default -RUN yum install iproute epel-release -y +RUN yum install iproute epel-release python-pip python-devel -y # we can has SSH EXPOSE 22 diff --git a/tests/Dockerfile.debian_8-builded b/tests/Dockerfile.debian_8-builded index 4055247..144b1f1 100644 --- a/tests/Dockerfile.debian_8-builded +++ b/tests/Dockerfile.debian_8-builded @@ -3,5 +3,5 @@ FROM debian:8 ARG DEBIAN_FRONTEND=noninteractive RUN apt update && \ - apt install --yes python-minimal && \ + apt install --yes python-minimal python-pip && \ rm /lib/systemd/system/getty@.service diff --git a/tests/Dockerfile.debian_9-builded b/tests/Dockerfile.debian_9-builded index 8397f5c..fb75ce1 100644 --- a/tests/Dockerfile.debian_9-builded +++ b/tests/Dockerfile.debian_9-builded @@ -3,7 +3,7 @@ FROM debian:9 ARG DEBIAN_FRONTEND=noninteractive RUN apt update && \ - apt install --yes python-minimal systemd gnupg + apt install --yes python-minimal systemd gnupg python-pip RUN cd /lib/systemd/system/sysinit.target.wants/ && \ ls | grep -v systemd-tmpfiles-setup.service | xargs rm -f && \ diff --git a/tests/Dockerfile.ubuntu-upstart_14_04-builded b/tests/Dockerfile.ubuntu-upstart_14_04-builded new file mode 100644 index 0000000..7f3f135 --- /dev/null +++ b/tests/Dockerfile.ubuntu-upstart_14_04-builded @@ -0,0 +1,7 @@ +FROM ubuntu-upstart + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install --yes python-minimal python-pip python-dev iproute2 + diff --git a/tests/Dockerfile.ubuntu_16_04-builded b/tests/Dockerfile.ubuntu_16_04-builded index d61d9cd..ea9038c 100644 --- a/tests/Dockerfile.ubuntu_16_04-builded +++ b/tests/Dockerfile.ubuntu_16_04-builded @@ -3,19 +3,6 @@ FROM ubuntu:16.04 ARG DEBIAN_FRONTEND=noninteractive RUN apt update && \ - apt install --yes python-pip curl jq && \ - pip install apt-select && \ - EXTERNAL_IP=`curl -s https://ipinfo.io/ip` && \ - echo "EXTERNAL_IP is ${EXTERNAL_IP}" && \ - DATA_IP=`curl -s https://json.geoiplookup.io/$EXTERNAL_IP` && \ - echo $DATA_IP && \ - COUNTRY_IP=`echo $DATA_IP | jq '.country_code' | tr -d '"'` && \ - apt-select -C $COUNTRY_IP -t 1 -m one-week-behind && \ - cat sources.list && \ - cp /etc/apt/sources.list /etc/apt/sources.list.backup && \ - mv -f sources.list /etc/apt/ - -RUN apt update && \ - apt install --yes python-minimal iproute2 && \ - rm -f /lib/systemd/system/getty@.service + apt install --yes python-minimal python-pip python-dev iproute2 && \ + rm /lib/systemd/system/getty@.service diff --git a/tests/Dockerfile.ubuntu_18_04-builded b/tests/Dockerfile.ubuntu_18_04-builded index 3c908e2..0c4a18d 100644 --- a/tests/Dockerfile.ubuntu_18_04-builded +++ b/tests/Dockerfile.ubuntu_18_04-builded @@ -3,20 +3,7 @@ FROM ubuntu:18.04 ARG DEBIAN_FRONTEND=noninteractive RUN apt update && \ - apt install --yes python-pip curl jq && \ - pip install apt-select && \ - EXTERNAL_IP=`curl -s https://ipinfo.io/ip` && \ - echo "EXTERNAL_IP is ${EXTERNAL_IP}" && \ - DATA_IP=`curl -s https://json.geoiplookup.io/$EXTERNAL_IP` && \ - echo $DATA_IP && \ - COUNTRY_IP=`echo $DATA_IP | jq '.country_code' | tr -d '"'` && \ - apt-select -C $COUNTRY_IP -t 1 -m one-week-behind && \ - cat sources.list && \ - cp /etc/apt/sources.list /etc/apt/sources.list.backup && \ - mv -f sources.list /etc/apt/ - -RUN apt update && \ - apt install --yes python-minimal systemd gnupg iproute2 + apt install --yes python-minimal python-pip python-dev systemd gnupg iproute2 RUN cd /lib/systemd/system/sysinit.target.wants/ && \ ls | grep -v systemd-tmpfiles-setup.service | xargs rm -f && \ diff --git a/tests/files/etc/apt/apt.conf.d/proxy.conf b/tests/files/etc/apt/apt.conf.d/proxy.conf new file mode 100644 index 0000000..1a3273b --- /dev/null +++ b/tests/files/etc/apt/apt.conf.d/proxy.conf @@ -0,0 +1,3 @@ +Acquire::http::Proxy "http://proxy:3128/"; +Acquire::https::Proxy "http://proxy:3128/"; + diff --git a/tests/files/etc/environment b/tests/files/etc/environment new file mode 100644 index 0000000..d8ef85f --- /dev/null +++ b/tests/files/etc/environment @@ -0,0 +1,2 @@ +PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" +HTTP_PROXY="http://proxy:3128" diff --git a/tests/files/etc/squid/squid.conf b/tests/files/etc/squid/squid.conf new file mode 100644 index 0000000..66e1616 --- /dev/null +++ b/tests/files/etc/squid/squid.conf @@ -0,0 +1,34 @@ +acl localhost src 127.0.0.1/32 ::1 +acl localnet src 10.0.0.0/8 # RFC1918 possible internal network +acl localnet src 172.16.0.0/12 # RFC1918 possible internal network +acl localnet src 192.168.0.0/16 # RFC1918 possible internal network +acl SSL_ports port 443 +acl Safe_ports port 80 # http +acl Safe_ports port 21 # ftp +acl Safe_ports port 443 # https +acl Safe_ports port 70 # gopher +acl Safe_ports port 210 # wais +acl Safe_ports port 1025-65535 # unregistered ports +acl Safe_ports port 280 # http-mgmt +acl Safe_ports port 488 # gss-http +acl Safe_ports port 591 # filemaker +acl Safe_ports port 777 # multiling http +acl CONNECT method CONNECT +http_access allow manager localhost +http_access deny manager +http_access deny !Safe_ports +http_access deny CONNECT !SSL_ports +http_access allow localnet +http_access allow localhost +http_access deny all +http_port 3128 +maximum_object_size 2048 MB +cache_dir aufs /var/spool/squid 5000 24 256 +coredump_dir /var/spool/squid +refresh_pattern ^ftp: 1440 20% 10080 +refresh_pattern ^gopher: 1440 0% 1440 +refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 +refresh_pattern (Release|Packages(.gz)*)$ 0 20% 2880 +refresh_pattern . 0 20% 4320 +refresh_all_ims on + diff --git a/tests/site.yml b/tests/site.yml index a1b624f..9a27506 100644 --- a/tests/site.yml +++ b/tests/site.yml @@ -5,6 +5,29 @@ become: no gather_facts: no tasks: + - name: Copy environmnet file + copy: + src: etc/environment + dest: /tmp/environment + - name: Copy Squid config + copy: + src: etc/squid/squid.conf + dest: /tmp/squid.conf + - name: Create a network + docker_network: + name: MongoDBTestingNet + - name: Run Proxy in Docker + docker_container: + name: proxy + image: sameersbn/squid + state: started + volumes: + - /tmp/squid.conf:/etc/squid/squid.conf:ro + networks: + - name: MongoDBTestingNet + aliases: + - proxy + networks_cli_compatible: yes - name: Run MongoDB cluster in Docker docker_container: name: "{{ item }}" @@ -23,11 +46,35 @@ target: /run - type: tmpfs target: /run/lock + volumes: + - /tmp/environment:/etc/environment:ro + networks: + - name: MongoDBTestingNet + aliases: + - "{{ item }}" + networks_cli_compatible: yes with_items: - mongo1 - mongo2 - mongo3 +- hosts: mongo + become: no + gather_facts: yes + tasks: + - name: Set proxy for apt + copy: + src: etc/apt/apt.conf.d/proxy.conf + dest: /etc/apt/apt.conf.d/proxy.conf + when: ansible_os_family == "Debian" + - name: Set proxy for yum + ini_file: + path: /etc/yum.conf + section: main + option: proxy + value: http://proxy:3128 + when: ansible_os_family == "RedHat" + - hosts: mongo become: no gather_facts: no From d0bacd7cd2522dfb785e8a4531e4ab75c7cc7262 Mon Sep 17 00:00:00 2001 From: Mikhail Konyakhin Date: Fri, 27 Dec 2019 13:40:19 +0300 Subject: [PATCH 8/9] Split replication configuration without auth to fix first run ansible on all hosts in MongoDB cluster in same time. --- tasks/replication.yml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/tasks/replication.yml b/tasks/replication.yml index 2464636..2dcd707 100644 --- a/tasks/replication.yml +++ b/tasks/replication.yml @@ -19,7 +19,7 @@ with_items: - "{{ mongodb_replication_params|default([]) }}" -- name: Replication configuration without auth +- name: Replication configuration without auth on master mongodb_replication: build_indexes: "{{ item.build_indexes|default(omit) }}" login_host: "{{ mongodb_login_host|default('localhost') }}" @@ -32,6 +32,26 @@ priority: "{{ item.priority|default(1.0) }}" votes: "{{ item.votes|default(omit) }}" ssl: "{{ True if mongodb_net_ssl_mode == 'requireSSL' else False }}" - when: mongodb_security_authorization == 'disabled' + when: ( mongodb_security_authorization == 'disabled' + and mongodb_master is defined and mongodb_master ) with_items: - "{{ mongodb_replication_params|default([]) }}" + +- name: Replication configuration without auth on replicas and arbiters + mongodb_replication: + build_indexes: "{{ item.build_indexes|default(omit) }}" + login_host: "{{ mongodb_login_host|default('localhost') }}" + login_port: "{{ mongodb_login_port|default(27017) }}" + replica_set: "{{ mongodb_replication_replset }}" + host_name: "{{ item.host_name }}" + host_port: "{{ item.host_port|default(27017) }}" + host_type: "{{ item.host_type|default('replica') }}" + hidden: "{{ item.hidden|default(false) }}" + priority: "{{ item.priority|default(1.0) }}" + votes: "{{ item.votes|default(omit) }}" + ssl: "{{ True if mongodb_net_ssl_mode == 'requireSSL' else False }}" + when: ( mongodb_security_authorization == 'disabled' + and ( mongodb_master is not defined or not mongodb_master) ) + with_items: + - "{{ mongodb_replication_params|default([]) }}" + From 26f5b616d8880fdc0b02e4d0ecbc9afc8e481927 Mon Sep 17 00:00:00 2001 From: Mikhail Konyakhin Date: Fri, 27 Dec 2019 13:43:00 +0300 Subject: [PATCH 9/9] Update data format in tests and one run for all hosts in MongoDB cluster in tests. --- tests/group_vars/all.yml | 2 +- tests/host_vars/mongo1.yml | 2 +- tests/host_vars/mongo2.yml | 2 +- tests/site.yml | 8 -------- 4 files changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/group_vars/all.yml b/tests/group_vars/all.yml index abd21b7..e77379b 100644 --- a/tests/group_vars/all.yml +++ b/tests/group_vars/all.yml @@ -1,6 +1,6 @@ --- -image_name: "ubuntu-upstart:18.04" +image_name: "ubuntu:18.04" mongodb_package: mongodb-org mongodb_version: "4.0" mongodb_storage_dbpath: /var/lib/mongodb diff --git a/tests/host_vars/mongo1.yml b/tests/host_vars/mongo1.yml index 337101b..d994009 100644 --- a/tests/host_vars/mongo1.yml +++ b/tests/host_vars/mongo1.yml @@ -2,4 +2,4 @@ mongodb_master: true mongodb_replication_params: - - { host_name: "{{ hostvars[inventory_hostname].ansible_default_ipv4.address }}" } + - host_name: "{{ hostvars[inventory_hostname].ansible_default_ipv4.address }}" diff --git a/tests/host_vars/mongo2.yml b/tests/host_vars/mongo2.yml index 4ca2aca..ed46ae0 100644 --- a/tests/host_vars/mongo2.yml +++ b/tests/host_vars/mongo2.yml @@ -1,4 +1,4 @@ --- mongodb_replication_params: - - { host_name: "{{ hostvars[inventory_hostname].ansible_default_ipv4.address }}" } + - host_name: "{{ hostvars[inventory_hostname].ansible_default_ipv4.address }}" diff --git a/tests/site.yml b/tests/site.yml index 9a27506..abd13ab 100644 --- a/tests/site.yml +++ b/tests/site.yml @@ -87,11 +87,3 @@ gather_facts: yes roles: - role: greendayonfire.mongodb - when: "'mongo_master' in group_names" - -- hosts: "{{ target | default('mongo') }}" - become: no - gather_facts: yes - roles: - - role: greendayonfire.mongodb - when: "'mongo_replicas' in group_names"