Merge pull request #198 from sportsru/fix_handlers_and_systemd_unit

Fix handlers and systemd unit, speed up tests

.gitignore

@@ -3,3 +3,4 @@ tests/*.retry
 \#*\#
 .\#*
 /.python-version
+*.pyc

.travis.yml

@@ -4,7 +4,7 @@ dist: trusty
 sudo: required
 language: python
 python:
-  - "2.7"
+  - "2.7.15"
 env:
   - >
     DISTRIBUTION=ubuntu
@@ -46,17 +46,17 @@ env:
     DOCKER_PRIVILEGED=true
   - >
     DISTRIBUTION=ubuntu-upstart
-    DIST_VERSION=14.04
+    DIST_VERSION=14_04-builded
     MONGODB_VERSION=4.0
     DOCKER_PRIVILEGED=true
   - >
     DISTRIBUTION=ubuntu-upstart
-    DIST_VERSION=14.04
+    DIST_VERSION=14_04-builded
     MONGODB_VERSION=3.6
     DOCKER_PRIVILEGED=true
   - >
     DISTRIBUTION=ubuntu-upstart
-    DIST_VERSION=14.04
+    DIST_VERSION=14_04-builded
     MONGODB_VERSION=3.4
     DOCKER_PRIVILEGED=true
   - >

defaults/main.yml

@@ -16,6 +16,8 @@ mongodb_pymongo_pip_version: 3.7.1
 mongodb_user_update_password: "on_create"        # MongoDB user password update default policy
 mongodb_manage_service: true
 mongodb_manage_systemd_unit: true
+mongodb_systemd_unit_limit_nofile: 64000
+mongodb_systemd_unit_limit_nproc: 64000
 
 mongodb_disable_transparent_hugepages: false
 

handlers/main.yml

@@ -3,9 +3,6 @@
 - name: reload systemd
   systemd:
     daemon_reload: yes
-  when:
-    - ansible_service_mgr == "systemd"
-    - mongodb_manage_service | bool
 
 - name: run disable-transparent-hugepages
   systemd:
@@ -13,13 +10,6 @@
     enabled: true
     state: started
     daemon_reload: yes
-  when: ansible_service_mgr == "systemd" and mongodb_disable_transparent_hugepages | bool
-
-- name: mongodb reload
-  service:
-    name: "{{ mongodb_daemon_name }}"
-    state: reloaded
-  when: mongodb_manage_service | bool
 
 - name: mongodb restart
   service:
@@ -38,11 +28,6 @@
     name: sysfsutils
     state: restarted
 
-- name: service started
-  service:
-    name: "{{ mongodb_daemon_name }}"
-    state: started
-
 - name: wait when mongodb is started
   wait_for:
     host: "{{ item }}"
@@ -56,3 +41,4 @@
     port: "{{ mongodb_net_port }}"
     delay: 5
     timeout: 120
+

tasks/configure.yml

@@ -81,7 +81,6 @@
     mode: 0644
   notify:
     - mongodb restart
-    - service started
     - wait when mongodb is started
 
 - name: Flush all handlers at this point

tasks/disable_transparent_hugepages.yml

@@ -10,4 +10,7 @@
     - "disable-transparent-hugepages.{{ ansible_distribution | lower }}.service.j2"
     - "disable-transparent-hugepages.{{ ansible_os_family | lower }}.service.j2"
   notify: "run disable-transparent-hugepages"
+  when:
+    - ansible_service_mgr == "systemd"
+    - mongodb_disable_transparent_hugepages | bool
 

tasks/install.debian.yml

@@ -59,8 +59,8 @@
     state: present
 
 - name: Add systemd configuration if present
-  copy:
-    src: mongodb.service
+  template:
+    src: mongodb.service.j2
     dest: "/lib/systemd/system/{{mongodb_daemon_name}}.service"
     owner: root
     group: root

tasks/main.yml

@@ -124,6 +124,12 @@
   no_log: false
   tags: [mongodb]
 
+- name: service started
+  service:
+    name: "{{ mongodb_daemon_name }}"
+    state: started
+    enabled: yes
+
 - name: Include MMS Agent configuration
   include: mms-agent.yml
   when: mongodb_mms_api_key | length > 0

tasks/mms-agent.yml

@@ -19,9 +19,14 @@
     mode: 0755
 
 - name: Configure the MMS agent pt. 2
-  template: src=monitoring-agent.config.j2 dest=/etc/mongodb-mms/monitoring-agent.config
+  template:
+   src: monitoring-agent.config.j2
+   dest: /etc/mongodb-mms/monitoring-agent.config
   notify: mongodb-mms-monitoring-agent restart
 
 - name: Ensure that the MMS agent is started
-  service: name=mongodb-mms-monitoring-agent state=started enabled=yes
-  when: mongodb_manage_service
+  service:
+    name: mongodb-mms-monitoring-agent
+    state: started
+    enabled: yes
+

tasks/replication.yml

@@ -19,7 +19,7 @@
   with_items:
     - "{{ mongodb_replication_params|default([]) }}"
 
-- name: Replication configuration without auth
+- name: Replication configuration without auth on master
   mongodb_replication:
     build_indexes: "{{ item.build_indexes|default(omit) }}"
     login_host: "{{ mongodb_login_host|default('localhost') }}"
@@ -32,6 +32,26 @@
     priority: "{{ item.priority|default(1.0) }}"
     votes: "{{ item.votes|default(omit) }}"
     ssl: "{{ True if mongodb_net_ssl_mode == 'requireSSL' else False }}"
-  when: mongodb_security_authorization == 'disabled'
+  when: ( mongodb_security_authorization == 'disabled'
+        and mongodb_master is defined and mongodb_master )
   with_items:
     - "{{ mongodb_replication_params|default([]) }}"
+
+- name: Replication configuration without auth on replicas and arbiters
+  mongodb_replication:
+    build_indexes: "{{ item.build_indexes|default(omit) }}"
+    login_host: "{{ mongodb_login_host|default('localhost') }}"
+    login_port: "{{ mongodb_login_port|default(27017) }}"
+    replica_set: "{{ mongodb_replication_replset }}"
+    host_name: "{{ item.host_name }}"
+    host_port: "{{ item.host_port|default(27017) }}"
+    host_type: "{{ item.host_type|default('replica') }}"
+    hidden: "{{ item.hidden|default(false) }}"
+    priority: "{{ item.priority|default(1.0) }}"
+    votes: "{{ item.votes|default(omit) }}"
+    ssl: "{{ True if mongodb_net_ssl_mode == 'requireSSL' else False }}"
+  when: ( mongodb_security_authorization == 'disabled'
+        and ( mongodb_master is not defined or not mongodb_master) )
+  with_items:
+    - "{{ mongodb_replication_params|default([]) }}"
+

templates/mongodb.service.j2

@@ -1,9 +1,10 @@
+# {{ ansible_managed }}
 [Unit]
 Description=An object/document-oriented database
 Documentation=man:mongod(1)
 
 [Service]
-User=mongodb
+User={{ mongodb_user }}
 ExecStart=/usr/bin/numactl --interleave=all /usr/bin/mongod --config /etc/mongod.conf
 # file size
 LimitFSIZE=infinity
@@ -12,9 +13,9 @@ LimitCPU=infinity
 # virtual memory size
 LimitAS=infinity
 # open files
-LimitNOFILE=64000
+LimitNOFILE={{ mongodb_systemd_unit_limit_nofile }}
 # processes/threads
-LimitNPROC=64000
+LimitNPROC={{ mongodb_systemd_unit_limit_nproc }}
 # locked memory
 LimitMEMLOCK=infinity
 # total threads (user+kernel)

tests/Dockerfile.amazonlinux_2-builded

@@ -1,14 +1,12 @@
 FROM amazonlinux:2
 
 # This is needed so that ansible managed to read "ansible_default_ipv4"
-RUN yum install iproute -y
+# This step is needed since standard CentOS docker image does not come with i
+# This package seems to be required for Mongo 3.2 and downwards
+RUN yum install iproute initscripts python-pip python-devel -y
 
 # we can has SSH
 EXPOSE 22
 
-# This step is needed since standard CentOS docker image does not come with i
-# This package seems to be required for Mongo 3.2 and downwards
-RUN yum install initscripts -y
-
 # pepare for takeoff
 CMD ["/usr/sbin/init"]

tests/Dockerfile.centos_6-builded

@@ -1,10 +1,8 @@
 FROM centos:6
 
 # This is needed so that ansible managed to read "ansible_default_ipv4"
-RUN yum install iproute -y 
-
 # This step is needed since standard CentOS docker image does not come with EPEL installed by default
-RUN yum install epel-release -y
+RUN yum install iproute epel-release python-pip python-devel -y 
 
 # we can has SSH
 EXPOSE 22

tests/Dockerfile.centos_7-builded

@@ -1,10 +1,8 @@
 FROM centos:7
 
 # This is needed so that ansible managed to read "ansible_default_ipv4"
-RUN yum install iproute -y 
-
 # This step is needed since standard CentOS docker image does not come with EPEL installed by default
-RUN yum install epel-release -y
+RUN yum install iproute epel-release python-pip python-devel -y 
 
 # we can has SSH
 EXPOSE 22

tests/Dockerfile.debian_8-builded

@@ -3,5 +3,5 @@ FROM debian:8
 ARG DEBIAN_FRONTEND=noninteractive
 
 RUN apt update && \
-    apt install --yes python-minimal && \
+    apt install --yes python-minimal python-pip && \
     rm /lib/systemd/system/getty@.service

tests/Dockerfile.debian_9-builded

@@ -3,7 +3,7 @@ FROM debian:9
 ARG DEBIAN_FRONTEND=noninteractive
 
 RUN apt update && \
-    apt install --yes python-minimal systemd gnupg
+    apt install --yes python-minimal systemd gnupg python-pip
 
 RUN cd /lib/systemd/system/sysinit.target.wants/ && \
 		ls | grep -v systemd-tmpfiles-setup.service | xargs rm -f && \

tests/Dockerfile.ubuntu-upstart_14_04-builded

@@ -0,0 +1,7 @@
+FROM ubuntu-upstart
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    apt-get install --yes python-minimal python-pip python-dev iproute2
+

tests/Dockerfile.ubuntu_16_04-builded

@@ -3,5 +3,6 @@ FROM ubuntu:16.04
 ARG DEBIAN_FRONTEND=noninteractive
 
 RUN apt update && \
-    apt install --yes python-minimal iproute2 && \
+    apt install --yes python-minimal python-pip python-dev iproute2 && \
     rm /lib/systemd/system/getty@.service
+

tests/Dockerfile.ubuntu_18_04-builded

@@ -3,7 +3,7 @@ FROM ubuntu:18.04
 ARG DEBIAN_FRONTEND=noninteractive
 
 RUN apt update && \
-    apt install --yes python-minimal systemd gnupg iproute2
+    apt install --yes python-minimal python-pip python-dev systemd gnupg iproute2
 
 RUN cd /lib/systemd/system/sysinit.target.wants/ && \
 		ls | grep -v systemd-tmpfiles-setup.service | xargs rm -f && \

tests/callback_plugins/profile_tasks.py

@@ -0,0 +1,74 @@
+# profile_tasks.py: an Ansible plugin for timing tasks
+
+# Copyright (C) 2014 Jharrod LaFon <jharrod.lafon@gmail.com>
+# https://github.com/jlafon/ansible-profile/
+# Included with permission
+
+
+# The MIT License (MIT)
+#
+# Copyright (c) 2014 Jharrod LaFon
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+from ansible.plugins.callback import CallbackBase
+import time
+
+
+class CallbackModule(CallbackBase):
+    """
+    A plugin for timing tasks
+    """
+    def __init__(self):
+        self.stats = {}
+        self.current = None
+
+    def playbook_on_task_start(self, name, is_conditional):
+        """
+        Logs the start of each task
+        """
+        if self.current is not None:
+            # Record the running time of the last executed task
+            self.stats[self.current] = time.time() - self.stats[self.current]
+
+        # Record the start time of the current task
+        self.current = name
+        self.stats[self.current] = time.time()
+
+    def playbook_on_stats(self, stats):
+        """
+        Prints the timings
+        """
+        # Record the timing of the very last task
+        if self.current is not None:
+            self.stats[self.current] = time.time() - self.stats[self.current]
+
+        # Sort the tasks by their running time
+        results = sorted(self.stats.items(),
+                         key=lambda value: value[1], reverse=True)
+
+        # Just keep the top 25
+        results = results[:25]
+
+        # Print the timings
+        for name, elapsed in results:
+            print("{0:-<70}{1:->9}".format(
+                    '{0} '.format(name),
+                    ' {0:.02f}s'.format(elapsed)))

tests/files/etc/apt/apt.conf.d/proxy.conf

@@ -0,0 +1,3 @@
+Acquire::http::Proxy "http://proxy:3128/";
+Acquire::https::Proxy "http://proxy:3128/";
+

tests/files/etc/environment

@@ -0,0 +1,2 @@
+PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games"
+HTTP_PROXY="http://proxy:3128"

tests/files/etc/squid/squid.conf

@@ -0,0 +1,34 @@
+acl localhost src 127.0.0.1/32 ::1
+acl localnet src 10.0.0.0/8 # RFC1918 possible internal network
+acl localnet src 172.16.0.0/12  # RFC1918 possible internal network
+acl localnet src 192.168.0.0/16 # RFC1918 possible internal network
+acl SSL_ports port 443
+acl Safe_ports port 80      # http
+acl Safe_ports port 21      # ftp
+acl Safe_ports port 443     # https
+acl Safe_ports port 70      # gopher
+acl Safe_ports port 210     # wais
+acl Safe_ports port 1025-65535  # unregistered ports
+acl Safe_ports port 280     # http-mgmt
+acl Safe_ports port 488     # gss-http
+acl Safe_ports port 591     # filemaker
+acl Safe_ports port 777     # multiling http
+acl CONNECT method CONNECT
+http_access allow manager localhost
+http_access deny manager
+http_access deny !Safe_ports
+http_access deny CONNECT !SSL_ports
+http_access allow localnet
+http_access allow localhost
+http_access deny all
+http_port 3128
+maximum_object_size 2048 MB
+cache_dir aufs /var/spool/squid 5000 24 256
+coredump_dir /var/spool/squid
+refresh_pattern ^ftp:       1440    20% 10080
+refresh_pattern ^gopher:    1440    0%  1440
+refresh_pattern -i (/cgi-bin/|\?) 0 0%  0
+refresh_pattern (Release|Packages(.gz)*)$      0       20%     2880
+refresh_pattern .       0   20% 4320
+refresh_all_ims on
+

tests/group_vars/all.yml

@@ -1,6 +1,6 @@
 ---
 
-image_name: "ubuntu-upstart:18.04"
+image_name: "ubuntu:18.04"
 mongodb_package: mongodb-org
 mongodb_version: "4.0"
 mongodb_storage_dbpath: /var/lib/mongodb

tests/host_vars/mongo1.yml

@@ -2,4 +2,4 @@
 
 mongodb_master: true
 mongodb_replication_params:
-  - { host_name: "{{ hostvars[inventory_hostname].ansible_default_ipv4.address }}" }
+  - host_name: "{{ hostvars[inventory_hostname].ansible_default_ipv4.address }}"

tests/host_vars/mongo2.yml

@@ -1,4 +1,4 @@
 ---
 
 mongodb_replication_params:
-  - { host_name: "{{ hostvars[inventory_hostname].ansible_default_ipv4.address }}" }
+  - host_name: "{{ hostvars[inventory_hostname].ansible_default_ipv4.address }}"

tests/scripts/before_install.sh

@@ -12,7 +12,14 @@ set -o errexit
 set -o pipefail
 
 # Latest Ansible install
-pip install docker ansible
+pip install docker ansible mitogen
+
+cat << EOF > ansible.cfg
+[defaults]
+pipelining = True
+strategy = mitogen_linear
+strategy_plugins = /home/travis/virtualenv/python2.7.15/lib/python2.7/site-packages/ansible_mitogen/plugins/strategy
+EOF
 
 # Pull docker image or build it
 if [ -f tests/Dockerfile.${DISTRIBUTION}_${DIST_VERSION} ]

tests/site.yml

@@ -5,6 +5,29 @@
   become: no
   gather_facts: no
   tasks:
+    - name: Copy environmnet file
+      copy:
+        src: etc/environment
+        dest: /tmp/environment
+    - name: Copy Squid config
+      copy:
+        src: etc/squid/squid.conf
+        dest: /tmp/squid.conf
+    - name: Create a network
+      docker_network:
+        name: MongoDBTestingNet
+    - name: Run Proxy in Docker
+      docker_container:
+        name: proxy
+        image: sameersbn/squid
+        state: started
+        volumes:
+          - /tmp/squid.conf:/etc/squid/squid.conf:ro
+        networks:
+          - name: MongoDBTestingNet
+            aliases:
+              - proxy
+        networks_cli_compatible: yes
     - name: Run MongoDB cluster in Docker
       docker_container:
         name: "{{ item }}"
@@ -23,11 +46,35 @@
             target: /run
           - type: tmpfs
             target: /run/lock
+        volumes:
+          - /tmp/environment:/etc/environment:ro
+        networks:
+          - name: MongoDBTestingNet
+            aliases:
+              - "{{ item }}"
+        networks_cli_compatible: yes
       with_items:
         - mongo1
         - mongo2
         - mongo3
 
+- hosts: mongo
+  become: no
+  gather_facts: yes
+  tasks:
+    - name: Set proxy for apt
+      copy:
+        src: etc/apt/apt.conf.d/proxy.conf
+        dest: /etc/apt/apt.conf.d/proxy.conf
+      when: ansible_os_family == "Debian"
+    - name: Set proxy for yum
+      ini_file:
+        path: /etc/yum.conf
+        section: main
+        option: proxy
+        value: http://proxy:3128
+      when: ansible_os_family == "RedHat"
+
 - hosts: mongo
   become: no
   gather_facts: no
@@ -40,11 +87,3 @@
   gather_facts: yes
   roles:
     - role: greendayonfire.mongodb
-      when: "'mongo_master' in group_names"
-
-- hosts: "{{ target | default('mongo') }}"
-  become: no
-  gather_facts: yes
-  roles:
-    - role: greendayonfire.mongodb
-      when: "'mongo_replicas' in group_names"