Merge pull request #198 from sportsru/fix_handlers_and_systemd_unit
Fix handlers and systemd unit, speed up tests
This commit is contained in:
commit
e3b6b7577f
|
@ -3,3 +3,4 @@ tests/*.retry
|
|||
\#*\#
|
||||
.\#*
|
||||
/.python-version
|
||||
*.pyc
|
||||
|
|
|
@ -4,7 +4,7 @@ dist: trusty
|
|||
sudo: required
|
||||
language: python
|
||||
python:
|
||||
- "2.7"
|
||||
- "2.7.15"
|
||||
env:
|
||||
- >
|
||||
DISTRIBUTION=ubuntu
|
||||
|
@ -46,17 +46,17 @@ env:
|
|||
DOCKER_PRIVILEGED=true
|
||||
- >
|
||||
DISTRIBUTION=ubuntu-upstart
|
||||
DIST_VERSION=14.04
|
||||
DIST_VERSION=14_04-builded
|
||||
MONGODB_VERSION=4.0
|
||||
DOCKER_PRIVILEGED=true
|
||||
- >
|
||||
DISTRIBUTION=ubuntu-upstart
|
||||
DIST_VERSION=14.04
|
||||
DIST_VERSION=14_04-builded
|
||||
MONGODB_VERSION=3.6
|
||||
DOCKER_PRIVILEGED=true
|
||||
- >
|
||||
DISTRIBUTION=ubuntu-upstart
|
||||
DIST_VERSION=14.04
|
||||
DIST_VERSION=14_04-builded
|
||||
MONGODB_VERSION=3.4
|
||||
DOCKER_PRIVILEGED=true
|
||||
- >
|
||||
|
|
|
@ -16,6 +16,8 @@ mongodb_pymongo_pip_version: 3.7.1
|
|||
mongodb_user_update_password: "on_create" # MongoDB user password update default policy
|
||||
mongodb_manage_service: true
|
||||
mongodb_manage_systemd_unit: true
|
||||
mongodb_systemd_unit_limit_nofile: 64000
|
||||
mongodb_systemd_unit_limit_nproc: 64000
|
||||
|
||||
mongodb_disable_transparent_hugepages: false
|
||||
|
||||
|
|
|
@ -3,9 +3,6 @@
|
|||
- name: reload systemd
|
||||
systemd:
|
||||
daemon_reload: yes
|
||||
when:
|
||||
- ansible_service_mgr == "systemd"
|
||||
- mongodb_manage_service | bool
|
||||
|
||||
- name: run disable-transparent-hugepages
|
||||
systemd:
|
||||
|
@ -13,13 +10,6 @@
|
|||
enabled: true
|
||||
state: started
|
||||
daemon_reload: yes
|
||||
when: ansible_service_mgr == "systemd" and mongodb_disable_transparent_hugepages | bool
|
||||
|
||||
- name: mongodb reload
|
||||
service:
|
||||
name: "{{ mongodb_daemon_name }}"
|
||||
state: reloaded
|
||||
when: mongodb_manage_service | bool
|
||||
|
||||
- name: mongodb restart
|
||||
service:
|
||||
|
@ -38,11 +28,6 @@
|
|||
name: sysfsutils
|
||||
state: restarted
|
||||
|
||||
- name: service started
|
||||
service:
|
||||
name: "{{ mongodb_daemon_name }}"
|
||||
state: started
|
||||
|
||||
- name: wait when mongodb is started
|
||||
wait_for:
|
||||
host: "{{ item }}"
|
||||
|
@ -56,3 +41,4 @@
|
|||
port: "{{ mongodb_net_port }}"
|
||||
delay: 5
|
||||
timeout: 120
|
||||
|
||||
|
|
|
@ -81,7 +81,6 @@
|
|||
mode: 0644
|
||||
notify:
|
||||
- mongodb restart
|
||||
- service started
|
||||
- wait when mongodb is started
|
||||
|
||||
- name: Flush all handlers at this point
|
||||
|
|
|
@ -10,4 +10,7 @@
|
|||
- "disable-transparent-hugepages.{{ ansible_distribution | lower }}.service.j2"
|
||||
- "disable-transparent-hugepages.{{ ansible_os_family | lower }}.service.j2"
|
||||
notify: "run disable-transparent-hugepages"
|
||||
when:
|
||||
- ansible_service_mgr == "systemd"
|
||||
- mongodb_disable_transparent_hugepages | bool
|
||||
|
||||
|
|
|
@ -59,8 +59,8 @@
|
|||
state: present
|
||||
|
||||
- name: Add systemd configuration if present
|
||||
copy:
|
||||
src: mongodb.service
|
||||
template:
|
||||
src: mongodb.service.j2
|
||||
dest: "/lib/systemd/system/{{mongodb_daemon_name}}.service"
|
||||
owner: root
|
||||
group: root
|
||||
|
|
|
@ -124,6 +124,12 @@
|
|||
no_log: false
|
||||
tags: [mongodb]
|
||||
|
||||
- name: service started
|
||||
service:
|
||||
name: "{{ mongodb_daemon_name }}"
|
||||
state: started
|
||||
enabled: yes
|
||||
|
||||
- name: Include MMS Agent configuration
|
||||
include: mms-agent.yml
|
||||
when: mongodb_mms_api_key | length > 0
|
||||
|
|
|
@ -19,9 +19,14 @@
|
|||
mode: 0755
|
||||
|
||||
- name: Configure the MMS agent pt. 2
|
||||
template: src=monitoring-agent.config.j2 dest=/etc/mongodb-mms/monitoring-agent.config
|
||||
template:
|
||||
src: monitoring-agent.config.j2
|
||||
dest: /etc/mongodb-mms/monitoring-agent.config
|
||||
notify: mongodb-mms-monitoring-agent restart
|
||||
|
||||
- name: Ensure that the MMS agent is started
|
||||
service: name=mongodb-mms-monitoring-agent state=started enabled=yes
|
||||
when: mongodb_manage_service
|
||||
service:
|
||||
name: mongodb-mms-monitoring-agent
|
||||
state: started
|
||||
enabled: yes
|
||||
|
||||
|
|
|
@ -19,7 +19,7 @@
|
|||
with_items:
|
||||
- "{{ mongodb_replication_params|default([]) }}"
|
||||
|
||||
- name: Replication configuration without auth
|
||||
- name: Replication configuration without auth on master
|
||||
mongodb_replication:
|
||||
build_indexes: "{{ item.build_indexes|default(omit) }}"
|
||||
login_host: "{{ mongodb_login_host|default('localhost') }}"
|
||||
|
@ -32,6 +32,26 @@
|
|||
priority: "{{ item.priority|default(1.0) }}"
|
||||
votes: "{{ item.votes|default(omit) }}"
|
||||
ssl: "{{ True if mongodb_net_ssl_mode == 'requireSSL' else False }}"
|
||||
when: mongodb_security_authorization == 'disabled'
|
||||
when: ( mongodb_security_authorization == 'disabled'
|
||||
and mongodb_master is defined and mongodb_master )
|
||||
with_items:
|
||||
- "{{ mongodb_replication_params|default([]) }}"
|
||||
|
||||
- name: Replication configuration without auth on replicas and arbiters
|
||||
mongodb_replication:
|
||||
build_indexes: "{{ item.build_indexes|default(omit) }}"
|
||||
login_host: "{{ mongodb_login_host|default('localhost') }}"
|
||||
login_port: "{{ mongodb_login_port|default(27017) }}"
|
||||
replica_set: "{{ mongodb_replication_replset }}"
|
||||
host_name: "{{ item.host_name }}"
|
||||
host_port: "{{ item.host_port|default(27017) }}"
|
||||
host_type: "{{ item.host_type|default('replica') }}"
|
||||
hidden: "{{ item.hidden|default(false) }}"
|
||||
priority: "{{ item.priority|default(1.0) }}"
|
||||
votes: "{{ item.votes|default(omit) }}"
|
||||
ssl: "{{ True if mongodb_net_ssl_mode == 'requireSSL' else False }}"
|
||||
when: ( mongodb_security_authorization == 'disabled'
|
||||
and ( mongodb_master is not defined or not mongodb_master) )
|
||||
with_items:
|
||||
- "{{ mongodb_replication_params|default([]) }}"
|
||||
|
||||
|
|
|
@ -1,9 +1,10 @@
|
|||
# {{ ansible_managed }}
|
||||
[Unit]
|
||||
Description=An object/document-oriented database
|
||||
Documentation=man:mongod(1)
|
||||
|
||||
[Service]
|
||||
User=mongodb
|
||||
User={{ mongodb_user }}
|
||||
ExecStart=/usr/bin/numactl --interleave=all /usr/bin/mongod --config /etc/mongod.conf
|
||||
# file size
|
||||
LimitFSIZE=infinity
|
||||
|
@ -12,9 +13,9 @@ LimitCPU=infinity
|
|||
# virtual memory size
|
||||
LimitAS=infinity
|
||||
# open files
|
||||
LimitNOFILE=64000
|
||||
LimitNOFILE={{ mongodb_systemd_unit_limit_nofile }}
|
||||
# processes/threads
|
||||
LimitNPROC=64000
|
||||
LimitNPROC={{ mongodb_systemd_unit_limit_nproc }}
|
||||
# locked memory
|
||||
LimitMEMLOCK=infinity
|
||||
# total threads (user+kernel)
|
|
@ -1,14 +1,12 @@
|
|||
FROM amazonlinux:2
|
||||
|
||||
# This is needed so that ansible managed to read "ansible_default_ipv4"
|
||||
RUN yum install iproute -y
|
||||
# This step is needed since standard CentOS docker image does not come with i
|
||||
# This package seems to be required for Mongo 3.2 and downwards
|
||||
RUN yum install iproute initscripts python-pip python-devel -y
|
||||
|
||||
# we can has SSH
|
||||
EXPOSE 22
|
||||
|
||||
# This step is needed since standard CentOS docker image does not come with i
|
||||
# This package seems to be required for Mongo 3.2 and downwards
|
||||
RUN yum install initscripts -y
|
||||
|
||||
# pepare for takeoff
|
||||
CMD ["/usr/sbin/init"]
|
||||
|
|
|
@ -1,10 +1,8 @@
|
|||
FROM centos:6
|
||||
|
||||
# This is needed so that ansible managed to read "ansible_default_ipv4"
|
||||
RUN yum install iproute -y
|
||||
|
||||
# This step is needed since standard CentOS docker image does not come with EPEL installed by default
|
||||
RUN yum install epel-release -y
|
||||
RUN yum install iproute epel-release python-pip python-devel -y
|
||||
|
||||
# we can has SSH
|
||||
EXPOSE 22
|
||||
|
|
|
@ -1,10 +1,8 @@
|
|||
FROM centos:7
|
||||
|
||||
# This is needed so that ansible managed to read "ansible_default_ipv4"
|
||||
RUN yum install iproute -y
|
||||
|
||||
# This step is needed since standard CentOS docker image does not come with EPEL installed by default
|
||||
RUN yum install epel-release -y
|
||||
RUN yum install iproute epel-release python-pip python-devel -y
|
||||
|
||||
# we can has SSH
|
||||
EXPOSE 22
|
||||
|
|
|
@ -3,5 +3,5 @@ FROM debian:8
|
|||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt update && \
|
||||
apt install --yes python-minimal && \
|
||||
apt install --yes python-minimal python-pip && \
|
||||
rm /lib/systemd/system/getty@.service
|
||||
|
|
|
@ -3,7 +3,7 @@ FROM debian:9
|
|||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt update && \
|
||||
apt install --yes python-minimal systemd gnupg
|
||||
apt install --yes python-minimal systemd gnupg python-pip
|
||||
|
||||
RUN cd /lib/systemd/system/sysinit.target.wants/ && \
|
||||
ls | grep -v systemd-tmpfiles-setup.service | xargs rm -f && \
|
||||
|
|
|
@ -0,0 +1,7 @@
|
|||
FROM ubuntu-upstart
|
||||
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install --yes python-minimal python-pip python-dev iproute2
|
||||
|
|
@ -3,5 +3,6 @@ FROM ubuntu:16.04
|
|||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt update && \
|
||||
apt install --yes python-minimal iproute2 && \
|
||||
apt install --yes python-minimal python-pip python-dev iproute2 && \
|
||||
rm /lib/systemd/system/getty@.service
|
||||
|
||||
|
|
|
@ -3,7 +3,7 @@ FROM ubuntu:18.04
|
|||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt update && \
|
||||
apt install --yes python-minimal systemd gnupg iproute2
|
||||
apt install --yes python-minimal python-pip python-dev systemd gnupg iproute2
|
||||
|
||||
RUN cd /lib/systemd/system/sysinit.target.wants/ && \
|
||||
ls | grep -v systemd-tmpfiles-setup.service | xargs rm -f && \
|
||||
|
|
|
@ -0,0 +1,74 @@
|
|||
# profile_tasks.py: an Ansible plugin for timing tasks
|
||||
|
||||
# Copyright (C) 2014 Jharrod LaFon <jharrod.lafon@gmail.com>
|
||||
# https://github.com/jlafon/ansible-profile/
|
||||
# Included with permission
|
||||
|
||||
|
||||
# The MIT License (MIT)
|
||||
#
|
||||
# Copyright (c) 2014 Jharrod LaFon
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
|
||||
from ansible.plugins.callback import CallbackBase
|
||||
import time
|
||||
|
||||
|
||||
class CallbackModule(CallbackBase):
|
||||
"""
|
||||
A plugin for timing tasks
|
||||
"""
|
||||
def __init__(self):
|
||||
self.stats = {}
|
||||
self.current = None
|
||||
|
||||
def playbook_on_task_start(self, name, is_conditional):
|
||||
"""
|
||||
Logs the start of each task
|
||||
"""
|
||||
if self.current is not None:
|
||||
# Record the running time of the last executed task
|
||||
self.stats[self.current] = time.time() - self.stats[self.current]
|
||||
|
||||
# Record the start time of the current task
|
||||
self.current = name
|
||||
self.stats[self.current] = time.time()
|
||||
|
||||
def playbook_on_stats(self, stats):
|
||||
"""
|
||||
Prints the timings
|
||||
"""
|
||||
# Record the timing of the very last task
|
||||
if self.current is not None:
|
||||
self.stats[self.current] = time.time() - self.stats[self.current]
|
||||
|
||||
# Sort the tasks by their running time
|
||||
results = sorted(self.stats.items(),
|
||||
key=lambda value: value[1], reverse=True)
|
||||
|
||||
# Just keep the top 25
|
||||
results = results[:25]
|
||||
|
||||
# Print the timings
|
||||
for name, elapsed in results:
|
||||
print("{0:-<70}{1:->9}".format(
|
||||
'{0} '.format(name),
|
||||
' {0:.02f}s'.format(elapsed)))
|
|
@ -0,0 +1,3 @@
|
|||
Acquire::http::Proxy "http://proxy:3128/";
|
||||
Acquire::https::Proxy "http://proxy:3128/";
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games"
|
||||
HTTP_PROXY="http://proxy:3128"
|
|
@ -0,0 +1,34 @@
|
|||
acl localhost src 127.0.0.1/32 ::1
|
||||
acl localnet src 10.0.0.0/8 # RFC1918 possible internal network
|
||||
acl localnet src 172.16.0.0/12 # RFC1918 possible internal network
|
||||
acl localnet src 192.168.0.0/16 # RFC1918 possible internal network
|
||||
acl SSL_ports port 443
|
||||
acl Safe_ports port 80 # http
|
||||
acl Safe_ports port 21 # ftp
|
||||
acl Safe_ports port 443 # https
|
||||
acl Safe_ports port 70 # gopher
|
||||
acl Safe_ports port 210 # wais
|
||||
acl Safe_ports port 1025-65535 # unregistered ports
|
||||
acl Safe_ports port 280 # http-mgmt
|
||||
acl Safe_ports port 488 # gss-http
|
||||
acl Safe_ports port 591 # filemaker
|
||||
acl Safe_ports port 777 # multiling http
|
||||
acl CONNECT method CONNECT
|
||||
http_access allow manager localhost
|
||||
http_access deny manager
|
||||
http_access deny !Safe_ports
|
||||
http_access deny CONNECT !SSL_ports
|
||||
http_access allow localnet
|
||||
http_access allow localhost
|
||||
http_access deny all
|
||||
http_port 3128
|
||||
maximum_object_size 2048 MB
|
||||
cache_dir aufs /var/spool/squid 5000 24 256
|
||||
coredump_dir /var/spool/squid
|
||||
refresh_pattern ^ftp: 1440 20% 10080
|
||||
refresh_pattern ^gopher: 1440 0% 1440
|
||||
refresh_pattern -i (/cgi-bin/|\?) 0 0% 0
|
||||
refresh_pattern (Release|Packages(.gz)*)$ 0 20% 2880
|
||||
refresh_pattern . 0 20% 4320
|
||||
refresh_all_ims on
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
---
|
||||
|
||||
image_name: "ubuntu-upstart:18.04"
|
||||
image_name: "ubuntu:18.04"
|
||||
mongodb_package: mongodb-org
|
||||
mongodb_version: "4.0"
|
||||
mongodb_storage_dbpath: /var/lib/mongodb
|
||||
|
|
|
@ -2,4 +2,4 @@
|
|||
|
||||
mongodb_master: true
|
||||
mongodb_replication_params:
|
||||
- { host_name: "{{ hostvars[inventory_hostname].ansible_default_ipv4.address }}" }
|
||||
- host_name: "{{ hostvars[inventory_hostname].ansible_default_ipv4.address }}"
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
---
|
||||
|
||||
mongodb_replication_params:
|
||||
- { host_name: "{{ hostvars[inventory_hostname].ansible_default_ipv4.address }}" }
|
||||
- host_name: "{{ hostvars[inventory_hostname].ansible_default_ipv4.address }}"
|
||||
|
|
|
@ -12,7 +12,14 @@ set -o errexit
|
|||
set -o pipefail
|
||||
|
||||
# Latest Ansible install
|
||||
pip install docker ansible
|
||||
pip install docker ansible mitogen
|
||||
|
||||
cat << EOF > ansible.cfg
|
||||
[defaults]
|
||||
pipelining = True
|
||||
strategy = mitogen_linear
|
||||
strategy_plugins = /home/travis/virtualenv/python2.7.15/lib/python2.7/site-packages/ansible_mitogen/plugins/strategy
|
||||
EOF
|
||||
|
||||
# Pull docker image or build it
|
||||
if [ -f tests/Dockerfile.${DISTRIBUTION}_${DIST_VERSION} ]
|
||||
|
|
|
@ -5,6 +5,29 @@
|
|||
become: no
|
||||
gather_facts: no
|
||||
tasks:
|
||||
- name: Copy environmnet file
|
||||
copy:
|
||||
src: etc/environment
|
||||
dest: /tmp/environment
|
||||
- name: Copy Squid config
|
||||
copy:
|
||||
src: etc/squid/squid.conf
|
||||
dest: /tmp/squid.conf
|
||||
- name: Create a network
|
||||
docker_network:
|
||||
name: MongoDBTestingNet
|
||||
- name: Run Proxy in Docker
|
||||
docker_container:
|
||||
name: proxy
|
||||
image: sameersbn/squid
|
||||
state: started
|
||||
volumes:
|
||||
- /tmp/squid.conf:/etc/squid/squid.conf:ro
|
||||
networks:
|
||||
- name: MongoDBTestingNet
|
||||
aliases:
|
||||
- proxy
|
||||
networks_cli_compatible: yes
|
||||
- name: Run MongoDB cluster in Docker
|
||||
docker_container:
|
||||
name: "{{ item }}"
|
||||
|
@ -23,11 +46,35 @@
|
|||
target: /run
|
||||
- type: tmpfs
|
||||
target: /run/lock
|
||||
volumes:
|
||||
- /tmp/environment:/etc/environment:ro
|
||||
networks:
|
||||
- name: MongoDBTestingNet
|
||||
aliases:
|
||||
- "{{ item }}"
|
||||
networks_cli_compatible: yes
|
||||
with_items:
|
||||
- mongo1
|
||||
- mongo2
|
||||
- mongo3
|
||||
|
||||
- hosts: mongo
|
||||
become: no
|
||||
gather_facts: yes
|
||||
tasks:
|
||||
- name: Set proxy for apt
|
||||
copy:
|
||||
src: etc/apt/apt.conf.d/proxy.conf
|
||||
dest: /etc/apt/apt.conf.d/proxy.conf
|
||||
when: ansible_os_family == "Debian"
|
||||
- name: Set proxy for yum
|
||||
ini_file:
|
||||
path: /etc/yum.conf
|
||||
section: main
|
||||
option: proxy
|
||||
value: http://proxy:3128
|
||||
when: ansible_os_family == "RedHat"
|
||||
|
||||
- hosts: mongo
|
||||
become: no
|
||||
gather_facts: no
|
||||
|
@ -40,11 +87,3 @@
|
|||
gather_facts: yes
|
||||
roles:
|
||||
- role: greendayonfire.mongodb
|
||||
when: "'mongo_master' in group_names"
|
||||
|
||||
- hosts: "{{ target | default('mongo') }}"
|
||||
become: no
|
||||
gather_facts: yes
|
||||
roles:
|
||||
- role: greendayonfire.mongodb
|
||||
when: "'mongo_replicas' in group_names"
|
||||
|
|
Loading…
Reference in New Issue