Changes in mongodb_replication module

library/mongodb_replication.py

@@ -106,6 +106,7 @@ try:
     from pymongo.errors import ConnectionFailure
     from pymongo.errors import OperationFailure
     from pymongo.errors import ConfigurationError
+    from pymongo.errors import AutoReconnect
     from pymongo import version as PyMongoVersion
     from pymongo import MongoClient
     from pymongo import MongoReplicaSetClient
@@ -145,7 +146,9 @@ def check_members(state, module, client, host_name, host_port, host_type):
                 if "{0}:{1}".format(host_name, host_port) not in member['host'] and member['arbiterOnly']:
                     module.exit_json(changed=False, host_name=host_name, host_port=host_port, host_type=host_type)
 
-def add_host(module, client, host_name, host_port, host_type):
+def add_host(module, client, host_name, host_port, host_type, timeout=180):
+    while True:
+        try:
             admin_db = client['admin']
             local_db = client['local']
 
@@ -164,8 +167,16 @@ def add_host(module, client, host_name, host_port, host_type):
 
             cfg['members'].append(new_host)
             admin_db.command('replSetReconfig', cfg)
+            return
+        except (OperationFailure, AutoReconnect) as e:
+            timeout = timeout - 5
+            if timeout <= 0:
+                module.fail_json(msg='reached timeout while waiting for rs.reconfig(): %s' % str(e))
+            time.sleep(5)
 
-def remove_host(module, client, host_name):
+def remove_host(module, client, host_name, timeout=180):
+    while True:
+        try:
             admin_db = client['admin']
             local_db = client['local']
 
@@ -186,7 +197,11 @@ def remove_host(module, client, host_name):
                 else:
                     fail_msg = "couldn't find member with hostname: {0} in replica set members list".format(host_name)
                     module.fail_json(msg=fail_msg)
-    admin_db.command('replSetReconfig', cfg)
+        except (OperationFailure, AutoReconnect) as e:
+            timeout = timeout - 5
+            if timeout <= 0:
+                module.fail_json(msg='reached timeout while waiting for rs.reconfig(): %s' % str(e))
+            time.sleep(5)
 
 def load_mongocnf():
     config = ConfigParser.RawConfigParser()
@@ -203,6 +218,30 @@ def load_mongocnf():
 
     return creds
 
+def wait_for_ok_and_master(module, client, timeout = 60):
+    while True:
+        status = client.admin.command('replSetGetStatus', check=False)
+        if status['ok'] == 1 and status['myState'] == 1:
+            return
+
+        timeout = timeout - 1
+        if timeout == 0:
+            module.fail_json(msg='reached timeout while waiting for rs.status() to become ok=1')
+
+        time.sleep(1)
+
+def reconfigure(module,client, rs_config, timeout = 180):
+    while True:
+        try:
+            client['admin'].command('replSetReconfig', rs_config)
+            return
+        except (OperationFailure, AutoReconnect) as e:
+            print e
+            timeout = timeout - 5
+            if timeout <= 0:
+                module.fail_json(msg='reached timeout while waiting for rs.reconfig()')
+            time.sleep(5)
+
 def authenticate(client, login_user, login_password):
     if login_user is None and login_password is None:
         mongocnf_creds = load_mongocnf()
@@ -267,7 +306,7 @@ def main():
             if state == 'present':
                 config = { '_id': "{0}".format(replica_set), 'members': [{ '_id': 0, 'host': "{0}:{1}".format(host_name, host_port)}] }
                 client['admin'].command('replSetInitiate', config)
-                time.sleep(3)
+                wait_for_ok_and_master(module, client)
                 replica_set_created = True
                 module.exit_json(changed=True, host_name=host_name, host_port=host_port, host_type=host_type)
         except OperationFailure, e: