Enabling connetion pool_size definition

requests has a default connections pool of 10. Creating multiple threads
will consume from that same pool, since the underlying implementation of
the requests pool is a singletown.

Let's make the pool_size configurable, so clients can set the proper
number for their use case when running multiple threads.

Signed-off-by: Amador Pahim <apahim@redhat.com>

github/Requester.py

@@ -82,7 +82,14 @@ class RequestsResponse:
 class HTTPSRequestsConnectionClass:
     # mimic the httplib connection object
     def __init__(
-        self, host, port=None, strict=False, timeout=None, retry=None, **kwargs
+        self,
+        host,
+        port=None,
+        strict=False,
+        timeout=None,
+        retry=None,
+        pool_size=None,
+        **kwargs,
     ):
         self.port = port if port else 443
         self.host = host
@@ -95,8 +102,16 @@ class HTTPSRequestsConnectionClass:
             self.retry = requests.adapters.DEFAULT_RETRIES
         else:
             self.retry = retry
+
+        if pool_size is None:
+            self.pool_size = requests.adapters.DEFAULT_POOLSIZE
+        else:
+            self.pool_size = pool_size
+
         self.adapter = requests.adapters.HTTPAdapter(
             max_retries=self.retry,
+            pool_connections=self.pool_size,
+            pool_maxsize=self.pool_size,
         )
         self.session.mount("https://", self.adapter)
 
@@ -126,7 +141,14 @@ class HTTPSRequestsConnectionClass:
 class HTTPRequestsConnectionClass:
     # mimic the httplib connection object
     def __init__(
-        self, host, port=None, strict=False, timeout=None, retry=None, **kwargs
+        self,
+        host,
+        port=None,
+        strict=False,
+        timeout=None,
+        retry=None,
+        pool_size=None,
+        **kwargs,
     ):
         self.port = port if port else 80
         self.host = host
@@ -139,8 +161,16 @@ class HTTPRequestsConnectionClass:
             self.retry = requests.adapters.DEFAULT_RETRIES
         else:
             self.retry = retry
+
+        if pool_size is None:
+            self.pool_size = requests.adapters.DEFAULT_POOLSIZE
+        else:
+            self.pool_size = pool_size
+
         self.adapter = requests.adapters.HTTPAdapter(
             max_retries=self.retry,
+            pool_connections=self.pool_size,
+            pool_maxsize=self.pool_size,
         )
         self.session.mount("http://", self.adapter)
 
@@ -272,6 +302,7 @@ class Requester:
         per_page,
         verify,
         retry,
+        pool_size,
     ):
         self._initializeDebugFeature()
 
@@ -295,6 +326,7 @@ class Requester:
         self.__prefix = o.path
         self.__timeout = timeout
         self.__retry = retry  # NOTE: retry can be either int or an urllib3 Retry object
+        self.__pool_size = pool_size
         self.__scheme = o.scheme
         if o.scheme == "https":
             self.__connectionClass = self.__httpsConnectionClass
@@ -362,11 +394,17 @@ class Requester:
             ):  # issue80
                 if o.scheme == "http":
                     cnx = self.__httpConnectionClass(
-                        o.hostname, o.port, retry=self.__retry
+                        o.hostname,
+                        o.port,
+                        retry=self.__retry,
+                        pool_size=self.__pool_size,
                     )
                 elif o.scheme == "https":
                     cnx = self.__httpsConnectionClass(
-                        o.hostname, o.port, retry=self.__retry
+                        o.hostname,
+                        o.port,
+                        retry=self.__retry,
+                        pool_size=self.__pool_size,
                     )
         return cnx
 
@@ -583,7 +621,11 @@ class Requester:
             return self.__connection
 
         self.__connection = self.__connectionClass(
-            self.__hostname, self.__port, retry=self.__retry, **kwds
+            self.__hostname,
+            self.__port,
+            retry=self.__retry,
+            pool_size=self.__pool_size,
+            **kwds,
         )
 
         return self.__connection