Enabling connetion pool_size definition

requests has a default connections pool of 10. Creating multiple threads
will consume from that same pool, since the underlying implementation of
the requests pool is a singletown.

Let's make the pool_size configurable, so clients can set the proper
number for their use case when running multiple threads.

Signed-off-by: Amador Pahim <apahim@redhat.com>

github/MainClass.py

@@ -106,6 +106,7 @@ class Github:
         per_page=DEFAULT_PER_PAGE,
         verify=True,
         retry=None,
+        pool_size=None,
     ):
         """
         :param login_or_token: string
@@ -118,6 +119,7 @@ class Github:
         :param per_page: int
         :param verify: boolean or string
         :param retry: int or urllib3.util.retry.Retry object
+        :param pool_size: int
         """
 
         assert login_or_token is None or isinstance(login_or_token, str), login_or_token
@@ -133,6 +135,8 @@ class Github:
             or isinstance(retry, (int))
             or isinstance(retry, (urllib3.util.Retry))
         )
+        assert pool_size is None or isinstance(pool_size, (int)), pool_size
+
         if client_id is not None or client_secret is not None:
             warnings.warn(
                 "client_id and client_secret are deprecated and will be removed in a future release, switch to token authentication",
@@ -151,6 +155,7 @@ class Github:
             per_page,
             verify,
             retry,
+            pool_size,
         )
 
     def __get_FIX_REPO_GET_GIT_REF(self):

github/Requester.py

@@ -82,7 +82,14 @@ class RequestsResponse:
 class HTTPSRequestsConnectionClass:
     # mimic the httplib connection object
     def __init__(
-        self, host, port=None, strict=False, timeout=None, retry=None, **kwargs
+        self,
+        host,
+        port=None,
+        strict=False,
+        timeout=None,
+        retry=None,
+        pool_size=None,
+        **kwargs,
     ):
         self.port = port if port else 443
         self.host = host
@@ -95,8 +102,16 @@ class HTTPSRequestsConnectionClass:
             self.retry = requests.adapters.DEFAULT_RETRIES
         else:
             self.retry = retry
+
+        if pool_size is None:
+            self.pool_size = requests.adapters.DEFAULT_POOLSIZE
+        else:
+            self.pool_size = pool_size
+
         self.adapter = requests.adapters.HTTPAdapter(
             max_retries=self.retry,
+            pool_connections=self.pool_size,
+            pool_maxsize=self.pool_size,
         )
         self.session.mount("https://", self.adapter)
 
@@ -126,7 +141,14 @@ class HTTPSRequestsConnectionClass:
 class HTTPRequestsConnectionClass:
     # mimic the httplib connection object
     def __init__(
-        self, host, port=None, strict=False, timeout=None, retry=None, **kwargs
+        self,
+        host,
+        port=None,
+        strict=False,
+        timeout=None,
+        retry=None,
+        pool_size=None,
+        **kwargs,
     ):
         self.port = port if port else 80
         self.host = host
@@ -139,8 +161,16 @@ class HTTPRequestsConnectionClass:
             self.retry = requests.adapters.DEFAULT_RETRIES
         else:
             self.retry = retry
+
+        if pool_size is None:
+            self.pool_size = requests.adapters.DEFAULT_POOLSIZE
+        else:
+            self.pool_size = pool_size
+
         self.adapter = requests.adapters.HTTPAdapter(
             max_retries=self.retry,
+            pool_connections=self.pool_size,
+            pool_maxsize=self.pool_size,
         )
         self.session.mount("http://", self.adapter)
 
@@ -272,6 +302,7 @@ class Requester:
         per_page,
         verify,
         retry,
+        pool_size,
     ):
         self._initializeDebugFeature()
 
@@ -295,6 +326,7 @@ class Requester:
         self.__prefix = o.path
         self.__timeout = timeout
         self.__retry = retry  # NOTE: retry can be either int or an urllib3 Retry object
+        self.__pool_size = pool_size
         self.__scheme = o.scheme
         if o.scheme == "https":
             self.__connectionClass = self.__httpsConnectionClass
@@ -362,11 +394,17 @@ class Requester:
             ):  # issue80
                 if o.scheme == "http":
                     cnx = self.__httpConnectionClass(
-                        o.hostname, o.port, retry=self.__retry
+                        o.hostname,
+                        o.port,
+                        retry=self.__retry,
+                        pool_size=self.__pool_size,
                     )
                 elif o.scheme == "https":
                     cnx = self.__httpsConnectionClass(
-                        o.hostname, o.port, retry=self.__retry
+                        o.hostname,
+                        o.port,
+                        retry=self.__retry,
+                        pool_size=self.__pool_size,
                     )
         return cnx
 
@@ -583,7 +621,11 @@ class Requester:
             return self.__connection
 
         self.__connection = self.__connectionClass(
-            self.__hostname, self.__port, retry=self.__retry, **kwds
+            self.__hostname,
+            self.__port,
+            retry=self.__retry,
+            pool_size=self.__pool_size,
+            **kwds,
         )
 
         return self.__connection

tests/Framework.py

@@ -244,6 +244,7 @@ class BasicTestCase(unittest.TestCase):
     tokenAuthMode = False
     jwtAuthMode = False
     retry = None
+    pool_size = None
     replayDataFolder = os.path.join(os.path.dirname(__file__), "ReplayData")
 
     def setUp(self):
@@ -352,11 +353,17 @@ class TestCase(BasicTestCase):
         github.Requester.Requester.setOnCheckMe(self.getFrameChecker())
 
         if self.tokenAuthMode:
-            self.g = github.Github(self.oauth_token, retry=self.retry)
+            self.g = github.Github(
+                self.oauth_token, retry=self.retry, pool_size=self.pool_size
+            )
         elif self.jwtAuthMode:
-            self.g = github.Github(jwt=self.jwt, retry=self.retry)
+            self.g = github.Github(
+                jwt=self.jwt, retry=self.retry, pool_size=self.pool_size
+            )
         else:
-            self.g = github.Github(self.login, self.password, retry=self.retry)
+            self.g = github.Github(
+                self.login, self.password, retry=self.retry, pool_size=self.pool_size
+            )
 
 
 def activateRecordMode():  # pragma no cover (Function useful only when recording new tests, not used during automated tests)
@@ -373,3 +380,7 @@ def activateJWTAuthMode():  # pragma no cover (Function useful only when recordi
 
 def enableRetry(retry):
     BasicTestCase.retry = retry
+
+
+def setPoolSize(pool_size):
+    BasicTestCase.pool_size = pool_size

tests/PoolSize.py

@@ -0,0 +1,27 @@
+import github
+
+from . import Framework
+
+REPO_NAME = "PyGithub/PyGithub"
+
+
+class PoolSize(Framework.TestCase):
+    def setUp(self):
+        Framework.setPoolSize(20)
+        super().setUp()
+
+    def testReturnsRepoAfterSettingPoolSize(self):
+        repository = self.g.get_repo(REPO_NAME)
+        self.assertIsInstance(repository, github.Repository.Repository)
+        self.assertEqual(repository.full_name, REPO_NAME)
+
+    def testReturnsRepoAfterSettingPoolSizeHttp(self):
+        g = github.Github(
+            self.login,
+            self.password,
+            base_url="http://my.enterprise.com",
+            pool_size=20,
+        )
+        repository = g.get_repo(REPO_NAME)
+        self.assertIsInstance(repository, github.Repository.Repository)
+        self.assertEqual(repository.full_name, REPO_NAME)