status-im/PyGithub
1
0
Fork 0
mirror of https://github.com/status-im/PyGithub.git synced 2026-08-31 10:51:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1818 from apahim/define_pool_size

Browse Source 
Make requests pool_size configurable
This commit is contained in:
Jimmy Zelinskie
2021-02-06 21:48:20 -05:00
committed by GitHub
parent 34d097ce47 a77d4f48cb
commit 6900f187b5
8 changed files with 153 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
github/MainClass.py
+5
View File
@@ -106,6 +106,7 @@ class Github:
per_page=DEFAULT_PER_PAGE,
verify=True,
retry=None,
pool_size=None,
):
"""
:param login_or_token: string
@@ -118,6 +119,7 @@ class Github:
:param per_page: int
:param verify: boolean or string
:param retry: int or urllib3.util.retry.Retry object
:param pool_size: int
"""
assert login_or_token is None or isinstance(login_or_token, str), login_or_token
@@ -133,6 +135,8 @@ class Github:
or isinstance(retry, (int))
or isinstance(retry, (urllib3.util.Retry))
)
assert pool_size is None or isinstance(pool_size, (int)), pool_size
if client_id is not None or client_secret is not None:
warnings.warn(
"client_id and client_secret are deprecated and will be removed in a future release, switch to token authentication",
@@ -151,6 +155,7 @@ class Github:
per_page,
verify,
retry,
pool_size,
)
def __get_FIX_REPO_GET_GIT_REF(self):
github/Requester.py
+63 -13
View File
@@ -82,7 +82,14 @@ class RequestsResponse:
class HTTPSRequestsConnectionClass:
# mimic the httplib connection object
def __init__(
self, host, port=None, strict=False, timeout=None, retry=None, **kwargs
self,
host,
port=None,
strict=False,
timeout=None,
retry=None,
pool_size=None,
**kwargs,
):
self.port = port if port else 443
self.host = host
@@ -90,11 +97,23 @@ class HTTPSRequestsConnectionClass:
self.timeout = timeout
self.verify = kwargs.get("verify", True)
self.session = requests.Session()
# Code to support retries
if retry:
if retry is None:
self.retry = requests.adapters.DEFAULT_RETRIES
else:
self.retry = retry
self.adapter = requests.adapters.HTTPAdapter(max_retries=self.retry)
self.session.mount("https://", self.adapter)
if pool_size is None:
self.pool_size = requests.adapters.DEFAULT_POOLSIZE
else:
self.pool_size = pool_size
self.adapter = requests.adapters.HTTPAdapter(
max_retries=self.retry,
pool_connections=self.pool_size,
pool_maxsize=self.pool_size,
)
self.session.mount("https://", self.adapter)
def request(self, verb, url, input, headers):
self.verb = verb
@@ -122,7 +141,14 @@ class HTTPSRequestsConnectionClass:
class HTTPRequestsConnectionClass:
# mimic the httplib connection object
def __init__(
self, host, port=None, strict=False, timeout=None, retry=None, **kwargs
self,
host,
port=None,
strict=False,
timeout=None,
retry=None,
pool_size=None,
**kwargs,
):
self.port = port if port else 80
self.host = host
@@ -130,11 +156,23 @@ class HTTPRequestsConnectionClass:
self.timeout = timeout
self.verify = kwargs.get("verify", True)
self.session = requests.Session()
# Code to support retries
if retry:
if retry is None:
self.retry = requests.adapters.DEFAULT_RETRIES
else:
self.retry = retry
self.adapter = requests.adapters.HTTPAdapter(max_retries=self.retry)
self.session.mount("http://", self.adapter)
if pool_size is None:
self.pool_size = requests.adapters.DEFAULT_POOLSIZE
else:
self.pool_size = pool_size
self.adapter = requests.adapters.HTTPAdapter(
max_retries=self.retry,
pool_connections=self.pool_size,
pool_maxsize=self.pool_size,
)
self.session.mount("http://", self.adapter)
def request(self, verb, url, input, headers):
self.verb = verb
@@ -264,6 +302,7 @@ class Requester:
per_page,
verify,
retry,
pool_size,
):
self._initializeDebugFeature()
@@ -287,6 +326,7 @@ class Requester:
self.__prefix = o.path
self.__timeout = timeout
self.__retry = retry # NOTE: retry can be either int or an urllib3 Retry object
self.__pool_size = pool_size
self.__scheme = o.scheme
if o.scheme == "https":
self.__connectionClass = self.__httpsConnectionClass
@@ -354,11 +394,17 @@ class Requester:
): # issue80
if o.scheme == "http":
cnx = self.__httpConnectionClass(
o.hostname, o.port, retry=self.__retry
o.hostname,
o.port,
retry=self.__retry,
pool_size=self.__pool_size,
)
elif o.scheme == "https":
cnx = self.__httpsConnectionClass(
o.hostname, o.port, retry=self.__retry
o.hostname,
o.port,
retry=self.__retry,
pool_size=self.__pool_size,
)
return cnx
@@ -575,7 +621,11 @@ class Requester:
return self.__connection
self.__connection = self.__connectionClass(
self.__hostname, self.__port, retry=self.__retry, **kwds
self.__hostname,
self.__port,
retry=self.__retry,
pool_size=self.__pool_size,
**kwds,
)
return self.__connection
tests/Framework.py
+14 -3
View File
@@ -244,6 +244,7 @@ class BasicTestCase(unittest.TestCase):
tokenAuthMode = False
jwtAuthMode = False
retry = None
pool_size = None
replayDataFolder = os.path.join(os.path.dirname(__file__), "ReplayData")
def setUp(self):
@@ -352,11 +353,17 @@ class TestCase(BasicTestCase):
github.Requester.Requester.setOnCheckMe(self.getFrameChecker())
if self.tokenAuthMode:
self.g = github.Github(self.oauth_token, retry=self.retry)
self.g = github.Github(
self.oauth_token, retry=self.retry, pool_size=self.pool_size
)
elif self.jwtAuthMode:
self.g = github.Github(jwt=self.jwt, retry=self.retry)
self.g = github.Github(
jwt=self.jwt, retry=self.retry, pool_size=self.pool_size
)
else:
self.g = github.Github(self.login, self.password, retry=self.retry)
self.g = github.Github(
self.login, self.password, retry=self.retry, pool_size=self.pool_size
)
def activateRecordMode(): # pragma no cover (Function useful only when recording new tests, not used during automated tests)
@@ -373,3 +380,7 @@ def activateJWTAuthMode(): # pragma no cover (Function useful only when recordi
def enableRetry(retry):
BasicTestCase.retry = retry
def setPoolSize(pool_size):
BasicTestCase.pool_size = pool_size
tests/PoolSize.py
+27
View File
@@ -0,0 +1,27 @@
import github
from . import Framework
REPO_NAME = "PyGithub/PyGithub"
class PoolSize(Framework.TestCase):
def setUp(self):
Framework.setPoolSize(20)
super().setUp()
def testReturnsRepoAfterSettingPoolSize(self):
repository = self.g.get_repo(REPO_NAME)
self.assertIsInstance(repository, github.Repository.Repository)
self.assertEqual(repository.full_name, REPO_NAME)
def testReturnsRepoAfterSettingPoolSizeHttp(self):
g = github.Github(
self.login,
self.password,
base_url="http://my.enterprise.com",
pool_size=20,
)
repository = g.get_repo(REPO_NAME)
self.assertIsInstance(repository, github.Repository.Repository)
self.assertEqual(repository.full_name, REPO_NAME)
tests/ReplayData/PoolSize.testReturnsRepoAfterSettingPoolSize.txt
+11
View File
File diff suppressed because one or more lines are too long
tests/ReplayData/PoolSize.testReturnsRepoAfterSettingPoolSizeHttp.txt
+11
View File
File diff suppressed because one or more lines are too long
tests/ReplayData/Retry.testReturnsRepoAfterSettingRetryHttp.txt
+11
View File
File diff suppressed because one or more lines are too long
tests/Retry.py
+11
View File
@@ -75,3 +75,14 @@ class Retry(Framework.TestCase):
self.assertEqual(len(httpretty.latest_requests), 4)
for request in httpretty.latest_requests:
self.assertEqual(request.path, "/repos/PyGithub/PyGithub")
def testReturnsRepoAfterSettingRetryHttp(self):
g = github.Github(
self.login,
self.password,
base_url="http://my.enterprise.com",
retry=0,
) # http here
repository = g.get_repo(REPO_NAME)
self.assertIsInstance(repository, github.Repository.Repository)
self.assertEqual(repository.full_name, REPO_NAME)