Make host_to_url support redirection and add another test
This commit is contained in:
parent
7c9eea0361
commit
6bb4559d18
|
@ -152,6 +152,7 @@ class TrackerIcons(Component):
|
||||||
self.icons[None] = None
|
self.icons[None] = None
|
||||||
|
|
||||||
self.pending = {}
|
self.pending = {}
|
||||||
|
self.redirects = {}
|
||||||
|
|
||||||
def get(self, host):
|
def get(self, host):
|
||||||
"""
|
"""
|
||||||
|
@ -202,7 +203,7 @@ class TrackerIcons(Component):
|
||||||
:rtype: Deferred
|
:rtype: Deferred
|
||||||
"""
|
"""
|
||||||
if not url:
|
if not url:
|
||||||
url = host_to_url(host)
|
url = self.host_to_url(host)
|
||||||
log.debug("Downloading %s", url)
|
log.debug("Downloading %s", url)
|
||||||
return download_file(url, mkstemp()[1], force_filename=True)
|
return download_file(url, mkstemp()[1], force_filename=True)
|
||||||
|
|
||||||
|
@ -235,7 +236,8 @@ class TrackerIcons(Component):
|
||||||
d = f
|
d = f
|
||||||
if f.check(error.PageRedirect):
|
if f.check(error.PageRedirect):
|
||||||
# Handle redirect errors
|
# Handle redirect errors
|
||||||
location = urljoin(host_to_url(host), error_msg.split(" to ")[1])
|
location = urljoin(self.host_to_url(host), error_msg.split(" to ")[1])
|
||||||
|
self.redirects[host] = url_to_host(location)
|
||||||
d = self.download_page(host, url=location)
|
d = self.download_page(host, url=location)
|
||||||
d.addCallbacks(self.on_download_page_complete, self.on_download_page_fail,
|
d.addCallbacks(self.on_download_page_complete, self.on_download_page_fail,
|
||||||
errbackArgs=(host,))
|
errbackArgs=(host,))
|
||||||
|
@ -275,7 +277,7 @@ class TrackerIcons(Component):
|
||||||
:rtype: list
|
:rtype: list
|
||||||
"""
|
"""
|
||||||
log.debug("Got icons for %s: %s", host, icons)
|
log.debug("Got icons for %s: %s", host, icons)
|
||||||
url = host_to_url(host)
|
url = self.host_to_url(host)
|
||||||
icons = [(urljoin(url, icon), mimetype) for icon, mimetype in icons]
|
icons = [(urljoin(url, icon), mimetype) for icon, mimetype in icons]
|
||||||
log.debug("Icon urls: %s", icons)
|
log.debug("Icon urls: %s", icons)
|
||||||
return icons
|
return icons
|
||||||
|
@ -345,7 +347,7 @@ class TrackerIcons(Component):
|
||||||
d = f
|
d = f
|
||||||
if f.check(error.PageRedirect):
|
if f.check(error.PageRedirect):
|
||||||
# Handle redirect errors
|
# Handle redirect errors
|
||||||
location = urljoin(host_to_url(host), error_msg.split(" to ")[1])
|
location = urljoin(self.host_to_url(host), error_msg.split(" to ")[1])
|
||||||
d = self.download_icon([(location, extension_to_mimetype(location.rpartition('.')[2]))] + icons, host)
|
d = self.download_icon([(location, extension_to_mimetype(location.rpartition('.')[2]))] + icons, host)
|
||||||
if not icons:
|
if not icons:
|
||||||
d.addCallbacks(self.on_download_icon_complete, self.on_download_icon_fail,
|
d.addCallbacks(self.on_download_icon_complete, self.on_download_icon_fail,
|
||||||
|
@ -354,7 +356,7 @@ class TrackerIcons(Component):
|
||||||
d = self.download_icon(icons, host)
|
d = self.download_icon(icons, host)
|
||||||
elif f.check(IndexError, HTMLParseError):
|
elif f.check(IndexError, HTMLParseError):
|
||||||
# No icons, try favicon.ico as an act of desperation
|
# No icons, try favicon.ico as an act of desperation
|
||||||
d = self.download_icon([(urljoin(host_to_url(host), "favicon.ico"), extension_to_mimetype("ico"))], host)
|
d = self.download_icon([(urljoin(self.host_to_url(host), "favicon.ico"), extension_to_mimetype("ico"))], host)
|
||||||
d.addCallbacks(self.on_download_icon_complete, self.on_download_icon_fail,
|
d.addCallbacks(self.on_download_icon_complete, self.on_download_icon_fail,
|
||||||
callbackArgs=(host,), errbackArgs=(host,))
|
callbackArgs=(host,), errbackArgs=(host,))
|
||||||
else:
|
else:
|
||||||
|
@ -404,6 +406,19 @@ class TrackerIcons(Component):
|
||||||
del self.pending[host]
|
del self.pending[host]
|
||||||
return icon
|
return icon
|
||||||
|
|
||||||
|
def host_to_url(self, host):
|
||||||
|
"""
|
||||||
|
Given a host, returns the URL to fetch
|
||||||
|
|
||||||
|
:param host: the tracker host
|
||||||
|
:type host: string
|
||||||
|
:returns: the url of the tracker
|
||||||
|
:rtype: string
|
||||||
|
"""
|
||||||
|
if host in self.redirects:
|
||||||
|
host = self.redirects[host]
|
||||||
|
return "http://%s/" % host
|
||||||
|
|
||||||
################################ HELPER CLASSES ###############################
|
################################ HELPER CLASSES ###############################
|
||||||
|
|
||||||
class FaviconParser(HTMLParser):
|
class FaviconParser(HTMLParser):
|
||||||
|
@ -450,17 +465,6 @@ class FaviconParser(HTMLParser):
|
||||||
|
|
||||||
############################### HELPER FUNCTIONS ##############################
|
############################### HELPER FUNCTIONS ##############################
|
||||||
|
|
||||||
def host_to_url(host):
|
|
||||||
"""
|
|
||||||
Given a host, returns the URL to fetch
|
|
||||||
|
|
||||||
:param host: the tracker host
|
|
||||||
:type host: string
|
|
||||||
:returns: the url of the tracker
|
|
||||||
:rtype: string
|
|
||||||
"""
|
|
||||||
return "http://%s/" % host
|
|
||||||
|
|
||||||
def url_to_host(url):
|
def url_to_host(url):
|
||||||
"""
|
"""
|
||||||
Given a URL, returns the host it belongs to
|
Given a URL, returns the host it belongs to
|
||||||
|
|
Binary file not shown.
After Width: | Height: | Size: 2.5 KiB |
|
@ -52,3 +52,10 @@ class TrackerIconsTestCase(unittest.TestCase):
|
||||||
d.addCallback(self.assertNotIdentical, None)
|
d.addCallback(self.assertNotIdentical, None)
|
||||||
d.addCallback(self.assertEquals, icon)
|
d.addCallback(self.assertEquals, icon)
|
||||||
return d
|
return d
|
||||||
|
|
||||||
|
def test_get_publicbt_ico(self):
|
||||||
|
icon = TrackerIcon("../publicbt.ico")
|
||||||
|
d = icons.get("publicbt.org")
|
||||||
|
d.addCallback(self.assertNotIdentical, None)
|
||||||
|
d.addCallback(self.assertEquals, icon)
|
||||||
|
return d
|
||||||
|
|
Loading…
Reference in New Issue