Make host_to_url support redirection and add another test

This commit is contained in:
John Garland 2010-05-08 16:24:16 +10:00
parent 7c9eea0361
commit 6bb4559d18
3 changed files with 27 additions and 16 deletions

View File

@ -152,6 +152,7 @@ class TrackerIcons(Component):
self.icons[None] = None self.icons[None] = None
self.pending = {} self.pending = {}
self.redirects = {}
def get(self, host): def get(self, host):
""" """
@ -202,7 +203,7 @@ class TrackerIcons(Component):
:rtype: Deferred :rtype: Deferred
""" """
if not url: if not url:
url = host_to_url(host) url = self.host_to_url(host)
log.debug("Downloading %s", url) log.debug("Downloading %s", url)
return download_file(url, mkstemp()[1], force_filename=True) return download_file(url, mkstemp()[1], force_filename=True)
@ -235,7 +236,8 @@ class TrackerIcons(Component):
d = f d = f
if f.check(error.PageRedirect): if f.check(error.PageRedirect):
# Handle redirect errors # Handle redirect errors
location = urljoin(host_to_url(host), error_msg.split(" to ")[1]) location = urljoin(self.host_to_url(host), error_msg.split(" to ")[1])
self.redirects[host] = url_to_host(location)
d = self.download_page(host, url=location) d = self.download_page(host, url=location)
d.addCallbacks(self.on_download_page_complete, self.on_download_page_fail, d.addCallbacks(self.on_download_page_complete, self.on_download_page_fail,
errbackArgs=(host,)) errbackArgs=(host,))
@ -275,7 +277,7 @@ class TrackerIcons(Component):
:rtype: list :rtype: list
""" """
log.debug("Got icons for %s: %s", host, icons) log.debug("Got icons for %s: %s", host, icons)
url = host_to_url(host) url = self.host_to_url(host)
icons = [(urljoin(url, icon), mimetype) for icon, mimetype in icons] icons = [(urljoin(url, icon), mimetype) for icon, mimetype in icons]
log.debug("Icon urls: %s", icons) log.debug("Icon urls: %s", icons)
return icons return icons
@ -345,7 +347,7 @@ class TrackerIcons(Component):
d = f d = f
if f.check(error.PageRedirect): if f.check(error.PageRedirect):
# Handle redirect errors # Handle redirect errors
location = urljoin(host_to_url(host), error_msg.split(" to ")[1]) location = urljoin(self.host_to_url(host), error_msg.split(" to ")[1])
d = self.download_icon([(location, extension_to_mimetype(location.rpartition('.')[2]))] + icons, host) d = self.download_icon([(location, extension_to_mimetype(location.rpartition('.')[2]))] + icons, host)
if not icons: if not icons:
d.addCallbacks(self.on_download_icon_complete, self.on_download_icon_fail, d.addCallbacks(self.on_download_icon_complete, self.on_download_icon_fail,
@ -354,7 +356,7 @@ class TrackerIcons(Component):
d = self.download_icon(icons, host) d = self.download_icon(icons, host)
elif f.check(IndexError, HTMLParseError): elif f.check(IndexError, HTMLParseError):
# No icons, try favicon.ico as an act of desperation # No icons, try favicon.ico as an act of desperation
d = self.download_icon([(urljoin(host_to_url(host), "favicon.ico"), extension_to_mimetype("ico"))], host) d = self.download_icon([(urljoin(self.host_to_url(host), "favicon.ico"), extension_to_mimetype("ico"))], host)
d.addCallbacks(self.on_download_icon_complete, self.on_download_icon_fail, d.addCallbacks(self.on_download_icon_complete, self.on_download_icon_fail,
callbackArgs=(host,), errbackArgs=(host,)) callbackArgs=(host,), errbackArgs=(host,))
else: else:
@ -404,6 +406,19 @@ class TrackerIcons(Component):
del self.pending[host] del self.pending[host]
return icon return icon
def host_to_url(self, host):
"""
Given a host, returns the URL to fetch
:param host: the tracker host
:type host: string
:returns: the url of the tracker
:rtype: string
"""
if host in self.redirects:
host = self.redirects[host]
return "http://%s/" % host
################################ HELPER CLASSES ############################### ################################ HELPER CLASSES ###############################
class FaviconParser(HTMLParser): class FaviconParser(HTMLParser):
@ -450,17 +465,6 @@ class FaviconParser(HTMLParser):
############################### HELPER FUNCTIONS ############################## ############################### HELPER FUNCTIONS ##############################
def host_to_url(host):
"""
Given a host, returns the URL to fetch
:param host: the tracker host
:type host: string
:returns: the url of the tracker
:rtype: string
"""
return "http://%s/" % host
def url_to_host(url): def url_to_host(url):
""" """
Given a URL, returns the host it belongs to Given a URL, returns the host it belongs to

BIN
tests/publicbt.ico Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.5 KiB

View File

@ -52,3 +52,10 @@ class TrackerIconsTestCase(unittest.TestCase):
d.addCallback(self.assertNotIdentical, None) d.addCallback(self.assertNotIdentical, None)
d.addCallback(self.assertEquals, icon) d.addCallback(self.assertEquals, icon)
return d return d
def test_get_publicbt_ico(self):
icon = TrackerIcon("../publicbt.ico")
d = icons.get("publicbt.org")
d.addCallback(self.assertNotIdentical, None)
d.addCallback(self.assertEquals, icon)
return d