[TrackerIcon] Fixed parse error on UTF-8 sites with non-english chars
When parsing the site's page in search for the FAVICON, the page gets opens. The default file encoding in dependent on the running OS, and might not be `UTF-8` on Windows. Therefor, some trackers might not get their icon downloaded at all because of an error: `UnicodeDecodeError: 'charmap' codec can't decode byte 0x90 in position 2158: character maps to <undefined>`. This fix adds a detection of file encoding using the optional `chardet` dependency, and also a test. Closes: deluge-torrent/deluge#333 Closes: https://dev.deluge-torrent.org/ticket/3479
This commit is contained in:
parent
44dcbee5f4
commit
517b2c653b
|
@ -14,7 +14,6 @@ from . import common
|
||||||
from .basetest import BaseTestCase
|
from .basetest import BaseTestCase
|
||||||
|
|
||||||
common.set_tmp_config_dir()
|
common.set_tmp_config_dir()
|
||||||
deluge.ui.tracker_icons.PIL_INSTALLED = False
|
|
||||||
common.disable_new_release_check()
|
common.disable_new_release_check()
|
||||||
|
|
||||||
|
|
||||||
|
@ -45,6 +44,14 @@ class TrackerIconsTestCase(BaseTestCase):
|
||||||
d.addCallback(self.assertEqual, icon)
|
d.addCallback(self.assertEqual, icon)
|
||||||
return d
|
return d
|
||||||
|
|
||||||
|
def test_get_google_ico_hebrew(self):
|
||||||
|
"""Test that Google.co.il page is read as UTF-8"""
|
||||||
|
icon = TrackerIcon(common.get_test_data_file('google.ico'))
|
||||||
|
d = self.icons.fetch('www.google.co.il')
|
||||||
|
d.addCallback(self.assertNotIdentical, None)
|
||||||
|
d.addCallback(self.assertEqual, icon)
|
||||||
|
return d
|
||||||
|
|
||||||
def test_get_google_ico_with_redirect(self):
|
def test_get_google_ico_with_redirect(self):
|
||||||
# google.com redirects to www.google.com
|
# google.com redirects to www.google.com
|
||||||
icon = TrackerIcon(common.get_test_data_file('google.ico'))
|
icon = TrackerIcon(common.get_test_data_file('google.ico'))
|
||||||
|
|
|
@ -21,6 +21,11 @@ from deluge.configmanager import get_config_dir
|
||||||
from deluge.decorators import proxy
|
from deluge.decorators import proxy
|
||||||
from deluge.httpdownloader import download_file
|
from deluge.httpdownloader import download_file
|
||||||
|
|
||||||
|
try:
|
||||||
|
import chardet
|
||||||
|
except ImportError:
|
||||||
|
chardet = None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
except ImportError:
|
except ImportError:
|
||||||
|
@ -289,7 +294,13 @@ class TrackerIcons(Component):
|
||||||
:returns: a Deferred which callbacks a list of available favicons (url, type)
|
:returns: a Deferred which callbacks a list of available favicons (url, type)
|
||||||
:rtype: Deferred
|
:rtype: Deferred
|
||||||
"""
|
"""
|
||||||
with open(page) as _file:
|
encoding = 'UTF-8'
|
||||||
|
if chardet:
|
||||||
|
with open(page, 'rb') as _file:
|
||||||
|
result = chardet.detect(_file.read())
|
||||||
|
encoding = result['encoding']
|
||||||
|
|
||||||
|
with open(page, encoding=encoding) as _file:
|
||||||
parser = FaviconParser()
|
parser = FaviconParser()
|
||||||
for line in _file:
|
for line in _file:
|
||||||
parser.feed(line)
|
parser.feed(line)
|
||||||
|
|
Loading…
Reference in New Issue