codex-storage
/
deluge
mirror of https://github.com/codex-storage/deluge.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[Common] Rename decode_string to decode_bytes 

- Switch to using Python 3 naming convention where str now refers to unicode
   and bytes are encoded strings.
 - Cleanup docs and code
 - Also rename convert_to_utf8 to utf8_encode_structure to clarify functionality.
This commit is contained in:
Calum Lind 2017-02-22 12:53:20 +00:00
parent 3b1eeb0818
commit beb4f8c8f9
14 changed files with 64 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
deluge/common.py
View File

@ -777,36 +777,36 @@ def is_ipv6(ip):
return True
else:
try:
return ipaddress.IPv6Address(decode_string(ip))
return ipaddress.IPv6Address(decode_bytes(ip))
except ipaddress.AddressValueError:
pass
return False
def decode_string(s, encoding='utf8'):
"""
Decodes a string and return unicode. If it cannot decode using
`:param:encoding` then it will try latin1, and if that fails,
try to detect the string encoding. If that fails, decode with
ignore.
def decode_bytes(byte_str, encoding='utf8'):
"""Decodes a byte string and return unicode.
:param s: string to decode
:type s: string
:param encoding: the encoding to use in the decoding
:type encoding: string
:returns: s converted to unicode
:rtype: unicode
If it cannot decode using `encoding` then it will try latin1,
and if that fails, try to detect the string encoding. If that fails,
decode with ignore.
Args:
byte_str (bytes): The byte string to decode.
encoding (str): The encoding to try first when decoding.
Returns:
str: A unicode string.
"""
if not s:
if not byte_str:
return ''
elif isinstance(s, unicode):
return s
elif isinstance(byte_str, unicode):
return byte_str
encodings = [lambda: ('utf8', 'strict'),
lambda: ('iso-8859-1', 'strict'),
lambda: (chardet.detect(s)['encoding'], 'strict'),
lambda: (chardet.detect(byte_str)['encoding'], 'strict'),
lambda: (encoding, 'ignore')]
if encoding is not 'utf8':
@ -814,7 +814,7 @@ def decode_string(s, encoding='utf8'):
for l in encodings:
try:
return s.decode(*l())
return byte_str.decode(*l())
except UnicodeDecodeError:
pass
return ''
@ -833,13 +833,13 @@ def utf8_encoded(s, encoding='utf8'):
"""
if isinstance(s, str):
s = decode_string(s, encoding).encode('utf8')
s = decode_bytes(s, encoding).encode('utf8')
elif isinstance(s, unicode):
s = s.encode('utf8')
return s
def convert_to_utf8(data):
def utf8_encode_structure(data):
"""Recursively convert all unicode keys and values in a data structure to utf8.
e.g. converting keys and values for a dict with nested dicts and lists etc.
@ -851,13 +851,12 @@ def convert_to_utf8(data):
input type: The data with unicode keys and values converted to utf8.
"""
if isinstance(data, unicode):
return data.encode('utf8')
elif isinstance(data, (list, tuple)):
return type(data)(map(convert_to_utf8, data))
return type(data)(map(utf8_encode_structure, data))
elif isinstance(data, dict):
return dict(map(convert_to_utf8, data.items()))
return dict(map(utf8_encode_structure, data.items()))
else:
return data

4
deluge/config.py
View File

@ -47,7 +47,7 @@ import logging
import os
import shutil
from deluge.common import decode_string, get_default_config_dir, utf8_encoded
from deluge.common import decode_bytes, get_default_config_dir, utf8_encoded
log = logging.getLogger(__name__)
callLater = None # Necessary for the config tests
@ -245,7 +245,7 @@ class Config(object):
"""
if isinstance(self.__config[key], basestring):
return decode_string(self.__config[key])
return decode_bytes(self.__config[key])
else:
return self.__config[key]

4
deluge/core/alertmanager.py
View File

@ -23,7 +23,7 @@ from twisted.internet import reactor
import deluge.component as component
from deluge._libtorrent import lt
from deluge.common import decode_string
from deluge.common import decode_bytes
log = logging.getLogger(__name__)
@ -112,7 +112,7 @@ class AlertManager(component.Component):
alert_type = type(alert).__name__
# Display the alert message
if log.isEnabledFor(logging.DEBUG):
log.debug('%s: %s', alert_type, decode_string(alert.message()))
log.debug('%s: %s', alert_type, decode_bytes(alert.message()))
# Call any handlers for this alert type
if alert_type in self.handlers:
for handler in self.handlers[alert_type]:

14
deluge/core/torrent.py
View File

@ -26,7 +26,7 @@ from twisted.internet.defer import Deferred, DeferredList
import deluge.component as component
from deluge._libtorrent import lt
from deluge.common import decode_string, utf8_encoded
from deluge.common import decode_bytes, utf8_encoded
from deluge.configmanager import ConfigManager, get_config_dir
from deluge.core.authmanager import AUTH_LEVEL_ADMIN
from deluge.decorators import deprecated
@ -612,7 +612,7 @@ class Torrent(object):
self.state = 'Error'
# auto-manage status will be reverted upon resuming.
self.handle.auto_managed(False)
self.set_status_message(decode_string(status_error))
self.set_status_message(decode_bytes(status_error))
elif status.moving_storage:
self.state = 'Moving'
elif not session_paused and status.paused and status.auto_managed:
@ -761,7 +761,7 @@ class Torrent(object):
if peer.flags & peer.connecting or peer.flags & peer.handshake:
continue
client = decode_string(peer.client)
client = decode_bytes(peer.client)
try:
country = component.get('Core').geoip_instance.country_code_by_addr(peer.ip[0])
@ -868,7 +868,7 @@ class Torrent(object):
if not self.options['name']:
handle_name = self.handle.name()
if handle_name:
name = decode_string(handle_name)
name = decode_bytes(handle_name)
else:
name = self.torrent_id
else:
@ -1013,7 +1013,7 @@ class Torrent(object):
'trackers': lambda: self.trackers,
'tracker_status': lambda: self.tracker_status,
'upload_payload_rate': lambda: self.status.upload_payload_rate,
'comment': lambda: decode_string(self.torrent_info.comment()) if self.has_metadata else '',
'comment': lambda: decode_bytes(self.torrent_info.comment()) if self.has_metadata else '',
'num_files': lambda: self.torrent_info.num_files() if self.has_metadata else 0,
'num_pieces': lambda: self.torrent_info.num_pieces() if self.has_metadata else 0,
'piece_length': lambda: self.torrent_info.piece_length() if self.has_metadata else 0,
@ -1114,7 +1114,7 @@ class Torrent(object):
bool: True if successful, otherwise False
"""
dest = decode_string(dest)
dest = decode_bytes(dest)
if not os.path.exists(dest):
try:
@ -1253,7 +1253,7 @@ class Torrent(object):
"""
for index, filename in filenames:
# Make sure filename is a unicode object
filename = sanitize_filepath(decode_string(filename))
filename = sanitize_filepath(decode_bytes(filename))
# libtorrent needs unicode object if wstrings are enabled, utf8 bytestring otherwise
try:
self.handle.rename_file(index, filename)

24
deluge/core/torrentmanager.py
View File

@ -24,7 +24,7 @@ from twisted.internet.task import LoopingCall
import deluge.component as component
from deluge._libtorrent import lt
from deluge.common import decode_string, get_magnet_info, utf8_encoded
from deluge.common import decode_bytes, get_magnet_info, utf8_encoded
from deluge.configmanager import ConfigManager, get_config_dir
from deluge.core.authmanager import AUTH_LEVEL_ADMIN
from deluge.core.torrent import Torrent, TorrentOptions, sanitize_filepath
@ -358,7 +358,7 @@ class TorrentManager(component.Component):
# Check for renamed files and if so, rename them in the torrent_info before adding.
if options['mapped_files'] and torrent_info:
for index, fname in options['mapped_files'].items():
fname = sanitize_filepath(decode_string(fname))
fname = sanitize_filepath(decode_bytes(fname))
if log.isEnabledFor(logging.DEBUG):
log.debug('renaming file index %s to %s', index, fname)
try:
@ -1045,7 +1045,7 @@ class TorrentManager(component.Component):
except (RuntimeError, KeyError):
return
# Set the tracker status for the torrent
torrent.set_tracker_status('Warning: %s' % decode_string(alert.message()))
torrent.set_tracker_status('Warning: %s' % decode_bytes(alert.message()))
def on_alert_tracker_error(self, alert):
"""Alert handler for libtorrent tracker_error_alert"""
@ -1054,10 +1054,10 @@ class TorrentManager(component.Component):
except (RuntimeError, KeyError):
return
error_message = decode_string(alert.error_message())
error_message = decode_bytes(alert.error_message())
if not error_message:
error_message = alert.error.message()
log.debug('Tracker Error Alert: %s [%s]', decode_string(alert.message()), error_message)
log.debug('Tracker Error Alert: %s [%s]', decode_bytes(alert.message()), error_message)
torrent.set_tracker_status('Error: ' + error_message)
def on_alert_storage_moved(self, alert):
@ -1085,9 +1085,9 @@ class TorrentManager(component.Component):
except (RuntimeError, KeyError):
return
log.warning('on_alert_storage_moved_failed: %s', decode_string(alert.message()))
log.warning('on_alert_storage_moved_failed: %s', decode_bytes(alert.message()))
# Set an Error message and pause the torrent
alert_msg = decode_string(alert.message()).split(':', 1)[1].strip()
alert_msg = decode_bytes(alert.message()).split(':', 1)[1].strip()
torrent.force_error_state('Failed to move download folder: %s' % alert_msg)
if torrent_id in self.waiting_on_finish_moving:
@ -1145,7 +1145,7 @@ class TorrentManager(component.Component):
return
if torrent_id in self.waiting_on_resume_data:
self.waiting_on_resume_data[torrent_id].errback(Exception(decode_string(alert.message())))
self.waiting_on_resume_data[torrent_id].errback(Exception(decode_bytes(alert.message())))
def on_alert_fastresume_rejected(self, alert):
"""Alert handler for libtorrent fastresume_rejected_alert"""
@ -1155,7 +1155,7 @@ class TorrentManager(component.Component):
except (RuntimeError, KeyError):
return
alert_msg = decode_string(alert.message())
alert_msg = decode_bytes(alert.message())
log.error('on_alert_fastresume_rejected: %s', alert_msg)
if alert.error.value() == 134:
if not os.path.isdir(torrent.options['download_location']):
@ -1179,7 +1179,7 @@ class TorrentManager(component.Component):
except (RuntimeError, KeyError):
return
new_name = decode_string(alert.new_name)
new_name = decode_bytes(alert.new_name)
log.debug('index: %s name: %s', alert.index, new_name)
# We need to see if this file index is in a waiting_on_folder dict
@ -1251,13 +1251,13 @@ class TorrentManager(component.Component):
'external IP received: 0:0:0:0:0:0:0:0'
"""
external_ip = decode_string(alert.message()).split(' ')[-1]
external_ip = decode_bytes(alert.message()).split(' ')[-1]
log.info('on_alert_external_ip: %s', external_ip)
component.get('EventManager').emit(ExternalIPEvent(external_ip))
def on_alert_performance(self, alert):
"""Alert handler for libtorrent performance_alert"""
log.warning('on_alert_performance: %s, %s', decode_string(alert.message()), alert.warning_code)
log.warning('on_alert_performance: %s, %s', decode_bytes(alert.message()), alert.warning_code)
if alert.warning_code == lt.performance_warning_t.send_buffer_watermark_too_low:
max_send_buffer_watermark = 3 * 1024 * 1024 # 3MiB
settings = self.session.get_settings()

4
deluge/httpdownloader.py
View File

@ -19,7 +19,7 @@ from twisted.python.failure import Failure
from twisted.web import client, http
from twisted.web.error import PageRedirect
from deluge.common import convert_to_utf8, get_version
from deluge.common import get_version, utf8_encode_structure
log = logging.getLogger(__name__)
@ -175,7 +175,7 @@ def _download_file(url, filename, callback=None, headers=None, force_filename=Fa
url = url.encode('utf8')
filename = filename.encode('utf8')
headers = convert_to_utf8(headers) if headers else headers
headers = utf8_encode_structure(headers) if headers else headers
factory = HTTPDownloader(url, filename, callback, headers, force_filename, allow_compression)
# In Twisted 13.1.0 _parse() function replaced by _URI class.

4
deluge/maketorrent.py
View File

@ -14,7 +14,7 @@ import sys
from hashlib import sha1 as sha
from deluge.bencode import bencode
from deluge.common import convert_to_utf8, get_path_size
from deluge.common import get_path_size, utf8_encode_structure
class InvalidPath(Exception):
@ -194,7 +194,7 @@ class TorrentMetadata(object):
# Write out the torrent file
with open(torrent_path, 'wb') as _file:
_file.write(bencode(convert_to_utf8(torrent)))
_file.write(bencode(utf8_encode_structure(torrent)))
def get_data_path(self):
"""Get the path to the files that the torrent will contain.

4
deluge/tests/test_torrent.py
View File

@ -18,7 +18,7 @@ import deluge.component as component
import deluge.core.torrent
import deluge.tests.common as common
from deluge._libtorrent import lt
from deluge.common import convert_to_utf8
from deluge.common import utf8_encode_structure
from deluge.core.core import Core
from deluge.core.rpcserver import RPCServer
from deluge.core.torrent import Torrent
@ -180,7 +180,7 @@ class TorrentTestCase(BaseTestCase):
filename = common.get_test_data_file('test_torrent.file.torrent')
with open(filename) as _file:
filedump = _file.read()
resume_data = convert_to_utf8(resume_data)
resume_data = utf8_encode_structure(resume_data)
torrent_id = yield self.core.torrentmanager.add(state=torrent_state, filedump=filedump,
resume_data=lt.bencode(resume_data))
torrent = self.core.torrentmanager.torrents[torrent_id]

4
deluge/tests/test_ui_entry.py
View File

@ -24,7 +24,7 @@ import deluge.ui.console
import deluge.ui.console.cmdline.commands.quit
import deluge.ui.console.main
import deluge.ui.web.server
from deluge.common import convert_to_utf8
from deluge.common import utf8_encode_structure
from deluge.ui import ui_entry
from deluge.ui.web.server import DelugeWeb
@ -168,7 +168,7 @@ class GtkUIDelugeScriptEntryTestCase(BaseTestCase, GtkUIBaseTestCase):
self.var['cmd_name'] = 'deluge gtk'
self.var['start_cmd'] = ui_entry.start_ui
self.var['sys_arg_cmd'] = convert_to_utf8(['./deluge', 'gtk'])
self.var['sys_arg_cmd'] = utf8_encode_structure(['./deluge', 'gtk'])
def set_up(self):
return GtkUIBaseTestCase.set_up(self)

4
deluge/tests/test_webserver.py
View File

@ -18,7 +18,7 @@ from twisted.trial.unittest import SkipTest
from twisted.web.client import Agent, FileBodyProducer
from twisted.web.http_headers import Headers
from deluge.common import convert_to_utf8
from deluge.common import utf8_encode_structure
from . import common
from .common import get_test_data_file
@ -46,7 +46,7 @@ class WebServerTestCase(WebServerTestBase, WebServerMockBase):
'Content-Type': ['application/json']}
url = 'http://127.0.0.1:%s/json' % self.webserver_listen_port
d = yield agent.request(b'POST', url.encode('utf-8'), Headers(convert_to_utf8(headers)),
d = yield agent.request(b'POST', url.encode('utf-8'), Headers(utf8_encode_structure(headers)),
FileBodyProducer(StringIO(input_file.encode('utf-8'))))
try:
body = yield twisted.web.client.readBody(d)

10
deluge/ui/common.py
View File

@ -20,7 +20,7 @@ from hashlib import sha1 as sha
import deluge.configmanager
from deluge import bencode
from deluge.common import decode_string
from deluge.common import decode_bytes
log = logging.getLogger(__name__)
@ -166,9 +166,9 @@ class TorrentInfo(object):
# Check if 'name.utf-8' is in the torrent and if not try to decode the string
# using the encoding found.
if 'name.utf-8' in self.__m_metadata['info']:
self.__m_name = decode_string(self.__m_metadata['info']['name.utf-8'])
self.__m_name = decode_bytes(self.__m_metadata['info']['name.utf-8'])
else:
self.__m_name = decode_string(self.__m_metadata['info']['name'], self.encoding)
self.__m_name = decode_bytes(self.__m_metadata['info']['name'], self.encoding)
# Get list of files from torrent info
paths = {}
@ -180,10 +180,10 @@ class TorrentInfo(object):
for index, f in enumerate(self.__m_metadata['info']['files']):
if 'path.utf-8' in f:
path = decode_string(os.path.join(prefix, *f['path.utf-8']))
path = decode_bytes(os.path.join(prefix, *f['path.utf-8']))
del f['path.utf-8']
else:
path = os.path.join(prefix, decode_string(os.path.join(*f['path']), self.encoding))
path = os.path.join(prefix, decode_bytes(os.path.join(*f['path']), self.encoding))
f['path'] = path
f['index'] = index
if 'sha1' in f and len(f['sha1']) == 20:

2
deluge/ui/console/modes/addtorrents.py
View File

@ -181,7 +181,7 @@ class AddTorrents(BaseMode):
self.formatted_rows = []
for row in self.raw_rows:
filename = deluge.common.decode_string(row[0])
filename = deluge.common.decode_bytes(row[0])
size = row[1]
time = row[2]

4
deluge/ui/gtkui/ipcinterface.py
View File

@ -23,7 +23,7 @@ from twisted.internet import reactor
from twisted.internet.protocol import ClientFactory, Factory, Protocol, connectionDone
import deluge.component as component
from deluge.common import decode_string, is_magnet, is_url, windows_check
from deluge.common import decode_bytes, is_magnet, is_url, windows_check
from deluge.configmanager import ConfigManager, get_config_dir
from deluge.ui.client import client
@ -206,7 +206,7 @@ def process_args(args):
log.debug('Attempting to add file (%s) from external source...', arg)
if urlparse(arg).scheme == 'file':
arg = url2pathname(urlparse(arg).path)
path = os.path.abspath(decode_string(arg))
path = os.path.abspath(decode_bytes(arg))
if not os.path.exists(path):
log.error('No such file: %s', path)

2
deluge/ui/gtkui/preferences.py
View File

@ -910,7 +910,7 @@ class Preferences(component.Component):
response = chooser.run()
if response == gtk.RESPONSE_OK:
filepath = deluge.common.decode_string(chooser.get_filename())
filepath = deluge.common.decode_bytes(chooser.get_filename())
else:
chooser.destroy()
return