diff --git a/ChangeLog b/ChangeLog index ce5acf3e1..f4cf546da 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,6 +2,7 @@ ==== Core ==== * Implement #1063 option to delete torrent file copy on torrent removal - patch from Ghent * Implement #457 progress bars for folders + * Implement #1012 httpdownloader supports gzip decoding * #496: Remove deprecated functions in favour of get_session_status() ==== Blocklist ==== diff --git a/deluge/httpdownloader.py b/deluge/httpdownloader.py index 6fc004ea3..bb0144d77 100644 --- a/deluge/httpdownloader.py +++ b/deluge/httpdownloader.py @@ -39,6 +39,7 @@ from twisted.internet import reactor from deluge.log import setupLogger, LOG as log from common import get_version import os.path +import zlib class HTTPDownloader(client.HTTPDownloader): """ @@ -56,8 +57,9 @@ class HTTPDownloader(client.HTTPDownloader): :param headers: any optional headers to send :type headers: dictionary """ - self.__part_callback = part_callback + self.part_callback = part_callback self.current_length = 0 + self.decoder = None self.value = filename self.force_filename = force_filename agent = "Deluge/%s (http://deluge-torrent.org)" % get_version() @@ -74,6 +76,11 @@ class HTTPDownloader(client.HTTPDownloader): else: self.total_length = 0 + if "content-encoding" in headers and headers["content-encoding"][0] in ("gzip", "x-gzip", "deflate"): + # Adding 32 to the wbits enables gzip & zlib decoding (with automatic header detection) + # Adding 16 just enables gzip decoding (no zlib) + self.decoder = zlib.decompressobj(zlib.MAX_WBITS + 32) + if "content-disposition" in headers and not self.force_filename: try: new_file_name = str(headers["content-disposition"][0]).split(";")[1].split("=")[1] @@ -95,11 +102,19 @@ class HTTPDownloader(client.HTTPDownloader): def pagePart(self, data): if self.code == http.OK: self.current_length += len(data) - if self.__part_callback: - self.__part_callback(data, self.current_length, self.total_length) + if self.decoder: + data = self.decoder.decompress(data) + if self.part_callback: + self.part_callback(data, self.current_length, self.total_length) return client.HTTPDownloader.pagePart(self, data) + def pageEnd(self): + if self.decoder: + client.HTTPDownloader.pagePart(self, self.decoder.flush()) + + return client.HTTPDownloader.pageEnd(self) + def sanitise_filename(filename): """ Sanitises a filename to use as a download destination file. @@ -132,7 +147,7 @@ def sanitise_filename(filename): return filename -def download_file(url, filename, callback=None, headers=None, force_filename=False): +def download_file(url, filename, callback=None, headers=None, force_filename=False, allow_compression=True): """ Downloads a file from a specific URL and returns a Deferred. You can also specify a callback function to be called as parts are received. @@ -149,6 +164,8 @@ def download_file(url, filename, callback=None, headers=None, force_filename=Fal :param force_filename: force us to use the filename specified rather than one the server may suggest :type force_filename: boolean + :param allow_compression: allows gzip & deflate decoding + :type allow_compression: boolean :returns: the filename of the downloaded file :rtype: Deferred @@ -163,6 +180,11 @@ def download_file(url, filename, callback=None, headers=None, force_filename=Fal for key, value in headers.items(): headers[str(key)] = str(value) + if allow_compression: + if not headers: + headers = {} + headers["accept-encoding"] = "gzip, deflate" + scheme, host, port, path = client._parse(url) factory = HTTPDownloader(url, filename, callback, headers, force_filename) if scheme == "https":