[#3244|Web] Add support for accept-encoding header

* Use EncodingResourceWrapper to replace compress function so that the
proper checks for accept-encoding header are made.
* Ensure only text is compressed and images are left uncompressed.

deluge/tests/common_web.py

@@ -67,10 +67,3 @@ class WebServerMockBase(object):
             pass
 
         self.patch(auth, 'check_request', check_request)
-
-    def mock_compress_body(self):
-        def compress(contents, request):
-            return contents
-
-        # Patch compress to avoid having to decompress output with zlib
-        self.patch(deluge.ui.web.json_api, 'compress', compress)

deluge/tests/test_core.py

@@ -16,8 +16,8 @@ from twisted.internet import defer, reactor, task
 from twisted.internet.error import CannotListenError
 from twisted.python.failure import Failure
 from twisted.web.http import FORBIDDEN
-from twisted.web.resource import Resource
+from twisted.web.resource import EncodingResourceWrapper, Resource
-from twisted.web.server import Site
+from twisted.web.server import GzipEncoderFactory, Site
 from twisted.web.static import File
 
 import deluge.common
@@ -26,7 +26,6 @@ import deluge.core.torrent
 from deluge.core.core import Core
 from deluge.core.rpcserver import RPCServer
 from deluge.error import AddTorrentError, InvalidTorrentError
-from deluge.ui.web.common import compress
 
 from . import common
 from .basetest import BaseTestCase
@@ -49,6 +48,9 @@ class CookieResource(Resource):
 
 
 class PartialDownload(Resource):
+    def getChild(self, path, request):  # NOQA: N802
+        return EncodingResourceWrapper(self, [GzipEncoderFactory()])
+
     def render(self, request):
         with open(
             common.get_test_data_file('ubuntu-9.04-desktop-i386.iso.torrent'), 'rb'
@@ -56,8 +58,6 @@ class PartialDownload(Resource):
             data = _file.read()
         request.setHeader(b'Content-Length', str(len(data)))
         request.setHeader(b'Content-Type', b'application/x-bittorrent')
-        if request.requestHeaders.hasHeader('accept-encoding'):
-            return compress(data, request)
         return data
 
 

deluge/tests/test_httpdownloader.py

@@ -16,14 +16,13 @@ from twisted.python.failure import Failure
 from twisted.trial import unittest
 from twisted.web.error import PageRedirect
 from twisted.web.http import NOT_MODIFIED
-from twisted.web.resource import Resource
+from twisted.web.resource import EncodingResourceWrapper, Resource
-from twisted.web.server import Site
+from twisted.web.server import GzipEncoderFactory, Site
 from twisted.web.util import redirectTo
 
 from deluge.common import windows_check
 from deluge.httpdownloader import download_file
 from deluge.log import setup_logger
-from deluge.ui.web.common import compress
 
 temp_dir = tempfile.mkdtemp()
 
@@ -66,10 +65,13 @@ class CookieResource(Resource):
 
 
 class GzipResource(Resource):
+    def getChild(self, path, request):  # NOQA: N802
+        return EncodingResourceWrapper(self, [GzipEncoderFactory()])
+
     def render(self, request):
         message = request.args.get(b'msg', [b'EFFICIENCY!'])[0]
         request.setHeader(b'Content-Type', b'text/plain')
-        return compress(message, request)
+        return message
 
 
 class PartialDownloadResource(Resource):
@@ -227,13 +229,9 @@ class DownloadFileTestCase(unittest.TestCase):
         return d
 
     def test_download_with_gzip_encoding_disabled(self):
-        url = self.get_url('gzip?msg=fail')
+        url = self.get_url('gzip?msg=unzip')
         d = download_file(url, fname('gzip_encoded'), allow_compression=False)
-
+        d.addCallback(self.assertContains, 'unzip')
-        def cb(result):
-            print(result)
-
-        d.addCallback(self.assertNotContains, b'fail', file_mode='rb')
         return d
 
     def test_page_redirect_unhandled(self):

deluge/tests/test_json_api.py

@@ -79,11 +79,6 @@ class JSONTestCase(JSONBase):
         request = MagicMock()
         request.method = b'POST'
 
-        def compress(contents, request):
-            return contents
-
-        self.patch(deluge.ui.web.json_api, 'compress', compress)
-
         def write(response_str):
             request.write_was_called = True
             response = json_lib.loads(response_str.decode())
@@ -267,7 +262,6 @@ class JSONRequestFailedTestCase(JSONBase, WebServerMockBase):
         # Circumvent authentication
         auth = Auth({})
         self.mock_authentication_ignore(auth)
-        self.mock_compress_body()
 
         def write(response_str):
             request.write_was_called = True

deluge/tests/test_webserver.py

@@ -31,7 +31,6 @@ class WebServerTestCase(WebServerTestBase, WebServerMockBase):
         agent = Agent(reactor)
 
         self.mock_authentication_ignore(self.deluge_web.auth)
-        self.mock_compress_body()
 
         # This torrent file contains an uncommon field 'filehash' which must be hex
         # encoded to allow dumping the torrent info to json. Otherwise it will fail with:

deluge/ui/web/common.py

@@ -10,7 +10,8 @@
 from __future__ import unicode_literals
 
 import gettext
-import zlib
+
+from mako.template import Template as MakoTemplate
 
 from deluge.common import PY2, get_version
 
@@ -34,22 +35,7 @@ def escape(text):
     return text
 
 
-def compress(contents, request):
+class Template(MakoTemplate):
-    request.setHeader(b'content-encoding', b'gzip')
-    compress_zlib = zlib.compressobj(
-        6, zlib.DEFLATED, zlib.MAX_WBITS + 16, zlib.DEF_MEM_LEVEL, 0
-    )
-    contents = compress_zlib.compress(contents)
-    contents += compress_zlib.flush()
-    return contents
-
-
-try:
-    # This is beeing done like this in order to allow tests to use the above
-    # `compress` without requiring Mako to be instaled
-    from mako.template import Template as MakoTemplate
-
-    class Template(MakoTemplate):
     """
     A template that adds some built-ins to the rendering
     """
@@ -60,13 +46,3 @@ try:
         data.update(self.builtins)
         rendered = MakoTemplate.render_unicode(self, *args, **data)
         return rendered.encode('utf-8')
-
-
-except ImportError:
-    import warnings
-
-    warnings.warn('The Mako library is required to run deluge.ui.web', RuntimeWarning)
-
-    class Template(object):
-        def __new__(cls, *args, **kwargs):
-            raise RuntimeError('The Mako library is required to run deluge.ui.web')

deluge/ui/web/json_api.py

@@ -32,7 +32,7 @@ from deluge.ui.coreconfig import CoreConfig
 from deluge.ui.hostlist import HostList
 from deluge.ui.sessionproxy import SessionProxy
 from deluge.ui.translations_util import get_languages
-from deluge.ui.web.common import _, compress
+from deluge.ui.web.common import _
 
 log = logging.getLogger(__name__)
 
@@ -231,7 +231,7 @@ class JSON(resource.Resource, component.Component):
             return ''
         response = json.dumps(response)
         request.setHeader(b'content-type', b'application/json')
-        request.write(compress(response.encode(), request))
+        request.write(response.encode())
         request.finish()
         return server.NOT_DONE_YET
 

deluge/ui/web/server.py

@@ -19,6 +19,7 @@ import tempfile
 from twisted.application import internet, service
 from twisted.internet import defer, reactor
 from twisted.web import http, resource, server, static
+from twisted.web.resource import EncodingResourceWrapper
 
 from deluge import common, component, configmanager
 from deluge.common import is_ipv6
@@ -27,7 +28,7 @@ from deluge.crypto_utils import get_context_factory
 from deluge.ui.tracker_icons import TrackerIcons
 from deluge.ui.translations_util import set_language, setup_translations
 from deluge.ui.web.auth import Auth
-from deluge.ui.web.common import Template, compress
+from deluge.ui.web.common import Template
 from deluge.ui.web.json_api import JSON, WebApi, WebUtils
 from deluge.ui.web.pluginmanager import PluginManager
 
@@ -80,7 +81,7 @@ class GetText(resource.Resource):
     def render(self, request):
         request.setHeader(b'content-type', b'text/javascript; encoding=utf-8')
         template = Template(filename=rpath('js', 'gettext.js'))
-        return compress(template.render(), request)
+        return template.render()
 
 
 class MockGetText(resource.Resource):
@@ -93,8 +94,7 @@ class MockGetText(resource.Resource):
 
     def render(self, request):
         request.setHeader(b'content-type', b'text/javascript; encoding=utf-8')
-        data = b'function _(string) { return string; }'
+        return b'function _(string) { return string; }'
-        return compress(data, request)
 
 
 class Upload(resource.Resource):
@@ -131,9 +131,8 @@ class Upload(resource.Resource):
 
         request.setHeader(b'content-type', b'text/html')
         request.setResponseCode(http.OK)
-        return compress(
+        return json.dumps({'success': bool(filenames), 'files': filenames}).encode(
-            json.dumps({'success': bool(filenames), 'files': filenames}).encode('utf8'),
+            'utf8'
-            request,
         )
 
 
@@ -145,7 +144,7 @@ class Render(resource.Resource):
 
     def getChild(self, path, request):  # NOQA: N802
         request.render_file = path
-        return self
+        return EncodingResourceWrapper(self, [server.GzipEncoderFactory()])
 
     def render(self, request):
         log.debug('Render template file: %s', request.render_file)
@@ -163,7 +162,7 @@ class Render(resource.Resource):
             tpl_file = '404.html'
 
         template = Template(filename=rpath(os.path.join('render', tpl_file)))
-        return compress(template.render(), request)
+        return template.render()
 
 
 class Tracker(resource.Resource):
@@ -242,6 +241,10 @@ class LookupResource(resource.Resource, component.Component):
             request.lookup_path = os.path.join(request.lookup_path, path)
         else:
             request.lookup_path = path
+
+        if request.uri.endswith(b'css'):
+            return EncodingResourceWrapper(self, [server.GzipEncoderFactory()])
+        else:
             return self
 
     def render(self, request):
@@ -258,12 +261,12 @@ class LookupResource(resource.Resource, component.Component):
                     request.setHeader(b'content-type', mime_type[0].encode())
                     with open(path, 'rb') as _file:
                         data = _file.read()
-                    return compress(data, request)
+                    return data
 
         request.setResponseCode(http.NOT_FOUND)
         request.setHeader(b'content-type', b'text/html')
         template = Template(filename=rpath(os.path.join('render', '404.html')))
-        return compress(template.render(), request)
+        return template.render()
 
 
 class ScriptResource(resource.Resource, component.Component):
@@ -404,7 +407,7 @@ class ScriptResource(resource.Resource, component.Component):
             request.lookup_path += b'/' + path
         else:
             request.lookup_path = path
-        return self
+        return EncodingResourceWrapper(self, [server.GzipEncoderFactory()])
 
     def render(self, request):
         log.debug('Requested path: %s', request.lookup_path)
@@ -429,12 +432,21 @@ class ScriptResource(resource.Resource, component.Component):
                 request.setHeader(b'content-type', mime_type[0].encode())
                 with open(path, 'rb') as _file:
                     data = _file.read()
-                return compress(data, request)
+                return data
 
         request.setResponseCode(http.NOT_FOUND)
         request.setHeader(b'content-type', b'text/html')
         template = Template(filename=rpath(os.path.join('render', '404.html')))
-        return compress(template.render(), request)
+        return template.render()
+
+
+class Themes(static.File):
+    def getChild(self, path, request):  # NOQA: N802
+        child = static.File.getChild(self, path, request)
+        if request.uri.endswith(b'css'):
+            return EncodingResourceWrapper(child, [server.GzipEncoderFactory()])
+        else:
+            return child
 
 
 class TopLevel(resource.Resource):
@@ -450,7 +462,10 @@ class TopLevel(resource.Resource):
 
         self.putChild(b'css', LookupResource('Css', rpath('css')))
         if os.path.isfile(rpath('js', 'gettext.js')):
-            self.putChild(b'gettext.js', GetText())
+            self.putChild(
+                b'gettext.js',
+                EncodingResourceWrapper(GetText(), [server.GzipEncoderFactory()]),
+            )
         else:
             log.warning(
                 'Cannot find "gettext.js" translation file!'
@@ -505,10 +520,14 @@ class TopLevel(resource.Resource):
 
         self.js = js
         self.putChild(b'js', js)
-        self.putChild(b'json', JSON())
+        self.putChild(
-        self.putChild(b'upload', Upload())
+            b'json', EncodingResourceWrapper(JSON(), [server.GzipEncoderFactory()])
+        )
+        self.putChild(
+            b'upload', EncodingResourceWrapper(Upload(), [server.GzipEncoderFactory()])
+        )
         self.putChild(b'render', Render())
-        self.putChild(b'themes', static.File(rpath('themes')))
+        self.putChild(b'themes', Themes(rpath('themes')))
         self.putChild(b'tracker', Tracker())
 
         theme = component.get('DelugeWeb').config['theme']

setup.cfg

@@ -26,7 +26,7 @@ known_third_party =
     cairo, gi,
 # Ignore other module dependencies for pre-commit isort.
     twisted, OpenSSL, pytest, recommonmark, chardet, pkg_resources, zope, mock,
-    sphinx, rencode, six
+    sphinx, rencode, six, mako
 known_first_party = msgfmt, deluge
 order_by_type = true
 not_skip = __init__.py