From c8718ad643d00c7d166dd1314f385572dc4788a9 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 15 Feb 2012 14:21:02 +0000 Subject: [PATCH] Implemented #1382: * Implemented whitelist support to both core and GTK UI. * Implemented ip filter cleaning before each update. Restarting the deluge daemon is no longer needed. * If "check_after_days" is 0(zero), the timer is not started anymore. It would keep updating one call after the other. If the value changed, the timer is now stopped and restarted using the new value. --- ChangeLog | 8 + deluge/plugins/Blocklist/deluge/__init__.py | 3 +- .../Blocklist/deluge/plugins/__init__.py | 3 +- .../deluge/plugins/blocklist/common.py | 100 ++++++++++ .../deluge/plugins/blocklist/core.py | 112 ++++++++++- .../blocklist/data/blocklist_pref.glade | 178 +++++++++++++++++- .../deluge/plugins/blocklist/gtkui.py | 92 +++++++-- .../deluge/plugins/blocklist/readers.py | 13 +- deluge/plugins/Blocklist/setup.py | 1 + 9 files changed, 472 insertions(+), 38 deletions(-) diff --git a/ChangeLog b/ChangeLog index deb3cc29b..a9ff6e9bc 100644 --- a/ChangeLog +++ b/ChangeLog @@ -33,6 +33,14 @@ * #378: Allow showing a pieces bar instead of a regular progress bar in a torrent's status tab. +=== Blocklist Plugin === + * #1382: Implemented whitelist support to both core and GTK UI. + * Implemented ip filter cleaning before each update. Restarting the deluge + daemon is no longer needed. + * If "check_after_days" is 0(zero), the timer is not started anymore. It + would keep updating one call after the other. If the value changed, the + timer is now stopped and restarted using the new value. + === Deluge 1.3.4 (In Development) === ==== Core ==== * #1921: Free disk space reporting incorrectly in FreeBSD diff --git a/deluge/plugins/Blocklist/deluge/__init__.py b/deluge/plugins/Blocklist/deluge/__init__.py index 94033e829..0786b820c 100644 --- a/deluge/plugins/Blocklist/deluge/__init__.py +++ b/deluge/plugins/Blocklist/deluge/__init__.py @@ -1,3 +1,2 @@ # this is a namespace package -import pkg_resources -pkg_resources.declare_namespace(__name__) +__import__('pkg_resources').declare_namespace(__name__) diff --git a/deluge/plugins/Blocklist/deluge/plugins/__init__.py b/deluge/plugins/Blocklist/deluge/plugins/__init__.py index 94033e829..0786b820c 100644 --- a/deluge/plugins/Blocklist/deluge/plugins/__init__.py +++ b/deluge/plugins/Blocklist/deluge/plugins/__init__.py @@ -1,3 +1,2 @@ # this is a namespace package -import pkg_resources -pkg_resources.declare_namespace(__name__) +__import__('pkg_resources').declare_namespace(__name__) diff --git a/deluge/plugins/Blocklist/deluge/plugins/blocklist/common.py b/deluge/plugins/Blocklist/deluge/plugins/blocklist/common.py index 147f77d5e..9a74d40da 100644 --- a/deluge/plugins/Blocklist/deluge/plugins/blocklist/common.py +++ b/deluge/plugins/Blocklist/deluge/plugins/blocklist/common.py @@ -85,3 +85,103 @@ def remove_zeros(ip): """ return ".".join([part.lstrip("0").zfill(1) for part in ip.split(".")]) + +class BadIP(Exception): + _message = None + def __init__(self, message): + self.message = message + + def __set_message(self, message): + self._message = message + + def __get_message(self): + return self._message + + message = property(__get_message, __set_message) + del __get_message, __set_message + +class IP(object): + __slots__ = ('q1', 'q2', 'q3', 'q4', '_long') + def __init__(self, q1, q2, q3, q4): + self.q1 = q1 + self.q2 = q2 + self.q3 = q3 + self.q4 = q4 + self._long = 0 + for q in self.quadrants(): + self._long = (self._long << 8) | int(q) + + @property + def address(self): + return '.'.join([str(q) for q in [self.q1, self.q2, self.q3, self.q4]]) + + @property + def long(self): + return self._long + + @classmethod + def parse(cls, ip): + try: + q1, q2, q3, q4 = [int(q) for q in ip.split('.')] + except ValueError: + raise BadIP(_("The IP address \"%s\" is badly formed" % ip)) + if q1<0 or q2<0 or q3<0 or q4<0: + raise BadIP(_("The IP address \"%s\" is badly formed" % ip)) + elif q1>255 or q2>255 or q3>255 or q4>255: + raise BadIP(_("The IP address \"%s\" is badly formed" % ip)) + return cls(q1, q2, q3, q4) + + def quadrants(self): + return (self.q1, self.q2, self.q3, self.q4) + +# def next_ip(self): +# (q1, q2, q3, q4) = self.quadrants() +# if q4 >= 255: +# if q3 >= 255: +# if q2 >= 255: +# if q1 >= 255: +# raise BadIP(_("There ain't a next IP address")) +# q1 += 1 +# else: +# q2 += 1 +# else: +# q3 += 1 +# else: +# q4 += 1 +# return IP(q1, q2, q3, q4) +# +# def previous_ip(self): +# (q1, q2, q3, q4) = self.quadrants() +# if q4 <= 1: +# if q3 <= 1: +# if q2 <= 1: +# if q1 <= 1: +# raise BadIP(_("There ain't a previous IP address")) +# q1 -= 1 +# else: +# q2 -= 1 +# else: +# q3 -= 1 +# else: +# q4 -= 1 +# return IP(q1, q2, q3, q4) + + def __lt__(self, other): + if isinstance(other, basestring): + other = IP.parse(other) + return self.long < other.long + + def __gt__(self, other): + if isinstance(other, basestring): + other = IP.parse(other) + return self.long > other.long + + def __eq__(self, other): + if isinstance(other, basestring): + other = IP.parse(other) + return self.long == other.long + + def __repr__(self): + return '<%s long=%s address="%s">' % ( + self.__class__.__name__, self.long, self.address + ) diff --git a/deluge/plugins/Blocklist/deluge/plugins/blocklist/core.py b/deluge/plugins/Blocklist/deluge/plugins/blocklist/core.py index 897333113..26be958f9 100644 --- a/deluge/plugins/Blocklist/deluge/plugins/blocklist/core.py +++ b/deluge/plugins/Blocklist/deluge/plugins/blocklist/core.py @@ -46,13 +46,13 @@ from twisted.internet.task import LoopingCall from twisted.internet import threads, defer from twisted.web import error - from deluge.plugins.pluginbase import CorePluginBase import deluge.component as component import deluge.configmanager from deluge.common import is_url from deluge.core.rpcserver import export from deluge.httpdownloader import download_file +from common import IP, BadIP from detect import detect_compression, detect_format, create_reader, UnknownFormatError from readers import ReaderParseError @@ -71,14 +71,16 @@ DEFAULT_PREFS = { "list_size": 0, "timeout": 180, "try_times": 3, + "whitelisted": [], } # Constants +ALLOW_RANGE = 0 BLOCK_RANGE = 1 class Core(CorePluginBase): def enable(self): - log.debug('Blocklist: Plugin enabled..') + log.debug('Blocklist: Plugin enabled...') self.is_url = True self.is_downloading = False @@ -86,11 +88,14 @@ class Core(CorePluginBase): self.has_imported = False self.up_to_date = False self.need_to_resume_session = False + self.num_whited = 0 self.num_blocked = 0 self.file_progress = 0.0 self.core = component.get("Core") self.config = deluge.configmanager.ConfigManager("blocklist.conf", DEFAULT_PREFS) + if "whitelisted" not in self.config: + self.config["whitelisted"] = [] self.reader = create_reader(self.config["list_type"], self.config["list_compression"]) @@ -114,10 +119,17 @@ class Core(CorePluginBase): # This function is called every 'check_after_days' days, to download # and import a new list if needed. self.update_timer = LoopingCall(self.check_import) - self.update_timer.start(self.config["check_after_days"] * 24 * 60 * 60, update_now) + if self.config["check_after_days"] > 0: + self.update_timer.start( + self.config["check_after_days"] * 24 * 60 * 60, update_now + ) def disable(self): self.config.save() + log.debug("Reset IP filter") + self.core.session.get_ip_filter().add_rule( + "0.0.0.0", "255.255.255.255", ALLOW_RANGE + ) log.debug('Blocklist: Plugin disabled') def update(self): @@ -136,7 +148,6 @@ class Core(CorePluginBase): :rtype: Deferred """ - # Reset variables self.filename = None self.force_download = force @@ -178,9 +189,67 @@ class Core(CorePluginBase): :param config: config to set :type config: dictionary """ + needs_blocklist_import = False for key in config.keys(): + if key == 'whitelisted': + saved = set(self.config[key]) + update = set(config[key]) + diff = saved.symmetric_difference(update) + if diff: + log.debug("Whitelist changed. Updating...") + added = update.intersection(diff) + removed = saved.intersection(diff) + if added: + for ip in added: + try: + ip = IP.parse(ip) + self.blocklist.add_rule( + ip.address, ip.address, ALLOW_RANGE + ) + saved.add(ip.address) + log.debug("Added %s to whitelisted", ip) + self.num_whited += 1 + except BadIP, e: + log.error("Bad IP: %s", e) + continue + if removed: + needs_blocklist_import = True + for ip in removed: + try: + ip = IP.parse(ip) + saved.remove(ip.address) + log.debug("Removed %s from whitelisted", ip) + except BadIP, e: + log.error("Bad IP: %s", e) + continue + + self.config[key] = list(saved) + continue + elif key == "check_after_days": + if self.config[key] != config[key]: + self.config[key] = config[key] + update_now = False + if self.config["last_update"]: + last_update = datetime.fromtimestamp(self.config["last_update"]) + check_period = timedelta(days=self.config["check_after_days"]) + if not self.config["last_update"] or last_update + check_period < datetime.now(): + update_now = True + self.update_timer.running and self.update_timer.stop() + if self.config["check_after_days"] > 0: + self.update_timer.start( + self.config["check_after_days"] * 24 * 60 * 60, update_now + ) + continue self.config[key] = config[key] + if needs_blocklist_import: + log.debug("IP addresses were removed from the whitelist. Since we " + "don't know if they were blocked before. Re-import " + "current blocklist and re-add whitelisted.") + self.has_imported = False + d = self.import_list(deluge.configmanager.get_config_dir("blocklist.cache")) + d.addCallbacks(self.on_import_complete, self.on_import_error) + @export def get_status(self): """ @@ -198,15 +267,16 @@ class Core(CorePluginBase): status["state"] = "Idle" status["up_to_date"] = self.up_to_date + status["num_whited"] = self.num_whited status["num_blocked"] = self.num_blocked status["file_progress"] = self.file_progress status["file_url"] = self.config["url"] status["file_size"] = self.config["list_size"] status["file_date"] = self.config["last_update"] status["file_type"] = self.config["list_type"] + status["whitelisted"] = self.config["whitelisted"] if self.config["list_compression"]: status["file_type"] += " (%s)" % self.config["list_compression"] - return status #### @@ -258,7 +328,10 @@ class Core(CorePluginBase): log.debug("Attempting to download blocklist %s", url) log.debug("Sending headers: %s", headers) self.is_downloading = True - return download_file(url, deluge.configmanager.get_config_dir("blocklist.download"), on_retrieve_data, headers) + return download_file( + url, deluge.configmanager.get_config_dir("blocklist.download"), + on_retrieve_data, headers + ) def on_download_complete(self, blocklist): """ @@ -318,13 +391,24 @@ class Core(CorePluginBase): :returns: a Deferred that fires when the blocklist has been imported :rtype: Deferred """ + log.trace("on import_list") def on_read_ip_range(start, end): """Add ip range to blocklist""" - self.blocklist.add_rule(start, end, BLOCK_RANGE) +# log.trace("Adding ip range %s - %s to ipfilter as blocked", start, end) + self.blocklist.add_rule(start.address, end.address, BLOCK_RANGE) self.num_blocked += 1 def on_finish_read(result): - """Add blocklist to session""" + """Add any whitelisted IP's and add the blocklist to session""" + # White listing happens last because the last rules added have + # priority + log.info("Added %d ranges to ipfilter as blocked", self.num_blocked) + for ip in self.config["whitelisted"]: + ip = IP.parse(ip) + self.blocklist.add_rule(ip.address, ip.address, ALLOW_RANGE) + self.num_whited += 1 + log.trace("Added %s to the ipfiler as white-listed", ip.address) + log.info("Added %d ranges to ipfilter as white-listed", self.num_whited) self.core.session.set_ip_filter(self.blocklist) return result @@ -335,6 +419,7 @@ class Core(CorePluginBase): self.is_importing = True self.num_blocked = 0 + self.num_whited = 0 self.blocklist = self.core.session.get_ip_filter() if not blocklist: @@ -344,10 +429,16 @@ class Core(CorePluginBase): self.auto_detect(blocklist) self.auto_detected = True + def on_reader_failure(failure): + log.error("Failed to read!!!!!!") + log.exception(failure) + log.debug("Importing using reader: %s", self.reader) log.debug("Reader type: %s compression: %s", self.config["list_type"], self.config["list_compression"]) + log.debug("Clearing current ip filtering") +# self.blocklist.add_rule("0.0.0.0", "255.255.255.255", ALLOW_RANGE) d = threads.deferToThread(self.reader(blocklist).read, on_read_ip_range) - d.addCallback(on_finish_read) + d.addCallback(on_finish_read).addErrback(on_reader_failure) return d @@ -360,6 +451,7 @@ class Core(CorePluginBase): :returns: a Deferred that fires when clean up is done :rtype: Deferred """ + log.trace("on_import_list_complete") d = blocklist self.is_importing = False self.has_imported = True @@ -384,6 +476,7 @@ class Core(CorePluginBase): else the original failure :rtype: Deferred or Failure """ + log.trace("on_import_error: %s", f) d = f self.is_importing = False try_again = False @@ -424,6 +517,7 @@ class Core(CorePluginBase): else: self.reader = create_reader(self.config["list_type"], self.config["list_compression"]) + def pause_session(self): if not self.core.session.is_paused(): self.core.session.pause() diff --git a/deluge/plugins/Blocklist/deluge/plugins/blocklist/data/blocklist_pref.glade b/deluge/plugins/Blocklist/deluge/plugins/blocklist/data/blocklist_pref.glade index 98de5500c..914587204 100644 --- a/deluge/plugins/Blocklist/deluge/plugins/blocklist/data/blocklist_pref.glade +++ b/deluge/plugins/Blocklist/deluge/plugins/blocklist/data/blocklist_pref.glade @@ -1,29 +1,34 @@ - + - + + False True - vertical + False 5 True + False 0 none True + False 12 True + False 5 True + False URL: @@ -36,9 +41,15 @@ True True - ● + ● + False + False + True + True + True + True 1 @@ -49,6 +60,7 @@ True + False 5 <b>General</b> True @@ -67,26 +79,30 @@ True + False 0 none True + False 12 True - vertical + False 5 True + False 3 5 5 True + False 0 Days @@ -101,7 +117,11 @@ True True - 2 0 100 1 10 0 + False + False + True + True + 1 1 100 1 10 0 1 @@ -113,6 +133,7 @@ True + False 0 Check for new list every: @@ -134,6 +155,7 @@ True True False + False True @@ -149,6 +171,7 @@ True + False 5 <b>Settings</b> True @@ -167,35 +190,41 @@ True + False 0 none True + False 0 0 12 True + False True - vertical + False True True True Download the blocklist file if necessary and import the file. - + False + True + False 5 True + False gtk-missing-image @@ -207,6 +236,7 @@ True + False Check Download and Import @@ -230,14 +260,17 @@ True True Download a new blocklist file and import it. - + False + True + False 5 True + False gtk-missing-image @@ -249,6 +282,7 @@ True + False Force Download and Import @@ -268,17 +302,22 @@ + True + True 0 + False Blocklist is up to date 0.15000000596046448 2 gtk-yes + True + True 1 @@ -289,6 +328,7 @@ True + False 5 <b>Options</b> True @@ -307,34 +347,41 @@ True + False 0 none True + False 5 12 True - vertical + False True + False + True + True 0 True + False 4 2 5 True + False 0 @@ -347,6 +394,7 @@ True + False 0 @@ -359,6 +407,7 @@ True + False 0 @@ -371,6 +420,7 @@ True + False 0 @@ -381,6 +431,7 @@ True + False 0 URL: @@ -393,6 +444,7 @@ True + False 0 Type: @@ -405,6 +457,7 @@ True + False 0 Date: @@ -417,6 +470,7 @@ True + False 0 File Size: @@ -438,6 +492,7 @@ True + False <b>Info</b> True @@ -452,6 +507,109 @@ 3 + + + True + False + 0 + none + + + True + False + 12 + + + True + False + + + True + True + automatic + automatic + + + True + True + False + False + + + + + True + True + 0 + + + + + True + False + True + start + + + gtk-add + True + True + True + False + True + + + + False + False + 0 + + + + + gtk-delete + True + True + True + False + True + + + + False + False + 1 + + + + + False + True + 1 + + + + + + + + + True + False + <b>Whitelist</b> + True + + + label_item + + + + + True + True + 4 + + diff --git a/deluge/plugins/Blocklist/deluge/plugins/blocklist/gtkui.py b/deluge/plugins/Blocklist/deluge/plugins/blocklist/gtkui.py index 7ac58370f..5bfcab410 100644 --- a/deluge/plugins/Blocklist/deluge/plugins/blocklist/gtkui.py +++ b/deluge/plugins/Blocklist/deluge/plugins/blocklist/gtkui.py @@ -51,13 +51,18 @@ class GtkUI(GtkPluginBase): log.debug("Blocklist GtkUI enable..") self.plugin = component.get("PluginManager") - self.load_preferences_page() + try: + self.load_preferences_page() + except Exception, err: + log.exception(err) + raise self.status_item = component.get("StatusBar").add_item( image=common.get_resource("blocklist16.png"), text="", callback=self._on_status_item_clicked, - tooltip="Blocked IP Ranges") + tooltip=_("Blocked IP Ranges /Whitelisted IP Ranges") + ) # Register some hooks self.plugin.register_hook("on_apply_prefs", self._on_apply_prefs) @@ -115,7 +120,8 @@ class GtkUI(GtkPluginBase): self.glade.get_widget("image_up_to_date").hide() self.table_info.show() - self.status_item.set_text(str(status["num_blocked"])) + self.status_item.set_text("%(num_blocked)s/%(num_whited)s" % status) + self.glade.get_widget("label_filesize").set_text( deluge.common.fsize(status["file_size"])) self.glade.get_widget("label_modified").set_text( @@ -128,14 +134,11 @@ class GtkUI(GtkPluginBase): def _on_show_prefs(self): def _on_get_config(config): - self.glade.get_widget("entry_url").set_text( - config["url"]) - - self.glade.get_widget("spin_check_days").set_value( - config["check_after_days"]) - - self.glade.get_widget("chk_import_on_start").set_active( - config["load_on_start"]) + log.trace("Loaded config: %s", config) + self.glade.get_widget("entry_url").set_text(config["url"]) + self.glade.get_widget("spin_check_days").set_value(config["check_after_days"]) + self.glade.get_widget("chk_import_on_start").set_active(config["load_on_start"]) + self.populate_whitelist(config["whitelisted"]) client.blocklist.get_config().addCallback(_on_get_config) @@ -144,6 +147,7 @@ class GtkUI(GtkPluginBase): config["url"] = self.glade.get_widget("entry_url").get_text() config["check_after_days"] = self.glade.get_widget("spin_check_days").get_value_as_int() config["load_on_start"] = self.glade.get_widget("chk_import_on_start").get_active() + config["whitelisted"] = [ip[0] for ip in self.whitelist_model if ip[0]!='IP HERE'] client.blocklist.set_config(config) def _on_button_check_download_clicked(self, widget): @@ -162,6 +166,7 @@ class GtkUI(GtkPluginBase): # Load the preferences page self.glade = gtk.glade.XML(common.get_resource("blocklist_pref.glade")) + self.whitelist_frame = self.glade.get_widget("whitelist_frame") self.progress_bar = self.glade.get_widget("progressbar") self.table_info = self.glade.get_widget("table_info") @@ -169,9 +174,16 @@ class GtkUI(GtkPluginBase): self.progress_bar.hide() self.table_info.show() + # Create the whitelisted model + self.build_whitelist_model_treeview() + self.glade.signal_autoconnect({ "on_button_check_download_clicked": self._on_button_check_download_clicked, - "on_button_force_download_clicked": self._on_button_force_download_clicked + "on_button_force_download_clicked": self._on_button_force_download_clicked, + 'on_whitelist_add_clicked': (self.on_add_button_clicked, + self.whitelist_treeview), + 'on_whitelist_remove_clicked': (self.on_delete_button_clicked, + self.whitelist_treeview), }) # Set button icons @@ -188,3 +200,59 @@ class GtkUI(GtkPluginBase): self.plugin.add_preferences_page( _("Blocklist"), self.glade.get_widget("blocklist_prefs_box")) + + def build_whitelist_model_treeview(self): + self.whitelist_treeview = self.glade.get_widget("whitelist_treeview") + treeview_selection = self.whitelist_treeview.get_selection() + treeview_selection.connect( + "changed", self.on_whitelist_treeview_selection_changed + ) + self.whitelist_model = gtk.ListStore(str, bool) + renderer = gtk.CellRendererText() + renderer.connect("edited", self.on_cell_edited, self.whitelist_model) + renderer.set_data("ip", 0) + + column = gtk.TreeViewColumn("IPs", renderer, text=0, editable=1) + column.set_expand(True) + self.whitelist_treeview.append_column(column) + self.whitelist_treeview.set_model(self.whitelist_model) + + def on_cell_edited(self, cell, path_string, new_text, model): +# iter = model.get_iter_from_string(path_string) +# path = model.get_path(iter)[0] + try: + ip = common.IP.parse(new_text) + model.set(model.get_iter_from_string(path_string), 0, ip.address) + except common.BadIP, e: + model.remove(model.get_iter_from_string(path_string)) + from deluge.ui.gtkui import dialogs + d = dialogs.ErrorDialog(_("Bad IP address"), e.message) + d.run() + + + def on_whitelist_treeview_selection_changed(self, selection): + model, selected_connection_iter = selection.get_selected() + if selected_connection_iter: + self.glade.get_widget("whitelist_delete").set_property('sensitive', + True) + else: + self.glade.get_widget("whitelist_delete").set_property('sensitive', + False) + + def on_add_button_clicked(self, widget, treeview): + model = treeview.get_model() + model.set(model.append(), 0, "IP HERE", 1, True) + + def on_delete_button_clicked(self, widget, treeview): + selection = treeview.get_selection() + model, iter = selection.get_selected() + if iter: +# path = model.get_path(iter)[0] + model.remove(iter) + + def populate_whitelist(self, whitelist): + self.whitelist_model.clear() + for ip in whitelist: + self.whitelist_model.set( + self.whitelist_model.append(),0, ip, 1, True + ) diff --git a/deluge/plugins/Blocklist/deluge/plugins/blocklist/readers.py b/deluge/plugins/Blocklist/deluge/plugins/blocklist/readers.py index f1d9a4da7..9cc5e2b55 100644 --- a/deluge/plugins/Blocklist/deluge/plugins/blocklist/readers.py +++ b/deluge/plugins/Blocklist/deluge/plugins/blocklist/readers.py @@ -33,9 +33,12 @@ # # -from common import raisesErrorsAs, remove_zeros +import logging +from common import raisesErrorsAs, IP, BadIP import re +log = logging.getLogger(__name__) + class ReaderParseError(Exception): pass @@ -51,12 +54,16 @@ class BaseReader(object): def parse(self, line): """Extracts ip range from given line""" - raise NotYetImplemented + raise NotImplementedError def read(self, callback): """Calls callback on each ip range in the file""" for start, end in self.readranges(): - callback(remove_zeros(start), remove_zeros(end)) + try: + callback(IP.parse(start), IP.parse(end)) + except BadIP, e: + log.error("Failed to parse IP: %s", e) +# log.exception(e) return self.file def is_ignored(self, line): diff --git a/deluge/plugins/Blocklist/setup.py b/deluge/plugins/Blocklist/setup.py index ee44297f4..d43edc2bd 100644 --- a/deluge/plugins/Blocklist/setup.py +++ b/deluge/plugins/Blocklist/setup.py @@ -51,6 +51,7 @@ setup( author_email=__author_email__, url=__url__, license=__license__, + zip_safe=False, long_description=__long_description__, packages=find_packages(),