codex-storage
/
deluge
mirror of https://github.com/codex-storage/deluge.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Begin rewrite of blocklist (core) using twisted.

This commit is contained in:
John Garland 2009-07-02 15:29:57 +00:00
parent d317d5c857
commit 8e69a82881
1 changed files with 124 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

295
deluge/plugins/blocklist/blocklist/core.py
View File

@ -2,6 +2,7 @@
# core.py # core.py
# #
# Copyright (C) 2008 Andrew Resch <andrewresch@gmail.com> # Copyright (C) 2008 Andrew Resch <andrewresch@gmail.com>
# Copyright (C) 2009 John Garland <johnnybg@gmail.com>
# #
# Deluge is free software. # Deluge is free software.
# #
@ -33,20 +34,20 @@
# #
# #
import urllib
import os import os
import datetime import datetime
import time
import shutil import shutil
from twisted.internet.task import LoopingCall from twisted.internet.task import LoopingCall
from twisted.internet import reactor from twisted.internet import reactor, threads
from twisted.web import error
from deluge.log import LOG as log from deluge.log import LOG as log
from deluge.plugins.pluginbase import CorePluginBase from deluge.plugins.pluginbase import CorePluginBase
import deluge.component as component import deluge.component as component
import deluge.configmanager import deluge.configmanager
from deluge.core.rpcserver import export from deluge.core.rpcserver import export
from deluge.httpdownloader import download_file
from peerguardian import PGReader, PGException from peerguardian import PGReader, PGException
from text import TextReader, GZMuleReader, PGZip, PGTextReaderGzip from text import TextReader, GZMuleReader, PGZip, PGTextReaderGzip
@ -55,13 +56,11 @@ DEFAULT_PREFS = {
"url": "http://deluge-torrent.org/blocklist/nipfilter.dat.gz", "url": "http://deluge-torrent.org/blocklist/nipfilter.dat.gz",
"load_on_start": False, "load_on_start": False,
"check_after_days": 4, "check_after_days": 4,
"listtype": "gzmule", "list_type": "gzmule",
"last_update": "",
"list_size": 0,
"timeout": 180, "timeout": 180,
"try_times": 3, "try_times": 3,
"file_type": "",
"file_url": "",
"file_date": "",
"file_size": 0,
} }
FORMATS = { FORMATS = {
@ -76,8 +75,6 @@ class Core(CorePluginBase):
def enable(self): def enable(self):
log.debug('Blocklist: Plugin enabled..') log.debug('Blocklist: Plugin enabled..')
self.is_downloading = False
self.is_importing = False
self.has_imported = False self.has_imported = False
self.up_to_date = False self.up_to_date = False
self.num_blocked = 0 self.num_blocked = 0
@ -87,11 +84,14 @@ class Core(CorePluginBase):
self.config = deluge.configmanager.ConfigManager("blocklist.conf", DEFAULT_PREFS) self.config = deluge.configmanager.ConfigManager("blocklist.conf", DEFAULT_PREFS)
if self.config["load_on_start"]: if self.config["load_on_start"]:
self.import_list() # TODO: Check if been more than check_after_days
self.use_cache = True
d = self.import_list()
d.addCallbacks(self.on_import_complete, self.on_import_error)
# This function is called every 'check_after_days' days, to download # This function is called every 'check_after_days' days, to download
# and import a new list if needed. # and import a new list if needed.
self.update_timer = LoopingCall(self.download_blocklist) self.update_timer = LoopingCall(self.check_import)
self.update_timer.start(self.config["check_after_days"] * 24 * 60 * 60) self.update_timer.start(self.config["check_after_days"] * 24 * 60 * 60)
def disable(self): def disable(self):
@ -105,15 +105,22 @@ class Core(CorePluginBase):
## Exported RPC methods ### ## Exported RPC methods ###
@export() @export()
def download_list(self, _import=False): def check_import(self, force=False):
"""Download the blocklist specified in the config as url""" """Imports latest blocklist specified by blocklist url.
self.download_blocklist(_import) Only downloads/imports if necessary or forced."""
@export() # Reset variables
def import_list(self, force=False): self.force_download = force
"""Import the blocklist from the blocklist.cache, if load is True, then self.use_cache = False
it will download the blocklist file if needed.""" self.failed_attempts = 0
reactor.callInThread(self.import_blocklist, force=force)
# Start callback chain
d = self.download_list()
d.addCallbacks(self.on_download_complete, self.on_download_error)
d.addCallback(self.import_list)
d.addCallbacks(self.on_import_complete, self.on_import_error)
return d
@export() @export()
def get_config(self): def get_config(self):
@ -137,97 +144,27 @@ class Core(CorePluginBase):
else: else:
status["state"] = "Idle" status["state"] = "Idle"
status["up_to_date"] = self.up_to_date
status["num_blocked"] = self.num_blocked status["num_blocked"] = self.num_blocked
status["file_progress"] = self.file_progress status["file_progress"] = self.file_progress
status["file_type"] = self.config["file_type"] status["file_type"] = self.config["list_type"]
status["file_url"] = self.config["file_url"] status["file_url"] = self.config["url"]
status["file_size"] = self.config["file_size"] status["file_size"] = self.config["list_size"]
status["file_date"] = self.config["file_date"] status["file_date"] = self.config["last_update"]
return status return status
#### ####
def update_info(self, blocklist):
"""Updates blocklist info"""
self.config["last_update"] = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT")
self.config["list_size"] = os.path.getsize(blocklist)
def on_download_blocklist(self, load): def download_list(self, url=None):
self.is_downloading = False
if load:
self.import_list()
def import_blocklist(self, force=False):
"""Imports the downloaded blocklist into the session"""
if self.is_downloading:
return
if force or self.need_new_blocklist():
self.download_blocklist(True)
return
# If we have a newly downloaded file, lets try that before the .cache
if os.path.exists(deluge.configmanager.get_config_dir("blocklist.download")):
bl_file = deluge.configmanager.get_config_dir("blocklist.download")
using_download = True
elif self.has_imported:
# Blocklist is up to date so doesn't need to be imported
log.debug("Latest blocklist is already imported")
return
else:
bl_file = deluge.configmanager.get_config_dir("blocklist.cache")
using_download = False
self.is_importing = True
log.debug("Reset IP Filter..")
component.get("Core").reset_ip_filter()
self.num_blocked = 0
# Open the file for reading
try:
read_list = FORMATS[self.config["listtype"]][1](bl_file)
except Exception, e:
log.debug("Unable to read blocklist file: %s", e)
self.is_importing = False
return
try:
log.debug("Blocklist import starting..")
ips = read_list.next()
while ips:
self.core.block_ip_range(ips)
self.num_blocked += 1
ips = read_list.next()
read_list.close()
except Exception, e:
log.debug("Exception during import: %s", e)
else:
log.debug("Blocklist import complete!")
# The import was successful so lets move this to blocklist.cache
if using_download:
log.debug("Moving blocklist.download to blocklist.cache")
shutil.move(bl_file, deluge.configmanager.get_config_dir("blocklist.cache"))
# Set information about the file
self.config["file_type"] = self.config["listtype"]
self.config["file_url"] = self.config["url"]
self.is_importing = False
self.has_imported = True
def download_blocklist(self, load=False):
"""Runs download_blocklist_thread() in a thread and calls on_download_blocklist
when finished. If load is True, then we will import the blocklist
upon download completion."""
if self.is_importing:
return
self.is_downloading = True
reactor.callInThread(self.download_blocklist_thread, self.on_download_blocklist, load)
def download_blocklist_thread(self, callback, load):
"""Downloads the blocklist specified by 'url' in the config""" """Downloads the blocklist specified by 'url' in the config"""
def on_retrieve_data(count, block_size, total_blocks): def on_retrieve_data(data, current_length, total_length):
if total_blocks: if total_length:
fp = float(count * block_size) / total_blocks fp = float(current_length) / total_length
if fp > 1.0: if fp > 1.0:
fp = 1.0 fp = 1.0
else: else:
@ -238,78 +175,100 @@ class Core(CorePluginBase):
import socket import socket
socket.setdefaulttimeout(self.config["timeout"]) socket.setdefaulttimeout(self.config["timeout"])
for i in xrange(self.config["try_times"]): headers = {}
log.debug("Attempting to download blocklist %s", self.config["url"]) if not url:
try: url = self.config["url"]
(filename, headers) = urllib.urlretrieve(
self.config["url"], blocklist = deluge.configmanager.get_config_dir("blocklist.cache")
deluge.configmanager.get_config_dir("blocklist.download"), if os.path.exists(blocklist) and not self.force_download:
on_retrieve_data) last_modified = datetime.datetime.utcfromtimestamp(os.path.getmtime(blocklist))
except Exception, e: headers['If-Modified-Since'] = last_modified.strftime("%a, %d %b %Y %H:%M:%S GMT")
log.debug("Error downloading blocklist: %s", e)
os.remove(deluge.configmanager.get_config_dir("blocklist.download")) log.debug("Attempting to download blocklist %s", url)
continue self.is_downloading = True
return download_file(url, deluge.configmanager.get_config_dir("blocklist.download"), headers)
def on_download_complete(self, result):
"""Runs any download clean up functions"""
log.debug("Blocklist download complete!")
self.is_downloading = False
return threads.deferToThread(self.update_info,
deluge.configmanager.ConfigManager("blocklist.download"))
def on_download_error(self, f):
"""Recovers from download error"""
self.is_downloading = False
error_msg = f.getErrorMessage()
d = None
if f.check(error.PageRedirect):
# Handle redirect errors
location = error_msg.split(" to ")[1]
if "Moved Permanently" in error:
log.debug("Setting blocklist url to %s" % location)
self.config["url"] = location
f.trap(f.type)
d = self.download_list(url=location)
d.addCallbacks(self.on_download_complete, self.on_download_error)
else: else:
log.debug("Blocklist successfully downloaded..") if "Not Modified" in error_msg:
self.config["file_date"] = datetime.datetime.strptime(headers["last-modified"],"%a, %d %b %Y %H:%M:%S GMT").ctime() log.debug("Blocklist is up-to-date!")
self.config["file_size"] = long(headers["content-length"]) d = threads.deferToThread(update_info,
reactor.callLater(0, callback, load) deluge.configmanager.ConfigManager("blocklist.cache"))
return self.use_cache = True
f.trap(f.type)
elif self.failed_attempts < self.config["try_times"]:
log.warning("Blocklist download failed!")
self.failed_attempts += 1
f.trap(f.type)
return d
def need_new_blocklist(self): def import_list(self, force=False):
"""Returns True if a new blocklist file should be downloaded""" """Imports the downloaded blocklist into the session"""
if self.use_cache and self.has_imported:
# Assume blocklist is not up to date log.debug("Latest blocklist is already imported")
self.up_to_date = False
# Check to see if we've just downloaded a new blocklist
if os.path.exists(deluge.configmanager.get_config_dir("blocklist.download")):
log.debug("New blocklist waiting to be imported")
return False
if os.path.exists(deluge.configmanager.get_config_dir("blocklist.cache")):
# Check current block lists time stamp and decide if it needs to be replaced
list_stats = os.stat(deluge.configmanager.get_config_dir("blocklist.cache"))
list_size = long(list_stats.st_size)
list_checked = datetime.datetime.fromtimestamp(list_stats.st_mtime)
try:
list_time = datetime.datetime.strptime(self.config["file_date"], "%a %b %d %H:%M:%S %Y")
except:
list_time = list_checked
current_time = datetime.datetime.today()
else:
log.debug("Blocklist doesn't exist")
return True return True
# If local blocklist file exists but nothing is in it self.is_importing = True
if list_size == 0: log.debug("Reset IP Filter..")
log.debug("Empty blocklist") # Does this return a deferred?
return True self.core.reset_ip_filter()
# If blocklist has just started up, check for updates if over x days self.num_blocked = 0
if not self.has_imported and current_time < (list_checked + datetime.timedelta(days=self.config["check_after_days"])):
log.debug("Blocklist doesn't need checking yet")
return False
import socket # TODO: Make non-blocking (use deferToThread)
socket.setdefaulttimeout(self.config["timeout"])
try: # Open the file for reading
# Get remote blocklist time stamp and size read_list = FORMATS[self.config["listtype"]][1](bl_file)
remote_stats = urllib.urlopen(self.config["url"]).info() log.debug("Blocklist import starting..")
remote_size = long(remote_stats["content-length"]) ips = read_list.next()
remote_time = datetime.datetime.strptime(remote_stats["last-modified"],"%a, %d %b %Y %H:%M:%S GMT") while ips:
except Exception, e: self.core.block_ip_range(ips)
log.debug("Unable to get blocklist stats: %s", e) self.num_blocked += 1
return False ips = read_list.next()
read_list.close()
# Check if remote blocklist is newer (in date or size) def on_import_complete(self, result):
if list_time < remote_time or list_size < remote_size: """Runs any import clean up functions"""
log.debug("Newer blocklist exists (%s & %d vs %s & %d)", remote_time, remote_size, list_time, list_size) d = None
return True self.is_importing = False
self.has_imported = True
log.debug("Blocklist import complete!")
# Move downloaded blocklist to cache
if not self.use_cache:
d = threads.deferToThread(shutil.move,
deluge.configmanager.ConfigManager("blocklist.download"),
deluge.configmanager.ConfigManager("blocklist.cache"))
return d
# Update last modified time of blocklist def on_import_error(self, f):
os.utime(deluge.configmanager.get_config_dir("blocklist.cache"), None) """Recovers from import error"""
self.up_to_date = True d = None
log.debug("Blocklist is up to date") self.is_importing = False
return False blocklist = deluge.configmanager.get_config_dir("blocklist.cache")
# If we have a backup and we haven't already used it
if os.path.exists(blocklist) and not self.use_cache:
e = f.trap(error.Error, IOError, TextException, PGException)
log.warning("Error reading blocklist: ", e)
d = self.import_list()
d.addCallbacks(on_import_complete, on_import_error)
return d