diff --git a/plugins/BlocklistImport/__init__.py b/plugins/BlocklistImport/__init__.py index 302f83029..c9e6a2de0 100644 --- a/plugins/BlocklistImport/__init__.py +++ b/plugins/BlocklistImport/__init__.py @@ -26,14 +26,15 @@ def enable(core, interface): import urllib, deluge.common, deluge.pref from peerguardian import PGReader, PGException -from text import TextReader, GZMuleReader +from text import TextReader, GZMuleReader, PGZip from ui import GTKConfig, GTKProgress # List of formats supported. This is used to generate the UI list and # specify the reader class readers = {'p2bgz':("PeerGuardian P2B (GZip)", PGReader), 'pgtext':("PeerGuardian Text (Uncompressed)", TextReader), - 'gzmule':("Emule IP list (GZip)", GZMuleReader)} + 'gzmule':("Emule IP list (GZip)", GZMuleReader), + 'spzip':("SafePeer Text (Zipped)", PGZip)} class BlocklistImport: @@ -96,15 +97,23 @@ class BlocklistImport: print "Starting import" ips = reader.next() curr = 0 - while ips and not self.cancelled: - self.core.add_range_to_ip_filter(*ips) - ips = reader.next() - curr += 1 - if curr % 100 == 0: - self.gtkprog.import_prog(text="Imported %s IPs"%curr) - else: + try: + while ips and not self.cancelled: + self.core.add_range_to_ip_filter(*ips) + ips = reader.next() + curr += 1 + if curr % 100 == 0: + self.gtkprog.import_prog(text="Imported %s IPs"%curr) + else: self.gtkprog.import_prog() + except FormatException, (ex): + err = ui.GTKError("Format error in blocklist: %s"%ex) + self.gtkprog.stop() + reader.close() + return + + reader.close() self.gtkprog.end_import() print "Import complete" diff --git a/plugins/BlocklistImport/listfile1.txt b/plugins/BlocklistImport/listfile1.txt new file mode 100644 index 000000000..62fa68b53 --- /dev/null +++ b/plugins/BlocklistImport/listfile1.txt @@ -0,0 +1 @@ +Bogon,:0.0.0.0-3.255.255.255 diff --git a/plugins/BlocklistImport/listfile2.txt b/plugins/BlocklistImport/listfile2.txt new file mode 100644 index 000000000..0875e7b77 --- /dev/null +++ b/plugins/BlocklistImport/listfile2.txt @@ -0,0 +1 @@ +Bogon,:1.1.1.1-3.255.255.255 diff --git a/plugins/BlocklistImport/peerguardian.py b/plugins/BlocklistImport/peerguardian.py index 38ff6536c..8fea503f0 100644 --- a/plugins/BlocklistImport/peerguardian.py +++ b/plugins/BlocklistImport/peerguardian.py @@ -37,16 +37,6 @@ class PGReader: raise PGException("Invalid version %d" % ver) - def numentries(self): - save = self.fd.tell() - self.fd.seek(8) - count = 0 - while self.next(): - count += 1 - self.fd.seek(save) - return count - - def next(self): # Skip over the string diff --git a/plugins/BlocklistImport/splist.txt b/plugins/BlocklistImport/splist.txt index c78ba864d..6b6ecc2af 100644 --- a/plugins/BlocklistImport/splist.txt +++ b/plugins/BlocklistImport/splist.txt @@ -1,4 +1,2 @@ Bogon,:0.0.0.0-3.255.255.255 - s0-0.c:4.0.25.146-4.0.25.148 - diff --git a/plugins/BlocklistImport/test.py b/plugins/BlocklistImport/test.py index a87acc3bc..96d77c254 100644 --- a/plugins/BlocklistImport/test.py +++ b/plugins/BlocklistImport/test.py @@ -1,20 +1,34 @@ import unittest -from text import TextReader, GZMuleReader +from text import TextReader, GZMuleReader, PGZip class ImportTests(unittest.TestCase): def testpgtext(self): - tr = TextReader("pg.txt") - ips = tr.next() + fr = TextReader("pg.txt") + ips = fr.next() self.assertEqual("3.0.0.0", ips[0]) self.assertEqual("3.255.255.255", ips[1]) def testMule(self): - mr = GZMuleReader("nipfilter.dat.gz") - ips = mr.next() + fr = GZMuleReader("nipfilter.dat.gz") + ips = fr.next() self.assertEqual("0.0.0.0", ips[0]) self.assertEqual("3.255.255.255", ips[1]) + def testZip(self): + fr = PGZip("splist.zip") + ips = fr.next() + print "wibble wibble",ips + self.assertEqual("1.1.1.1", ips[0]) + self.assertEqual("3.255.255.255", ips[1]) + + ips = fr.next() + self.assertEqual("0.0.0.0", ips[0]) + self.assertEqual("3.255.255.255", ips[1]) + + ips = fr.next() + self.assertEqual(ips, False) + if __name__ == '__main__': unittest.main() diff --git a/plugins/BlocklistImport/text.py b/plugins/BlocklistImport/text.py index ad55d3080..90cd38c9e 100644 --- a/plugins/BlocklistImport/text.py +++ b/plugins/BlocklistImport/text.py @@ -5,9 +5,10 @@ from exceptions import Exception -import re, gzip +import re, gzip, os from socket import inet_aton from struct import unpack +from zipfile import ZipFile class TextException(Exception): @@ -24,16 +25,6 @@ class TextBase: self.fd = fd self.re = re.compile(regexp) - def numentries(self): - print "Scanning" - save = self.fd.tell() - self.fd.seek(0) - count = 0 - for l in self.fd: - count += 1 - self.fd.seek(save) - return count - def next(self): self.count += 1 @@ -43,7 +34,7 @@ class TextBase: match = self.re.search(txt) if not match: - raise FormatException("Couldn't match on line %d: %s (%s)"%(self.count,txt,self.re.pattern)) + raise FormatException("Couldn't match on line %d: %s (%s)"%(self.count,txt,txt)) g = match.groups() start = ".".join(g[0:4]) @@ -87,3 +78,49 @@ class GZMuleReader(MuleReader): MuleReader.__init__(self, gzip.open(filename, "r")) +# Reads zip files from SafePeer style files +class PGZip(TextBase): + + def __init__(self, filename): + # Open zip and extract first file + self.zfd = ZipFile(filename, 'r') + self.files = self.zfd.namelist() + + self.opennext() + + def opennext(self): + self.tmp = os.tmpfile() + f = self.files.pop() + print "Loading file",f + self.tmp.write(self.zfd.read(f)) + self.tmp.seek(0) + self.reader = PGTextReader(self.tmp) + + def next(self): + try: + ret = self.reader.next() + if ret == False: + # This bit is repeated below and could be moved into a + # new procedure. However I'm not clear on how this + # would effect tail recursion, so it remains + # broken-out for now. + if len(self.files) > 0: + self.opennext() + return self.next() + else: + # End of zip + return False + return ret + + except FormatException, (e): + print "Got format exception for zipfile:",e + # Just skip + if len(self.files) > 0: + self.opennext() + return self.next() + else: + return False + + def close(self): + self.zfd.close() +