blocklist update -tarka

plugins/BlocklistImport/__init__.py

@@ -26,14 +26,15 @@ def enable(core, interface):
 
 import urllib, deluge.common, deluge.pref
 from peerguardian import PGReader, PGException
-from text import TextReader, GZMuleReader
+from text import TextReader, GZMuleReader, PGZip
 from ui import GTKConfig, GTKProgress
 
 # List of formats supported.  This is used to generate the UI list and
 # specify the reader class
 readers = {'p2bgz':("PeerGuardian P2B (GZip)", PGReader),
            'pgtext':("PeerGuardian Text (Uncompressed)", TextReader),
-           'gzmule':("Emule IP list (GZip)", GZMuleReader)}
+           'gzmule':("Emule IP list (GZip)", GZMuleReader),
+           'spzip':("SafePeer Text (Zipped)", PGZip)}
 
 class BlocklistImport:
 
@@ -96,15 +97,23 @@ class BlocklistImport:
         print "Starting import"
         ips = reader.next()
         curr = 0
-        while ips and not self.cancelled:
+        try:
-            self.core.add_range_to_ip_filter(*ips)
+            while ips and not self.cancelled:
-            ips = reader.next()
+                self.core.add_range_to_ip_filter(*ips)
-            curr += 1
+                ips = reader.next()
-            if curr % 100 == 0:
+                curr += 1
-                self.gtkprog.import_prog(text="Imported %s IPs"%curr)
+                if curr % 100 == 0:
-            else:
+                    self.gtkprog.import_prog(text="Imported %s IPs"%curr)
+                else:
 	            self.gtkprog.import_prog()
 
+        except FormatException, (ex):
+            err = ui.GTKError("Format error in blocklist: %s"%ex)
+            self.gtkprog.stop()
+            reader.close()
+            return
+            
+
         reader.close()
         self.gtkprog.end_import()
         print "Import complete"

plugins/BlocklistImport/listfile1.txt

@@ -0,0 +1 @@
+Bogon,:0.0.0.0-3.255.255.255

plugins/BlocklistImport/listfile2.txt

@@ -0,0 +1 @@
+Bogon,:1.1.1.1-3.255.255.255

plugins/BlocklistImport/peerguardian.py

@@ -37,16 +37,6 @@ class PGReader:
             raise PGException("Invalid version %d" % ver)
 
 
-    def numentries(self):
-        save = self.fd.tell()
-        self.fd.seek(8)
-        count = 0
-        while self.next():
-            count += 1
-        self.fd.seek(save)
-        return count
-
-
     def next(self):
 
         # Skip over the string

plugins/BlocklistImport/splist.txt

@@ -1,4 +1,2 @@
 Bogon,:0.0.0.0-3.255.255.255
-
 s0-0.c:4.0.25.146-4.0.25.148
-

plugins/BlocklistImport/test.py

@@ -1,20 +1,34 @@
 import unittest
-from text import TextReader, GZMuleReader
+from text import TextReader, GZMuleReader, PGZip
 
 
 class ImportTests(unittest.TestCase):
 
     def testpgtext(self):
-        tr = TextReader("pg.txt")
+        fr = TextReader("pg.txt")
-        ips = tr.next()
+        ips = fr.next()
         self.assertEqual("3.0.0.0", ips[0])
         self.assertEqual("3.255.255.255", ips[1])
 
     def testMule(self):
-        mr = GZMuleReader("nipfilter.dat.gz")
+        fr = GZMuleReader("nipfilter.dat.gz")
-        ips = mr.next()
+        ips = fr.next()
         self.assertEqual("0.0.0.0", ips[0])
         self.assertEqual("3.255.255.255", ips[1])
 
+    def testZip(self):
+        fr = PGZip("splist.zip")
+        ips = fr.next()
+        print "wibble wibble",ips
+        self.assertEqual("1.1.1.1", ips[0])
+        self.assertEqual("3.255.255.255", ips[1])
+
+        ips = fr.next()
+        self.assertEqual("0.0.0.0", ips[0])
+        self.assertEqual("3.255.255.255", ips[1])
+
+        ips = fr.next()
+        self.assertEqual(ips, False)
+
 if __name__ == '__main__':
     unittest.main()

plugins/BlocklistImport/text.py

@@ -5,9 +5,10 @@
 
 
 from exceptions import Exception
-import re, gzip
+import re, gzip, os
 from socket import inet_aton
 from struct import unpack
+from zipfile import ZipFile
 
 
 class TextException(Exception):
@@ -24,16 +25,6 @@ class TextBase:
         self.fd = fd
         self.re = re.compile(regexp)
 
-    def numentries(self):
-        print "Scanning"
-        save = self.fd.tell()
-        self.fd.seek(0)
-        count = 0
-        for l in self.fd:
-            count += 1
-        self.fd.seek(save)
-        return count
-
     def next(self):
         self.count += 1
         
@@ -43,7 +34,7 @@ class TextBase:
 
         match = self.re.search(txt)
         if not match:
-            raise FormatException("Couldn't match on line %d: %s (%s)"%(self.count,txt,self.re.pattern))
+            raise FormatException("Couldn't match on line %d: %s (%s)"%(self.count,txt,txt))
 
         g = match.groups()
         start = ".".join(g[0:4])
@@ -87,3 +78,49 @@ class GZMuleReader(MuleReader):
         MuleReader.__init__(self, gzip.open(filename, "r"))
 
 
+# Reads zip files from SafePeer style files 
+class PGZip(TextBase):
+
+    def __init__(self, filename):
+        # Open zip and extract first file
+        self.zfd = ZipFile(filename, 'r')
+        self.files = self.zfd.namelist()
+
+        self.opennext()
+
+    def opennext(self):
+        self.tmp = os.tmpfile()
+        f = self.files.pop()
+        print "Loading file",f
+        self.tmp.write(self.zfd.read(f))
+        self.tmp.seek(0)
+        self.reader = PGTextReader(self.tmp)
+
+    def next(self):
+        try:
+            ret = self.reader.next()
+            if ret == False:
+                # This bit is repeated below and could be moved into a
+                # new procedure.  However I'm not clear on how this
+                # would effect tail recursion, so it remains
+                # broken-out for now.
+                if len(self.files) > 0:
+                    self.opennext()
+                    return self.next()
+                else:
+                    # End of zip
+                    return False
+            return ret
+        
+        except FormatException, (e):
+            print "Got format exception for zipfile:",e
+            # Just skip
+            if len(self.files) > 0:
+                self.opennext()
+                return self.next()
+            else:
+                return False
+
+    def close(self):
+        self.zfd.close()
+