mule list support for blocklist - tarka

plugins/BlocklistImport/__init__.py

@@ -26,13 +26,14 @@ def enable(core, interface):
 
 import urllib, deluge.common, deluge.pref
 from peerguardian import PGReader, PGException
-from text import TextReader
+from text import TextReader, GZMuleReader
 from ui import GTKConfig, GTKProgress
 
 # List of formats supported.  This is used to generate the UI list and
 # specify the reader class
 readers = {'p2bgz':("PeerGuardian P2B (GZip)", PGReader),
-           'text':("PeerGuardian Text", TextReader)}
+           'pgtext':("PeerGuardian Text (Uncompressed)", TextReader),
+           'gzmule':("Emule IP list (GZip)", GZMuleReader)}
 
 class BlocklistImport:
 
@@ -64,7 +65,6 @@ class BlocklistImport:
         self.gtkprog.download_prog(curr/incs)
 
     def loadlist(self, fetch=False):
-        # FIXME
         self.gtkprog.start()
         
         # Attempt initial import
@@ -78,16 +78,17 @@ class BlocklistImport:
         self.gtkprog.start_import()
 
         self.core.reset_ip_filter()
-        #reader = PGReader(self.blockfile)
         ltype = self.config.get('listtype')
         print "importing with",ltype
         reader = readers[ltype][1](self.blockfile)
 
         ips = reader.next()
+        curr = 0
         while ips and not self.cancelled:
             self.core.add_range_to_ip_filter(*ips)
-            self.gtkprog.import_prog()
             ips = reader.next()
+            curr += 1
+            self.gtkprog.import_prog()
 
         reader.close()
         self.gtkprog.end_import()

plugins/BlocklistImport/peerguardian.py

@@ -36,6 +36,17 @@ class PGReader:
         if ver != 1 and ver != 2:
             raise PGException("Invalid version %d" % ver)
 
+
+    def numentries(self):
+        save = self.fd.tell()
+        self.fd.seek(8)
+        count = 0
+        while self.next():
+            count += 1
+        self.fd.seek(save)
+        return count
+
+
     def next(self):
 
         # Skip over the string

plugins/BlocklistImport/text.py

@@ -5,7 +5,10 @@
 
 
 from exceptions import Exception
-import re
+import re, gzip
+from socket import inet_aton
+from struct import unpack
+
 
 class TextException(Exception):
     pass
@@ -13,17 +16,23 @@ class TextException(Exception):
 class FormatException(TextException):
     pass
 
-# This reads text-formatted block-lists
+class TextBase:
-class TextReader:
     
-    def __init__(self, filename):
+    def __init__(self, fd, regexp):
-        print "TextReader loading",filename
+        print "TextBase loading"
         self.count = 0
+        self.fd = fd
+        self.re = re.compile(regexp)
 
-        # FIXME: Catch or just leave?
+    def numentries(self):
-        self.fd = open(filename, 'r')
+        print "Scanning"
-        
+        save = self.fd.tell()
-        self.re = re.compile(':(\d+\.\d+\.\d+\.\d+)-(\d+\.\d+\.\d+\.\d+)\s*$')
+        self.fd.seek(0)
+        count = 0
+        for l in self.fd:
+            count += 1
+        self.fd.seek(save)
+        return count
 
     def next(self):
         self.count += 1
@@ -34,10 +43,47 @@ class TextReader:
 
         match = self.re.search(txt)
         if not match:
-            raise FormatException("Couldn't match on line %d"%self.count)
+            raise FormatException("Couldn't match on line %d: %s (%s)"%(self.count,txt,self.re.pattern))
 
-        return match.groups()
+        g = match.groups()
+        start = ".".join(g[0:4])
+        end = ".".join(g[4:8])
+
+        return (start, end)
 
     def close(self):
         self.fd.close()
 
+
+# This reads PeerGuardian text-formatted block-lists
+class PGTextReader(TextBase):
+
+    def __init__(self, fd):
+        print "PGTextReader loading"
+        regexp = ':(\d+)\.(\d+)\.(\d+)\.(\d+)-(\d+)\.(\d+)\.(\d+)\.(\d+)\s*$'
+        TextBase.__init__(self, fd, regexp)
+
+
+# This reads uncompressed PG text list
+class TextReader(PGTextReader):
+
+    def __init__(self, filename):
+        print "TextReader loading",filename
+        PGTextReader.__init__(self, open(filename, 'r'))
+
+
+# Reads Emule style blocklists (aka nipfilter)
+class MuleReader(TextBase):
+
+    def __init__(self, fd):
+        print "MuleReader loading"
+        regexp = '0*(\d+)\.0*(\d+)\.0*(\d+)\.0*(\d+)\s*-\s*0*(\d+)\.0*(\d+)\.0*(\d+)\.0*(\d+)\s*,'
+        TextBase.__init__(self, fd, regexp)
+
+class GZMuleReader(MuleReader):
+
+    def __init__(self, filename):
+        print "GZMuleReader loading",filename
+        MuleReader.__init__(self, gzip.open(filename, "r"))
+
+