style: normalize indentation and improve code style

- convert tabs to 4-space indentation
- add docstrings to modules and classes
- remove unused import (copy)
- use explicit object inheritance
- use 'while True' over 'while 1'
- use 'while args' over 'while len(args)'
- use '{}' over 'dict()'
- consistent string formatting
- Python 2/3 compatible Queue import

comboparse.py

@@ -1,22 +1,36 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+"""Combined config file and argument parser."""
+
 from ConfigParser import SafeConfigParser, NoOptionError
 from argparse import ArgumentParser
 import sys
 
-class _Dummy():
+
+class _Dummy(object):
+    """Placeholder for config sections."""
     pass
 
+
 class ComboParser(object):
+    """Parse configuration from INI file and command-line arguments.
+
+    Command-line arguments override INI file values.
+    """
+
     def __init__(self, ini):
         self.items = []
         self.cparser = SafeConfigParser()
         self.aparser = ArgumentParser()
         self.ini = ini
-		self.items = []
         self.loaded = False
+        self.args = None
 
     def add_item(self, section, name, type, default, desc, required):
+        """Add a configuration item."""
         def str2bool(val):
-			return val in ['True', 'true', '1', 'yes']
+            return val.lower() in ('true', '1', 'yes')
+
         self.items.append({
             'section': section,
             'name': name,
@@ -26,49 +40,71 @@ class ComboParser(object):
         })
         self.aparser.add_argument(
             '--%s.%s' % (section, name),
-			help='%s, default: (%s)'%(desc, str(default)),
+            help='%s (default: %s)' % (desc, default),
             type=type if type is not bool else str2bool,
             default=None,
             required=False
         )
+
     def load(self):
-		if self.loaded: return
+        """Load configuration from file and command-line."""
+        if self.loaded:
+            return
         self.loaded = True
 
         try:
             self.cparser.read(self.ini)
         except Exception:
-			pass  # config file missing or unreadable, use defaults
+            pass  # Config file missing or unreadable, use defaults
-		args = self.aparser.parse_args()
+
+        self.args = self.aparser.parse_args()
+
         for item in self.items:
-			try:
+            section = item['section']
-				obj = getattr(self, item['section'])
+            name = item['name']
-			except AttributeError:
-				setattr(self, item['section'], _Dummy())
-				obj = getattr(self, item['section'])
 
-			setattr(obj, item['name'], item['default'])
+            # Ensure section object exists
-			inner = getattr(obj, item['name'])
+            if not hasattr(self, section):
+                setattr(self, section, _Dummy())
+            obj = getattr(self, section)
 
-			item['found'] = True
+            # Start with default value
+            value = item['default']
+            found = False
+
+            # Try to read from config file
             try:
-				if   item['type'] is bool : inner = self.cparser.getboolean(item['section'], item['name'])
+                if item['type'] is bool:
-				elif item['type'] is float: inner = self.cparser.getfloat(item['section'], item['name'])
+                    value = self.cparser.getboolean(section, name)
-				elif item['type'] is int  : inner = self.cparser.getint(item['section'], item['name'])
+                elif item['type'] is float:
-				elif item['type'] is str  : inner = self.cparser.get(item['section'], item['name'])
+                    value = self.cparser.getfloat(section, name)
+                elif item['type'] is int:
+                    value = self.cparser.getint(section, name)
+                elif item['type'] is str:
+                    value = self.cparser.get(section, name)
+                found = True
             except NoOptionError:
-				item['found'] = False
+                pass
-			try:
+
-				arg = getattr(args, '%s.%s'%(item['section'], item['name']))
+            # Command-line overrides config file
-				if arg is not None:
+            arg_name = '%s.%s' % (section, name)
-					inner = arg
+            arg_value = getattr(self.args, arg_name, None)
-					item['found'] = True
+            if arg_value is not None:
-			except AttributeError:
+                value = arg_value
-				pass  # arg not provided on command line
+                found = True
-			if not item['found']:
+
+            # Handle missing required items
+            if not found:
                 if item['required']:
-					sys.stderr.write('error: required config item "%s" not found in section "%s" of "%s"!\n'%(item['name'], item['section'], self.ini))
+                    sys.stderr.write(
+                        'error: required config item "%s" not found in section "%s" of "%s"\n'
+                        % (name, section, self.ini)
+                    )
                     sys.exit(1)
                 else:
-					sys.stderr.write('warning: assigned default value of "%s" to "%s.%s"\n'%(str(item['default']), item['section'], item['name']))
+                    sys.stderr.write(
-			setattr(obj, item['name'], inner)
+                        'warning: assigned default value of "%s" to "%s.%s"\n'
+                        % (item['default'], section, name)
+                    )
+
+            setattr(obj, name, value)

dbs.py

@@ -1,7 +1,13 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+"""Database table creation and insertion utilities."""
+
 import time
 from misc import _log
 
+
 def create_table_if_not_exists(sqlite, dbname):
+    """Create database table with indexes if it doesn't exist."""
     if dbname == 'proxylist':
         sqlite.execute("""CREATE TABLE IF NOT EXISTS proxylist (
             proxy BLOB UNIQUE,
@@ -17,7 +23,7 @@ def create_table_if_not_exists(sqlite, dbname):
             port INT,
             consecutive_success INT,
             total_duration INT)""")
-		# indexes for common query patterns
+        # Indexes for common query patterns
         sqlite.execute('CREATE INDEX IF NOT EXISTS idx_proxylist_failed ON proxylist(failed)')
         sqlite.execute('CREATE INDEX IF NOT EXISTS idx_proxylist_tested ON proxylist(tested)')
         sqlite.execute('CREATE INDEX IF NOT EXISTS idx_proxylist_proto ON proxylist(proto)')
@@ -31,31 +37,44 @@ def create_table_if_not_exists(sqlite, dbname):
             stale_count INT,
             retrievals INT,
             proxies_added INT,
-			added INT
+            added INT)""")
-			)""")
+        # Indexes for common query patterns
-		# indexes for common query patterns
         sqlite.execute('CREATE INDEX IF NOT EXISTS idx_uris_error ON uris(error)')
         sqlite.execute('CREATE INDEX IF NOT EXISTS idx_uris_checktime ON uris(check_time)')
 
     sqlite.commit()
 
+
 def insert_proxies(proxydb, proxies, url):
-	if not proxies: return
+    """Insert new proxies into database."""
+    if not proxies:
+        return
     timestamp = int(time.time())
     rows = []
     for p in proxies:
         ip, port = p.split(':')
         rows.append((timestamp, p, ip, port, 3, 0, 0, 0, 0, 0))
-	proxydb.executemany('INSERT OR IGNORE INTO proxylist (added,proxy,ip,port,failed,tested,success_count,total_duration,mitm,consecutive_success) VALUES (?,?,?,?,?,?,?,?,?,?)', rows)
+    proxydb.executemany(
+        'INSERT OR IGNORE INTO proxylist '
+        '(added,proxy,ip,port,failed,tested,success_count,total_duration,mitm,consecutive_success) '
+        'VALUES (?,?,?,?,?,?,?,?,?,?)',
+        rows
+    )
     proxydb.commit()
     _log('+%d proxy/ies from %s' % (len(proxies), url), 'added')
 
 
 def insert_urls(urls, search, sqlite):
-	if not urls: return
+    """Insert new URLs into database."""
-	time_now = int(time.time())
+    if not urls:
-	rows = [ (time_now,u,0,1,0,0,0) for u in urls ]
+        return
-	sqlite.executemany('INSERT OR IGNORE INTO uris (added,url,check_time,error,stale_count,retrievals,proxies_added) values(?,?,?,?,?,?,?)', rows)
+    timestamp = int(time.time())
+    rows = [(timestamp, u, 0, 1, 0, 0, 0) for u in urls]
+    sqlite.executemany(
+        'INSERT OR IGNORE INTO uris '
+        '(added,url,check_time,error,stale_count,retrievals,proxies_added) '
+        'VALUES (?,?,?,?,?,?,?)',
+        rows
+    )
     sqlite.commit()
     _log('+%d url(s) from %s' % (len(urls), search), 'added')
-

mysqlite.py

@@ -1,9 +1,29 @@
-import time, random, sys
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+"""SQLite wrapper with retry logic and WAL mode."""
+
+import time
+import random
+import sys
 import sqlite3
 
-class mysqlite:
+
+class mysqlite(object):
+    """SQLite connection wrapper with automatic retry on lock."""
+
+    def __init__(self, database, factory=None):
+        self.handle = sqlite3.connect(database)
+        if factory is not None:
+            self.handle.text_factory = factory
+        self.cursor = self.handle.cursor()
+        self.dbname = database
+        # Enable WAL mode for better concurrency
+        self.cursor.execute('PRAGMA journal_mode=WAL')
+        self.cursor.execute('PRAGMA synchronous=NORMAL')
+
     def _try_op(self, op, query, args=None, rmin=1.5, rmax=7.0):
-		while 1:
+        """Execute operation with retry on database lock."""
+        while True:
             try:
                 if query is None:
                     return op()
@@ -12,33 +32,31 @@ class mysqlite:
                 else:
                     return op(query, args)
             except sqlite3.OperationalError as e:
-				if e.message == 'database is locked':
+                err_msg = str(e)
-					print "zzZzzZZ: db is locked (%s)"%self.dbname
+                if 'database is locked' in err_msg:
+                    sys.stderr.write('zzZzzZZ: db is locked (%s)\n' % self.dbname)
                     time.sleep(random.uniform(rmin, rmax))
                     continue
                 else:
-					print '%s\nquery: %s\nargs: %s' % (str(sys.exc_info()), str(query), str(args))
+                    sys.stderr.write('%s\nquery: %s\nargs: %s\n' % (
-					raise e
+                        str(sys.exc_info()), str(query), str(args)))
+                    raise
 
     def execute(self, query, args=None, rmin=1.5, rmax=7.0):
+        """Execute a single query with retry."""
         return self._try_op(self.cursor.execute, query, args, rmin, rmax)
 
     def executemany(self, query, args, rmin=1.5, rmax=7.0):
-		while len(args):
+        """Execute query for multiple argument sets, batched."""
-			self._try_op(self.cursor.executemany, query, args[:500], rmin, rmax)
+        while args:
+            batch = args[:500]
+            self._try_op(self.cursor.executemany, query, batch, rmin, rmax)
             args = args[500:]
 
     def commit(self, rmin=1.5, rmax=7.0):
+        """Commit transaction with retry."""
         return self._try_op(self.handle.commit, None, None, rmin, rmax)
 
     def close(self):
+        """Close database connection."""
         self.handle.close()
-
-	def __init__(self, database, factory = None):
-		self.handle = sqlite3.connect(database)
-		if factory: self.handle.text_factory = factory
-		self.cursor = self.handle.cursor()
-		self.dbname = database
-		# enable WAL mode for better concurrency
-		self.cursor.execute('PRAGMA journal_mode=WAL')
-		self.cursor.execute('PRAGMA synchronous=NORMAL')

proxywatchd.py

@@ -1,9 +1,16 @@
 #!/usr/bin/env python2
 
 import threading
-import time, random, string, re, copy
+import time
-import Queue
+import random
+import string
+import re
 import heapq
+
+try:
+    import Queue
+except ImportError:
+    import queue as Queue
 try:
     import IP2Location
     import os
@@ -22,7 +29,7 @@ import connection_pool
 config = Config()
 
 _run_standalone = False
-cached_dns = dict()
+cached_dns = {}
 
 regexes = {
     'www.facebook.com': 'X-FB-Debug',

soup_parser.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python2
-# HTML parsing with optional BeautifulSoup or stdlib fallback
+# -*- coding: utf-8 -*-
+"""HTML parsing with optional BeautifulSoup or stdlib fallback."""
 
 from HTMLParser import HTMLParser
 import sys
@@ -14,7 +15,9 @@ except ImportError:
     _bs4_available = False
 
 
-class Tag():
+class Tag(object):
+    """Simple tag representation for stdlib parser."""
+
     def __init__(self, name, attrs):
         self.name = name
         self.attrs = dict(attrs)
@@ -26,12 +29,15 @@ class Tag():
         return self.attrs.get(key, default)
 
 
-class SoupResult():
+class SoupResult(object):
+    """BeautifulSoup-like result wrapper for stdlib parser."""
+
     def __init__(self, tags):
         self._tags = tags
         self.body = self
 
     def find_all(self, tag_name, **kwargs):
+        """Find all tags matching criteria."""
         results = []
         for tag in self._tags:
             if tag.name != tag_name:
@@ -46,6 +52,8 @@ class SoupResult():
 
 
 class LinkExtractor(HTMLParser):
+    """Extract tags from HTML using stdlib."""
+
     def __init__(self):
         HTMLParser.__init__(self)
         self.tags = []
@@ -58,15 +66,17 @@ class LinkExtractor(HTMLParser):
 
 
 def _parse_stdlib(html):
+    """Parse HTML using stdlib HTMLParser."""
     parser = LinkExtractor()
     try:
         parser.feed(html)
     except Exception:
-		pass  # malformed HTML, return partial results
+        pass  # Malformed HTML, return partial results
     return SoupResult(parser.tags)
 
 
 def _parse_bs4(html):
+    """Parse HTML using BeautifulSoup."""
     try:
         return BeautifulSoup(html, 'lxml')
     except (FeatureNotFound, Exception):
@@ -74,6 +84,7 @@ def _parse_bs4(html):
 
 
 def set_nobs(enabled):
+    """Disable BeautifulSoup and use stdlib instead."""
     global _use_bs4
     _use_bs4 = not enabled
     if enabled and _bs4_available:
@@ -83,12 +94,13 @@ def set_nobs(enabled):
 
 
 def soupify(html, nohtml=False):
-	htm = html if nohtml else '<html><body>%s</body></html>' % (html)
+    """Parse HTML content, returning BeautifulSoup-like object."""
+    htm = html if nohtml else '<html><body>%s</body></html>' % html
     if _use_bs4 and _bs4_available:
         return _parse_bs4(htm)
-	else:
     return _parse_stdlib(htm)
 
 
 def is_available():
+    """Check if BeautifulSoup is available."""
     return _bs4_available