httpd: pass url database to api server

ppf.py

@@ -5,10 +5,10 @@ __version__ = '2.0.0'
 import sys
 import os
 
-# Worker mode requires gevent - must monkey-patch before other imports
-if '--worker' in sys.argv or '--register' in sys.argv:
-    from gevent import monkey
-    monkey.patch_all()
+# Gevent monkey-patch MUST happen before any other imports
+# Both master (httpd) and worker modes use gevent for async I/O
+from gevent import monkey
+monkey.patch_all()
 
 import cProfile
 import pstats
@@ -598,6 +598,7 @@ def worker_main(config):
                 port = proxy_info['port']
                 proto = proxy_info.get('proto', 'http')
                 failed = proxy_info.get('failed', 0)
+                source_proto = proxy_info.get('source_proto')
                 proxy_str = '%s:%d' % (ip, port)
 
                 # Create state for this proxy
@@ -607,7 +608,7 @@ def worker_main(config):
                     country=None, mitm=0, consecutive_success=0,
                     asn=None, oldies=False,
                     completion_queue=completion_queue,
-                    proxy_full=proxy_str
+                    proxy_full=proxy_str, source_proto=source_proto
                 )
                 pending_states[proxy_str] = state
 
@@ -780,6 +781,7 @@ def main():
                 config.watchd.database,
                 stats_provider=httpd_stats_provider,
                 profiling=profiling,
+                url_database=config.ppf.database,
             )
             httpd_server.start()
 
@@ -838,9 +840,6 @@ def main():
                 else:
                     _log('handing %d job(s) to %d thread(s)' % ( len(rows), config.ppf.threads ), 'ppf')
 
-            _proxylist = [ '%s://%s' % (p[0], p[1]) for p in proxydb.execute("SELECT proto,proxy from proxylist where failed=0 AND tested IS NOT NULL AND proto IN ('http','socks4','socks5')").fetchall() ]
-            if not _proxylist: _proxylist = None
-
             for thread in threads:
                 if thread.status == 'ok':
                     url, proxylist, stale_count, error, retrievals, content_type, proxies_added, execute = thread.retrieve()
@@ -857,6 +856,9 @@ def main():
 
             threads = [ thread for thread in threads if thread.is_alive() ]
             if len(threads) < config.ppf.threads and rows:
+                # Only query proxydb when actually starting a new thread (reduces GIL blocking)
+                _proxylist = [ '%s://%s' % (p[0], p[1]) for p in proxydb.execute("SELECT proto,proxy from proxylist where failed=0 AND tested IS NOT NULL AND proto IN ('http','socks4','socks5')").fetchall() ]
+                if not _proxylist: _proxylist = None
                 p = random.sample(_proxylist, min(5, len(_proxylist))) if _proxylist else None
                 row = random.choice(rows)
                 urldb.execute('UPDATE uris SET check_time=? where url=?', (time.time(), row[0]))