0d7d2dce70
Extract focused modules to reduce proxywatchd.py complexity: - stats.py: JudgeStats, Stats, regexes, ssl_targets (557 lines) - mitm.py: MITMCertStats, cert extraction functions (239 lines) - dns.py: socks4_resolve with TTL caching (86 lines) - job.py: PriorityJobQueue, calculate_priority (103 lines) proxywatchd.py reduced from 2488 to 1591 lines (-36%).
104 lines
3.0 KiB
Python
104 lines
3.0 KiB
Python
#!/usr/bin/env python2
|
|
"""Job queue and priority handling for PPF proxy testing."""
|
|
from __future__ import division
|
|
|
|
import heapq
|
|
import threading
|
|
try:
|
|
import Queue
|
|
except ImportError:
|
|
import queue as Queue
|
|
|
|
|
|
class PriorityJobQueue(object):
|
|
"""Priority queue for proxy test jobs.
|
|
|
|
Lower priority number = higher priority.
|
|
Priority 0: New proxies (never tested)
|
|
Priority 1: Recently working (no failures, has successes)
|
|
Priority 2: Low fail count (< 3 failures)
|
|
Priority 3: Medium fail count
|
|
Priority 4: High fail count
|
|
"""
|
|
|
|
def __init__(self):
|
|
self.heap = []
|
|
self.lock = threading.Lock()
|
|
self.not_empty = threading.Condition(self.lock)
|
|
self.counter = 0 # tie-breaker for equal priorities
|
|
|
|
def put(self, job, priority=3):
|
|
"""Add job with priority (lower = higher priority)."""
|
|
with self.lock:
|
|
# Reset counter when queue was empty to prevent unbounded growth
|
|
if not self.heap:
|
|
self.counter = 0
|
|
heapq.heappush(self.heap, (priority, self.counter, job))
|
|
self.counter += 1
|
|
self.not_empty.notify()
|
|
|
|
def get(self, timeout=None):
|
|
"""Get highest priority job. Raises Queue.Empty on timeout."""
|
|
with self.not_empty:
|
|
if timeout is None:
|
|
while not self.heap:
|
|
self.not_empty.wait()
|
|
else:
|
|
end_time = __import__('time').time() + timeout
|
|
while not self.heap:
|
|
remaining = end_time - __import__('time').time()
|
|
if remaining <= 0:
|
|
raise Queue.Empty()
|
|
self.not_empty.wait(remaining)
|
|
_, _, job = heapq.heappop(self.heap)
|
|
return job
|
|
|
|
def get_nowait(self):
|
|
"""Get job without waiting. Raises Queue.Empty if empty."""
|
|
with self.lock:
|
|
if not self.heap:
|
|
raise Queue.Empty()
|
|
_, _, job = heapq.heappop(self.heap)
|
|
return job
|
|
|
|
def empty(self):
|
|
"""Check if queue is empty."""
|
|
with self.lock:
|
|
return len(self.heap) == 0
|
|
|
|
def qsize(self):
|
|
"""Return queue size."""
|
|
with self.lock:
|
|
return len(self.heap)
|
|
|
|
def task_done(self):
|
|
"""Compatibility method (no-op for heap queue)."""
|
|
pass
|
|
|
|
|
|
def calculate_priority(failcount, success_count, max_fail):
|
|
"""Calculate job priority based on proxy state.
|
|
|
|
Args:
|
|
failcount: Current failure count
|
|
success_count: Lifetime success count
|
|
max_fail: Maximum failures before considered dead
|
|
|
|
Returns:
|
|
int: Priority 0-4 (lower = higher priority)
|
|
"""
|
|
# New proxy (never successfully tested)
|
|
if success_count == 0 and failcount == 0:
|
|
return 0
|
|
# Recently working (no current failures)
|
|
if failcount == 0:
|
|
return 1
|
|
# Low fail count
|
|
if failcount < 3:
|
|
return 2
|
|
# Medium fail count
|
|
if failcount < max_fail // 2:
|
|
return 3
|
|
# High fail count
|
|
return 4
|