refactor: extract modules from proxywatchd.py
Extract focused modules to reduce proxywatchd.py complexity: - stats.py: JudgeStats, Stats, regexes, ssl_targets (557 lines) - mitm.py: MITMCertStats, cert extraction functions (239 lines) - dns.py: socks4_resolve with TTL caching (86 lines) - job.py: PriorityJobQueue, calculate_priority (103 lines) proxywatchd.py reduced from 2488 to 1591 lines (-36%).
This commit is contained in:
103
job.py
Normal file
103
job.py
Normal file
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env python2
|
||||
"""Job queue and priority handling for PPF proxy testing."""
|
||||
from __future__ import division
|
||||
|
||||
import heapq
|
||||
import threading
|
||||
try:
|
||||
import Queue
|
||||
except ImportError:
|
||||
import queue as Queue
|
||||
|
||||
|
||||
class PriorityJobQueue(object):
|
||||
"""Priority queue for proxy test jobs.
|
||||
|
||||
Lower priority number = higher priority.
|
||||
Priority 0: New proxies (never tested)
|
||||
Priority 1: Recently working (no failures, has successes)
|
||||
Priority 2: Low fail count (< 3 failures)
|
||||
Priority 3: Medium fail count
|
||||
Priority 4: High fail count
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.heap = []
|
||||
self.lock = threading.Lock()
|
||||
self.not_empty = threading.Condition(self.lock)
|
||||
self.counter = 0 # tie-breaker for equal priorities
|
||||
|
||||
def put(self, job, priority=3):
|
||||
"""Add job with priority (lower = higher priority)."""
|
||||
with self.lock:
|
||||
# Reset counter when queue was empty to prevent unbounded growth
|
||||
if not self.heap:
|
||||
self.counter = 0
|
||||
heapq.heappush(self.heap, (priority, self.counter, job))
|
||||
self.counter += 1
|
||||
self.not_empty.notify()
|
||||
|
||||
def get(self, timeout=None):
|
||||
"""Get highest priority job. Raises Queue.Empty on timeout."""
|
||||
with self.not_empty:
|
||||
if timeout is None:
|
||||
while not self.heap:
|
||||
self.not_empty.wait()
|
||||
else:
|
||||
end_time = __import__('time').time() + timeout
|
||||
while not self.heap:
|
||||
remaining = end_time - __import__('time').time()
|
||||
if remaining <= 0:
|
||||
raise Queue.Empty()
|
||||
self.not_empty.wait(remaining)
|
||||
_, _, job = heapq.heappop(self.heap)
|
||||
return job
|
||||
|
||||
def get_nowait(self):
|
||||
"""Get job without waiting. Raises Queue.Empty if empty."""
|
||||
with self.lock:
|
||||
if not self.heap:
|
||||
raise Queue.Empty()
|
||||
_, _, job = heapq.heappop(self.heap)
|
||||
return job
|
||||
|
||||
def empty(self):
|
||||
"""Check if queue is empty."""
|
||||
with self.lock:
|
||||
return len(self.heap) == 0
|
||||
|
||||
def qsize(self):
|
||||
"""Return queue size."""
|
||||
with self.lock:
|
||||
return len(self.heap)
|
||||
|
||||
def task_done(self):
|
||||
"""Compatibility method (no-op for heap queue)."""
|
||||
pass
|
||||
|
||||
|
||||
def calculate_priority(failcount, success_count, max_fail):
|
||||
"""Calculate job priority based on proxy state.
|
||||
|
||||
Args:
|
||||
failcount: Current failure count
|
||||
success_count: Lifetime success count
|
||||
max_fail: Maximum failures before considered dead
|
||||
|
||||
Returns:
|
||||
int: Priority 0-4 (lower = higher priority)
|
||||
"""
|
||||
# New proxy (never successfully tested)
|
||||
if success_count == 0 and failcount == 0:
|
||||
return 0
|
||||
# Recently working (no current failures)
|
||||
if failcount == 0:
|
||||
return 1
|
||||
# Low fail count
|
||||
if failcount < 3:
|
||||
return 2
|
||||
# Medium fail count
|
||||
if failcount < max_fail // 2:
|
||||
return 3
|
||||
# High fail count
|
||||
return 4
|
||||
Reference in New Issue
Block a user