scraper: remove dead InstanceTracker class
This commit is contained in:
44
scraper.py
44
scraper.py
@@ -25,50 +25,6 @@ if os.path.exists('searx.instances'):
|
||||
if line.lower().startswith('http')]
|
||||
|
||||
|
||||
class InstanceTracker(object):
|
||||
"""Track instance health with exponential backoff."""
|
||||
|
||||
def __init__(self, instances, base_delay=30, max_delay=3600):
|
||||
self.instances = list(instances)
|
||||
self.base_delay = base_delay
|
||||
self.max_delay = max_delay
|
||||
self.failures = {}
|
||||
self.backoff_until = {}
|
||||
self.success_count = {}
|
||||
|
||||
def get_available(self):
|
||||
"""Return instances not currently in backoff."""
|
||||
now = time.time()
|
||||
available = []
|
||||
for inst in self.instances:
|
||||
if inst not in self.backoff_until or now >= self.backoff_until[inst]:
|
||||
available.append(inst)
|
||||
return available
|
||||
|
||||
def mark_success(self, instance):
|
||||
"""Reset failure count on success."""
|
||||
self.failures[instance] = 0
|
||||
self.success_count[instance] = self.success_count.get(instance, 0) + 1
|
||||
if instance in self.backoff_until:
|
||||
del self.backoff_until[instance]
|
||||
|
||||
def mark_failure(self, instance):
|
||||
"""Increment failure count and set exponential backoff."""
|
||||
count = self.failures.get(instance, 0) + 1
|
||||
self.failures[instance] = count
|
||||
delay = min(self.base_delay * (2 ** (count - 1)), self.max_delay)
|
||||
self.backoff_until[instance] = time.time() + delay
|
||||
name = instance.split('/')[2] if '/' in instance else instance
|
||||
_log('%s: backoff %ds (failures: %d)' % (name, delay, count), 'rate')
|
||||
return delay
|
||||
|
||||
def get_status(self):
|
||||
"""Return status summary."""
|
||||
available = len(self.get_available())
|
||||
in_backoff = len(self.instances) - available
|
||||
return available, in_backoff, len(self.instances)
|
||||
|
||||
|
||||
class EngineTracker(object):
|
||||
"""Track multiple search engine instances with rate limiting."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user