scraper: remove dead InstanceTracker class
This commit is contained in:
44
scraper.py
44
scraper.py
@@ -25,50 +25,6 @@ if os.path.exists('searx.instances'):
|
|||||||
if line.lower().startswith('http')]
|
if line.lower().startswith('http')]
|
||||||
|
|
||||||
|
|
||||||
class InstanceTracker(object):
|
|
||||||
"""Track instance health with exponential backoff."""
|
|
||||||
|
|
||||||
def __init__(self, instances, base_delay=30, max_delay=3600):
|
|
||||||
self.instances = list(instances)
|
|
||||||
self.base_delay = base_delay
|
|
||||||
self.max_delay = max_delay
|
|
||||||
self.failures = {}
|
|
||||||
self.backoff_until = {}
|
|
||||||
self.success_count = {}
|
|
||||||
|
|
||||||
def get_available(self):
|
|
||||||
"""Return instances not currently in backoff."""
|
|
||||||
now = time.time()
|
|
||||||
available = []
|
|
||||||
for inst in self.instances:
|
|
||||||
if inst not in self.backoff_until or now >= self.backoff_until[inst]:
|
|
||||||
available.append(inst)
|
|
||||||
return available
|
|
||||||
|
|
||||||
def mark_success(self, instance):
|
|
||||||
"""Reset failure count on success."""
|
|
||||||
self.failures[instance] = 0
|
|
||||||
self.success_count[instance] = self.success_count.get(instance, 0) + 1
|
|
||||||
if instance in self.backoff_until:
|
|
||||||
del self.backoff_until[instance]
|
|
||||||
|
|
||||||
def mark_failure(self, instance):
|
|
||||||
"""Increment failure count and set exponential backoff."""
|
|
||||||
count = self.failures.get(instance, 0) + 1
|
|
||||||
self.failures[instance] = count
|
|
||||||
delay = min(self.base_delay * (2 ** (count - 1)), self.max_delay)
|
|
||||||
self.backoff_until[instance] = time.time() + delay
|
|
||||||
name = instance.split('/')[2] if '/' in instance else instance
|
|
||||||
_log('%s: backoff %ds (failures: %d)' % (name, delay, count), 'rate')
|
|
||||||
return delay
|
|
||||||
|
|
||||||
def get_status(self):
|
|
||||||
"""Return status summary."""
|
|
||||||
available = len(self.get_available())
|
|
||||||
in_backoff = len(self.instances) - available
|
|
||||||
return available, in_backoff, len(self.instances)
|
|
||||||
|
|
||||||
|
|
||||||
class EngineTracker(object):
|
class EngineTracker(object):
|
||||||
"""Track multiple search engine instances with rate limiting."""
|
"""Track multiple search engine instances with rate limiting."""
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user