diff --git a/scraper.py b/scraper.py index b9e6eaf..19e82c3 100755 --- a/scraper.py +++ b/scraper.py @@ -25,50 +25,6 @@ if os.path.exists('searx.instances'): if line.lower().startswith('http')] -class InstanceTracker(object): - """Track instance health with exponential backoff.""" - - def __init__(self, instances, base_delay=30, max_delay=3600): - self.instances = list(instances) - self.base_delay = base_delay - self.max_delay = max_delay - self.failures = {} - self.backoff_until = {} - self.success_count = {} - - def get_available(self): - """Return instances not currently in backoff.""" - now = time.time() - available = [] - for inst in self.instances: - if inst not in self.backoff_until or now >= self.backoff_until[inst]: - available.append(inst) - return available - - def mark_success(self, instance): - """Reset failure count on success.""" - self.failures[instance] = 0 - self.success_count[instance] = self.success_count.get(instance, 0) + 1 - if instance in self.backoff_until: - del self.backoff_until[instance] - - def mark_failure(self, instance): - """Increment failure count and set exponential backoff.""" - count = self.failures.get(instance, 0) + 1 - self.failures[instance] = count - delay = min(self.base_delay * (2 ** (count - 1)), self.max_delay) - self.backoff_until[instance] = time.time() + delay - name = instance.split('/')[2] if '/' in instance else instance - _log('%s: backoff %ds (failures: %d)' % (name, delay, count), 'rate') - return delay - - def get_status(self): - """Return status summary.""" - available = len(self.get_available()) - in_backoff = len(self.instances) - available - return available, in_backoff, len(self.instances) - - class EngineTracker(object): """Track multiple search engine instances with rate limiting."""