ppf: add periodic re-seeding of proxy source URLs
Seed sources that error out are permanently excluded from claiming. Over time this starves the pipeline. Re-seed every 6 hours with error reset for exhausted sources, preventing the starvation loop that caused the previous outage.
This commit is contained in:
7
ppf.py
7
ppf.py
@@ -1045,8 +1045,15 @@ def main():
|
||||
statusmsg = time.time()
|
||||
list_max_age_seconds = config.ppf.list_max_age_days * 86400
|
||||
last_skip_log = 0
|
||||
last_reseed = time.time()
|
||||
reseed_interval = 6 * 3600 # re-seed sources every 6 hours
|
||||
while True:
|
||||
try:
|
||||
# Periodic re-seeding: reset errored-out seed sources
|
||||
if time.time() - last_reseed >= reseed_interval:
|
||||
dbs.seed_proxy_sources(urldb, reset_errors=True)
|
||||
last_reseed = time.time()
|
||||
|
||||
# When ppf threads = 0, skip URL fetching (workers handle it via /api/claim-urls)
|
||||
if config.ppf.threads == 0:
|
||||
time.sleep(60)
|
||||
|
||||
Reference in New Issue
Block a user