feat: dynamic health test concurrency

Auto-scale test concurrency to ~10% of proxy count, capped by
test_concurrency config ceiling (default raised from 5 to 25).
Prevents saturating upstream Tor when pool size varies.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
user
2026-02-18 10:09:44 +01:00
parent d4e3638143
commit b3966c9a9f
6 changed files with 109 additions and 6 deletions

View File

@@ -155,6 +155,102 @@ class TestProxyPoolWeight:
pool.report_failure(hop) # should not raise
class TestDynamicConcurrency:
"""Test dynamic health test concurrency scaling."""
def test_scales_to_ten_percent(self):
import asyncio
from unittest.mock import AsyncMock, patch
cfg = ProxyPoolConfig(sources=[], test_concurrency=25)
pool = ProxyPool(cfg, [], timeout=10.0)
now = time.time()
# add 100 proxies -> effective concurrency = max(3, min(100//10, 25)) = 10
for i in range(100):
hop = ChainHop(proto="socks5", host=f"10.0.{i // 256}.{i % 256}", port=1080)
key = f"socks5://10.0.{i // 256}.{i % 256}:1080"
pool._proxies[key] = ProxyEntry(hop=hop, alive=False, last_seen=now)
captured = {}
original_semaphore = asyncio.Semaphore
def capture_semaphore(value):
captured["concurrency"] = value
return original_semaphore(value)
with (
patch.object(pool, "_test_proxy", new_callable=AsyncMock, return_value=True),
patch("s5p.pool.asyncio.Semaphore", side_effect=capture_semaphore),
):
asyncio.run(pool._run_health_tests())
assert captured["concurrency"] == 10
def test_minimum_of_three(self):
import asyncio
from unittest.mock import AsyncMock, patch
cfg = ProxyPoolConfig(sources=[], test_concurrency=25)
pool = ProxyPool(cfg, [], timeout=10.0)
now = time.time()
# 5 proxies -> 5//10=0, but min is 3
for i in range(5):
hop = ChainHop(proto="socks5", host=f"10.0.0.{i}", port=1080)
pool._proxies[f"socks5://10.0.0.{i}:1080"] = ProxyEntry(
hop=hop, alive=False, last_seen=now,
)
captured = {}
original_semaphore = asyncio.Semaphore
def capture_semaphore(value):
captured["concurrency"] = value
return original_semaphore(value)
with (
patch.object(pool, "_test_proxy", new_callable=AsyncMock, return_value=True),
patch("s5p.pool.asyncio.Semaphore", side_effect=capture_semaphore),
):
asyncio.run(pool._run_health_tests())
assert captured["concurrency"] == 3
def test_capped_by_config(self):
import asyncio
from unittest.mock import AsyncMock, patch
cfg = ProxyPoolConfig(sources=[], test_concurrency=5)
pool = ProxyPool(cfg, [], timeout=10.0)
now = time.time()
# 1000 proxies -> 1000//10=100, capped at 5
for i in range(1000):
h = f"10.{i // 65536}.{(i // 256) % 256}.{i % 256}"
hop = ChainHop(proto="socks5", host=h, port=1080)
key = str(hop)
pool._proxies[key] = ProxyEntry(hop=hop, alive=False, last_seen=now)
captured = {}
original_semaphore = asyncio.Semaphore
def capture_semaphore(value):
captured["concurrency"] = value
return original_semaphore(value)
with (
patch.object(pool, "_test_proxy", new_callable=AsyncMock, return_value=True),
patch("s5p.pool.asyncio.Semaphore", side_effect=capture_semaphore),
):
asyncio.run(pool._run_health_tests())
assert captured["concurrency"] == 5
class TestProxyPoolHealthTests:
"""Test selective health testing."""