feat: dynamic health test concurrency
Auto-scale test concurrency to ~10% of proxy count, capped by test_concurrency config ceiling (default raised from 5 to 25). Prevents saturating upstream Tor when pool size varies. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -155,6 +155,102 @@ class TestProxyPoolWeight:
|
||||
pool.report_failure(hop) # should not raise
|
||||
|
||||
|
||||
class TestDynamicConcurrency:
|
||||
"""Test dynamic health test concurrency scaling."""
|
||||
|
||||
def test_scales_to_ten_percent(self):
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
cfg = ProxyPoolConfig(sources=[], test_concurrency=25)
|
||||
pool = ProxyPool(cfg, [], timeout=10.0)
|
||||
|
||||
now = time.time()
|
||||
# add 100 proxies -> effective concurrency = max(3, min(100//10, 25)) = 10
|
||||
for i in range(100):
|
||||
hop = ChainHop(proto="socks5", host=f"10.0.{i // 256}.{i % 256}", port=1080)
|
||||
key = f"socks5://10.0.{i // 256}.{i % 256}:1080"
|
||||
pool._proxies[key] = ProxyEntry(hop=hop, alive=False, last_seen=now)
|
||||
|
||||
captured = {}
|
||||
|
||||
original_semaphore = asyncio.Semaphore
|
||||
|
||||
def capture_semaphore(value):
|
||||
captured["concurrency"] = value
|
||||
return original_semaphore(value)
|
||||
|
||||
with (
|
||||
patch.object(pool, "_test_proxy", new_callable=AsyncMock, return_value=True),
|
||||
patch("s5p.pool.asyncio.Semaphore", side_effect=capture_semaphore),
|
||||
):
|
||||
asyncio.run(pool._run_health_tests())
|
||||
|
||||
assert captured["concurrency"] == 10
|
||||
|
||||
def test_minimum_of_three(self):
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
cfg = ProxyPoolConfig(sources=[], test_concurrency=25)
|
||||
pool = ProxyPool(cfg, [], timeout=10.0)
|
||||
|
||||
now = time.time()
|
||||
# 5 proxies -> 5//10=0, but min is 3
|
||||
for i in range(5):
|
||||
hop = ChainHop(proto="socks5", host=f"10.0.0.{i}", port=1080)
|
||||
pool._proxies[f"socks5://10.0.0.{i}:1080"] = ProxyEntry(
|
||||
hop=hop, alive=False, last_seen=now,
|
||||
)
|
||||
|
||||
captured = {}
|
||||
|
||||
original_semaphore = asyncio.Semaphore
|
||||
|
||||
def capture_semaphore(value):
|
||||
captured["concurrency"] = value
|
||||
return original_semaphore(value)
|
||||
|
||||
with (
|
||||
patch.object(pool, "_test_proxy", new_callable=AsyncMock, return_value=True),
|
||||
patch("s5p.pool.asyncio.Semaphore", side_effect=capture_semaphore),
|
||||
):
|
||||
asyncio.run(pool._run_health_tests())
|
||||
|
||||
assert captured["concurrency"] == 3
|
||||
|
||||
def test_capped_by_config(self):
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
cfg = ProxyPoolConfig(sources=[], test_concurrency=5)
|
||||
pool = ProxyPool(cfg, [], timeout=10.0)
|
||||
|
||||
now = time.time()
|
||||
# 1000 proxies -> 1000//10=100, capped at 5
|
||||
for i in range(1000):
|
||||
h = f"10.{i // 65536}.{(i // 256) % 256}.{i % 256}"
|
||||
hop = ChainHop(proto="socks5", host=h, port=1080)
|
||||
key = str(hop)
|
||||
pool._proxies[key] = ProxyEntry(hop=hop, alive=False, last_seen=now)
|
||||
|
||||
captured = {}
|
||||
|
||||
original_semaphore = asyncio.Semaphore
|
||||
|
||||
def capture_semaphore(value):
|
||||
captured["concurrency"] = value
|
||||
return original_semaphore(value)
|
||||
|
||||
with (
|
||||
patch.object(pool, "_test_proxy", new_callable=AsyncMock, return_value=True),
|
||||
patch("s5p.pool.asyncio.Semaphore", side_effect=capture_semaphore),
|
||||
):
|
||||
asyncio.run(pool._run_health_tests())
|
||||
|
||||
assert captured["concurrency"] == 5
|
||||
|
||||
|
||||
class TestProxyPoolHealthTests:
|
||||
"""Test selective health testing."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user