feat: connection pooling via urllib3 + batch OG fetching
Replace per-request SOCKS5+TLS handshakes with urllib3 SOCKSProxyManager connection pool (20 pools, 4 conns/host). Batch _fetch_og calls via ThreadPoolExecutor to parallelize OG tag enrichment in alert polling. Cache flaskpaste SSL context at module level. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
"""Tests for the SOCKS5 proxy HTTP/TCP module."""
|
||||
|
||||
import ssl
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
@@ -12,20 +13,46 @@ from derp.http import (
|
||||
_PROXY_ADDR,
|
||||
_PROXY_PORT,
|
||||
_get_opener,
|
||||
_get_pool,
|
||||
_ProxyHandler,
|
||||
build_opener,
|
||||
create_connection,
|
||||
urlopen,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_opener_cache():
|
||||
"""Clear cached opener between tests."""
|
||||
def _reset_caches():
|
||||
"""Clear cached opener and pool between tests."""
|
||||
derp.http._default_opener = None
|
||||
derp.http._pool = None
|
||||
yield
|
||||
derp.http._default_opener = None
|
||||
derp.http._pool = None
|
||||
|
||||
|
||||
# -- Connection pool ---------------------------------------------------------
|
||||
|
||||
class TestConnectionPool:
|
||||
def test_pool_lazy_init(self):
|
||||
assert derp.http._pool is None
|
||||
pool = _get_pool()
|
||||
assert pool is not None
|
||||
assert derp.http._pool is pool
|
||||
|
||||
def test_pool_cached(self):
|
||||
a = _get_pool()
|
||||
b = _get_pool()
|
||||
assert a is b
|
||||
|
||||
def test_pool_is_socks_manager(self):
|
||||
from urllib3.contrib.socks import SOCKSProxyManager
|
||||
pool = _get_pool()
|
||||
assert isinstance(pool, SOCKSProxyManager)
|
||||
|
||||
|
||||
# -- Legacy opener -----------------------------------------------------------
|
||||
|
||||
class TestProxyHandler:
|
||||
def test_uses_socks5(self):
|
||||
handler = _ProxyHandler()
|
||||
@@ -103,6 +130,106 @@ class TestOpenerCache:
|
||||
assert a is not b
|
||||
|
||||
|
||||
# -- urlopen (pooled path) --------------------------------------------------
|
||||
|
||||
class TestUrlopen:
|
||||
@patch.object(derp.http, "_get_pool")
|
||||
def test_extracts_request_fields(self, mock_pool_fn):
|
||||
pool = MagicMock()
|
||||
resp = MagicMock()
|
||||
resp.status = 200
|
||||
pool.request.return_value = resp
|
||||
mock_pool_fn.return_value = pool
|
||||
|
||||
req = urllib.request.Request(
|
||||
"https://example.com/test",
|
||||
headers={"X-Custom": "val"},
|
||||
method="POST",
|
||||
)
|
||||
req.data = b"body"
|
||||
urlopen(req, timeout=10)
|
||||
|
||||
pool.request.assert_called_once()
|
||||
call_kw = pool.request.call_args
|
||||
assert call_kw[0][0] == "POST"
|
||||
assert call_kw[0][1] == "https://example.com/test"
|
||||
assert call_kw[1]["body"] == b"body"
|
||||
|
||||
@patch.object(derp.http, "_get_pool")
|
||||
def test_string_url(self, mock_pool_fn):
|
||||
pool = MagicMock()
|
||||
resp = MagicMock()
|
||||
resp.status = 200
|
||||
pool.request.return_value = resp
|
||||
mock_pool_fn.return_value = pool
|
||||
|
||||
urlopen("https://example.com/")
|
||||
call_args = pool.request.call_args
|
||||
assert call_args[0] == ("GET", "https://example.com/")
|
||||
|
||||
@patch.object(derp.http, "_get_pool")
|
||||
def test_raises_http_error_on_4xx(self, mock_pool_fn):
|
||||
pool = MagicMock()
|
||||
resp = MagicMock()
|
||||
resp.status = 404
|
||||
resp.reason = "Not Found"
|
||||
resp.headers = {}
|
||||
resp.read.return_value = b""
|
||||
pool.request.return_value = resp
|
||||
mock_pool_fn.return_value = pool
|
||||
|
||||
with pytest.raises(urllib.error.HTTPError) as exc_info:
|
||||
urlopen("https://example.com/missing")
|
||||
assert exc_info.value.code == 404
|
||||
|
||||
@patch.object(derp.http, "_get_pool")
|
||||
def test_raises_http_error_on_5xx(self, mock_pool_fn):
|
||||
pool = MagicMock()
|
||||
resp = MagicMock()
|
||||
resp.status = 500
|
||||
resp.reason = "Internal Server Error"
|
||||
resp.headers = {}
|
||||
resp.read.return_value = b""
|
||||
pool.request.return_value = resp
|
||||
mock_pool_fn.return_value = pool
|
||||
|
||||
with pytest.raises(urllib.error.HTTPError) as exc_info:
|
||||
urlopen("https://example.com/error")
|
||||
assert exc_info.value.code == 500
|
||||
|
||||
@patch.object(derp.http, "_get_pool")
|
||||
def test_returns_response_on_2xx(self, mock_pool_fn):
|
||||
pool = MagicMock()
|
||||
resp = MagicMock()
|
||||
resp.status = 200
|
||||
pool.request.return_value = resp
|
||||
mock_pool_fn.return_value = pool
|
||||
|
||||
result = urlopen("https://example.com/")
|
||||
assert result is resp
|
||||
|
||||
@patch.object(derp.http, "_get_pool")
|
||||
def test_context_falls_back_to_opener(self, mock_pool_fn):
|
||||
"""Custom SSL context should use legacy opener, not pool."""
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
|
||||
with patch.object(derp.http, "_get_opener") as mock_opener_fn:
|
||||
opener = MagicMock()
|
||||
resp = MagicMock()
|
||||
opener.open.return_value = resp
|
||||
mock_opener_fn.return_value = opener
|
||||
|
||||
result = urlopen("https://example.com/", context=ctx)
|
||||
|
||||
mock_pool_fn.assert_not_called()
|
||||
mock_opener_fn.assert_called_once_with(ctx)
|
||||
assert result is resp
|
||||
|
||||
|
||||
# -- create_connection -------------------------------------------------------
|
||||
|
||||
class TestCreateConnection:
|
||||
@patch("derp.http.socks.socksocket")
|
||||
def test_sets_socks5_proxy(self, mock_cls):
|
||||
|
||||
Reference in New Issue
Block a user