perf: cache default HTTP opener at module level
Avoid rebuilding _ProxyHandler + build_opener() on every request. Default-context callers (16 of 18 plugins) reuse one cached opener; custom-context callers still get a fresh one. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -18,6 +18,18 @@ _RETRY_ERRORS = (ssl.SSLError, ConnectionError, TimeoutError, OSError)
|
|||||||
|
|
||||||
_log = logging.getLogger(__name__)
|
_log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_default_opener: urllib.request.OpenerDirector | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def _get_opener(context=None):
|
||||||
|
"""Return cached opener for default context, fresh for custom."""
|
||||||
|
global _default_opener
|
||||||
|
if context is not None:
|
||||||
|
return urllib.request.build_opener(_ProxyHandler(context=context))
|
||||||
|
if _default_opener is None:
|
||||||
|
_default_opener = urllib.request.build_opener(_ProxyHandler())
|
||||||
|
return _default_opener
|
||||||
|
|
||||||
|
|
||||||
class _ProxyHandler(SocksiPyHandler, urllib.request.HTTPSHandler):
|
class _ProxyHandler(SocksiPyHandler, urllib.request.HTTPSHandler):
|
||||||
"""SOCKS5 handler that forwards SSL context to HTTPS connections."""
|
"""SOCKS5 handler that forwards SSL context to HTTPS connections."""
|
||||||
@@ -45,8 +57,7 @@ def urlopen(req, *, timeout=None, context=None):
|
|||||||
|
|
||||||
Retries on transient SSL/connection errors with exponential backoff.
|
Retries on transient SSL/connection errors with exponential backoff.
|
||||||
"""
|
"""
|
||||||
handler = _ProxyHandler(context=context)
|
opener = _get_opener(context)
|
||||||
opener = urllib.request.build_opener(handler)
|
|
||||||
kwargs = {}
|
kwargs = {}
|
||||||
if timeout is not None:
|
if timeout is not None:
|
||||||
kwargs["timeout"] = timeout
|
kwargs["timeout"] = timeout
|
||||||
@@ -64,6 +75,8 @@ def urlopen(req, *, timeout=None, context=None):
|
|||||||
|
|
||||||
def build_opener(*handlers, context=None):
|
def build_opener(*handlers, context=None):
|
||||||
"""Proxy-aware drop-in for urllib.request.build_opener."""
|
"""Proxy-aware drop-in for urllib.request.build_opener."""
|
||||||
|
if not handlers and context is None:
|
||||||
|
return _get_opener()
|
||||||
proxy = _ProxyHandler(context=context)
|
proxy = _ProxyHandler(context=context)
|
||||||
return urllib.request.build_opener(proxy, *handlers)
|
return urllib.request.build_opener(proxy, *handlers)
|
||||||
|
|
||||||
|
|||||||
@@ -4,18 +4,28 @@ import ssl
|
|||||||
import urllib.request
|
import urllib.request
|
||||||
from unittest.mock import MagicMock, patch
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
import socks
|
import pytest
|
||||||
from socks import SOCKS5
|
from socks import SOCKS5
|
||||||
|
|
||||||
|
import derp.http
|
||||||
from derp.http import (
|
from derp.http import (
|
||||||
_PROXY_ADDR,
|
_PROXY_ADDR,
|
||||||
_PROXY_PORT,
|
_PROXY_PORT,
|
||||||
|
_get_opener,
|
||||||
_ProxyHandler,
|
_ProxyHandler,
|
||||||
build_opener,
|
build_opener,
|
||||||
create_connection,
|
create_connection,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _reset_opener_cache():
|
||||||
|
"""Clear cached opener between tests."""
|
||||||
|
derp.http._default_opener = None
|
||||||
|
yield
|
||||||
|
derp.http._default_opener = None
|
||||||
|
|
||||||
|
|
||||||
class TestProxyHandler:
|
class TestProxyHandler:
|
||||||
def test_uses_socks5(self):
|
def test_uses_socks5(self):
|
||||||
handler = _ProxyHandler()
|
handler = _ProxyHandler()
|
||||||
@@ -67,6 +77,32 @@ class TestBuildOpener:
|
|||||||
assert proxy._ssl_context is ctx
|
assert proxy._ssl_context is ctx
|
||||||
|
|
||||||
|
|
||||||
|
class TestOpenerCache:
|
||||||
|
def test_default_opener_cached(self):
|
||||||
|
a = _get_opener()
|
||||||
|
b = _get_opener()
|
||||||
|
assert a is b
|
||||||
|
|
||||||
|
def test_custom_context_not_cached(self):
|
||||||
|
ctx = ssl.create_default_context()
|
||||||
|
a = _get_opener(context=ctx)
|
||||||
|
b = _get_opener(context=ctx)
|
||||||
|
assert a is not b
|
||||||
|
|
||||||
|
def test_build_opener_no_args_returns_cached(self):
|
||||||
|
a = build_opener()
|
||||||
|
b = build_opener()
|
||||||
|
assert a is b
|
||||||
|
|
||||||
|
def test_build_opener_with_handlers_returns_fresh(self):
|
||||||
|
class Custom(urllib.request.HTTPRedirectHandler):
|
||||||
|
pass
|
||||||
|
|
||||||
|
a = build_opener()
|
||||||
|
b = build_opener(Custom)
|
||||||
|
assert a is not b
|
||||||
|
|
||||||
|
|
||||||
class TestCreateConnection:
|
class TestCreateConnection:
|
||||||
@patch("derp.http.socks.socksocket")
|
@patch("derp.http.socks.socksocket")
|
||||||
def test_sets_socks5_proxy(self, mock_cls):
|
def test_sets_socks5_proxy(self, mock_cls):
|
||||||
|
|||||||
Reference in New Issue
Block a user