perf: cache default HTTP opener at module level

Avoid rebuilding _ProxyHandler + build_opener() on every request.
Default-context callers (16 of 18 plugins) reuse one cached opener;
custom-context callers still get a fresh one.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
user
2026-02-17 10:15:20 +01:00
parent 3c505dd825
commit 933d9e1ddd
2 changed files with 52 additions and 3 deletions

View File

@@ -18,6 +18,18 @@ _RETRY_ERRORS = (ssl.SSLError, ConnectionError, TimeoutError, OSError)
_log = logging.getLogger(__name__) _log = logging.getLogger(__name__)
_default_opener: urllib.request.OpenerDirector | None = None
def _get_opener(context=None):
"""Return cached opener for default context, fresh for custom."""
global _default_opener
if context is not None:
return urllib.request.build_opener(_ProxyHandler(context=context))
if _default_opener is None:
_default_opener = urllib.request.build_opener(_ProxyHandler())
return _default_opener
class _ProxyHandler(SocksiPyHandler, urllib.request.HTTPSHandler): class _ProxyHandler(SocksiPyHandler, urllib.request.HTTPSHandler):
"""SOCKS5 handler that forwards SSL context to HTTPS connections.""" """SOCKS5 handler that forwards SSL context to HTTPS connections."""
@@ -45,8 +57,7 @@ def urlopen(req, *, timeout=None, context=None):
Retries on transient SSL/connection errors with exponential backoff. Retries on transient SSL/connection errors with exponential backoff.
""" """
handler = _ProxyHandler(context=context) opener = _get_opener(context)
opener = urllib.request.build_opener(handler)
kwargs = {} kwargs = {}
if timeout is not None: if timeout is not None:
kwargs["timeout"] = timeout kwargs["timeout"] = timeout
@@ -64,6 +75,8 @@ def urlopen(req, *, timeout=None, context=None):
def build_opener(*handlers, context=None): def build_opener(*handlers, context=None):
"""Proxy-aware drop-in for urllib.request.build_opener.""" """Proxy-aware drop-in for urllib.request.build_opener."""
if not handlers and context is None:
return _get_opener()
proxy = _ProxyHandler(context=context) proxy = _ProxyHandler(context=context)
return urllib.request.build_opener(proxy, *handlers) return urllib.request.build_opener(proxy, *handlers)

View File

@@ -4,18 +4,28 @@ import ssl
import urllib.request import urllib.request
from unittest.mock import MagicMock, patch from unittest.mock import MagicMock, patch
import socks import pytest
from socks import SOCKS5 from socks import SOCKS5
import derp.http
from derp.http import ( from derp.http import (
_PROXY_ADDR, _PROXY_ADDR,
_PROXY_PORT, _PROXY_PORT,
_get_opener,
_ProxyHandler, _ProxyHandler,
build_opener, build_opener,
create_connection, create_connection,
) )
@pytest.fixture(autouse=True)
def _reset_opener_cache():
"""Clear cached opener between tests."""
derp.http._default_opener = None
yield
derp.http._default_opener = None
class TestProxyHandler: class TestProxyHandler:
def test_uses_socks5(self): def test_uses_socks5(self):
handler = _ProxyHandler() handler = _ProxyHandler()
@@ -67,6 +77,32 @@ class TestBuildOpener:
assert proxy._ssl_context is ctx assert proxy._ssl_context is ctx
class TestOpenerCache:
def test_default_opener_cached(self):
a = _get_opener()
b = _get_opener()
assert a is b
def test_custom_context_not_cached(self):
ctx = ssl.create_default_context()
a = _get_opener(context=ctx)
b = _get_opener(context=ctx)
assert a is not b
def test_build_opener_no_args_returns_cached(self):
a = build_opener()
b = build_opener()
assert a is b
def test_build_opener_with_handlers_returns_fresh(self):
class Custom(urllib.request.HTTPRedirectHandler):
pass
a = build_opener()
b = build_opener(Custom)
assert a is not b
class TestCreateConnection: class TestCreateConnection:
@patch("derp.http.socks.socksocket") @patch("derp.http.socks.socksocket")
def test_sets_socks5_proxy(self, mock_cls): def test_sets_socks5_proxy(self, mock_cls):