feat: async HTTP client and parallel source fetching

Replace blocking urllib with a minimal async HTTP/1.1 client (http.py)
using asyncio streams. Pool source fetches now run in parallel via
asyncio.gather. Dead proxy reporting uses async POST. Handles
Content-Length, chunked transfer-encoding, and connection-close bodies.
No new dependencies.
This commit is contained in:
user
2026-02-15 17:55:56 +01:00
parent 714e8efb3d
commit 903cb38b9f
5 changed files with 338 additions and 58 deletions

View File

@@ -1,6 +1,5 @@
"""Tests for the managed proxy pool."""
import json
import time
import pytest
@@ -304,21 +303,20 @@ class TestProxyPoolReport:
asyncio.run(pool._run_health_tests())
mock_report.assert_not_called()
def test_report_sync_payload(self):
from unittest.mock import MagicMock, patch
def test_report_async_payload(self):
import asyncio
from unittest.mock import AsyncMock, patch
cfg = ProxyPoolConfig(sources=[], report_url="http://api:8081/report")
pool = ProxyPool(cfg, [], timeout=10.0)
dead = [{"proto": "socks5", "proxy": "10.0.0.1:1080"}]
with patch("s5p.pool.urllib.request.urlopen", new_callable=MagicMock) as mock_open:
mock_open.return_value.__enter__ = MagicMock()
mock_open.return_value.__exit__ = MagicMock(return_value=False)
pool._report_sync(dead)
req = mock_open.call_args[0][0]
assert req.method == "POST"
assert req.full_url == "http://api:8081/report"
assert b'"dead"' in req.data
with patch("s5p.pool.http_post_json", new_callable=AsyncMock) as mock_post:
asyncio.run(pool._report_dead(["socks5://10.0.0.1:1080"]))
mock_post.assert_called_once()
url = mock_post.call_args[0][0]
payload = mock_post.call_args[0][1]
assert url == "http://api:8081/report"
assert payload == {"dead": [{"proto": "socks5", "proxy": "10.0.0.1:1080"}]}
class TestProxyPoolStaleExpiry: