feat: async HTTP client and parallel source fetching
Replace blocking urllib with a minimal async HTTP/1.1 client (http.py) using asyncio streams. Pool source fetches now run in parallel via asyncio.gather. Dead proxy reporting uses async POST. Handles Content-Length, chunked transfer-encoding, and connection-close bodies. No new dependencies.
This commit is contained in:
153
tests/test_http.py
Normal file
153
tests/test_http.py
Normal file
@@ -0,0 +1,153 @@
|
||||
"""Tests for the async HTTP client."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from s5p.http import http_get_json, http_post_json
|
||||
|
||||
|
||||
async def _run_mock_server(handler, host="127.0.0.1", port=0):
|
||||
"""Start a mock TCP server, return (server, port)."""
|
||||
server = await asyncio.start_server(handler, host, port)
|
||||
port = server.sockets[0].getsockname()[1]
|
||||
return server, port
|
||||
|
||||
|
||||
class TestHttpGetJson:
|
||||
"""Test async HTTP GET."""
|
||||
|
||||
def test_basic_get(self):
|
||||
payload = {"proxies": [{"proto": "socks5", "proxy": "1.2.3.4:1080"}]}
|
||||
body = json.dumps(payload).encode()
|
||||
|
||||
async def handler(reader, writer):
|
||||
await reader.readline() # request line
|
||||
while (await reader.readline()) not in (b"\r\n", b"\n", b""):
|
||||
pass
|
||||
writer.write(
|
||||
f"HTTP/1.1 200 OK\r\n"
|
||||
f"Content-Length: {len(body)}\r\n"
|
||||
f"Connection: close\r\n"
|
||||
f"\r\n".encode()
|
||||
+ body
|
||||
)
|
||||
await writer.drain()
|
||||
writer.close()
|
||||
|
||||
async def run():
|
||||
server, port = await _run_mock_server(handler)
|
||||
async with server:
|
||||
result = await http_get_json(f"http://127.0.0.1:{port}/test")
|
||||
assert result == payload
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
def test_chunked_response(self):
|
||||
payload = {"status": "ok"}
|
||||
body = json.dumps(payload).encode()
|
||||
|
||||
async def handler(reader, writer):
|
||||
await reader.readline()
|
||||
while (await reader.readline()) not in (b"\r\n", b"\n", b""):
|
||||
pass
|
||||
writer.write(
|
||||
b"HTTP/1.1 200 OK\r\n"
|
||||
b"Transfer-Encoding: chunked\r\n"
|
||||
b"Connection: close\r\n"
|
||||
b"\r\n"
|
||||
)
|
||||
# Send body as a single chunk
|
||||
writer.write(f"{len(body):x}\r\n".encode() + body + b"\r\n")
|
||||
writer.write(b"0\r\n\r\n")
|
||||
await writer.drain()
|
||||
writer.close()
|
||||
|
||||
async def run():
|
||||
server, port = await _run_mock_server(handler)
|
||||
async with server:
|
||||
result = await http_get_json(f"http://127.0.0.1:{port}/chunked")
|
||||
assert result == payload
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
def test_error_status(self):
|
||||
async def handler(reader, writer):
|
||||
await reader.readline()
|
||||
while (await reader.readline()) not in (b"\r\n", b"\n", b""):
|
||||
pass
|
||||
writer.write(b"HTTP/1.1 500 Internal Server Error\r\nConnection: close\r\n\r\n")
|
||||
await writer.drain()
|
||||
writer.close()
|
||||
|
||||
async def run():
|
||||
server, port = await _run_mock_server(handler)
|
||||
async with server:
|
||||
with pytest.raises(OSError, match="HTTP 500"):
|
||||
await http_get_json(f"http://127.0.0.1:{port}/fail")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
def test_connection_close_body(self):
|
||||
"""Server sends no Content-Length, just closes connection."""
|
||||
payload = {"key": "value"}
|
||||
body = json.dumps(payload).encode()
|
||||
|
||||
async def handler(reader, writer):
|
||||
await reader.readline()
|
||||
while (await reader.readline()) not in (b"\r\n", b"\n", b""):
|
||||
pass
|
||||
writer.write(
|
||||
b"HTTP/1.1 200 OK\r\n"
|
||||
b"Connection: close\r\n"
|
||||
b"\r\n"
|
||||
+ body
|
||||
)
|
||||
await writer.drain()
|
||||
writer.close()
|
||||
await writer.wait_closed()
|
||||
|
||||
async def run():
|
||||
server, port = await _run_mock_server(handler)
|
||||
async with server:
|
||||
result = await http_get_json(f"http://127.0.0.1:{port}/nosize")
|
||||
assert result == payload
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
class TestHttpPostJson:
|
||||
"""Test async HTTP POST."""
|
||||
|
||||
def test_basic_post(self):
|
||||
received = {}
|
||||
|
||||
async def handler(reader, writer):
|
||||
request_line = await reader.readline()
|
||||
received["method"] = request_line.decode().split()[0]
|
||||
content_length = 0
|
||||
while True:
|
||||
line = await reader.readline()
|
||||
if line in (b"\r\n", b"\n", b""):
|
||||
break
|
||||
header = line.decode().lower()
|
||||
if header.startswith("content-length:"):
|
||||
content_length = int(header.split(":", 1)[1].strip())
|
||||
body = await reader.readexactly(content_length)
|
||||
received["body"] = json.loads(body)
|
||||
writer.write(b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n")
|
||||
await writer.drain()
|
||||
writer.close()
|
||||
|
||||
async def run():
|
||||
server, port = await _run_mock_server(handler)
|
||||
async with server:
|
||||
await http_post_json(
|
||||
f"http://127.0.0.1:{port}/report",
|
||||
{"dead": [{"proto": "socks5", "proxy": "1.2.3.4:1080"}]},
|
||||
)
|
||||
assert received["method"] == "POST"
|
||||
assert received["body"]["dead"][0]["proto"] == "socks5"
|
||||
|
||||
asyncio.run(run())
|
||||
@@ -1,6 +1,5 @@
|
||||
"""Tests for the managed proxy pool."""
|
||||
|
||||
import json
|
||||
import time
|
||||
|
||||
import pytest
|
||||
@@ -304,21 +303,20 @@ class TestProxyPoolReport:
|
||||
asyncio.run(pool._run_health_tests())
|
||||
mock_report.assert_not_called()
|
||||
|
||||
def test_report_sync_payload(self):
|
||||
from unittest.mock import MagicMock, patch
|
||||
def test_report_async_payload(self):
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
cfg = ProxyPoolConfig(sources=[], report_url="http://api:8081/report")
|
||||
pool = ProxyPool(cfg, [], timeout=10.0)
|
||||
|
||||
dead = [{"proto": "socks5", "proxy": "10.0.0.1:1080"}]
|
||||
with patch("s5p.pool.urllib.request.urlopen", new_callable=MagicMock) as mock_open:
|
||||
mock_open.return_value.__enter__ = MagicMock()
|
||||
mock_open.return_value.__exit__ = MagicMock(return_value=False)
|
||||
pool._report_sync(dead)
|
||||
req = mock_open.call_args[0][0]
|
||||
assert req.method == "POST"
|
||||
assert req.full_url == "http://api:8081/report"
|
||||
assert b'"dead"' in req.data
|
||||
with patch("s5p.pool.http_post_json", new_callable=AsyncMock) as mock_post:
|
||||
asyncio.run(pool._report_dead(["socks5://10.0.0.1:1080"]))
|
||||
mock_post.assert_called_once()
|
||||
url = mock_post.call_args[0][0]
|
||||
payload = mock_post.call_args[0][1]
|
||||
assert url == "http://api:8081/report"
|
||||
assert payload == {"dead": [{"proto": "socks5", "proxy": "10.0.0.1:1080"}]}
|
||||
|
||||
|
||||
class TestProxyPoolStaleExpiry:
|
||||
|
||||
Reference in New Issue
Block a user