feat: add proxy pool config dataclasses
Add PoolSourceConfig and ProxyPoolConfig for multi-source proxy pool with health testing. Config supports both HTTP API and file sources. Backward compat: legacy proxy_source YAML key auto-converts to proxy_pool. CLI -S flag creates ProxyPoolConfig with single source. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -7,7 +7,7 @@ import asyncio
|
||||
import logging
|
||||
|
||||
from . import __version__
|
||||
from .config import Config, ProxySourceConfig, load_config, parse_proxy_url
|
||||
from .config import Config, PoolSourceConfig, ProxyPoolConfig, load_config, parse_proxy_url
|
||||
from .server import serve
|
||||
|
||||
|
||||
@@ -81,7 +81,9 @@ def main(argv: list[str] | None = None) -> int:
|
||||
config.retries = args.retries
|
||||
|
||||
if args.proxy_source:
|
||||
config.proxy_source = ProxySourceConfig(url=args.proxy_source)
|
||||
config.proxy_pool = ProxyPoolConfig(
|
||||
sources=[PoolSourceConfig(url=args.proxy_source)],
|
||||
)
|
||||
|
||||
if args.verbose:
|
||||
config.log_level = "debug"
|
||||
|
||||
@@ -8,7 +8,6 @@ from urllib.parse import urlparse
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
DEFAULT_PORTS = {"socks5": 1080, "socks4": 1080, "http": 8080}
|
||||
|
||||
|
||||
@@ -29,7 +28,7 @@ class ChainHop:
|
||||
|
||||
@dataclass
|
||||
class ProxySourceConfig:
|
||||
"""Configuration for the dynamic proxy source API."""
|
||||
"""Configuration for the dynamic proxy source API (legacy)."""
|
||||
|
||||
url: str = ""
|
||||
proto: str | None = None
|
||||
@@ -38,6 +37,31 @@ class ProxySourceConfig:
|
||||
refresh: float = 300.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class PoolSourceConfig:
|
||||
"""A single proxy source: HTTP API or text file."""
|
||||
|
||||
url: str | None = None
|
||||
file: str | None = None
|
||||
proto: str | None = None
|
||||
country: str | None = None
|
||||
limit: int | None = 1000
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProxyPoolConfig:
|
||||
"""Configuration for the managed proxy pool."""
|
||||
|
||||
sources: list[PoolSourceConfig] = field(default_factory=list)
|
||||
refresh: float = 300.0
|
||||
test_interval: float = 120.0
|
||||
test_url: str = "http://httpbin.org/ip"
|
||||
test_timeout: float = 15.0
|
||||
test_concurrency: int = 5
|
||||
max_fails: int = 3
|
||||
state_file: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
"""Server configuration."""
|
||||
@@ -49,6 +73,7 @@ class Config:
|
||||
retries: int = 3
|
||||
log_level: str = "info"
|
||||
proxy_source: ProxySourceConfig | None = None
|
||||
proxy_pool: ProxyPoolConfig | None = None
|
||||
|
||||
|
||||
def parse_proxy_url(url: str) -> ChainHop:
|
||||
@@ -114,17 +139,51 @@ def load_config(path: str | Path) -> Config:
|
||||
)
|
||||
)
|
||||
|
||||
if "proxy_source" in raw:
|
||||
if "proxy_pool" in raw:
|
||||
pp = raw["proxy_pool"]
|
||||
sources = []
|
||||
for src in pp.get("sources", []):
|
||||
sources.append(
|
||||
PoolSourceConfig(
|
||||
url=src.get("url"),
|
||||
file=src.get("file"),
|
||||
proto=src.get("proto"),
|
||||
country=src.get("country"),
|
||||
limit=src.get("limit", 1000),
|
||||
)
|
||||
)
|
||||
config.proxy_pool = ProxyPoolConfig(
|
||||
sources=sources,
|
||||
refresh=float(pp.get("refresh", 300)),
|
||||
test_interval=float(pp.get("test_interval", 120)),
|
||||
test_url=pp.get("test_url", "http://httpbin.org/ip"),
|
||||
test_timeout=float(pp.get("test_timeout", 15)),
|
||||
test_concurrency=int(pp.get("test_concurrency", 5)),
|
||||
max_fails=int(pp.get("max_fails", 3)),
|
||||
state_file=pp.get("state_file", ""),
|
||||
)
|
||||
elif "proxy_source" in raw:
|
||||
# backward compat: convert legacy proxy_source to proxy_pool
|
||||
ps = raw["proxy_source"]
|
||||
if isinstance(ps, str):
|
||||
config.proxy_source = ProxySourceConfig(url=ps)
|
||||
url, proto, country, limit, refresh = ps, None, None, 1000, 300.0
|
||||
elif isinstance(ps, dict):
|
||||
url = ps.get("url", "")
|
||||
proto = ps.get("proto")
|
||||
country = ps.get("country")
|
||||
limit = ps.get("limit", 1000)
|
||||
refresh = float(ps.get("refresh", 300))
|
||||
else:
|
||||
url, proto, country, limit, refresh = "", None, None, 1000, 300.0
|
||||
|
||||
if url:
|
||||
config.proxy_pool = ProxyPoolConfig(
|
||||
sources=[PoolSourceConfig(url=url, proto=proto, country=country, limit=limit)],
|
||||
refresh=refresh,
|
||||
)
|
||||
# keep legacy field for source.py compat during transition
|
||||
config.proxy_source = ProxySourceConfig(
|
||||
url=ps.get("url", ""),
|
||||
proto=ps.get("proto"),
|
||||
country=ps.get("country"),
|
||||
limit=ps.get("limit", 1000),
|
||||
refresh=float(ps.get("refresh", 300)),
|
||||
url=url, proto=proto, country=country, limit=limit, refresh=refresh,
|
||||
)
|
||||
|
||||
return config
|
||||
|
||||
Reference in New Issue
Block a user