add configurable thread scaling and queue counter reset
- Add scale_cooldown and scale_threshold config options - ThreadScaler now reads scaling params from config - Reset priority queue counter when queue empties to prevent unbounded growth
This commit is contained in:
@@ -67,6 +67,10 @@ source_file = servers.txt
|
||||
# Enable Tor circuit health monitoring
|
||||
tor_safeguard = 1
|
||||
|
||||
# Thread scaling
|
||||
scale_cooldown = 10
|
||||
scale_threshold = 10.0
|
||||
|
||||
# Debug output
|
||||
debug = 0
|
||||
|
||||
@@ -95,6 +99,10 @@ max_fail = 5
|
||||
# Only extract URLs from same domain
|
||||
extract_samedomain = 0
|
||||
|
||||
# Max age in days for proxy list URLs
|
||||
# URLs older than this that never produced proxies are skipped
|
||||
list_max_age_days = 7
|
||||
|
||||
# Debug output
|
||||
debug = 0
|
||||
|
||||
@@ -140,3 +148,15 @@ listenip = 127.0.0.1
|
||||
|
||||
# Listen port
|
||||
port = 8081
|
||||
|
||||
[worker]
|
||||
# Distributed worker settings (for --worker mode)
|
||||
|
||||
# Proxies per work batch
|
||||
batch_size = 100
|
||||
|
||||
# Heartbeat interval in seconds
|
||||
heartbeat = 60
|
||||
|
||||
# Seconds before unclaimed work is released
|
||||
claim_timeout = 300
|
||||
|
||||
Reference in New Issue
Block a user