rename --worker-v2 to --worker
No V1 means no need for the suffix. Update flag, function name, compose command, log messages, and docs.
This commit is contained in:
2
TODO.md
2
TODO.md
@@ -55,7 +55,7 @@ Optimize only if memory becomes a constraint.
|
||||
|
||||
Completed. Removed `--worker` flag, `worker_main()`, `claim_work()`,
|
||||
`submit_results()`, `/api/work`, `/api/results`, and related config
|
||||
options. V2 (`--worker-v2`) is the only worker protocol.
|
||||
options. `--worker` now routes to the URL-driven protocol.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -35,4 +35,4 @@ services:
|
||||
- ./data:/app/data:Z
|
||||
- ./config.ini:/app/config.ini:ro,Z
|
||||
- ./servers.txt:/app/servers.txt:ro,Z
|
||||
command: python -u ppf.py --worker-v2 --server ${PPF_MASTER_URL:-http://10.200.1.250:8081}
|
||||
command: python -u ppf.py --worker --server ${PPF_MASTER_URL:-http://10.200.1.250:8081}
|
||||
|
||||
@@ -180,4 +180,4 @@ class Config(ComboParser):
|
||||
self.aparser.add_argument("--worker-key", help="worker authentication key", type=str, default='')
|
||||
self.aparser.add_argument("--register", help="register as worker with master server", action='store_true', default=False)
|
||||
self.aparser.add_argument("--worker-name", help="worker name for registration (default: hostname)", type=str, default='')
|
||||
self.aparser.add_argument("--worker-v2", help="run as worker node (URL-driven fetching)", action='store_true', default=False)
|
||||
self.aparser.add_argument("--worker", help="run as worker node", action='store_true', default=False)
|
||||
|
||||
16
ppf.py
16
ppf.py
@@ -328,7 +328,7 @@ def worker_send_heartbeat(server_url, worker_key, tor_ok, tor_ip=None, profiling
|
||||
|
||||
|
||||
def worker_claim_urls(server_url, worker_key, count=5):
|
||||
"""Claim batch of URLs for V2 worker mode."""
|
||||
"""Claim batch of URLs for worker mode."""
|
||||
url = '%s/api/claim-urls?key=%s&count=%d' % (server_url.rstrip('/'), worker_key, count)
|
||||
|
||||
try:
|
||||
@@ -428,8 +428,8 @@ def check_tor_connectivity(tor_hosts):
|
||||
return working, tor_ip
|
||||
|
||||
|
||||
def worker_v2_main(config):
|
||||
"""V2 worker mode -- URL-driven discovery.
|
||||
def worker_main(config):
|
||||
"""Worker mode -- URL-driven discovery.
|
||||
|
||||
Claims URLs from master, fetches through Tor, extracts and tests proxies,
|
||||
reports working proxies back to master.
|
||||
@@ -470,7 +470,7 @@ def worker_v2_main(config):
|
||||
if config.args.register:
|
||||
return
|
||||
|
||||
_log('starting worker V2 mode (URL-driven)', 'info')
|
||||
_log('starting worker mode (URL-driven)', 'info')
|
||||
_log(' server: %s' % server_url, 'info')
|
||||
_log(' threads: %d' % num_threads, 'info')
|
||||
_log(' url batch: %d' % url_batch_size, 'info')
|
||||
@@ -823,13 +823,13 @@ def worker_v2_main(config):
|
||||
|
||||
except KeyboardInterrupt:
|
||||
elapsed = time.time() - start_time
|
||||
_log('worker V2 stopping...', 'info')
|
||||
_log('worker stopping...', 'info')
|
||||
session.close()
|
||||
for wt in threads:
|
||||
wt.stop()
|
||||
for wt in threads:
|
||||
wt.term()
|
||||
_log('worker V2 stopped after %s' % format_duration(int(elapsed)), 'info')
|
||||
_log('worker stopped after %s' % format_duration(int(elapsed)), 'info')
|
||||
_log(' cycles: %d' % cycles, 'info')
|
||||
_log(' urls fetched: %d' % urls_fetched, 'info')
|
||||
_log(' proxies found: %d' % proxies_found, 'info')
|
||||
@@ -849,8 +849,8 @@ def main():
|
||||
sys.exit(1)
|
||||
|
||||
# Worker mode: URL-driven discovery
|
||||
if config.args.worker_v2 or config.args.register:
|
||||
worker_v2_main(config)
|
||||
if config.args.worker or config.args.register:
|
||||
worker_main(config)
|
||||
return
|
||||
|
||||
proxydb = mysqlite.mysqlite(config.watchd.database, str)
|
||||
|
||||
Reference in New Issue
Block a user