diff --git a/TODO.md b/TODO.md index ed1089d..2a10e81 100644 --- a/TODO.md +++ b/TODO.md @@ -55,7 +55,7 @@ Optimize only if memory becomes a constraint. Completed. Removed `--worker` flag, `worker_main()`, `claim_work()`, `submit_results()`, `/api/work`, `/api/results`, and related config -options. V2 (`--worker-v2`) is the only worker protocol. +options. `--worker` now routes to the URL-driven protocol. --- diff --git a/compose.worker.yml b/compose.worker.yml index cac7fa5..5d9d66e 100644 --- a/compose.worker.yml +++ b/compose.worker.yml @@ -35,4 +35,4 @@ services: - ./data:/app/data:Z - ./config.ini:/app/config.ini:ro,Z - ./servers.txt:/app/servers.txt:ro,Z - command: python -u ppf.py --worker-v2 --server ${PPF_MASTER_URL:-http://10.200.1.250:8081} + command: python -u ppf.py --worker --server ${PPF_MASTER_URL:-http://10.200.1.250:8081} diff --git a/config.py b/config.py index f2dac31..5c4eae1 100644 --- a/config.py +++ b/config.py @@ -180,4 +180,4 @@ class Config(ComboParser): self.aparser.add_argument("--worker-key", help="worker authentication key", type=str, default='') self.aparser.add_argument("--register", help="register as worker with master server", action='store_true', default=False) self.aparser.add_argument("--worker-name", help="worker name for registration (default: hostname)", type=str, default='') - self.aparser.add_argument("--worker-v2", help="run as worker node (URL-driven fetching)", action='store_true', default=False) + self.aparser.add_argument("--worker", help="run as worker node", action='store_true', default=False) diff --git a/ppf.py b/ppf.py index dca207b..2fad42e 100644 --- a/ppf.py +++ b/ppf.py @@ -328,7 +328,7 @@ def worker_send_heartbeat(server_url, worker_key, tor_ok, tor_ip=None, profiling def worker_claim_urls(server_url, worker_key, count=5): - """Claim batch of URLs for V2 worker mode.""" + """Claim batch of URLs for worker mode.""" url = '%s/api/claim-urls?key=%s&count=%d' % (server_url.rstrip('/'), worker_key, count) try: @@ -428,8 +428,8 @@ def check_tor_connectivity(tor_hosts): return working, tor_ip -def worker_v2_main(config): - """V2 worker mode -- URL-driven discovery. +def worker_main(config): + """Worker mode -- URL-driven discovery. Claims URLs from master, fetches through Tor, extracts and tests proxies, reports working proxies back to master. @@ -470,7 +470,7 @@ def worker_v2_main(config): if config.args.register: return - _log('starting worker V2 mode (URL-driven)', 'info') + _log('starting worker mode (URL-driven)', 'info') _log(' server: %s' % server_url, 'info') _log(' threads: %d' % num_threads, 'info') _log(' url batch: %d' % url_batch_size, 'info') @@ -823,13 +823,13 @@ def worker_v2_main(config): except KeyboardInterrupt: elapsed = time.time() - start_time - _log('worker V2 stopping...', 'info') + _log('worker stopping...', 'info') session.close() for wt in threads: wt.stop() for wt in threads: wt.term() - _log('worker V2 stopped after %s' % format_duration(int(elapsed)), 'info') + _log('worker stopped after %s' % format_duration(int(elapsed)), 'info') _log(' cycles: %d' % cycles, 'info') _log(' urls fetched: %d' % urls_fetched, 'info') _log(' proxies found: %d' % proxies_found, 'info') @@ -849,8 +849,8 @@ def main(): sys.exit(1) # Worker mode: URL-driven discovery - if config.args.worker_v2 or config.args.register: - worker_v2_main(config) + if config.args.worker or config.args.register: + worker_main(config) return proxydb = mysqlite.mysqlite(config.watchd.database, str)