diff --git a/ppf.py b/ppf.py index 07d2b6b..f2b4663 100644 --- a/ppf.py +++ b/ppf.py @@ -369,6 +369,71 @@ def worker_send_heartbeat(server_url, worker_key, tor_ok, tor_ip=None, profiling return False +def worker_claim_urls(server_url, worker_key, count=5): + """Claim batch of URLs for V2 worker mode.""" + url = '%s/api/claim-urls?key=%s&count=%d' % (server_url.rstrip('/'), worker_key, count) + + try: + resp = urllib2.urlopen(url, timeout=30) + result = json.loads(resp.read()) + return result.get('urls', []) + except urllib2.HTTPError as e: + if e.code == 403: + _log('worker key rejected (403), need to re-register', 'warn') + raise NeedReregister() + _log('failed to claim urls: %s' % e, 'error') + return [] + except Exception as e: + _log('failed to claim urls: %s' % e, 'error') + return [] + + +def worker_report_urls(server_url, worker_key, reports): + """Report URL fetch results to master.""" + url = '%s/api/report-urls?key=%s' % (server_url.rstrip('/'), worker_key) + data = json.dumps({'reports': reports}) + + req = urllib2.Request(url, data) + req.add_header('Content-Type', 'application/json') + + try: + resp = urllib2.urlopen(req, timeout=30) + result = json.loads(resp.read()) + return result.get('processed', 0) + except urllib2.HTTPError as e: + if e.code == 403: + _log('worker key rejected (403), need to re-register', 'warn') + raise NeedReregister() + _log('failed to report urls: %s' % e, 'error') + return 0 + except Exception as e: + _log('failed to report urls: %s' % e, 'error') + return 0 + + +def worker_report_proxies(server_url, worker_key, proxies): + """Report working proxies to master.""" + url = '%s/api/report-proxies?key=%s' % (server_url.rstrip('/'), worker_key) + data = json.dumps({'proxies': proxies}) + + req = urllib2.Request(url, data) + req.add_header('Content-Type', 'application/json') + + try: + resp = urllib2.urlopen(req, timeout=30) + result = json.loads(resp.read()) + return result.get('processed', 0) + except urllib2.HTTPError as e: + if e.code == 403: + _log('worker key rejected (403), need to re-register', 'warn') + raise NeedReregister() + _log('failed to report proxies: %s' % e, 'error') + return 0 + except Exception as e: + _log('failed to report proxies: %s' % e, 'error') + return 0 + + def check_tor_connectivity(tor_hosts): """Test Tor connectivity. Returns (working_hosts, tor_ip).""" import socket