username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ppf: handle confidence field in proxy tuples

Browse Source
This commit is contained in:
Username
2025-12-26 19:34:22 +01:00
parent 481dc514fb
commit a20b5525f0
1 changed files with 5 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
ppf.py
View File

@@ -12,7 +12,6 @@ from config import Config
import fetch import fetch
import sys import sys
from soup_parser import set_nobs from soup_parser import set_nobs
import re
import threading import threading
import random import random
@@ -153,8 +152,8 @@ class Leechered(threading.Thread):
# Content changed or first fetch - reset stale_count, proceed with normal processing # Content changed or first fetch - reset stale_count, proceed with normal processing
self.stale_count = 0 self.stale_count = 0
# unique is list of (address, proto) tuples; filter by address, keep tuple # unique is list of (address, proto, confidence) tuples; filter by address, keep tuple
self.proxylist = [(addr, pr) for addr, pr in unique if not fetch.is_known_proxy(addr)] self.proxylist = [(addr, pr, conf) for addr, pr, conf in unique if not fetch.is_known_proxy(addr)]
proxy_count = len(self.proxylist) proxy_count = len(self.proxylist)
if self.retrievals == 0: # new site if self.retrievals == 0: # new site
@@ -267,10 +266,10 @@ def main():
for thread in threads: for thread in threads:
if thread.status == 'ok': if thread.status == 'ok':
url, proxylist, stale_count, error, retrievals, content_type, proxies_added, execute = thread.retrieve() url, proxylist, stale_count, error, retrievals, content_type, proxies_added, execute = thread.retrieve()
# proxylist is list of (address, proto) tuples # proxylist is list of (address, proto, confidence) tuples
new = [(addr, pr) for addr, pr in proxylist if not fetch.is_known_proxy(addr)] new = [(addr, pr, conf) for addr, pr, conf in proxylist if not fetch.is_known_proxy(addr)]
if new: if new:
fetch.add_known_proxies([addr for addr, pr in new]) fetch.add_known_proxies([addr for addr, pr, conf in new])
# Update content_hash if we have a new one # Update content_hash if we have a new one
new_hash = thread.new_hash new_hash = thread.new_hash
execute = (error, stale_count, int(time.time()), retrievals, proxies_added+len(new), content_type, new_hash, url) execute = (error, stale_count, int(time.time()), retrievals, proxies_added+len(new), content_type, new_hash, url)