username/ppf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dbs: ensure items are unique

Browse Source
This commit is contained in:
Your Name
2021-02-06 23:22:39 +01:00
parent f6749393ae
commit d658309ee4
1 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
dbs.py
View File

@@ -4,7 +4,7 @@ from misc import _log
def create_table_if_not_exists(sqlite, dbname): def create_table_if_not_exists(sqlite, dbname):
if dbname == 'proxylist': if dbname == 'proxylist':
sqlite.execute("""CREATE TABLE IF NOT EXISTS proxylist ( sqlite.execute("""CREATE TABLE IF NOT EXISTS proxylist (
proxy BLOB, proxy BLOB UNIQUE,
country BLOB, country BLOB,
added INT, added INT,
failed INT, failed INT,
@@ -16,7 +16,7 @@ def create_table_if_not_exists(sqlite, dbname):
elif dbname == 'uris': elif dbname == 'uris':
sqlite.execute("""CREATE TABLE IF NOT EXISTS uris ( sqlite.execute("""CREATE TABLE IF NOT EXISTS uris (
url TEXT, url TEXT UNIQUE,
content_type TEXT, content_type TEXT,
check_time INT, check_time INT,
error INT, error INT,
@@ -35,7 +35,7 @@ def insert_proxies(proxydb, proxies, url):
for p in proxies: for p in proxies:
new.append((timestamp,p,3,0,0,0)) new.append((timestamp,p,3,0,0,0))
proxydb.executemany('INSERT INTO proxylist (added,proxy,failed,tested,success_count,total_duration) VALUES (?,?,?,?,?,?)', new) proxydb.executemany('INSERT OR IGNORE INTO proxylist (added,proxy,failed,tested,success_count,total_duration) VALUES (?,?,?,?,?,?)', new)
proxydb.commit() proxydb.commit()
_log('+%d proxy/ies from %s' % (len(proxies), url), 'added') _log('+%d proxy/ies from %s' % (len(proxies), url), 'added')
@@ -47,7 +47,7 @@ def insert_urls(urls, search, sqlite):
time_now = int(time.time()) time_now = int(time.time())
new = [ (time_now,i,0,1,0,0,0) for i in urls if not i in known ] new = [ (time_now,i,0,1,0,0,0) for i in urls if not i in known ]
if not len(new): return if not len(new): return
sqlite.executemany('INSERT INTO uris (added,url,check_time,error,stale_count,retrievals,proxies_added) values(?,?,?,?,?,?,?)', new) sqlite.executemany('INSERT OR IGNORE INTO uris (added,url,check_time,error,stale_count,retrievals,proxies_added) values(?,?,?,?,?,?,?)', new)
sqlite.commit() sqlite.commit()
_log('+%d url(s) from %s' % (len(new), search), 'added') _log('+%d url(s) from %s' % (len(new), search), 'added')