dbs: add url scoring columns to uris table

Migration functions for check_interval, working_ratio, avg_fetch_time,
last_worker, and yield_rate columns with sensible defaults.
This commit is contained in:
Username
2026-02-17 15:19:59 +01:00
parent 862eeed5c8
commit 66441f9292

50
dbs.py
View File

@@ -98,6 +98,51 @@ def _migrate_last_seen(sqlite):
sqlite.commit()
def _migrate_uri_check_interval(sqlite):
"""Add adaptive check_interval column to uris table."""
try:
sqlite.execute('SELECT check_interval FROM uris LIMIT 1')
except Exception:
sqlite.execute('ALTER TABLE uris ADD COLUMN check_interval INT DEFAULT 3600')
sqlite.commit()
def _migrate_uri_working_ratio(sqlite):
"""Add working_ratio column to uris table for proxy quality tracking."""
try:
sqlite.execute('SELECT working_ratio FROM uris LIMIT 1')
except Exception:
sqlite.execute('ALTER TABLE uris ADD COLUMN working_ratio REAL DEFAULT 0.0')
sqlite.commit()
def _migrate_uri_avg_fetch_time(sqlite):
"""Add avg_fetch_time column to uris table for fetch latency EMA."""
try:
sqlite.execute('SELECT avg_fetch_time FROM uris LIMIT 1')
except Exception:
sqlite.execute('ALTER TABLE uris ADD COLUMN avg_fetch_time INT DEFAULT 0')
sqlite.commit()
def _migrate_uri_last_worker(sqlite):
"""Add last_worker column to uris table."""
try:
sqlite.execute('SELECT last_worker FROM uris LIMIT 1')
except Exception:
sqlite.execute('ALTER TABLE uris ADD COLUMN last_worker TEXT')
sqlite.commit()
def _migrate_uri_yield_rate(sqlite):
"""Add yield_rate column to uris table for proxy yield EMA."""
try:
sqlite.execute('SELECT yield_rate FROM uris LIMIT 1')
except Exception:
sqlite.execute('ALTER TABLE uris ADD COLUMN yield_rate REAL DEFAULT 0.0')
sqlite.commit()
def compute_proxy_list_hash(proxies):
"""Compute MD5 hash of sorted proxy list for change detection.
@@ -356,6 +401,11 @@ def create_table_if_not_exists(sqlite, dbname):
content_hash TEXT)""")
# Migration for existing databases
_migrate_content_hash_column(sqlite)
_migrate_uri_check_interval(sqlite)
_migrate_uri_working_ratio(sqlite)
_migrate_uri_avg_fetch_time(sqlite)
_migrate_uri_last_worker(sqlite)
_migrate_uri_yield_rate(sqlite)
# Indexes for common query patterns
sqlite.execute('CREATE INDEX IF NOT EXISTS idx_uris_error ON uris(error)')
sqlite.execute('CREATE INDEX IF NOT EXISTS idx_uris_checktime ON uris(check_time)')