dbs: add url scoring columns to uris table
Migration functions for check_interval, working_ratio, avg_fetch_time, last_worker, and yield_rate columns with sensible defaults.
This commit is contained in:
50
dbs.py
50
dbs.py
@@ -98,6 +98,51 @@ def _migrate_last_seen(sqlite):
|
||||
sqlite.commit()
|
||||
|
||||
|
||||
def _migrate_uri_check_interval(sqlite):
|
||||
"""Add adaptive check_interval column to uris table."""
|
||||
try:
|
||||
sqlite.execute('SELECT check_interval FROM uris LIMIT 1')
|
||||
except Exception:
|
||||
sqlite.execute('ALTER TABLE uris ADD COLUMN check_interval INT DEFAULT 3600')
|
||||
sqlite.commit()
|
||||
|
||||
|
||||
def _migrate_uri_working_ratio(sqlite):
|
||||
"""Add working_ratio column to uris table for proxy quality tracking."""
|
||||
try:
|
||||
sqlite.execute('SELECT working_ratio FROM uris LIMIT 1')
|
||||
except Exception:
|
||||
sqlite.execute('ALTER TABLE uris ADD COLUMN working_ratio REAL DEFAULT 0.0')
|
||||
sqlite.commit()
|
||||
|
||||
|
||||
def _migrate_uri_avg_fetch_time(sqlite):
|
||||
"""Add avg_fetch_time column to uris table for fetch latency EMA."""
|
||||
try:
|
||||
sqlite.execute('SELECT avg_fetch_time FROM uris LIMIT 1')
|
||||
except Exception:
|
||||
sqlite.execute('ALTER TABLE uris ADD COLUMN avg_fetch_time INT DEFAULT 0')
|
||||
sqlite.commit()
|
||||
|
||||
|
||||
def _migrate_uri_last_worker(sqlite):
|
||||
"""Add last_worker column to uris table."""
|
||||
try:
|
||||
sqlite.execute('SELECT last_worker FROM uris LIMIT 1')
|
||||
except Exception:
|
||||
sqlite.execute('ALTER TABLE uris ADD COLUMN last_worker TEXT')
|
||||
sqlite.commit()
|
||||
|
||||
|
||||
def _migrate_uri_yield_rate(sqlite):
|
||||
"""Add yield_rate column to uris table for proxy yield EMA."""
|
||||
try:
|
||||
sqlite.execute('SELECT yield_rate FROM uris LIMIT 1')
|
||||
except Exception:
|
||||
sqlite.execute('ALTER TABLE uris ADD COLUMN yield_rate REAL DEFAULT 0.0')
|
||||
sqlite.commit()
|
||||
|
||||
|
||||
def compute_proxy_list_hash(proxies):
|
||||
"""Compute MD5 hash of sorted proxy list for change detection.
|
||||
|
||||
@@ -356,6 +401,11 @@ def create_table_if_not_exists(sqlite, dbname):
|
||||
content_hash TEXT)""")
|
||||
# Migration for existing databases
|
||||
_migrate_content_hash_column(sqlite)
|
||||
_migrate_uri_check_interval(sqlite)
|
||||
_migrate_uri_working_ratio(sqlite)
|
||||
_migrate_uri_avg_fetch_time(sqlite)
|
||||
_migrate_uri_last_worker(sqlite)
|
||||
_migrate_uri_yield_rate(sqlite)
|
||||
# Indexes for common query patterns
|
||||
sqlite.execute('CREATE INDEX IF NOT EXISTS idx_uris_error ON uris(error)')
|
||||
sqlite.execute('CREATE INDEX IF NOT EXISTS idx_uris_checktime ON uris(check_time)')
|
||||
|
||||
Reference in New Issue
Block a user