dbs: add url scoring columns to uris table
Migration functions for check_interval, working_ratio, avg_fetch_time, last_worker, and yield_rate columns with sensible defaults.
This commit is contained in:
50
dbs.py
50
dbs.py
@@ -98,6 +98,51 @@ def _migrate_last_seen(sqlite):
|
|||||||
sqlite.commit()
|
sqlite.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def _migrate_uri_check_interval(sqlite):
|
||||||
|
"""Add adaptive check_interval column to uris table."""
|
||||||
|
try:
|
||||||
|
sqlite.execute('SELECT check_interval FROM uris LIMIT 1')
|
||||||
|
except Exception:
|
||||||
|
sqlite.execute('ALTER TABLE uris ADD COLUMN check_interval INT DEFAULT 3600')
|
||||||
|
sqlite.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def _migrate_uri_working_ratio(sqlite):
|
||||||
|
"""Add working_ratio column to uris table for proxy quality tracking."""
|
||||||
|
try:
|
||||||
|
sqlite.execute('SELECT working_ratio FROM uris LIMIT 1')
|
||||||
|
except Exception:
|
||||||
|
sqlite.execute('ALTER TABLE uris ADD COLUMN working_ratio REAL DEFAULT 0.0')
|
||||||
|
sqlite.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def _migrate_uri_avg_fetch_time(sqlite):
|
||||||
|
"""Add avg_fetch_time column to uris table for fetch latency EMA."""
|
||||||
|
try:
|
||||||
|
sqlite.execute('SELECT avg_fetch_time FROM uris LIMIT 1')
|
||||||
|
except Exception:
|
||||||
|
sqlite.execute('ALTER TABLE uris ADD COLUMN avg_fetch_time INT DEFAULT 0')
|
||||||
|
sqlite.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def _migrate_uri_last_worker(sqlite):
|
||||||
|
"""Add last_worker column to uris table."""
|
||||||
|
try:
|
||||||
|
sqlite.execute('SELECT last_worker FROM uris LIMIT 1')
|
||||||
|
except Exception:
|
||||||
|
sqlite.execute('ALTER TABLE uris ADD COLUMN last_worker TEXT')
|
||||||
|
sqlite.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def _migrate_uri_yield_rate(sqlite):
|
||||||
|
"""Add yield_rate column to uris table for proxy yield EMA."""
|
||||||
|
try:
|
||||||
|
sqlite.execute('SELECT yield_rate FROM uris LIMIT 1')
|
||||||
|
except Exception:
|
||||||
|
sqlite.execute('ALTER TABLE uris ADD COLUMN yield_rate REAL DEFAULT 0.0')
|
||||||
|
sqlite.commit()
|
||||||
|
|
||||||
|
|
||||||
def compute_proxy_list_hash(proxies):
|
def compute_proxy_list_hash(proxies):
|
||||||
"""Compute MD5 hash of sorted proxy list for change detection.
|
"""Compute MD5 hash of sorted proxy list for change detection.
|
||||||
|
|
||||||
@@ -356,6 +401,11 @@ def create_table_if_not_exists(sqlite, dbname):
|
|||||||
content_hash TEXT)""")
|
content_hash TEXT)""")
|
||||||
# Migration for existing databases
|
# Migration for existing databases
|
||||||
_migrate_content_hash_column(sqlite)
|
_migrate_content_hash_column(sqlite)
|
||||||
|
_migrate_uri_check_interval(sqlite)
|
||||||
|
_migrate_uri_working_ratio(sqlite)
|
||||||
|
_migrate_uri_avg_fetch_time(sqlite)
|
||||||
|
_migrate_uri_last_worker(sqlite)
|
||||||
|
_migrate_uri_yield_rate(sqlite)
|
||||||
# Indexes for common query patterns
|
# Indexes for common query patterns
|
||||||
sqlite.execute('CREATE INDEX IF NOT EXISTS idx_uris_error ON uris(error)')
|
sqlite.execute('CREATE INDEX IF NOT EXISTS idx_uris_error ON uris(error)')
|
||||||
sqlite.execute('CREATE INDEX IF NOT EXISTS idx_uris_checktime ON uris(check_time)')
|
sqlite.execute('CREATE INDEX IF NOT EXISTS idx_uris_checktime ON uris(check_time)')
|
||||||
|
|||||||
Reference in New Issue
Block a user