e24f68500c
- convert tabs to 4-space indentation
- add docstrings to modules and classes
- remove unused import (copy)
- use explicit object inheritance
- use 'while True' over 'while 1'
- use 'while args' over 'while len(args)'
- use '{}' over 'dict()'
- consistent string formatting
- Python 2/3 compatible Queue import
63 lines
2.2 KiB
Python
63 lines
2.2 KiB
Python
#!/usr/bin/env python2
|
|
# -*- coding: utf-8 -*-
|
|
"""SQLite wrapper with retry logic and WAL mode."""
|
|
|
|
import time
|
|
import random
|
|
import sys
|
|
import sqlite3
|
|
|
|
|
|
class mysqlite(object):
|
|
"""SQLite connection wrapper with automatic retry on lock."""
|
|
|
|
def __init__(self, database, factory=None):
|
|
self.handle = sqlite3.connect(database)
|
|
if factory is not None:
|
|
self.handle.text_factory = factory
|
|
self.cursor = self.handle.cursor()
|
|
self.dbname = database
|
|
# Enable WAL mode for better concurrency
|
|
self.cursor.execute('PRAGMA journal_mode=WAL')
|
|
self.cursor.execute('PRAGMA synchronous=NORMAL')
|
|
|
|
def _try_op(self, op, query, args=None, rmin=1.5, rmax=7.0):
|
|
"""Execute operation with retry on database lock."""
|
|
while True:
|
|
try:
|
|
if query is None:
|
|
return op()
|
|
elif args is None:
|
|
return op(query)
|
|
else:
|
|
return op(query, args)
|
|
except sqlite3.OperationalError as e:
|
|
err_msg = str(e)
|
|
if 'database is locked' in err_msg:
|
|
sys.stderr.write('zzZzzZZ: db is locked (%s)\n' % self.dbname)
|
|
time.sleep(random.uniform(rmin, rmax))
|
|
continue
|
|
else:
|
|
sys.stderr.write('%s\nquery: %s\nargs: %s\n' % (
|
|
str(sys.exc_info()), str(query), str(args)))
|
|
raise
|
|
|
|
def execute(self, query, args=None, rmin=1.5, rmax=7.0):
|
|
"""Execute a single query with retry."""
|
|
return self._try_op(self.cursor.execute, query, args, rmin, rmax)
|
|
|
|
def executemany(self, query, args, rmin=1.5, rmax=7.0):
|
|
"""Execute query for multiple argument sets, batched."""
|
|
while args:
|
|
batch = args[:500]
|
|
self._try_op(self.cursor.executemany, query, batch, rmin, rmax)
|
|
args = args[500:]
|
|
|
|
def commit(self, rmin=1.5, rmax=7.0):
|
|
"""Commit transaction with retry."""
|
|
return self._try_op(self.handle.commit, None, None, rmin, rmax)
|
|
|
|
def close(self):
|
|
"""Close database connection."""
|
|
self.handle.close()
|