style: normalize indentation and improve code style
- convert tabs to 4-space indentation
- add docstrings to modules and classes
- remove unused import (copy)
- use explicit object inheritance
- use 'while True' over 'while 1'
- use 'while args' over 'while len(args)'
- use '{}' over 'dict()'
- consistent string formatting
- Python 2/3 compatible Queue import
This commit is contained in:
92
mysqlite.py
92
mysqlite.py
@@ -1,44 +1,62 @@
|
||||
import time, random, sys
|
||||
#!/usr/bin/env python2
|
||||
# -*- coding: utf-8 -*-
|
||||
"""SQLite wrapper with retry logic and WAL mode."""
|
||||
|
||||
import time
|
||||
import random
|
||||
import sys
|
||||
import sqlite3
|
||||
|
||||
class mysqlite:
|
||||
def _try_op(self, op, query, args=None, rmin=1.5, rmax=7.0):
|
||||
while 1:
|
||||
try:
|
||||
if query is None:
|
||||
return op()
|
||||
elif args is None:
|
||||
return op(query)
|
||||
else:
|
||||
return op(query, args)
|
||||
except sqlite3.OperationalError as e:
|
||||
if e.message == 'database is locked':
|
||||
print "zzZzzZZ: db is locked (%s)"%self.dbname
|
||||
time.sleep(random.uniform(rmin, rmax))
|
||||
continue
|
||||
else:
|
||||
print '%s\nquery: %s\nargs: %s' % (str(sys.exc_info()), str(query), str(args))
|
||||
raise e
|
||||
|
||||
def execute(self, query, args = None, rmin=1.5, rmax=7.0):
|
||||
return self._try_op(self.cursor.execute, query, args, rmin, rmax)
|
||||
class mysqlite(object):
|
||||
"""SQLite connection wrapper with automatic retry on lock."""
|
||||
|
||||
def executemany(self, query, args, rmin=1.5, rmax=7.0):
|
||||
while len(args):
|
||||
self._try_op(self.cursor.executemany, query, args[:500], rmin, rmax)
|
||||
args = args[500:]
|
||||
def __init__(self, database, factory=None):
|
||||
self.handle = sqlite3.connect(database)
|
||||
if factory is not None:
|
||||
self.handle.text_factory = factory
|
||||
self.cursor = self.handle.cursor()
|
||||
self.dbname = database
|
||||
# Enable WAL mode for better concurrency
|
||||
self.cursor.execute('PRAGMA journal_mode=WAL')
|
||||
self.cursor.execute('PRAGMA synchronous=NORMAL')
|
||||
|
||||
def commit(self, rmin=1.5, rmax=7.0):
|
||||
return self._try_op(self.handle.commit, None, None, rmin, rmax)
|
||||
def _try_op(self, op, query, args=None, rmin=1.5, rmax=7.0):
|
||||
"""Execute operation with retry on database lock."""
|
||||
while True:
|
||||
try:
|
||||
if query is None:
|
||||
return op()
|
||||
elif args is None:
|
||||
return op(query)
|
||||
else:
|
||||
return op(query, args)
|
||||
except sqlite3.OperationalError as e:
|
||||
err_msg = str(e)
|
||||
if 'database is locked' in err_msg:
|
||||
sys.stderr.write('zzZzzZZ: db is locked (%s)\n' % self.dbname)
|
||||
time.sleep(random.uniform(rmin, rmax))
|
||||
continue
|
||||
else:
|
||||
sys.stderr.write('%s\nquery: %s\nargs: %s\n' % (
|
||||
str(sys.exc_info()), str(query), str(args)))
|
||||
raise
|
||||
|
||||
def close(self):
|
||||
self.handle.close()
|
||||
def execute(self, query, args=None, rmin=1.5, rmax=7.0):
|
||||
"""Execute a single query with retry."""
|
||||
return self._try_op(self.cursor.execute, query, args, rmin, rmax)
|
||||
|
||||
def __init__(self, database, factory = None):
|
||||
self.handle = sqlite3.connect(database)
|
||||
if factory: self.handle.text_factory = factory
|
||||
self.cursor = self.handle.cursor()
|
||||
self.dbname = database
|
||||
# enable WAL mode for better concurrency
|
||||
self.cursor.execute('PRAGMA journal_mode=WAL')
|
||||
self.cursor.execute('PRAGMA synchronous=NORMAL')
|
||||
def executemany(self, query, args, rmin=1.5, rmax=7.0):
|
||||
"""Execute query for multiple argument sets, batched."""
|
||||
while args:
|
||||
batch = args[:500]
|
||||
self._try_op(self.cursor.executemany, query, batch, rmin, rmax)
|
||||
args = args[500:]
|
||||
|
||||
def commit(self, rmin=1.5, rmax=7.0):
|
||||
"""Commit transaction with retry."""
|
||||
return self._try_op(self.handle.commit, None, None, rmin, rmax)
|
||||
|
||||
def close(self):
|
||||
"""Close database connection."""
|
||||
self.handle.close()
|
||||
|
||||
Reference in New Issue
Block a user