Initial commit: RF Mapper v0.3.0-dev

WiFi & Bluetooth signal mapping tool for Raspberry Pi with:
- WiFi scanning via iw command
- Bluetooth Classic/BLE device discovery
- RSSI-based distance estimation
- OUI manufacturer lookup
- Web dashboard with multiple views:
  - Radar view (polar plot)
  - 2D Map (Leaflet/OpenStreetMap)
  - 3D Map (MapLibre GL JS with building extrusion)
- Floor-based device positioning
- Live BT tracking mode (auto-starts on page load)
- SQLite database for historical device tracking:
  - RSSI time-series history
  - Device statistics (avg/min/max)
  - Movement detection and velocity estimation
  - Activity patterns (hourly/daily)
  - New device alerts
  - Automatic data retention/cleanup
- REST API for all functionality

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
User
2026-02-01 00:08:21 +01:00
commit 52df6421be
33 changed files with 8939 additions and 0 deletions

643
src/rf_mapper/database.py Normal file
View File

@@ -0,0 +1,643 @@
"""SQLite database for RF Mapper historical data and device tracking"""
import sqlite3
import json
from datetime import datetime, timedelta
from pathlib import Path
from dataclasses import dataclass
from typing import Optional
import threading
@dataclass
class DeviceStats:
"""Statistics for a device"""
device_id: str
device_type: str # 'wifi' or 'bluetooth'
name: str
manufacturer: str
first_seen: str
last_seen: str
total_observations: int
avg_rssi: float
min_rssi: int
max_rssi: int
avg_distance_m: float
min_distance_m: float
max_distance_m: float
@dataclass
class RSSIObservation:
"""Single RSSI observation"""
timestamp: str
rssi: int
distance_m: float
floor: Optional[int] = None
class DeviceDatabase:
"""SQLite database for device history and statistics"""
def __init__(self, db_path: Path | str):
self.db_path = Path(db_path)
self.db_path.parent.mkdir(parents=True, exist_ok=True)
self._local = threading.local()
self._init_schema()
def _get_connection(self) -> sqlite3.Connection:
"""Get thread-local database connection"""
if not hasattr(self._local, 'conn') or self._local.conn is None:
self._local.conn = sqlite3.connect(str(self.db_path))
self._local.conn.row_factory = sqlite3.Row
return self._local.conn
def _init_schema(self):
"""Initialize database schema"""
conn = self._get_connection()
cursor = conn.cursor()
# Devices table - master record for each unique device
cursor.execute("""
CREATE TABLE IF NOT EXISTS devices (
device_id TEXT PRIMARY KEY,
device_type TEXT NOT NULL, -- 'wifi' or 'bluetooth'
name TEXT,
ssid TEXT, -- For WiFi only
manufacturer TEXT,
device_class TEXT, -- For Bluetooth
bt_device_type TEXT, -- For Bluetooth
encryption TEXT, -- For WiFi
channel INTEGER, -- For WiFi
frequency INTEGER, -- For WiFi
first_seen TEXT NOT NULL,
last_seen TEXT NOT NULL,
total_observations INTEGER DEFAULT 0,
custom_label TEXT, -- User-assigned name
is_favorite INTEGER DEFAULT 0,
notes TEXT,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT DEFAULT CURRENT_TIMESTAMP
)
""")
# RSSI observations - time series data
cursor.execute("""
CREATE TABLE IF NOT EXISTS rssi_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
device_id TEXT NOT NULL,
timestamp TEXT NOT NULL,
rssi INTEGER NOT NULL,
distance_m REAL,
floor INTEGER,
scan_id TEXT,
FOREIGN KEY (device_id) REFERENCES devices(device_id)
)
""")
# Scans table - record of each scan
cursor.execute("""
CREATE TABLE IF NOT EXISTS scans (
scan_id TEXT PRIMARY KEY,
timestamp TEXT NOT NULL,
location_label TEXT,
lat REAL,
lon REAL,
wifi_count INTEGER DEFAULT 0,
bt_count INTEGER DEFAULT 0,
created_at TEXT DEFAULT CURRENT_TIMESTAMP
)
""")
# Device statistics - pre-computed for performance
cursor.execute("""
CREATE TABLE IF NOT EXISTS device_stats (
device_id TEXT PRIMARY KEY,
avg_rssi REAL,
min_rssi INTEGER,
max_rssi INTEGER,
avg_distance_m REAL,
min_distance_m REAL,
max_distance_m REAL,
appearance_count INTEGER DEFAULT 0,
last_computed TEXT,
FOREIGN KEY (device_id) REFERENCES devices(device_id)
)
""")
# Movement events - detected motion
cursor.execute("""
CREATE TABLE IF NOT EXISTS movement_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
device_id TEXT NOT NULL,
timestamp TEXT NOT NULL,
rssi_delta INTEGER,
distance_delta_m REAL,
direction TEXT, -- 'approaching', 'receding', 'stationary'
velocity_m_s REAL,
FOREIGN KEY (device_id) REFERENCES devices(device_id)
)
""")
# Alerts table - for new device detection, absence alerts
cursor.execute("""
CREATE TABLE IF NOT EXISTS alerts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
alert_type TEXT NOT NULL, -- 'new_device', 'device_absent', 'rssi_threshold'
device_id TEXT,
timestamp TEXT NOT NULL,
message TEXT,
acknowledged INTEGER DEFAULT 0,
FOREIGN KEY (device_id) REFERENCES devices(device_id)
)
""")
# Create indexes for performance
cursor.execute("CREATE INDEX IF NOT EXISTS idx_rssi_device_time ON rssi_history(device_id, timestamp)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_rssi_timestamp ON rssi_history(timestamp)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_devices_type ON devices(device_type)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_devices_last_seen ON devices(last_seen)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_movement_device ON movement_events(device_id, timestamp)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_alerts_type ON alerts(alert_type, acknowledged)")
conn.commit()
def record_scan(self, scan_id: str, timestamp: str, location_label: str,
lat: float, lon: float, wifi_count: int, bt_count: int):
"""Record a scan event"""
conn = self._get_connection()
cursor = conn.cursor()
cursor.execute("""
INSERT OR REPLACE INTO scans (scan_id, timestamp, location_label, lat, lon, wifi_count, bt_count)
VALUES (?, ?, ?, ?, ?, ?, ?)
""", (scan_id, timestamp, location_label, lat, lon, wifi_count, bt_count))
conn.commit()
def record_wifi_observation(self, bssid: str, ssid: str, rssi: int, distance_m: float,
channel: int, frequency: int, encryption: str,
manufacturer: str, floor: Optional[int] = None,
scan_id: Optional[str] = None):
"""Record a WiFi network observation"""
conn = self._get_connection()
cursor = conn.cursor()
timestamp = datetime.now().isoformat()
# Insert or update device
cursor.execute("""
INSERT INTO devices (device_id, device_type, name, ssid, manufacturer, encryption, channel, frequency, first_seen, last_seen, total_observations)
VALUES (?, 'wifi', ?, ?, ?, ?, ?, ?, ?, ?, 1)
ON CONFLICT(device_id) DO UPDATE SET
name = COALESCE(excluded.name, devices.name),
ssid = COALESCE(excluded.ssid, devices.ssid),
manufacturer = COALESCE(excluded.manufacturer, devices.manufacturer),
encryption = COALESCE(excluded.encryption, devices.encryption),
channel = COALESCE(excluded.channel, devices.channel),
frequency = COALESCE(excluded.frequency, devices.frequency),
last_seen = excluded.last_seen,
total_observations = devices.total_observations + 1,
updated_at = CURRENT_TIMESTAMP
""", (bssid, ssid, ssid, manufacturer, encryption, channel, frequency, timestamp, timestamp))
# Insert RSSI observation
cursor.execute("""
INSERT INTO rssi_history (device_id, timestamp, rssi, distance_m, floor, scan_id)
VALUES (?, ?, ?, ?, ?, ?)
""", (bssid, timestamp, rssi, distance_m, floor, scan_id))
conn.commit()
# Check if this is a new device
cursor.execute("SELECT total_observations FROM devices WHERE device_id = ?", (bssid,))
row = cursor.fetchone()
if row and row['total_observations'] == 1:
self._create_alert('new_device', bssid, f"New WiFi network detected: {ssid} ({manufacturer})")
def record_bluetooth_observation(self, address: str, name: str, rssi: int, distance_m: float,
device_class: str, device_type: str, manufacturer: str,
floor: Optional[int] = None, scan_id: Optional[str] = None):
"""Record a Bluetooth device observation"""
conn = self._get_connection()
cursor = conn.cursor()
timestamp = datetime.now().isoformat()
# Get previous observation for movement detection
cursor.execute("""
SELECT rssi, distance_m, timestamp FROM rssi_history
WHERE device_id = ? ORDER BY timestamp DESC LIMIT 1
""", (address,))
prev = cursor.fetchone()
# Insert or update device
cursor.execute("""
INSERT INTO devices (device_id, device_type, name, manufacturer, device_class, bt_device_type, first_seen, last_seen, total_observations)
VALUES (?, 'bluetooth', ?, ?, ?, ?, ?, ?, 1)
ON CONFLICT(device_id) DO UPDATE SET
name = CASE WHEN excluded.name != '<unknown>' AND excluded.name != '' THEN excluded.name ELSE devices.name END,
manufacturer = COALESCE(NULLIF(excluded.manufacturer, ''), devices.manufacturer),
device_class = COALESCE(excluded.device_class, devices.device_class),
bt_device_type = COALESCE(excluded.bt_device_type, devices.bt_device_type),
last_seen = excluded.last_seen,
total_observations = devices.total_observations + 1,
updated_at = CURRENT_TIMESTAMP
""", (address, name, manufacturer, device_class, device_type, timestamp, timestamp))
# Insert RSSI observation
cursor.execute("""
INSERT INTO rssi_history (device_id, timestamp, rssi, distance_m, floor, scan_id)
VALUES (?, ?, ?, ?, ?, ?)
""", (address, timestamp, rssi, distance_m, floor, scan_id))
conn.commit()
# Movement detection
if prev:
rssi_delta = rssi - prev['rssi']
distance_delta = distance_m - prev['distance_m']
prev_time = datetime.fromisoformat(prev['timestamp'])
time_delta = (datetime.now() - prev_time).total_seconds()
if abs(distance_delta) > 0.5 and time_delta > 0: # More than 0.5m movement
velocity = distance_delta / time_delta if time_delta > 0 else 0
direction = 'approaching' if distance_delta < 0 else 'receding'
cursor.execute("""
INSERT INTO movement_events (device_id, timestamp, rssi_delta, distance_delta_m, direction, velocity_m_s)
VALUES (?, ?, ?, ?, ?, ?)
""", (address, timestamp, rssi_delta, distance_delta, direction, velocity))
conn.commit()
# Check if this is a new device
cursor.execute("SELECT total_observations FROM devices WHERE device_id = ?", (address,))
row = cursor.fetchone()
if row and row['total_observations'] == 1:
self._create_alert('new_device', address, f"New Bluetooth device detected: {name} ({manufacturer})")
def _create_alert(self, alert_type: str, device_id: str, message: str):
"""Create an alert"""
conn = self._get_connection()
cursor = conn.cursor()
timestamp = datetime.now().isoformat()
cursor.execute("""
INSERT INTO alerts (alert_type, device_id, timestamp, message)
VALUES (?, ?, ?, ?)
""", (alert_type, device_id, timestamp, message))
conn.commit()
def get_device(self, device_id: str) -> Optional[dict]:
"""Get device details"""
conn = self._get_connection()
cursor = conn.cursor()
cursor.execute("SELECT * FROM devices WHERE device_id = ?", (device_id,))
row = cursor.fetchone()
return dict(row) if row else None
def get_all_devices(self, device_type: Optional[str] = None,
since: Optional[str] = None,
limit: int = 100) -> list[dict]:
"""Get all devices with optional filtering"""
conn = self._get_connection()
cursor = conn.cursor()
query = "SELECT * FROM devices WHERE 1=1"
params = []
if device_type:
query += " AND device_type = ?"
params.append(device_type)
if since:
query += " AND last_seen >= ?"
params.append(since)
query += " ORDER BY last_seen DESC LIMIT ?"
params.append(limit)
cursor.execute(query, params)
return [dict(row) for row in cursor.fetchall()]
def get_device_rssi_history(self, device_id: str,
since: Optional[str] = None,
limit: int = 1000) -> list[RSSIObservation]:
"""Get RSSI history for a device"""
conn = self._get_connection()
cursor = conn.cursor()
query = "SELECT timestamp, rssi, distance_m, floor FROM rssi_history WHERE device_id = ?"
params = [device_id]
if since:
query += " AND timestamp >= ?"
params.append(since)
query += " ORDER BY timestamp DESC LIMIT ?"
params.append(limit)
cursor.execute(query, params)
return [RSSIObservation(
timestamp=row['timestamp'],
rssi=row['rssi'],
distance_m=row['distance_m'],
floor=row['floor']
) for row in cursor.fetchall()]
def get_device_stats(self, device_id: str) -> Optional[DeviceStats]:
"""Get computed statistics for a device"""
conn = self._get_connection()
cursor = conn.cursor()
# Get device info
cursor.execute("SELECT * FROM devices WHERE device_id = ?", (device_id,))
device = cursor.fetchone()
if not device:
return None
# Compute stats from RSSI history
cursor.execute("""
SELECT
AVG(rssi) as avg_rssi,
MIN(rssi) as min_rssi,
MAX(rssi) as max_rssi,
AVG(distance_m) as avg_distance_m,
MIN(distance_m) as min_distance_m,
MAX(distance_m) as max_distance_m
FROM rssi_history WHERE device_id = ?
""", (device_id,))
stats = cursor.fetchone()
return DeviceStats(
device_id=device_id,
device_type=device['device_type'],
name=device['custom_label'] or device['name'] or device['ssid'] or device_id,
manufacturer=device['manufacturer'] or '',
first_seen=device['first_seen'],
last_seen=device['last_seen'],
total_observations=device['total_observations'],
avg_rssi=round(stats['avg_rssi'], 1) if stats['avg_rssi'] else 0,
min_rssi=stats['min_rssi'] or 0,
max_rssi=stats['max_rssi'] or 0,
avg_distance_m=round(stats['avg_distance_m'], 2) if stats['avg_distance_m'] else 0,
min_distance_m=round(stats['min_distance_m'], 2) if stats['min_distance_m'] else 0,
max_distance_m=round(stats['max_distance_m'], 2) if stats['max_distance_m'] else 0
)
def get_movement_events(self, device_id: Optional[str] = None,
since: Optional[str] = None,
limit: int = 100) -> list[dict]:
"""Get movement events"""
conn = self._get_connection()
cursor = conn.cursor()
query = "SELECT * FROM movement_events WHERE 1=1"
params = []
if device_id:
query += " AND device_id = ?"
params.append(device_id)
if since:
query += " AND timestamp >= ?"
params.append(since)
query += " ORDER BY timestamp DESC LIMIT ?"
params.append(limit)
cursor.execute(query, params)
return [dict(row) for row in cursor.fetchall()]
def get_alerts(self, acknowledged: Optional[bool] = None,
alert_type: Optional[str] = None,
limit: int = 50) -> list[dict]:
"""Get alerts"""
conn = self._get_connection()
cursor = conn.cursor()
query = "SELECT * FROM alerts WHERE 1=1"
params = []
if acknowledged is not None:
query += " AND acknowledged = ?"
params.append(1 if acknowledged else 0)
if alert_type:
query += " AND alert_type = ?"
params.append(alert_type)
query += " ORDER BY timestamp DESC LIMIT ?"
params.append(limit)
cursor.execute(query, params)
return [dict(row) for row in cursor.fetchall()]
def acknowledge_alert(self, alert_id: int):
"""Mark an alert as acknowledged"""
conn = self._get_connection()
cursor = conn.cursor()
cursor.execute("UPDATE alerts SET acknowledged = 1 WHERE id = ?", (alert_id,))
conn.commit()
def set_device_label(self, device_id: str, label: str):
"""Set a custom label for a device"""
conn = self._get_connection()
cursor = conn.cursor()
cursor.execute("""
UPDATE devices SET custom_label = ?, updated_at = CURRENT_TIMESTAMP
WHERE device_id = ?
""", (label, device_id))
conn.commit()
def set_device_favorite(self, device_id: str, is_favorite: bool):
"""Mark a device as favorite"""
conn = self._get_connection()
cursor = conn.cursor()
cursor.execute("""
UPDATE devices SET is_favorite = ?, updated_at = CURRENT_TIMESTAMP
WHERE device_id = ?
""", (1 if is_favorite else 0, device_id))
conn.commit()
def get_recent_activity(self, hours: int = 24) -> dict:
"""Get activity summary for the last N hours"""
conn = self._get_connection()
cursor = conn.cursor()
since = (datetime.now() - timedelta(hours=hours)).isoformat()
# Count active devices
cursor.execute("""
SELECT device_type, COUNT(*) as count
FROM devices WHERE last_seen >= ?
GROUP BY device_type
""", (since,))
active_counts = {row['device_type']: row['count'] for row in cursor.fetchall()}
# Count observations
cursor.execute("""
SELECT COUNT(*) as count FROM rssi_history WHERE timestamp >= ?
""", (since,))
observation_count = cursor.fetchone()['count']
# Count movement events
cursor.execute("""
SELECT COUNT(*) as count FROM movement_events WHERE timestamp >= ?
""", (since,))
movement_count = cursor.fetchone()['count']
# Count new devices
cursor.execute("""
SELECT COUNT(*) as count FROM devices WHERE first_seen >= ?
""", (since,))
new_device_count = cursor.fetchone()['count']
# Count scans
cursor.execute("""
SELECT COUNT(*) as count FROM scans WHERE timestamp >= ?
""", (since,))
scan_count = cursor.fetchone()['count']
return {
"period_hours": hours,
"since": since,
"active_wifi_devices": active_counts.get('wifi', 0),
"active_bt_devices": active_counts.get('bluetooth', 0),
"total_observations": observation_count,
"movement_events": movement_count,
"new_devices": new_device_count,
"scan_count": scan_count
}
def get_device_activity_pattern(self, device_id: str, days: int = 7) -> dict:
"""Get hourly activity pattern for a device over the last N days"""
conn = self._get_connection()
cursor = conn.cursor()
since = (datetime.now() - timedelta(days=days)).isoformat()
# Count observations per hour of day
cursor.execute("""
SELECT
CAST(strftime('%H', timestamp) AS INTEGER) as hour,
COUNT(*) as count,
AVG(rssi) as avg_rssi
FROM rssi_history
WHERE device_id = ? AND timestamp >= ?
GROUP BY hour
ORDER BY hour
""", (device_id, since))
hourly = {row['hour']: {'count': row['count'], 'avg_rssi': round(row['avg_rssi'], 1)}
for row in cursor.fetchall()}
# Count observations per day of week (0=Monday, 6=Sunday)
cursor.execute("""
SELECT
CAST(strftime('%w', timestamp) AS INTEGER) as dow,
COUNT(*) as count
FROM rssi_history
WHERE device_id = ? AND timestamp >= ?
GROUP BY dow
ORDER BY dow
""", (device_id, since))
daily = {row['dow']: row['count'] for row in cursor.fetchall()}
return {
"device_id": device_id,
"period_days": days,
"hourly_pattern": hourly,
"daily_pattern": daily
}
def cleanup_old_data(self, retention_days: int = 30):
"""Remove data older than retention period"""
conn = self._get_connection()
cursor = conn.cursor()
cutoff = (datetime.now() - timedelta(days=retention_days)).isoformat()
# Delete old RSSI history (keep summary in devices table)
cursor.execute("DELETE FROM rssi_history WHERE timestamp < ?", (cutoff,))
# Delete old movement events
cursor.execute("DELETE FROM movement_events WHERE timestamp < ?", (cutoff,))
# Delete old acknowledged alerts
cursor.execute("DELETE FROM alerts WHERE timestamp < ? AND acknowledged = 1", (cutoff,))
# Delete old scans
cursor.execute("DELETE FROM scans WHERE timestamp < ?", (cutoff,))
conn.commit()
return {
"retention_days": retention_days,
"cutoff": cutoff,
"cleaned_at": datetime.now().isoformat()
}
def get_database_stats(self) -> dict:
"""Get database statistics"""
conn = self._get_connection()
cursor = conn.cursor()
cursor.execute("SELECT COUNT(*) as count FROM devices")
device_count = cursor.fetchone()['count']
cursor.execute("SELECT COUNT(*) as count FROM rssi_history")
observation_count = cursor.fetchone()['count']
cursor.execute("SELECT COUNT(*) as count FROM scans")
scan_count = cursor.fetchone()['count']
cursor.execute("SELECT COUNT(*) as count FROM movement_events")
movement_count = cursor.fetchone()['count']
cursor.execute("SELECT COUNT(*) as count FROM alerts WHERE acknowledged = 0")
unread_alerts = cursor.fetchone()['count']
# Get database file size
db_size = self.db_path.stat().st_size if self.db_path.exists() else 0
return {
"total_devices": device_count,
"total_observations": observation_count,
"total_scans": scan_count,
"total_movement_events": movement_count,
"unread_alerts": unread_alerts,
"database_size_bytes": db_size,
"database_size_mb": round(db_size / 1024 / 1024, 2)
}
def close(self):
"""Close database connection"""
if hasattr(self._local, 'conn') and self._local.conn:
self._local.conn.close()
self._local.conn = None
# Global database instance
_db: DeviceDatabase | None = None
def get_database(db_path: Path | str | None = None) -> DeviceDatabase:
"""Get the global database instance"""
global _db
if _db is None:
if db_path is None:
db_path = Path.home() / "git" / "rf-mapper" / "data" / "devices.db"
_db = DeviceDatabase(db_path)
return _db
def init_database(db_path: Path | str) -> DeviceDatabase:
"""Initialize the global database instance"""
global _db
_db = DeviceDatabase(db_path)
return _db