feat: pagination totals, request logging, data retention

Add shared paginate() helper with total count to all list endpoints.
Add request logging middleware (method, path, status, duration, IP).
Add data retention service with configurable thresholds and CLI command.

src/esp32_web/api/__init__.py

@@ -1,6 +1,27 @@
 """API Blueprint."""
-from flask import Blueprint
+from flask import Blueprint, request
+from ..extensions import db
 
 bp = Blueprint('api', __name__)
 
+
+def paginate(query, schema_fn):
+    """Apply limit/offset pagination to a query and return items with metadata.
+
+    Returns dict with 'items', 'total', 'limit', 'offset'.
+    """
+    limit = min(request.args.get('limit', 100, type=int), 1000)
+    offset = request.args.get('offset', 0, type=int)
+    total = db.session.scalar(
+        db.select(db.func.count()).select_from(query.subquery())
+    )
+    results = db.session.scalars(query.limit(limit).offset(offset)).all()
+    return {
+        'items': [schema_fn(r) for r in results],
+        'total': total,
+        'limit': limit,
+        'offset': offset,
+    }
+
+
 from . import sensors, devices, alerts, events, probes, stats, export  # noqa: E402, F401

src/esp32_web/api/alerts.py

@@ -1,7 +1,7 @@
 """Alert endpoints."""
 from datetime import datetime, timedelta, UTC
 from flask import request
-from . import bp
+from . import bp, paginate
 from ..models import Alert
 from ..extensions import db
 
@@ -12,8 +12,6 @@ def list_alerts():
     alert_type = request.args.get('type')
     sensor_id = request.args.get('sensor_id', type=int)
     hours = request.args.get('hours', 24, type=int)
-    limit = min(int(request.args.get('limit', 100)), 1000)
-    offset = int(request.args.get('offset', 0))
 
     since = datetime.now(UTC) - timedelta(hours=hours)
     query = db.select(Alert).where(Alert.timestamp >= since).order_by(Alert.timestamp.desc())
@@ -23,7 +21,6 @@ def list_alerts():
     if sensor_id:
         query = query.where(Alert.sensor_id == sensor_id)
 
-    query = query.limit(limit).offset(offset)
-    alerts = db.session.scalars(query).all()
-
-    return {'alerts': [a.to_dict() for a in alerts], 'limit': limit, 'offset': offset}
+    result = paginate(query, Alert.to_dict)
+    return {'alerts': result['items'], 'total': result['total'],
+            'limit': result['limit'], 'offset': result['offset']}

src/esp32_web/api/devices.py

@@ -1,6 +1,6 @@
 """Device endpoints."""
 from flask import request
-from . import bp
+from . import bp, paginate
 from ..models import Device, Sighting
 from ..extensions import db
 from ..services.device_service import enrich_device
@@ -10,16 +10,14 @@ from ..services.device_service import enrich_device
 def list_devices():
     """List all devices."""
     device_type = request.args.get('type')  # 'ble' or 'wifi'
-    limit = min(int(request.args.get('limit', 100)), 1000)
-    offset = int(request.args.get('offset', 0))
 
     query = db.select(Device).order_by(Device.last_seen.desc())
     if device_type:
         query = query.where(Device.device_type == device_type)
-    query = query.limit(limit).offset(offset)
 
-    devices = db.session.scalars(query).all()
-    return {'devices': [enrich_device(d) for d in devices], 'limit': limit, 'offset': offset}
+    result = paginate(query, enrich_device)
+    return {'devices': result['items'], 'total': result['total'],
+            'limit': result['limit'], 'offset': result['offset']}
 
 
 @bp.route('/devices/<mac>')

src/esp32_web/api/events.py

@@ -1,7 +1,7 @@
 """Event endpoints."""
 from datetime import datetime, timedelta, UTC
 from flask import request
-from . import bp
+from . import bp, paginate
 from ..models import Event
 from ..extensions import db
 
@@ -12,8 +12,6 @@ def list_events():
     event_type = request.args.get('type')
     sensor_id = request.args.get('sensor_id', type=int)
     hours = request.args.get('hours', 24, type=int)
-    limit = min(int(request.args.get('limit', 100)), 1000)
-    offset = int(request.args.get('offset', 0))
 
     since = datetime.now(UTC) - timedelta(hours=hours)
     query = db.select(Event).where(Event.timestamp >= since).order_by(Event.timestamp.desc())
@@ -23,7 +21,6 @@ def list_events():
     if sensor_id:
         query = query.where(Event.sensor_id == sensor_id)
 
-    query = query.limit(limit).offset(offset)
-    events = db.session.scalars(query).all()
-
-    return {'events': [e.to_dict() for e in events], 'limit': limit, 'offset': offset}
+    result = paginate(query, Event.to_dict)
+    return {'events': result['items'], 'total': result['total'],
+            'limit': result['limit'], 'offset': result['offset']}

src/esp32_web/api/probes.py

@@ -2,7 +2,7 @@
 from datetime import datetime, timedelta, UTC
 from flask import request
 from sqlalchemy import func
-from . import bp
+from . import bp, paginate
 from ..models import Probe, Device
 from ..extensions import db
 
@@ -12,8 +12,6 @@ def list_probes():
     """List probe requests."""
     ssid = request.args.get('ssid')
     hours = request.args.get('hours', 24, type=int)
-    limit = min(int(request.args.get('limit', 100)), 1000)
-    offset = int(request.args.get('offset', 0))
 
     since = datetime.now(UTC) - timedelta(hours=hours)
     query = db.select(Probe).where(Probe.timestamp >= since).order_by(Probe.timestamp.desc())
@@ -21,10 +19,9 @@ def list_probes():
     if ssid:
         query = query.where(Probe.ssid == ssid)
 
-    query = query.limit(limit).offset(offset)
-    probes = db.session.scalars(query).all()
-
-    return {'probes': [p.to_dict() for p in probes], 'limit': limit, 'offset': offset}
+    result = paginate(query, Probe.to_dict)
+    return {'probes': result['items'], 'total': result['total'],
+            'limit': result['limit'], 'offset': result['offset']}
 
 
 @bp.route('/probes/ssids')

src/esp32_web/api/sensors.py

@@ -3,7 +3,7 @@ import json
 import socket
 from datetime import datetime, timedelta, UTC
 from flask import request, current_app
-from . import bp
+from . import bp, paginate
 from ..models import Sensor, Event, Sighting, Alert
 from ..extensions import db
 from ..services.heartbeat import get_heartbeat_summary, update_all_heartbeats
@@ -12,8 +12,10 @@ from ..services.heartbeat import get_heartbeat_summary, update_all_heartbeats
 @bp.route('/sensors')
 def list_sensors():
     """List all sensors."""
-    sensors = db.session.scalars(db.select(Sensor).order_by(Sensor.hostname)).all()
-    return {'sensors': [s.to_dict() for s in sensors]}
+    query = db.select(Sensor).order_by(Sensor.hostname)
+    result = paginate(query, Sensor.to_dict)
+    return {'sensors': result['items'], 'total': result['total'],
+            'limit': result['limit'], 'offset': result['offset']}
 
 
 @bp.route('/sensors/<hostname>')