feat: heartbeat detection and sensor metrics
- Heartbeat service: check_sensor_status (online/stale/offline) - GET /sensors/heartbeat: status summary for all sensors - POST /sensors/heartbeat: refresh heartbeat status - GET /sensors/<hostname>/metrics: activity counts, recent events - CLI command: flask check-heartbeats - Added 7 new tests (34 total)
This commit is contained in:
4
TASKS.md
4
TASKS.md
@@ -11,8 +11,8 @@
|
||||
- [x] `POST /api/v1/sensors/<id>/calibrate` — trigger calibration
|
||||
|
||||
### P2 - Normal
|
||||
- [ ] Sensor heartbeat timeout detection
|
||||
- [ ] Sensor metrics history endpoint
|
||||
- [x] Sensor heartbeat timeout detection
|
||||
- [x] Sensor metrics history endpoint
|
||||
|
||||
### P3 - Low
|
||||
- [ ] Add pagination to all list endpoints
|
||||
|
||||
@@ -62,4 +62,11 @@ def create_app(config_class=Config):
|
||||
else:
|
||||
click.echo('Failed to download OUI database', err=True)
|
||||
|
||||
@app.cli.command('check-heartbeats')
|
||||
def check_heartbeats_cmd():
|
||||
"""Check and update sensor heartbeat status."""
|
||||
from .services.heartbeat import update_all_heartbeats
|
||||
counts = update_all_heartbeats()
|
||||
click.echo(f"Sensors: {counts['online']} online, {counts['stale']} stale, {counts['offline']} offline")
|
||||
|
||||
return app
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
"""Sensor endpoints."""
|
||||
import json
|
||||
import socket
|
||||
from datetime import datetime, timedelta, UTC
|
||||
from flask import request, current_app
|
||||
from . import bp
|
||||
from ..models import Sensor
|
||||
from ..models import Sensor, Event, Sighting, Alert
|
||||
from ..extensions import db
|
||||
from ..services.heartbeat import get_heartbeat_summary, update_all_heartbeats
|
||||
|
||||
|
||||
@bp.route('/sensors')
|
||||
@@ -213,3 +215,73 @@ def trigger_calibrate(hostname):
|
||||
return {'error': f'Socket error: {e}'}, 500
|
||||
|
||||
return {'status': 'calibration_started', 'seconds': seconds}
|
||||
|
||||
|
||||
@bp.route('/sensors/heartbeat')
|
||||
def get_heartbeat_status():
|
||||
"""Get heartbeat status for all sensors."""
|
||||
return get_heartbeat_summary()
|
||||
|
||||
|
||||
@bp.route('/sensors/heartbeat', methods=['POST'])
|
||||
def refresh_heartbeats():
|
||||
"""Update heartbeat status for all sensors."""
|
||||
counts = update_all_heartbeats()
|
||||
return {
|
||||
'status': 'updated',
|
||||
'online': counts['online'],
|
||||
'stale': counts['stale'],
|
||||
'offline': counts['offline']
|
||||
}
|
||||
|
||||
|
||||
@bp.route('/sensors/<hostname>/metrics')
|
||||
def get_sensor_metrics(hostname):
|
||||
"""Get sensor activity metrics and recent events."""
|
||||
sensor = db.session.scalar(db.select(Sensor).where(Sensor.hostname == hostname))
|
||||
if not sensor:
|
||||
return {'error': 'Sensor not found'}, 404
|
||||
|
||||
# Time range (default: last 24 hours)
|
||||
hours = request.args.get('hours', 24, type=int)
|
||||
if hours < 1 or hours > 168: # max 1 week
|
||||
hours = 24
|
||||
since = datetime.now(UTC) - timedelta(hours=hours)
|
||||
|
||||
# Count activity
|
||||
sightings_count = db.session.scalar(
|
||||
db.select(db.func.count(Sighting.id))
|
||||
.where(Sighting.sensor_id == sensor.id)
|
||||
.where(Sighting.timestamp >= since)
|
||||
) or 0
|
||||
|
||||
alerts_count = db.session.scalar(
|
||||
db.select(db.func.count(Alert.id))
|
||||
.where(Alert.sensor_id == sensor.id)
|
||||
.where(Alert.timestamp >= since)
|
||||
) or 0
|
||||
|
||||
events_count = db.session.scalar(
|
||||
db.select(db.func.count(Event.id))
|
||||
.where(Event.sensor_id == sensor.id)
|
||||
.where(Event.timestamp >= since)
|
||||
) or 0
|
||||
|
||||
# Recent events (last 20)
|
||||
recent_events = db.session.scalars(
|
||||
db.select(Event)
|
||||
.where(Event.sensor_id == sensor.id)
|
||||
.order_by(Event.timestamp.desc())
|
||||
.limit(20)
|
||||
).all()
|
||||
|
||||
return {
|
||||
'hostname': sensor.hostname,
|
||||
'hours': hours,
|
||||
'activity': {
|
||||
'sightings': sightings_count,
|
||||
'alerts': alerts_count,
|
||||
'events': events_count,
|
||||
},
|
||||
'recent_events': [e.to_dict() for e in recent_events]
|
||||
}
|
||||
|
||||
83
src/esp32_web/services/heartbeat.py
Normal file
83
src/esp32_web/services/heartbeat.py
Normal file
@@ -0,0 +1,83 @@
|
||||
"""Sensor heartbeat service."""
|
||||
from datetime import datetime, UTC, timedelta
|
||||
from ..extensions import db
|
||||
from ..models import Sensor
|
||||
|
||||
|
||||
# Default thresholds in seconds
|
||||
ONLINE_THRESHOLD = 60 # < 1 minute = online
|
||||
STALE_THRESHOLD = 300 # 1-5 minutes = stale
|
||||
# > 5 minutes = offline
|
||||
|
||||
|
||||
def check_sensor_status(sensor: Sensor, now: datetime | None = None) -> str:
|
||||
"""Determine sensor status based on last_seen timestamp."""
|
||||
if now is None:
|
||||
now = datetime.now(UTC)
|
||||
|
||||
# Handle timezone-naive datetimes from DB
|
||||
last_seen = sensor.last_seen
|
||||
if last_seen.tzinfo is None:
|
||||
last_seen = last_seen.replace(tzinfo=UTC)
|
||||
|
||||
delta = (now - last_seen).total_seconds()
|
||||
|
||||
if delta < ONLINE_THRESHOLD:
|
||||
return 'online'
|
||||
elif delta < STALE_THRESHOLD:
|
||||
return 'stale'
|
||||
else:
|
||||
return 'offline'
|
||||
|
||||
|
||||
def update_all_heartbeats() -> dict:
|
||||
"""Update status for all sensors based on last_seen.
|
||||
|
||||
Returns dict with counts: {'online': n, 'stale': n, 'offline': n}
|
||||
"""
|
||||
now = datetime.now(UTC)
|
||||
sensors = db.session.scalars(db.select(Sensor)).all()
|
||||
|
||||
counts = {'online': 0, 'stale': 0, 'offline': 0}
|
||||
|
||||
for sensor in sensors:
|
||||
new_status = check_sensor_status(sensor, now)
|
||||
if sensor.status != new_status:
|
||||
sensor.status = new_status
|
||||
counts[new_status] += 1
|
||||
|
||||
db.session.commit()
|
||||
return counts
|
||||
|
||||
|
||||
def get_heartbeat_summary() -> dict:
|
||||
"""Get summary of sensor heartbeat status."""
|
||||
now = datetime.now(UTC)
|
||||
sensors = db.session.scalars(db.select(Sensor)).all()
|
||||
|
||||
summary = {
|
||||
'total': len(sensors),
|
||||
'online': 0,
|
||||
'stale': 0,
|
||||
'offline': 0,
|
||||
'sensors': []
|
||||
}
|
||||
|
||||
for sensor in sensors:
|
||||
status = check_sensor_status(sensor, now)
|
||||
summary[status] += 1
|
||||
|
||||
# Handle timezone-naive datetimes from DB
|
||||
last_seen = sensor.last_seen
|
||||
if last_seen.tzinfo is None:
|
||||
last_seen = last_seen.replace(tzinfo=UTC)
|
||||
|
||||
summary['sensors'].append({
|
||||
'hostname': sensor.hostname,
|
||||
'ip': sensor.ip,
|
||||
'status': status,
|
||||
'last_seen': last_seen.isoformat(),
|
||||
'seconds_ago': int((now - last_seen).total_seconds())
|
||||
})
|
||||
|
||||
return summary
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Sensor API tests."""
|
||||
from unittest.mock import patch, MagicMock
|
||||
from esp32_web.extensions import db
|
||||
from esp32_web.models import Sensor
|
||||
from esp32_web.models import Sensor, Event
|
||||
|
||||
|
||||
def test_list_sensors_empty(client):
|
||||
@@ -215,3 +215,123 @@ def test_trigger_calibrate_default_seconds(client, app):
|
||||
json={})
|
||||
assert response.status_code == 200
|
||||
assert response.json['seconds'] == 10
|
||||
|
||||
|
||||
# Heartbeat Tests
|
||||
|
||||
def test_heartbeat_status_empty(client):
|
||||
"""Test heartbeat status with no sensors."""
|
||||
response = client.get('/api/v1/sensors/heartbeat')
|
||||
assert response.status_code == 200
|
||||
assert response.json['total'] == 0
|
||||
assert response.json['sensors'] == []
|
||||
|
||||
|
||||
def test_heartbeat_status_with_sensors(client, app):
|
||||
"""Test heartbeat status with sensors."""
|
||||
from datetime import datetime, UTC, timedelta
|
||||
|
||||
with app.app_context():
|
||||
# Online sensor (just now)
|
||||
s1 = Sensor(hostname='sensor-online', ip='192.168.1.1',
|
||||
last_seen=datetime.now(UTC))
|
||||
# Stale sensor (3 minutes ago)
|
||||
s2 = Sensor(hostname='sensor-stale', ip='192.168.1.2',
|
||||
last_seen=datetime.now(UTC) - timedelta(minutes=3))
|
||||
# Offline sensor (10 minutes ago)
|
||||
s3 = Sensor(hostname='sensor-offline', ip='192.168.1.3',
|
||||
last_seen=datetime.now(UTC) - timedelta(minutes=10))
|
||||
db.session.add_all([s1, s2, s3])
|
||||
db.session.commit()
|
||||
|
||||
response = client.get('/api/v1/sensors/heartbeat')
|
||||
assert response.status_code == 200
|
||||
assert response.json['total'] == 3
|
||||
assert response.json['online'] == 1
|
||||
assert response.json['stale'] == 1
|
||||
assert response.json['offline'] == 1
|
||||
|
||||
|
||||
def test_refresh_heartbeats(client, app):
|
||||
"""Test refreshing heartbeat status."""
|
||||
from datetime import datetime, UTC, timedelta
|
||||
|
||||
with app.app_context():
|
||||
# Offline sensor but status still says 'online'
|
||||
sensor = Sensor(hostname='test-sensor', ip='192.168.1.1',
|
||||
last_seen=datetime.now(UTC) - timedelta(minutes=10),
|
||||
status='online')
|
||||
db.session.add(sensor)
|
||||
db.session.commit()
|
||||
|
||||
response = client.post('/api/v1/sensors/heartbeat')
|
||||
assert response.status_code == 200
|
||||
assert response.json['status'] == 'updated'
|
||||
assert response.json['offline'] == 1
|
||||
|
||||
# Verify status was updated
|
||||
with app.app_context():
|
||||
sensor = db.session.scalar(db.select(Sensor).where(Sensor.hostname == 'test-sensor'))
|
||||
assert sensor.status == 'offline'
|
||||
|
||||
|
||||
# Metrics Tests
|
||||
|
||||
def test_metrics_not_found(client):
|
||||
"""Test metrics for non-existent sensor."""
|
||||
response = client.get('/api/v1/sensors/nonexistent/metrics')
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
def test_metrics_empty(client, app):
|
||||
"""Test metrics for sensor with no activity."""
|
||||
with app.app_context():
|
||||
sensor = Sensor(hostname='test-sensor', ip='192.168.1.100')
|
||||
db.session.add(sensor)
|
||||
db.session.commit()
|
||||
|
||||
response = client.get('/api/v1/sensors/test-sensor/metrics')
|
||||
assert response.status_code == 200
|
||||
assert response.json['hostname'] == 'test-sensor'
|
||||
assert response.json['hours'] == 24
|
||||
assert response.json['activity']['sightings'] == 0
|
||||
assert response.json['activity']['alerts'] == 0
|
||||
assert response.json['activity']['events'] == 0
|
||||
assert response.json['recent_events'] == []
|
||||
|
||||
|
||||
def test_metrics_with_events(client, app):
|
||||
"""Test metrics with sensor events."""
|
||||
from datetime import datetime, UTC
|
||||
|
||||
with app.app_context():
|
||||
sensor = Sensor(hostname='test-sensor', ip='192.168.1.100')
|
||||
db.session.add(sensor)
|
||||
db.session.flush()
|
||||
|
||||
# Add some events
|
||||
event1 = Event(sensor_id=sensor.id, event_type='presence',
|
||||
payload_json='{"state": "detected"}',
|
||||
timestamp=datetime.now(UTC))
|
||||
event2 = Event(sensor_id=sensor.id, event_type='calibration',
|
||||
payload_json='{"nsub": 52}',
|
||||
timestamp=datetime.now(UTC))
|
||||
db.session.add_all([event1, event2])
|
||||
db.session.commit()
|
||||
|
||||
response = client.get('/api/v1/sensors/test-sensor/metrics')
|
||||
assert response.status_code == 200
|
||||
assert response.json['activity']['events'] == 2
|
||||
assert len(response.json['recent_events']) == 2
|
||||
|
||||
|
||||
def test_metrics_custom_hours(client, app):
|
||||
"""Test metrics with custom time range."""
|
||||
with app.app_context():
|
||||
sensor = Sensor(hostname='test-sensor', ip='192.168.1.100')
|
||||
db.session.add(sensor)
|
||||
db.session.commit()
|
||||
|
||||
response = client.get('/api/v1/sensors/test-sensor/metrics?hours=48')
|
||||
assert response.status_code == 200
|
||||
assert response.json['hours'] == 48
|
||||
|
||||
Reference in New Issue
Block a user