feat: heartbeat detection and sensor metrics
- Heartbeat service: check_sensor_status (online/stale/offline) - GET /sensors/heartbeat: status summary for all sensors - POST /sensors/heartbeat: refresh heartbeat status - GET /sensors/<hostname>/metrics: activity counts, recent events - CLI command: flask check-heartbeats - Added 7 new tests (34 total)
This commit is contained in:
4
TASKS.md
4
TASKS.md
@@ -11,8 +11,8 @@
|
|||||||
- [x] `POST /api/v1/sensors/<id>/calibrate` — trigger calibration
|
- [x] `POST /api/v1/sensors/<id>/calibrate` — trigger calibration
|
||||||
|
|
||||||
### P2 - Normal
|
### P2 - Normal
|
||||||
- [ ] Sensor heartbeat timeout detection
|
- [x] Sensor heartbeat timeout detection
|
||||||
- [ ] Sensor metrics history endpoint
|
- [x] Sensor metrics history endpoint
|
||||||
|
|
||||||
### P3 - Low
|
### P3 - Low
|
||||||
- [ ] Add pagination to all list endpoints
|
- [ ] Add pagination to all list endpoints
|
||||||
|
|||||||
@@ -62,4 +62,11 @@ def create_app(config_class=Config):
|
|||||||
else:
|
else:
|
||||||
click.echo('Failed to download OUI database', err=True)
|
click.echo('Failed to download OUI database', err=True)
|
||||||
|
|
||||||
|
@app.cli.command('check-heartbeats')
|
||||||
|
def check_heartbeats_cmd():
|
||||||
|
"""Check and update sensor heartbeat status."""
|
||||||
|
from .services.heartbeat import update_all_heartbeats
|
||||||
|
counts = update_all_heartbeats()
|
||||||
|
click.echo(f"Sensors: {counts['online']} online, {counts['stale']} stale, {counts['offline']} offline")
|
||||||
|
|
||||||
return app
|
return app
|
||||||
|
|||||||
@@ -1,10 +1,12 @@
|
|||||||
"""Sensor endpoints."""
|
"""Sensor endpoints."""
|
||||||
import json
|
import json
|
||||||
import socket
|
import socket
|
||||||
|
from datetime import datetime, timedelta, UTC
|
||||||
from flask import request, current_app
|
from flask import request, current_app
|
||||||
from . import bp
|
from . import bp
|
||||||
from ..models import Sensor
|
from ..models import Sensor, Event, Sighting, Alert
|
||||||
from ..extensions import db
|
from ..extensions import db
|
||||||
|
from ..services.heartbeat import get_heartbeat_summary, update_all_heartbeats
|
||||||
|
|
||||||
|
|
||||||
@bp.route('/sensors')
|
@bp.route('/sensors')
|
||||||
@@ -213,3 +215,73 @@ def trigger_calibrate(hostname):
|
|||||||
return {'error': f'Socket error: {e}'}, 500
|
return {'error': f'Socket error: {e}'}, 500
|
||||||
|
|
||||||
return {'status': 'calibration_started', 'seconds': seconds}
|
return {'status': 'calibration_started', 'seconds': seconds}
|
||||||
|
|
||||||
|
|
||||||
|
@bp.route('/sensors/heartbeat')
|
||||||
|
def get_heartbeat_status():
|
||||||
|
"""Get heartbeat status for all sensors."""
|
||||||
|
return get_heartbeat_summary()
|
||||||
|
|
||||||
|
|
||||||
|
@bp.route('/sensors/heartbeat', methods=['POST'])
|
||||||
|
def refresh_heartbeats():
|
||||||
|
"""Update heartbeat status for all sensors."""
|
||||||
|
counts = update_all_heartbeats()
|
||||||
|
return {
|
||||||
|
'status': 'updated',
|
||||||
|
'online': counts['online'],
|
||||||
|
'stale': counts['stale'],
|
||||||
|
'offline': counts['offline']
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@bp.route('/sensors/<hostname>/metrics')
|
||||||
|
def get_sensor_metrics(hostname):
|
||||||
|
"""Get sensor activity metrics and recent events."""
|
||||||
|
sensor = db.session.scalar(db.select(Sensor).where(Sensor.hostname == hostname))
|
||||||
|
if not sensor:
|
||||||
|
return {'error': 'Sensor not found'}, 404
|
||||||
|
|
||||||
|
# Time range (default: last 24 hours)
|
||||||
|
hours = request.args.get('hours', 24, type=int)
|
||||||
|
if hours < 1 or hours > 168: # max 1 week
|
||||||
|
hours = 24
|
||||||
|
since = datetime.now(UTC) - timedelta(hours=hours)
|
||||||
|
|
||||||
|
# Count activity
|
||||||
|
sightings_count = db.session.scalar(
|
||||||
|
db.select(db.func.count(Sighting.id))
|
||||||
|
.where(Sighting.sensor_id == sensor.id)
|
||||||
|
.where(Sighting.timestamp >= since)
|
||||||
|
) or 0
|
||||||
|
|
||||||
|
alerts_count = db.session.scalar(
|
||||||
|
db.select(db.func.count(Alert.id))
|
||||||
|
.where(Alert.sensor_id == sensor.id)
|
||||||
|
.where(Alert.timestamp >= since)
|
||||||
|
) or 0
|
||||||
|
|
||||||
|
events_count = db.session.scalar(
|
||||||
|
db.select(db.func.count(Event.id))
|
||||||
|
.where(Event.sensor_id == sensor.id)
|
||||||
|
.where(Event.timestamp >= since)
|
||||||
|
) or 0
|
||||||
|
|
||||||
|
# Recent events (last 20)
|
||||||
|
recent_events = db.session.scalars(
|
||||||
|
db.select(Event)
|
||||||
|
.where(Event.sensor_id == sensor.id)
|
||||||
|
.order_by(Event.timestamp.desc())
|
||||||
|
.limit(20)
|
||||||
|
).all()
|
||||||
|
|
||||||
|
return {
|
||||||
|
'hostname': sensor.hostname,
|
||||||
|
'hours': hours,
|
||||||
|
'activity': {
|
||||||
|
'sightings': sightings_count,
|
||||||
|
'alerts': alerts_count,
|
||||||
|
'events': events_count,
|
||||||
|
},
|
||||||
|
'recent_events': [e.to_dict() for e in recent_events]
|
||||||
|
}
|
||||||
|
|||||||
83
src/esp32_web/services/heartbeat.py
Normal file
83
src/esp32_web/services/heartbeat.py
Normal file
@@ -0,0 +1,83 @@
|
|||||||
|
"""Sensor heartbeat service."""
|
||||||
|
from datetime import datetime, UTC, timedelta
|
||||||
|
from ..extensions import db
|
||||||
|
from ..models import Sensor
|
||||||
|
|
||||||
|
|
||||||
|
# Default thresholds in seconds
|
||||||
|
ONLINE_THRESHOLD = 60 # < 1 minute = online
|
||||||
|
STALE_THRESHOLD = 300 # 1-5 minutes = stale
|
||||||
|
# > 5 minutes = offline
|
||||||
|
|
||||||
|
|
||||||
|
def check_sensor_status(sensor: Sensor, now: datetime | None = None) -> str:
|
||||||
|
"""Determine sensor status based on last_seen timestamp."""
|
||||||
|
if now is None:
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
|
||||||
|
# Handle timezone-naive datetimes from DB
|
||||||
|
last_seen = sensor.last_seen
|
||||||
|
if last_seen.tzinfo is None:
|
||||||
|
last_seen = last_seen.replace(tzinfo=UTC)
|
||||||
|
|
||||||
|
delta = (now - last_seen).total_seconds()
|
||||||
|
|
||||||
|
if delta < ONLINE_THRESHOLD:
|
||||||
|
return 'online'
|
||||||
|
elif delta < STALE_THRESHOLD:
|
||||||
|
return 'stale'
|
||||||
|
else:
|
||||||
|
return 'offline'
|
||||||
|
|
||||||
|
|
||||||
|
def update_all_heartbeats() -> dict:
|
||||||
|
"""Update status for all sensors based on last_seen.
|
||||||
|
|
||||||
|
Returns dict with counts: {'online': n, 'stale': n, 'offline': n}
|
||||||
|
"""
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
sensors = db.session.scalars(db.select(Sensor)).all()
|
||||||
|
|
||||||
|
counts = {'online': 0, 'stale': 0, 'offline': 0}
|
||||||
|
|
||||||
|
for sensor in sensors:
|
||||||
|
new_status = check_sensor_status(sensor, now)
|
||||||
|
if sensor.status != new_status:
|
||||||
|
sensor.status = new_status
|
||||||
|
counts[new_status] += 1
|
||||||
|
|
||||||
|
db.session.commit()
|
||||||
|
return counts
|
||||||
|
|
||||||
|
|
||||||
|
def get_heartbeat_summary() -> dict:
|
||||||
|
"""Get summary of sensor heartbeat status."""
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
sensors = db.session.scalars(db.select(Sensor)).all()
|
||||||
|
|
||||||
|
summary = {
|
||||||
|
'total': len(sensors),
|
||||||
|
'online': 0,
|
||||||
|
'stale': 0,
|
||||||
|
'offline': 0,
|
||||||
|
'sensors': []
|
||||||
|
}
|
||||||
|
|
||||||
|
for sensor in sensors:
|
||||||
|
status = check_sensor_status(sensor, now)
|
||||||
|
summary[status] += 1
|
||||||
|
|
||||||
|
# Handle timezone-naive datetimes from DB
|
||||||
|
last_seen = sensor.last_seen
|
||||||
|
if last_seen.tzinfo is None:
|
||||||
|
last_seen = last_seen.replace(tzinfo=UTC)
|
||||||
|
|
||||||
|
summary['sensors'].append({
|
||||||
|
'hostname': sensor.hostname,
|
||||||
|
'ip': sensor.ip,
|
||||||
|
'status': status,
|
||||||
|
'last_seen': last_seen.isoformat(),
|
||||||
|
'seconds_ago': int((now - last_seen).total_seconds())
|
||||||
|
})
|
||||||
|
|
||||||
|
return summary
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
"""Sensor API tests."""
|
"""Sensor API tests."""
|
||||||
from unittest.mock import patch, MagicMock
|
from unittest.mock import patch, MagicMock
|
||||||
from esp32_web.extensions import db
|
from esp32_web.extensions import db
|
||||||
from esp32_web.models import Sensor
|
from esp32_web.models import Sensor, Event
|
||||||
|
|
||||||
|
|
||||||
def test_list_sensors_empty(client):
|
def test_list_sensors_empty(client):
|
||||||
@@ -215,3 +215,123 @@ def test_trigger_calibrate_default_seconds(client, app):
|
|||||||
json={})
|
json={})
|
||||||
assert response.status_code == 200
|
assert response.status_code == 200
|
||||||
assert response.json['seconds'] == 10
|
assert response.json['seconds'] == 10
|
||||||
|
|
||||||
|
|
||||||
|
# Heartbeat Tests
|
||||||
|
|
||||||
|
def test_heartbeat_status_empty(client):
|
||||||
|
"""Test heartbeat status with no sensors."""
|
||||||
|
response = client.get('/api/v1/sensors/heartbeat')
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.json['total'] == 0
|
||||||
|
assert response.json['sensors'] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_heartbeat_status_with_sensors(client, app):
|
||||||
|
"""Test heartbeat status with sensors."""
|
||||||
|
from datetime import datetime, UTC, timedelta
|
||||||
|
|
||||||
|
with app.app_context():
|
||||||
|
# Online sensor (just now)
|
||||||
|
s1 = Sensor(hostname='sensor-online', ip='192.168.1.1',
|
||||||
|
last_seen=datetime.now(UTC))
|
||||||
|
# Stale sensor (3 minutes ago)
|
||||||
|
s2 = Sensor(hostname='sensor-stale', ip='192.168.1.2',
|
||||||
|
last_seen=datetime.now(UTC) - timedelta(minutes=3))
|
||||||
|
# Offline sensor (10 minutes ago)
|
||||||
|
s3 = Sensor(hostname='sensor-offline', ip='192.168.1.3',
|
||||||
|
last_seen=datetime.now(UTC) - timedelta(minutes=10))
|
||||||
|
db.session.add_all([s1, s2, s3])
|
||||||
|
db.session.commit()
|
||||||
|
|
||||||
|
response = client.get('/api/v1/sensors/heartbeat')
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.json['total'] == 3
|
||||||
|
assert response.json['online'] == 1
|
||||||
|
assert response.json['stale'] == 1
|
||||||
|
assert response.json['offline'] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_refresh_heartbeats(client, app):
|
||||||
|
"""Test refreshing heartbeat status."""
|
||||||
|
from datetime import datetime, UTC, timedelta
|
||||||
|
|
||||||
|
with app.app_context():
|
||||||
|
# Offline sensor but status still says 'online'
|
||||||
|
sensor = Sensor(hostname='test-sensor', ip='192.168.1.1',
|
||||||
|
last_seen=datetime.now(UTC) - timedelta(minutes=10),
|
||||||
|
status='online')
|
||||||
|
db.session.add(sensor)
|
||||||
|
db.session.commit()
|
||||||
|
|
||||||
|
response = client.post('/api/v1/sensors/heartbeat')
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.json['status'] == 'updated'
|
||||||
|
assert response.json['offline'] == 1
|
||||||
|
|
||||||
|
# Verify status was updated
|
||||||
|
with app.app_context():
|
||||||
|
sensor = db.session.scalar(db.select(Sensor).where(Sensor.hostname == 'test-sensor'))
|
||||||
|
assert sensor.status == 'offline'
|
||||||
|
|
||||||
|
|
||||||
|
# Metrics Tests
|
||||||
|
|
||||||
|
def test_metrics_not_found(client):
|
||||||
|
"""Test metrics for non-existent sensor."""
|
||||||
|
response = client.get('/api/v1/sensors/nonexistent/metrics')
|
||||||
|
assert response.status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
def test_metrics_empty(client, app):
|
||||||
|
"""Test metrics for sensor with no activity."""
|
||||||
|
with app.app_context():
|
||||||
|
sensor = Sensor(hostname='test-sensor', ip='192.168.1.100')
|
||||||
|
db.session.add(sensor)
|
||||||
|
db.session.commit()
|
||||||
|
|
||||||
|
response = client.get('/api/v1/sensors/test-sensor/metrics')
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.json['hostname'] == 'test-sensor'
|
||||||
|
assert response.json['hours'] == 24
|
||||||
|
assert response.json['activity']['sightings'] == 0
|
||||||
|
assert response.json['activity']['alerts'] == 0
|
||||||
|
assert response.json['activity']['events'] == 0
|
||||||
|
assert response.json['recent_events'] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_metrics_with_events(client, app):
|
||||||
|
"""Test metrics with sensor events."""
|
||||||
|
from datetime import datetime, UTC
|
||||||
|
|
||||||
|
with app.app_context():
|
||||||
|
sensor = Sensor(hostname='test-sensor', ip='192.168.1.100')
|
||||||
|
db.session.add(sensor)
|
||||||
|
db.session.flush()
|
||||||
|
|
||||||
|
# Add some events
|
||||||
|
event1 = Event(sensor_id=sensor.id, event_type='presence',
|
||||||
|
payload_json='{"state": "detected"}',
|
||||||
|
timestamp=datetime.now(UTC))
|
||||||
|
event2 = Event(sensor_id=sensor.id, event_type='calibration',
|
||||||
|
payload_json='{"nsub": 52}',
|
||||||
|
timestamp=datetime.now(UTC))
|
||||||
|
db.session.add_all([event1, event2])
|
||||||
|
db.session.commit()
|
||||||
|
|
||||||
|
response = client.get('/api/v1/sensors/test-sensor/metrics')
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.json['activity']['events'] == 2
|
||||||
|
assert len(response.json['recent_events']) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_metrics_custom_hours(client, app):
|
||||||
|
"""Test metrics with custom time range."""
|
||||||
|
with app.app_context():
|
||||||
|
sensor = Sensor(hostname='test-sensor', ip='192.168.1.100')
|
||||||
|
db.session.add(sensor)
|
||||||
|
db.session.commit()
|
||||||
|
|
||||||
|
response = client.get('/api/v1/sensors/test-sensor/metrics?hours=48')
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.json['hours'] == 48
|
||||||
|
|||||||
Reference in New Issue
Block a user