diff --git a/Makefile b/Makefile index c896863..e791c6b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help build run dev stop logs test migrate clean install start restart status +.PHONY: help build run dev stop logs test migrate clean install start restart status oui APP_NAME := esp32-web PORT := 5500 @@ -25,6 +25,7 @@ help: @echo "Development:" @echo " make install Install with dev dependencies" @echo " make test Run tests" + @echo " make oui Download OUI database" @echo " make clean Remove cache files" @echo "" @echo "Container:" @@ -81,6 +82,9 @@ dev: test: pytest -v +oui: + flask --app src/esp32_web download-oui + migrate: flask --app src/esp32_web db upgrade diff --git a/pyproject.toml b/pyproject.toml index d51fe7e..e29462e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "esp32-web" -version = "0.1.1" +version = "0.1.2" description = "REST API backend for ESP32 sensor fleet" requires-python = ">=3.11" dependencies = [ diff --git a/src/esp32_web/__init__.py b/src/esp32_web/__init__.py index 5c42d0e..850611a 100644 --- a/src/esp32_web/__init__.py +++ b/src/esp32_web/__init__.py @@ -1,6 +1,8 @@ """ESP32-Web Flask Application.""" +import click from datetime import datetime, UTC from flask import Flask +from pathlib import Path from .config import Config from .extensions import db, migrate @@ -47,4 +49,17 @@ def create_app(config_class=Config): with app.app_context(): collector.start() + # Register CLI commands + @app.cli.command('download-oui') + @click.option('--path', default=None, help='Path to save OUI database') + def download_oui_cmd(path): + """Download IEEE OUI database.""" + from .utils.oui import download_oui_db, load_oui_db, OUI_DB_PATH + target = Path(path) if path else OUI_DB_PATH + if download_oui_db(target): + count = load_oui_db(target) + click.echo(f'Downloaded and loaded {count} OUI entries') + else: + click.echo('Failed to download OUI database', err=True) + return app diff --git a/src/esp32_web/api/__init__.py b/src/esp32_web/api/__init__.py index c30c059..dd4221e 100644 --- a/src/esp32_web/api/__init__.py +++ b/src/esp32_web/api/__init__.py @@ -3,4 +3,4 @@ from flask import Blueprint bp = Blueprint('api', __name__) -from . import sensors, devices, alerts, events, probes, stats # noqa: E402, F401 +from . import sensors, devices, alerts, events, probes, stats, export # noqa: E402, F401 diff --git a/src/esp32_web/api/devices.py b/src/esp32_web/api/devices.py index f85bf72..3752d19 100644 --- a/src/esp32_web/api/devices.py +++ b/src/esp32_web/api/devices.py @@ -3,6 +3,7 @@ from flask import request from . import bp from ..models import Device, Sighting from ..extensions import db +from ..services.device_service import enrich_device @bp.route('/devices') @@ -18,7 +19,7 @@ def list_devices(): query = query.limit(limit).offset(offset) devices = db.session.scalars(query).all() - return {'devices': [d.to_dict() for d in devices], 'limit': limit, 'offset': offset} + return {'devices': [enrich_device(d) for d in devices], 'limit': limit, 'offset': offset} @bp.route('/devices/') @@ -37,6 +38,6 @@ def get_device(mac): .limit(20) ).all() - result = device.to_dict() + result = enrich_device(device) result['sightings'] = [s.to_dict() for s in sightings] return result diff --git a/src/esp32_web/api/export.py b/src/esp32_web/api/export.py new file mode 100644 index 0000000..bd8309f --- /dev/null +++ b/src/esp32_web/api/export.py @@ -0,0 +1,100 @@ +"""Export endpoints.""" +import csv +import io +import json +from flask import Response, request +from . import bp +from ..models import Device, Alert, Probe +from ..extensions import db +from ..services.device_service import enrich_device + + +@bp.route('/export/devices.csv') +def export_devices_csv(): + """Export devices as CSV.""" + devices = db.session.scalars(db.select(Device).order_by(Device.last_seen.desc())).all() + + output = io.StringIO() + writer = csv.writer(output) + writer.writerow(['mac', 'type', 'vendor', 'name', 'company_id', 'first_seen', 'last_seen']) + + for d in devices: + writer.writerow([ + d.mac, d.device_type, d.vendor or '', d.name or '', + d.company_id or '', d.first_seen.isoformat(), d.last_seen.isoformat() + ]) + + return Response( + output.getvalue(), + mimetype='text/csv', + headers={'Content-Disposition': 'attachment; filename=devices.csv'} + ) + + +@bp.route('/export/devices.json') +def export_devices_json(): + """Export devices as JSON.""" + devices = db.session.scalars(db.select(Device).order_by(Device.last_seen.desc())).all() + data = [enrich_device(d) for d in devices] + + return Response( + json.dumps(data, indent=2), + mimetype='application/json', + headers={'Content-Disposition': 'attachment; filename=devices.json'} + ) + + +@bp.route('/export/alerts.csv') +def export_alerts_csv(): + """Export alerts as CSV.""" + hours = request.args.get('hours', 24, type=int) + from datetime import datetime, timedelta, UTC + since = datetime.now(UTC) - timedelta(hours=hours) + + alerts = db.session.scalars( + db.select(Alert).where(Alert.timestamp >= since).order_by(Alert.timestamp.desc()) + ).all() + + output = io.StringIO() + writer = csv.writer(output) + writer.writerow(['timestamp', 'sensor_id', 'type', 'source_mac', 'target_mac', 'rssi']) + + for a in alerts: + writer.writerow([ + a.timestamp.isoformat(), a.sensor_id, a.alert_type, + a.source_mac or '', a.target_mac or '', a.rssi or '' + ]) + + return Response( + output.getvalue(), + mimetype='text/csv', + headers={'Content-Disposition': 'attachment; filename=alerts.csv'} + ) + + +@bp.route('/export/probes.csv') +def export_probes_csv(): + """Export probe requests as CSV.""" + hours = request.args.get('hours', 24, type=int) + from datetime import datetime, timedelta, UTC + since = datetime.now(UTC) - timedelta(hours=hours) + + probes = db.session.scalars( + db.select(Probe).where(Probe.timestamp >= since).order_by(Probe.timestamp.desc()) + ).all() + + output = io.StringIO() + writer = csv.writer(output) + writer.writerow(['timestamp', 'sensor_id', 'device_id', 'ssid', 'rssi', 'channel']) + + for p in probes: + writer.writerow([ + p.timestamp.isoformat(), p.sensor_id, p.device_id, + p.ssid, p.rssi, p.channel + ]) + + return Response( + output.getvalue(), + mimetype='text/csv', + headers={'Content-Disposition': 'attachment; filename=probes.csv'} + ) diff --git a/src/esp32_web/collector/parsers.py b/src/esp32_web/collector/parsers.py index 3c78fb0..dc68022 100644 --- a/src/esp32_web/collector/parsers.py +++ b/src/esp32_web/collector/parsers.py @@ -6,6 +6,7 @@ from datetime import datetime, UTC from ..extensions import db from ..models import Sensor, Device, Sighting, Alert, Event, Probe +from ..utils.oui import lookup_vendor log = logging.getLogger(__name__) @@ -29,11 +30,15 @@ def get_or_create_device(mac: str, device_type: str) -> Device: mac = mac.lower() device = db.session.scalar(db.select(Device).where(Device.mac == mac)) if not device: - device = Device(mac=mac, device_type=device_type) + vendor = lookup_vendor(mac) + device = Device(mac=mac, device_type=device_type, vendor=vendor) db.session.add(device) db.session.flush() else: device.last_seen = datetime.now(UTC) + # Update vendor if not set + if not device.vendor: + device.vendor = lookup_vendor(mac) return device diff --git a/src/esp32_web/services/device_service.py b/src/esp32_web/services/device_service.py new file mode 100644 index 0000000..6be3633 --- /dev/null +++ b/src/esp32_web/services/device_service.py @@ -0,0 +1,33 @@ +"""Device enrichment service.""" +from ..models import Device +from ..utils.oui import lookup_vendor +from ..utils.ble_companies import lookup_ble_company + + +def enrich_device(device: Device) -> dict: + """Enrich device with vendor and company info.""" + data = device.to_dict() + + # Add vendor from OUI if not set + if not device.vendor: + vendor = lookup_vendor(device.mac) + if vendor: + data['vendor'] = vendor + + # Add BLE company name if company_id present + if device.company_id: + company_name = lookup_ble_company(device.company_id) + if company_name: + data['company_name'] = company_name + + return data + + +def get_vendor_for_mac(mac: str) -> str | None: + """Get vendor for MAC address.""" + return lookup_vendor(mac) + + +def get_company_for_id(company_id: int) -> str | None: + """Get company name for BLE company ID.""" + return lookup_ble_company(company_id) diff --git a/src/esp32_web/utils/__init__.py b/src/esp32_web/utils/__init__.py index 183c974..9c3cf32 100644 --- a/src/esp32_web/utils/__init__.py +++ b/src/esp32_web/utils/__init__.py @@ -1 +1,5 @@ """Utility modules.""" +from .oui import lookup_vendor, load_oui_db, download_oui_db +from .ble_companies import lookup_ble_company + +__all__ = ['lookup_vendor', 'load_oui_db', 'download_oui_db', 'lookup_ble_company'] diff --git a/src/esp32_web/utils/ble_companies.py b/src/esp32_web/utils/ble_companies.py new file mode 100644 index 0000000..4add8a6 --- /dev/null +++ b/src/esp32_web/utils/ble_companies.py @@ -0,0 +1,46 @@ +"""BLE Company ID lookup.""" + +# Common BLE Company IDs (Bluetooth SIG assigned) +# https://www.bluetooth.com/specifications/assigned-numbers/company-identifiers/ +BLE_COMPANIES: dict[int, str] = { + 0x0000: 'Ericsson Technology Licensing', + 0x0001: 'Nokia Mobile Phones', + 0x0002: 'Intel Corp.', + 0x0003: 'IBM Corp.', + 0x0004: 'Toshiba Corp.', + 0x0006: 'Microsoft', + 0x000A: 'Qualcomm', + 0x000D: 'Texas Instruments', + 0x000F: 'Broadcom', + 0x0010: 'Mitel Semiconductor', + 0x0013: 'Atmel', + 0x001D: 'Qualcomm Technologies', + 0x004C: 'Apple, Inc.', + 0x0059: 'Nordic Semiconductor', + 0x005D: 'Realtek Semiconductor', + 0x0075: 'Samsung Electronics', + 0x0087: 'Garmin International', + 0x00E0: 'Google', + 0x00D2: 'Dialog Semiconductor', + 0x0157: 'Anhui Huami Information Technology', # Xiaomi/Amazfit + 0x0171: 'Amazon.com Services', + 0x01A3: 'Facebook Technologies', + 0x0224: 'Xiaomi Inc.', + 0x02E5: 'Shenzhen Goodix Technology', + 0x0310: 'Tile, Inc.', + 0x038F: 'Bose Corporation', + 0x0499: 'Ruuvi Innovations', + 0x04E7: 'Sonos, Inc.', + 0x0591: 'Espressif Inc.', + 0x0822: 'Nothing Technology', +} + + +def lookup_ble_company(company_id: int) -> str | None: + """Lookup BLE company by ID.""" + return BLE_COMPANIES.get(company_id) + + +def get_all_companies() -> dict[int, str]: + """Get all known BLE companies.""" + return BLE_COMPANIES.copy() diff --git a/src/esp32_web/utils/oui.py b/src/esp32_web/utils/oui.py new file mode 100644 index 0000000..7a5466c --- /dev/null +++ b/src/esp32_web/utils/oui.py @@ -0,0 +1,87 @@ +"""MAC OUI vendor lookup.""" +import csv +import logging +import os +from pathlib import Path + +log = logging.getLogger(__name__) + +# OUI database path +OUI_DB_PATH = Path(os.environ.get('OUI_DB_PATH', '/var/lib/esp32-web/oui.csv')) +OUI_DB_URL = 'https://standards-oui.ieee.org/oui/oui.csv' + +# In-memory cache +_oui_cache: dict[str, str] = {} +_loaded = False + + +def _normalize_mac(mac: str) -> str: + """Extract OUI prefix (first 6 hex chars) from MAC.""" + clean = mac.upper().replace(':', '').replace('-', '').replace('.', '') + return clean[:6] if len(clean) >= 6 else '' + + +def load_oui_db(path: Path | None = None) -> int: + """Load OUI database from CSV file.""" + global _oui_cache, _loaded + + if path is None: + path = OUI_DB_PATH + + if not path.exists(): + log.warning('OUI database not found: %s', path) + return 0 + + _oui_cache.clear() + count = 0 + + try: + with open(path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + for row in reader: + # IEEE CSV format: Registry,Assignment,Organization Name,... + assignment = row.get('Assignment', '').strip().upper() + org = row.get('Organization Name', '').strip() + if assignment and org: + _oui_cache[assignment] = org + count += 1 + except Exception as e: + log.exception('Error loading OUI database: %s', e) + return 0 + + _loaded = True + log.info('Loaded %d OUI entries from %s', count, path) + return count + + +def lookup_vendor(mac: str) -> str | None: + """Lookup vendor by MAC address.""" + global _loaded + + if not _loaded: + load_oui_db() + + oui = _normalize_mac(mac) + if not oui: + return None + + return _oui_cache.get(oui) + + +def download_oui_db(path: Path | None = None) -> bool: + """Download OUI database from IEEE.""" + import urllib.request + + if path is None: + path = OUI_DB_PATH + + path.parent.mkdir(parents=True, exist_ok=True) + + try: + log.info('Downloading OUI database from %s', OUI_DB_URL) + urllib.request.urlretrieve(OUI_DB_URL, path) + log.info('OUI database saved to %s', path) + return True + except Exception as e: + log.exception('Error downloading OUI database: %s', e) + return False diff --git a/tests/test_api/test_export.py b/tests/test_api/test_export.py new file mode 100644 index 0000000..06d322f --- /dev/null +++ b/tests/test_api/test_export.py @@ -0,0 +1,17 @@ +"""Export endpoint tests.""" + + +def test_export_devices_csv_empty(client): + """Test exporting devices CSV when empty.""" + response = client.get('/api/v1/export/devices.csv') + assert response.status_code == 200 + assert response.content_type == 'text/csv; charset=utf-8' + assert b'mac,type,vendor' in response.data + + +def test_export_devices_json_empty(client): + """Test exporting devices JSON when empty.""" + response = client.get('/api/v1/export/devices.json') + assert response.status_code == 200 + assert response.content_type == 'application/json' + assert response.json == [] diff --git a/tests/test_api/test_sensors.py b/tests/test_api/test_sensors.py index 8adb1ab..003f2b5 100644 --- a/tests/test_api/test_sensors.py +++ b/tests/test_api/test_sensors.py @@ -18,4 +18,6 @@ def test_health_check(client): """Test health endpoint.""" response = client.get('/health') assert response.status_code == 200 - assert response.json == {'status': 'ok'} + assert response.json['status'] == 'ok' + assert 'uptime' in response.json + assert 'uptime_seconds' in response.json diff --git a/tests/test_utils/__init__.py b/tests/test_utils/__init__.py new file mode 100644 index 0000000..8dc6e5d --- /dev/null +++ b/tests/test_utils/__init__.py @@ -0,0 +1 @@ +"""Utils tests.""" diff --git a/tests/test_utils/test_ble_companies.py b/tests/test_utils/test_ble_companies.py new file mode 100644 index 0000000..b315bf3 --- /dev/null +++ b/tests/test_utils/test_ble_companies.py @@ -0,0 +1,25 @@ +"""BLE company lookup tests.""" +from esp32_web.utils.ble_companies import lookup_ble_company, get_all_companies + + +def test_lookup_apple(): + """Test Apple company ID lookup.""" + assert lookup_ble_company(0x004C) == 'Apple, Inc.' + + +def test_lookup_google(): + """Test Google company ID lookup.""" + assert lookup_ble_company(0x00E0) == 'Google' + + +def test_lookup_unknown(): + """Test unknown company ID lookup.""" + assert lookup_ble_company(0xFFFF) is None + + +def test_get_all_companies(): + """Test getting all companies.""" + companies = get_all_companies() + assert isinstance(companies, dict) + assert len(companies) > 0 + assert 0x004C in companies diff --git a/tests/test_utils/test_oui.py b/tests/test_utils/test_oui.py new file mode 100644 index 0000000..888caaf --- /dev/null +++ b/tests/test_utils/test_oui.py @@ -0,0 +1,18 @@ +"""OUI lookup tests.""" +from esp32_web.utils.oui import _normalize_mac, lookup_vendor + + +def test_normalize_mac(): + """Test MAC normalization.""" + assert _normalize_mac('aa:bb:cc:dd:ee:ff') == 'AABBCC' + assert _normalize_mac('AA-BB-CC-DD-EE-FF') == 'AABBCC' + assert _normalize_mac('aabbccddeeff') == 'AABBCC' + assert _normalize_mac('short') == '' + + +def test_lookup_vendor_no_db(): + """Test vendor lookup without database.""" + # Should return None when no database loaded + result = lookup_vendor('aa:bb:cc:dd:ee:ff') + # Result depends on whether OUI db exists + assert result is None or isinstance(result, str)