feat: Add OSINT features (v0.1.2)

- MAC vendor lookup (IEEE OUI database)
- BLE company_id to manufacturer mapping
- Device profile enrichment in API responses
- Export endpoints: devices.csv, devices.json, alerts.csv, probes.csv
- Auto-populate vendor on device creation
- CLI command: flask download-oui
- Makefile target: make oui

13 tests passing

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help build run dev stop logs test migrate clean install start restart status
+.PHONY: help build run dev stop logs test migrate clean install start restart status oui
 
 APP_NAME := esp32-web
 PORT := 5500
@@ -25,6 +25,7 @@ help:
 	@echo "Development:"
 	@echo "  make install   Install with dev dependencies"
 	@echo "  make test      Run tests"
+	@echo "  make oui       Download OUI database"
 	@echo "  make clean     Remove cache files"
 	@echo ""
 	@echo "Container:"
@@ -81,6 +82,9 @@ dev:
 test:
 	pytest -v
 
+oui:
+	flask --app src/esp32_web download-oui
+
 migrate:
 	flask --app src/esp32_web db upgrade
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "esp32-web"
-version = "0.1.1"
+version = "0.1.2"
 description = "REST API backend for ESP32 sensor fleet"
 requires-python = ">=3.11"
 dependencies = [

src/esp32_web/__init__.py

@@ -1,6 +1,8 @@
 """ESP32-Web Flask Application."""
+import click
 from datetime import datetime, UTC
 from flask import Flask
+from pathlib import Path
 
 from .config import Config
 from .extensions import db, migrate
@@ -47,4 +49,17 @@ def create_app(config_class=Config):
         with app.app_context():
             collector.start()
 
+    # Register CLI commands
+    @app.cli.command('download-oui')
+    @click.option('--path', default=None, help='Path to save OUI database')
+    def download_oui_cmd(path):
+        """Download IEEE OUI database."""
+        from .utils.oui import download_oui_db, load_oui_db, OUI_DB_PATH
+        target = Path(path) if path else OUI_DB_PATH
+        if download_oui_db(target):
+            count = load_oui_db(target)
+            click.echo(f'Downloaded and loaded {count} OUI entries')
+        else:
+            click.echo('Failed to download OUI database', err=True)
+
     return app

src/esp32_web/api/__init__.py

@@ -3,4 +3,4 @@ from flask import Blueprint
 
 bp = Blueprint('api', __name__)
 
-from . import sensors, devices, alerts, events, probes, stats  # noqa: E402, F401
+from . import sensors, devices, alerts, events, probes, stats, export  # noqa: E402, F401

src/esp32_web/api/devices.py

@@ -3,6 +3,7 @@ from flask import request
 from . import bp
 from ..models import Device, Sighting
 from ..extensions import db
+from ..services.device_service import enrich_device
 
 
 @bp.route('/devices')
@@ -18,7 +19,7 @@ def list_devices():
     query = query.limit(limit).offset(offset)
 
     devices = db.session.scalars(query).all()
-    return {'devices': [d.to_dict() for d in devices], 'limit': limit, 'offset': offset}
+    return {'devices': [enrich_device(d) for d in devices], 'limit': limit, 'offset': offset}
 
 
 @bp.route('/devices/<mac>')
@@ -37,6 +38,6 @@ def get_device(mac):
         .limit(20)
     ).all()
 
-    result = device.to_dict()
+    result = enrich_device(device)
     result['sightings'] = [s.to_dict() for s in sightings]
     return result

src/esp32_web/api/export.py

@@ -0,0 +1,100 @@
+"""Export endpoints."""
+import csv
+import io
+import json
+from flask import Response, request
+from . import bp
+from ..models import Device, Alert, Probe
+from ..extensions import db
+from ..services.device_service import enrich_device
+
+
+@bp.route('/export/devices.csv')
+def export_devices_csv():
+    """Export devices as CSV."""
+    devices = db.session.scalars(db.select(Device).order_by(Device.last_seen.desc())).all()
+
+    output = io.StringIO()
+    writer = csv.writer(output)
+    writer.writerow(['mac', 'type', 'vendor', 'name', 'company_id', 'first_seen', 'last_seen'])
+
+    for d in devices:
+        writer.writerow([
+            d.mac, d.device_type, d.vendor or '', d.name or '',
+            d.company_id or '', d.first_seen.isoformat(), d.last_seen.isoformat()
+        ])
+
+    return Response(
+        output.getvalue(),
+        mimetype='text/csv',
+        headers={'Content-Disposition': 'attachment; filename=devices.csv'}
+    )
+
+
+@bp.route('/export/devices.json')
+def export_devices_json():
+    """Export devices as JSON."""
+    devices = db.session.scalars(db.select(Device).order_by(Device.last_seen.desc())).all()
+    data = [enrich_device(d) for d in devices]
+
+    return Response(
+        json.dumps(data, indent=2),
+        mimetype='application/json',
+        headers={'Content-Disposition': 'attachment; filename=devices.json'}
+    )
+
+
+@bp.route('/export/alerts.csv')
+def export_alerts_csv():
+    """Export alerts as CSV."""
+    hours = request.args.get('hours', 24, type=int)
+    from datetime import datetime, timedelta, UTC
+    since = datetime.now(UTC) - timedelta(hours=hours)
+
+    alerts = db.session.scalars(
+        db.select(Alert).where(Alert.timestamp >= since).order_by(Alert.timestamp.desc())
+    ).all()
+
+    output = io.StringIO()
+    writer = csv.writer(output)
+    writer.writerow(['timestamp', 'sensor_id', 'type', 'source_mac', 'target_mac', 'rssi'])
+
+    for a in alerts:
+        writer.writerow([
+            a.timestamp.isoformat(), a.sensor_id, a.alert_type,
+            a.source_mac or '', a.target_mac or '', a.rssi or ''
+        ])
+
+    return Response(
+        output.getvalue(),
+        mimetype='text/csv',
+        headers={'Content-Disposition': 'attachment; filename=alerts.csv'}
+    )
+
+
+@bp.route('/export/probes.csv')
+def export_probes_csv():
+    """Export probe requests as CSV."""
+    hours = request.args.get('hours', 24, type=int)
+    from datetime import datetime, timedelta, UTC
+    since = datetime.now(UTC) - timedelta(hours=hours)
+
+    probes = db.session.scalars(
+        db.select(Probe).where(Probe.timestamp >= since).order_by(Probe.timestamp.desc())
+    ).all()
+
+    output = io.StringIO()
+    writer = csv.writer(output)
+    writer.writerow(['timestamp', 'sensor_id', 'device_id', 'ssid', 'rssi', 'channel'])
+
+    for p in probes:
+        writer.writerow([
+            p.timestamp.isoformat(), p.sensor_id, p.device_id,
+            p.ssid, p.rssi, p.channel
+        ])
+
+    return Response(
+        output.getvalue(),
+        mimetype='text/csv',
+        headers={'Content-Disposition': 'attachment; filename=probes.csv'}
+    )

src/esp32_web/collector/parsers.py

@@ -6,6 +6,7 @@ from datetime import datetime, UTC
 
 from ..extensions import db
 from ..models import Sensor, Device, Sighting, Alert, Event, Probe
+from ..utils.oui import lookup_vendor
 
 log = logging.getLogger(__name__)
 
@@ -29,11 +30,15 @@ def get_or_create_device(mac: str, device_type: str) -> Device:
     mac = mac.lower()
     device = db.session.scalar(db.select(Device).where(Device.mac == mac))
     if not device:
-        device = Device(mac=mac, device_type=device_type)
+        vendor = lookup_vendor(mac)
+        device = Device(mac=mac, device_type=device_type, vendor=vendor)
         db.session.add(device)
         db.session.flush()
     else:
         device.last_seen = datetime.now(UTC)
+        # Update vendor if not set
+        if not device.vendor:
+            device.vendor = lookup_vendor(mac)
     return device
 
 

src/esp32_web/services/device_service.py

@@ -0,0 +1,33 @@
+"""Device enrichment service."""
+from ..models import Device
+from ..utils.oui import lookup_vendor
+from ..utils.ble_companies import lookup_ble_company
+
+
+def enrich_device(device: Device) -> dict:
+    """Enrich device with vendor and company info."""
+    data = device.to_dict()
+
+    # Add vendor from OUI if not set
+    if not device.vendor:
+        vendor = lookup_vendor(device.mac)
+        if vendor:
+            data['vendor'] = vendor
+
+    # Add BLE company name if company_id present
+    if device.company_id:
+        company_name = lookup_ble_company(device.company_id)
+        if company_name:
+            data['company_name'] = company_name
+
+    return data
+
+
+def get_vendor_for_mac(mac: str) -> str | None:
+    """Get vendor for MAC address."""
+    return lookup_vendor(mac)
+
+
+def get_company_for_id(company_id: int) -> str | None:
+    """Get company name for BLE company ID."""
+    return lookup_ble_company(company_id)

src/esp32_web/utils/__init__.py

@@ -1 +1,5 @@
 """Utility modules."""
+from .oui import lookup_vendor, load_oui_db, download_oui_db
+from .ble_companies import lookup_ble_company
+
+__all__ = ['lookup_vendor', 'load_oui_db', 'download_oui_db', 'lookup_ble_company']

src/esp32_web/utils/ble_companies.py

@@ -0,0 +1,46 @@
+"""BLE Company ID lookup."""
+
+# Common BLE Company IDs (Bluetooth SIG assigned)
+# https://www.bluetooth.com/specifications/assigned-numbers/company-identifiers/
+BLE_COMPANIES: dict[int, str] = {
+    0x0000: 'Ericsson Technology Licensing',
+    0x0001: 'Nokia Mobile Phones',
+    0x0002: 'Intel Corp.',
+    0x0003: 'IBM Corp.',
+    0x0004: 'Toshiba Corp.',
+    0x0006: 'Microsoft',
+    0x000A: 'Qualcomm',
+    0x000D: 'Texas Instruments',
+    0x000F: 'Broadcom',
+    0x0010: 'Mitel Semiconductor',
+    0x0013: 'Atmel',
+    0x001D: 'Qualcomm Technologies',
+    0x004C: 'Apple, Inc.',
+    0x0059: 'Nordic Semiconductor',
+    0x005D: 'Realtek Semiconductor',
+    0x0075: 'Samsung Electronics',
+    0x0087: 'Garmin International',
+    0x00E0: 'Google',
+    0x00D2: 'Dialog Semiconductor',
+    0x0157: 'Anhui Huami Information Technology',  # Xiaomi/Amazfit
+    0x0171: 'Amazon.com Services',
+    0x01A3: 'Facebook Technologies',
+    0x0224: 'Xiaomi Inc.',
+    0x02E5: 'Shenzhen Goodix Technology',
+    0x0310: 'Tile, Inc.',
+    0x038F: 'Bose Corporation',
+    0x0499: 'Ruuvi Innovations',
+    0x04E7: 'Sonos, Inc.',
+    0x0591: 'Espressif Inc.',
+    0x0822: 'Nothing Technology',
+}
+
+
+def lookup_ble_company(company_id: int) -> str | None:
+    """Lookup BLE company by ID."""
+    return BLE_COMPANIES.get(company_id)
+
+
+def get_all_companies() -> dict[int, str]:
+    """Get all known BLE companies."""
+    return BLE_COMPANIES.copy()

src/esp32_web/utils/oui.py

@@ -0,0 +1,87 @@
+"""MAC OUI vendor lookup."""
+import csv
+import logging
+import os
+from pathlib import Path
+
+log = logging.getLogger(__name__)
+
+# OUI database path
+OUI_DB_PATH = Path(os.environ.get('OUI_DB_PATH', '/var/lib/esp32-web/oui.csv'))
+OUI_DB_URL = 'https://standards-oui.ieee.org/oui/oui.csv'
+
+# In-memory cache
+_oui_cache: dict[str, str] = {}
+_loaded = False
+
+
+def _normalize_mac(mac: str) -> str:
+    """Extract OUI prefix (first 6 hex chars) from MAC."""
+    clean = mac.upper().replace(':', '').replace('-', '').replace('.', '')
+    return clean[:6] if len(clean) >= 6 else ''
+
+
+def load_oui_db(path: Path | None = None) -> int:
+    """Load OUI database from CSV file."""
+    global _oui_cache, _loaded
+
+    if path is None:
+        path = OUI_DB_PATH
+
+    if not path.exists():
+        log.warning('OUI database not found: %s', path)
+        return 0
+
+    _oui_cache.clear()
+    count = 0
+
+    try:
+        with open(path, 'r', encoding='utf-8') as f:
+            reader = csv.DictReader(f)
+            for row in reader:
+                # IEEE CSV format: Registry,Assignment,Organization Name,...
+                assignment = row.get('Assignment', '').strip().upper()
+                org = row.get('Organization Name', '').strip()
+                if assignment and org:
+                    _oui_cache[assignment] = org
+                    count += 1
+    except Exception as e:
+        log.exception('Error loading OUI database: %s', e)
+        return 0
+
+    _loaded = True
+    log.info('Loaded %d OUI entries from %s', count, path)
+    return count
+
+
+def lookup_vendor(mac: str) -> str | None:
+    """Lookup vendor by MAC address."""
+    global _loaded
+
+    if not _loaded:
+        load_oui_db()
+
+    oui = _normalize_mac(mac)
+    if not oui:
+        return None
+
+    return _oui_cache.get(oui)
+
+
+def download_oui_db(path: Path | None = None) -> bool:
+    """Download OUI database from IEEE."""
+    import urllib.request
+
+    if path is None:
+        path = OUI_DB_PATH
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+    try:
+        log.info('Downloading OUI database from %s', OUI_DB_URL)
+        urllib.request.urlretrieve(OUI_DB_URL, path)
+        log.info('OUI database saved to %s', path)
+        return True
+    except Exception as e:
+        log.exception('Error downloading OUI database: %s', e)
+        return False

tests/test_api/test_export.py

@@ -0,0 +1,17 @@
+"""Export endpoint tests."""
+
+
+def test_export_devices_csv_empty(client):
+    """Test exporting devices CSV when empty."""
+    response = client.get('/api/v1/export/devices.csv')
+    assert response.status_code == 200
+    assert response.content_type == 'text/csv; charset=utf-8'
+    assert b'mac,type,vendor' in response.data
+
+
+def test_export_devices_json_empty(client):
+    """Test exporting devices JSON when empty."""
+    response = client.get('/api/v1/export/devices.json')
+    assert response.status_code == 200
+    assert response.content_type == 'application/json'
+    assert response.json == []

tests/test_api/test_sensors.py

@@ -18,4 +18,6 @@ def test_health_check(client):
     """Test health endpoint."""
     response = client.get('/health')
     assert response.status_code == 200
-    assert response.json == {'status': 'ok'}
+    assert response.json['status'] == 'ok'
+    assert 'uptime' in response.json
+    assert 'uptime_seconds' in response.json

tests/test_utils/__init__.py

@@ -0,0 +1 @@
+"""Utils tests."""

tests/test_utils/test_ble_companies.py

@@ -0,0 +1,25 @@
+"""BLE company lookup tests."""
+from esp32_web.utils.ble_companies import lookup_ble_company, get_all_companies
+
+
+def test_lookup_apple():
+    """Test Apple company ID lookup."""
+    assert lookup_ble_company(0x004C) == 'Apple, Inc.'
+
+
+def test_lookup_google():
+    """Test Google company ID lookup."""
+    assert lookup_ble_company(0x00E0) == 'Google'
+
+
+def test_lookup_unknown():
+    """Test unknown company ID lookup."""
+    assert lookup_ble_company(0xFFFF) is None
+
+
+def test_get_all_companies():
+    """Test getting all companies."""
+    companies = get_all_companies()
+    assert isinstance(companies, dict)
+    assert len(companies) > 0
+    assert 0x004C in companies

tests/test_utils/test_oui.py

@@ -0,0 +1,18 @@
+"""OUI lookup tests."""
+from esp32_web.utils.oui import _normalize_mac, lookup_vendor
+
+
+def test_normalize_mac():
+    """Test MAC normalization."""
+    assert _normalize_mac('aa:bb:cc:dd:ee:ff') == 'AABBCC'
+    assert _normalize_mac('AA-BB-CC-DD-EE-FF') == 'AABBCC'
+    assert _normalize_mac('aabbccddeeff') == 'AABBCC'
+    assert _normalize_mac('short') == ''
+
+
+def test_lookup_vendor_no_db():
+    """Test vendor lookup without database."""
+    # Should return None when no database loaded
+    result = lookup_vendor('aa:bb:cc:dd:ee:ff')
+    # Result depends on whether OUI db exists
+    assert result is None or isinstance(result, str)