feat: add wave 3 local database plugins
GeoIP and ASN lookup via MaxMind GeoLite2 mmdb, Tor exit node check against local bulk exit list, IP reputation via Firehol/ET blocklist feeds, and CVE lookup against local NVD JSON mirror. Includes cron-friendly update script (scripts/update-data.sh) for all data sources and make update-data target. GeoLite2 requires a free MaxMind license key; all other sources are freely downloadable. Plugins: geoip, asn, torcheck, iprep, cve Commands: !geoip, !asn, !tor, !iprep, !cve Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
15
.gitignore
vendored
Normal file
15
.gitignore
vendored
Normal file
@@ -0,0 +1,15 @@
|
||||
.gitignore
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.egg-info/
|
||||
dist/
|
||||
build/
|
||||
.venv/
|
||||
.eggs/
|
||||
*.egg
|
||||
.ruff_cache/
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
config/derp.toml
|
||||
data/
|
||||
5
Makefile
5
Makefile
@@ -1,4 +1,4 @@
|
||||
.PHONY: install dev test lint clean help build container-run container-stop container-logs up down logs
|
||||
.PHONY: install dev test lint clean help build container-run container-stop container-logs update-data up down logs
|
||||
|
||||
APP_NAME := derp
|
||||
VENV := .venv
|
||||
@@ -54,6 +54,9 @@ container-stop: ## Stop and remove container
|
||||
container-logs: ## Follow container logs
|
||||
podman logs -f $(APP_NAME)
|
||||
|
||||
update-data: ## Download/refresh local data files
|
||||
./scripts/update-data.sh
|
||||
|
||||
up: ## Start with podman-compose (build + detach)
|
||||
podman-compose up -d --build
|
||||
|
||||
|
||||
@@ -50,6 +50,11 @@ make down # Stop
|
||||
| blacklist | blacklist | DNSBL/RBL IP reputation check |
|
||||
| rand | rand | Passwords, hex, UUIDs, dice rolls |
|
||||
| timer | timer | Countdown timers with notification |
|
||||
| geoip | geoip | GeoIP city/country lookup (MaxMind mmdb) |
|
||||
| asn | asn | AS number + organization lookup (MaxMind mmdb) |
|
||||
| torcheck | tor | Tor exit node check (local list) |
|
||||
| iprep | iprep | IP reputation (Firehol/ET blocklists) |
|
||||
| cve | cve | CVE lookup + search (local NVD mirror) |
|
||||
| example | echo | Demo plugin |
|
||||
|
||||
## Writing Plugins
|
||||
|
||||
15
TASKS.md
15
TASKS.md
@@ -4,18 +4,19 @@
|
||||
|
||||
| Pri | Status | Task |
|
||||
|-----|--------|------|
|
||||
| P0 | [ ] | GeoIP plugin (GeoLite2-City mmdb) |
|
||||
| P0 | [ ] | ASN plugin (GeoLite2-ASN mmdb) |
|
||||
| P0 | [ ] | Tor exit node check plugin |
|
||||
| P0 | [ ] | IP reputation plugin (Firehol blocklists) |
|
||||
| P0 | [ ] | CVE lookup plugin (NVD JSON feed) |
|
||||
| P0 | [ ] | Data update script (scripts/update-data.sh) |
|
||||
| P1 | [ ] | Documentation update (all docs current) |
|
||||
| P0 | [x] | GeoIP plugin (GeoLite2-City mmdb) |
|
||||
| P0 | [x] | ASN plugin (GeoLite2-ASN mmdb) |
|
||||
| P0 | [x] | Tor exit node check plugin |
|
||||
| P0 | [x] | IP reputation plugin (Firehol blocklists) |
|
||||
| P0 | [x] | CVE lookup plugin (NVD JSON feed) |
|
||||
| P0 | [x] | Data update script (scripts/update-data.sh) |
|
||||
| P0 | [x] | Documentation update (all docs current) |
|
||||
|
||||
## Completed
|
||||
|
||||
| Date | Task |
|
||||
|------|------|
|
||||
| 2026-02-15 | Wave 3 plugins (geoip, asn, torcheck, iprep, cve) + update script |
|
||||
| 2026-02-15 | Admin/owner permission system (hostmask + IRCOP) |
|
||||
| 2026-02-15 | SASL PLAIN, rate limiting, CTCP responses |
|
||||
| 2026-02-15 | Wave 2 plugins (whois, portcheck, httpcheck, tlscheck, blacklist, rand, timer) |
|
||||
|
||||
@@ -121,6 +121,28 @@ IRC operators are auto-detected via WHO. Hostmask patterns use fnmatch.
|
||||
!blacklist 1.2.3.4 # DNSBL reputation check
|
||||
```
|
||||
|
||||
## Intelligence (local databases)
|
||||
|
||||
```
|
||||
!geoip 8.8.8.8 # GeoIP: city, country, coords, tz
|
||||
!asn 8.8.8.8 # ASN: number + organization
|
||||
!tor 1.2.3.4 # Check Tor exit node
|
||||
!tor update # Download exit list
|
||||
!iprep 1.2.3.4 # Firehol/ET blocklist check
|
||||
!iprep update # Download blocklist feeds
|
||||
!cve CVE-2024-1234 # Lookup specific CVE
|
||||
!cve search apache rce # Search CVE descriptions
|
||||
!cve update # Download NVD feed (slow)
|
||||
!cve stats # Show index size
|
||||
```
|
||||
|
||||
### Data Setup
|
||||
|
||||
```bash
|
||||
./scripts/update-data.sh # Update tor + iprep
|
||||
MAXMIND_LICENSE_KEY=xxx ./scripts/update-data.sh # + GeoLite2
|
||||
```
|
||||
|
||||
## Random
|
||||
|
||||
```
|
||||
|
||||
@@ -85,6 +85,11 @@ level = "info" # Logging level: debug, info, warning, error
|
||||
| `!timer <duration> [label]` | Set countdown timer with notification |
|
||||
| `!timer list` | Show active timers |
|
||||
| `!timer cancel <label>` | Cancel a running timer |
|
||||
| `!geoip <ip>` | GeoIP lookup (city, country, coords, timezone) |
|
||||
| `!asn <ip>` | ASN lookup (AS number, organization) |
|
||||
| `!tor <ip\|update>` | Check IP against Tor exit nodes |
|
||||
| `!iprep <ip\|update>` | Check IP against Firehol/ET blocklists |
|
||||
| `!cve <id\|search>` | CVE lookup from local NVD mirror |
|
||||
|
||||
### Command Shorthand
|
||||
|
||||
@@ -155,6 +160,48 @@ async def cmd_dangerous(bot, message):
|
||||
...
|
||||
```
|
||||
|
||||
## Local Databases (Wave 3)
|
||||
|
||||
Several plugins rely on local data files in the `data/` directory. Use the
|
||||
update script or in-bot commands to populate them.
|
||||
|
||||
### Data Update Script
|
||||
|
||||
```bash
|
||||
./scripts/update-data.sh # Update all feeds
|
||||
MAXMIND_LICENSE_KEY=xxx ./scripts/update-data.sh # Include GeoLite2
|
||||
```
|
||||
|
||||
The script is cron-friendly (exit 0/1, quiet unless `NO_COLOR` is unset).
|
||||
|
||||
### In-Bot Updates
|
||||
|
||||
```
|
||||
!tor update # Download Tor exit node list
|
||||
!iprep update # Download Firehol/ET blocklist feeds
|
||||
!cve update # Download NVD CVE feed (slow, paginated)
|
||||
```
|
||||
|
||||
### Data Directory Layout
|
||||
|
||||
```
|
||||
data/
|
||||
GeoLite2-City.mmdb # MaxMind GeoIP (requires license key)
|
||||
GeoLite2-ASN.mmdb # MaxMind ASN (requires license key)
|
||||
tor-exit-nodes.txt # Tor exit node IPs
|
||||
iprep/ # Firehol/ET blocklist feeds
|
||||
firehol_level1.netset
|
||||
firehol_level2.netset
|
||||
et_compromised.ipset
|
||||
...
|
||||
nvd/ # NVD CVE JSON files
|
||||
nvd_0000.json
|
||||
...
|
||||
```
|
||||
|
||||
GeoLite2 databases require a free MaxMind license key. Set
|
||||
`MAXMIND_LICENSE_KEY` when running the update script.
|
||||
|
||||
## Plugin Management
|
||||
|
||||
Plugins can be loaded, unloaded, and reloaded at runtime without
|
||||
|
||||
85
plugins/asn.py
Normal file
85
plugins/asn.py
Normal file
@@ -0,0 +1,85 @@
|
||||
"""Plugin: ASN lookup using MaxMind GeoLite2-ASN mmdb."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from derp.plugin import command
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_DB_PATHS = [
|
||||
Path("data/GeoLite2-ASN.mmdb"),
|
||||
Path("/usr/share/GeoIP/GeoLite2-ASN.mmdb"),
|
||||
Path.home() / ".local" / "share" / "GeoIP" / "GeoLite2-ASN.mmdb",
|
||||
]
|
||||
|
||||
_reader = None
|
||||
|
||||
|
||||
def _get_reader():
|
||||
"""Lazy-load the mmdb reader."""
|
||||
global _reader
|
||||
if _reader is not None:
|
||||
return _reader
|
||||
try:
|
||||
import maxminddb
|
||||
except ImportError:
|
||||
log.error("maxminddb package not installed")
|
||||
return None
|
||||
for path in _DB_PATHS:
|
||||
if path.is_file():
|
||||
_reader = maxminddb.open_database(str(path))
|
||||
log.info("asn: loaded %s", path)
|
||||
return _reader
|
||||
log.warning("asn: no GeoLite2-ASN.mmdb found")
|
||||
return None
|
||||
|
||||
|
||||
@command("asn", help="ASN lookup: !asn <ip>")
|
||||
async def cmd_asn(bot, message):
|
||||
"""Look up the Autonomous System Number for an IP address.
|
||||
|
||||
Usage:
|
||||
!asn 8.8.8.8
|
||||
"""
|
||||
parts = message.text.split(None, 2)
|
||||
if len(parts) < 2:
|
||||
await bot.reply(message, "Usage: !asn <ip>")
|
||||
return
|
||||
|
||||
addr = parts[1]
|
||||
try:
|
||||
ip = ipaddress.ip_address(addr)
|
||||
except ValueError:
|
||||
await bot.reply(message, f"Invalid IP address: {addr}")
|
||||
return
|
||||
|
||||
if ip.is_private or ip.is_loopback:
|
||||
await bot.reply(message, f"{addr}: private/loopback address")
|
||||
return
|
||||
|
||||
reader = _get_reader()
|
||||
if reader is None:
|
||||
await bot.reply(message, "ASN database not available (run update-data)")
|
||||
return
|
||||
|
||||
try:
|
||||
rec = reader.get(str(ip))
|
||||
except Exception as exc:
|
||||
await bot.reply(message, f"Lookup error: {exc}")
|
||||
return
|
||||
|
||||
if not rec:
|
||||
await bot.reply(message, f"{addr}: no ASN data")
|
||||
return
|
||||
|
||||
asn = rec.get("autonomous_system_number", "")
|
||||
org = rec.get("autonomous_system_organization", "")
|
||||
|
||||
if asn:
|
||||
await bot.reply(message, f"{addr}: AS{asn} ({org})" if org else f"{addr}: AS{asn}")
|
||||
else:
|
||||
await bot.reply(message, f"{addr}: no ASN data")
|
||||
258
plugins/cve.py
Normal file
258
plugins/cve.py
Normal file
@@ -0,0 +1,258 @@
|
||||
"""Plugin: CVE lookup against local NVD JSON feed."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from derp.plugin import command
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_DATA_DIR = Path("data/nvd")
|
||||
_MAX_AGE = 86400
|
||||
_CVE_RE = re.compile(r"^CVE-\d{4}-\d{4,}$", re.IGNORECASE)
|
||||
_MAX_RESULTS = 5
|
||||
|
||||
# In-memory index: cve_id -> {description, severity, score, published}
|
||||
_index: dict[str, dict] = {}
|
||||
_loaded_at: float = 0
|
||||
|
||||
|
||||
def _load_index() -> dict[str, dict]:
|
||||
"""Load NVD JSON files into a searchable index."""
|
||||
idx: dict[str, dict] = {}
|
||||
if not _DATA_DIR.is_dir():
|
||||
return idx
|
||||
|
||||
for path in sorted(_DATA_DIR.glob("*.json")):
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError) as exc:
|
||||
log.warning("cve: skipping %s: %s", path.name, exc)
|
||||
continue
|
||||
|
||||
vulns = data.get("vulnerabilities", [])
|
||||
for entry in vulns:
|
||||
cve = entry.get("cve", {})
|
||||
cve_id = cve.get("id", "")
|
||||
if not cve_id:
|
||||
continue
|
||||
|
||||
# Extract English description
|
||||
descs = cve.get("descriptions", [])
|
||||
desc = ""
|
||||
for d in descs:
|
||||
if d.get("lang") == "en":
|
||||
desc = d.get("value", "")
|
||||
break
|
||||
if not desc and descs:
|
||||
desc = descs[0].get("value", "")
|
||||
|
||||
# Extract CVSS score (prefer v3.1, then v3.0, then v2)
|
||||
metrics = cve.get("metrics", {})
|
||||
score = ""
|
||||
severity = ""
|
||||
for key in ("cvssMetricV31", "cvssMetricV30", "cvssMetricV2"):
|
||||
metric_list = metrics.get(key, [])
|
||||
if metric_list:
|
||||
cvss = metric_list[0].get("cvssData", {})
|
||||
score = cvss.get("baseScore", "")
|
||||
severity = cvss.get("baseSeverity", "")
|
||||
break
|
||||
|
||||
published = cve.get("published", "")[:10]
|
||||
|
||||
idx[cve_id.upper()] = {
|
||||
"description": desc,
|
||||
"severity": severity,
|
||||
"score": score,
|
||||
"published": published,
|
||||
}
|
||||
|
||||
log.info("cve: indexed %d CVEs from %s", len(idx), _DATA_DIR)
|
||||
return idx
|
||||
|
||||
|
||||
def _refresh_if_stale() -> None:
|
||||
"""Reload the index if stale."""
|
||||
global _index, _loaded_at
|
||||
now = time.monotonic()
|
||||
if _index and (now - _loaded_at) < _MAX_AGE:
|
||||
return
|
||||
idx = _load_index()
|
||||
if idx:
|
||||
_index = idx
|
||||
_loaded_at = now
|
||||
|
||||
|
||||
def _format_cve(cve_id: str, rec: dict) -> str:
|
||||
"""Format a single CVE entry for IRC output."""
|
||||
parts = [cve_id]
|
||||
if rec["score"]:
|
||||
sev = f" {rec['severity']}" if rec["severity"] else ""
|
||||
parts.append(f"CVSS {rec['score']}{sev}")
|
||||
if rec["published"]:
|
||||
parts.append(rec["published"])
|
||||
desc = rec["description"]
|
||||
if len(desc) > 200:
|
||||
desc = desc[:197] + "..."
|
||||
parts.append(desc)
|
||||
return " | ".join(parts)
|
||||
|
||||
|
||||
async def _download_nvd() -> tuple[int, str]:
|
||||
"""Download NVD CVE JSON feed. Returns (count, error)."""
|
||||
import asyncio
|
||||
import urllib.request
|
||||
|
||||
_DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
# NVD 2.0 API: paginated, 2000 per request
|
||||
base_url = "https://services.nvd.nist.gov/rest/json/cves/2.0"
|
||||
page_size = 2000
|
||||
start_index = 0
|
||||
total = 0
|
||||
file_num = 0
|
||||
|
||||
def _fetch(url):
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "derp-bot"})
|
||||
with urllib.request.urlopen(req, timeout=120) as resp: # noqa: S310
|
||||
return resp.read()
|
||||
|
||||
try:
|
||||
while True:
|
||||
url = f"{base_url}?startIndex={start_index}&resultsPerPage={page_size}"
|
||||
data = await loop.run_in_executor(None, _fetch, url)
|
||||
parsed = json.loads(data)
|
||||
|
||||
total_results = parsed.get("totalResults", 0)
|
||||
vulns = parsed.get("vulnerabilities", [])
|
||||
if not vulns:
|
||||
break
|
||||
|
||||
dest = _DATA_DIR / f"nvd_{file_num:04d}.json"
|
||||
dest.write_bytes(data)
|
||||
total += len(vulns)
|
||||
file_num += 1
|
||||
|
||||
start_index += page_size
|
||||
if start_index >= total_results:
|
||||
break
|
||||
|
||||
# Rate limit: NVD allows ~5 req/30s without API key
|
||||
await asyncio.sleep(6)
|
||||
|
||||
except Exception as exc:
|
||||
if total > 0:
|
||||
return total, f"partial ({exc})"
|
||||
return 0, str(exc)
|
||||
|
||||
global _index, _loaded_at
|
||||
_index = {}
|
||||
_loaded_at = 0
|
||||
return total, ""
|
||||
|
||||
|
||||
@command("cve", help="CVE lookup: !cve <id|search term>")
|
||||
async def cmd_cve(bot, message):
|
||||
"""Look up CVE details or search by keyword.
|
||||
|
||||
Usage:
|
||||
!cve CVE-2024-1234 Lookup specific CVE
|
||||
!cve search apache rce Search descriptions
|
||||
!cve update Download NVD feed (slow)
|
||||
!cve stats Show index statistics
|
||||
"""
|
||||
parts = message.text.split(None, 2)
|
||||
if len(parts) < 2:
|
||||
await bot.reply(message, "Usage: !cve <CVE-ID|search <term>|update|stats>")
|
||||
return
|
||||
|
||||
arg = parts[1].strip()
|
||||
|
||||
if arg == "update":
|
||||
await bot.reply(message, "Downloading NVD feed (this takes a while)...")
|
||||
count, err = await _download_nvd()
|
||||
if err and count == 0:
|
||||
await bot.reply(message, f"Failed: {err}")
|
||||
elif err:
|
||||
await bot.reply(message, f"Downloaded {count} CVEs ({err})")
|
||||
else:
|
||||
await bot.reply(message, f"Downloaded {count} CVEs")
|
||||
return
|
||||
|
||||
if arg == "stats":
|
||||
_refresh_if_stale()
|
||||
if not _index:
|
||||
await bot.reply(message, "No CVE data loaded (run !cve update)")
|
||||
else:
|
||||
await bot.reply(message, f"CVE index: {len(_index)} entries")
|
||||
return
|
||||
|
||||
if arg.lower() == "search":
|
||||
term = parts[2].strip() if len(parts) > 2 else ""
|
||||
if not term:
|
||||
await bot.reply(message, "Usage: !cve search <term>")
|
||||
return
|
||||
|
||||
_refresh_if_stale()
|
||||
if not _index:
|
||||
await bot.reply(message, "No CVE data loaded (run !cve update)")
|
||||
return
|
||||
|
||||
term_lower = term.lower()
|
||||
matches = []
|
||||
for cve_id, rec in _index.items():
|
||||
if term_lower in rec["description"].lower() or term_lower in cve_id.lower():
|
||||
matches.append((cve_id, rec))
|
||||
if len(matches) >= _MAX_RESULTS:
|
||||
break
|
||||
|
||||
if not matches:
|
||||
await bot.reply(message, f"No CVEs matching '{term}'")
|
||||
else:
|
||||
for cve_id, rec in matches:
|
||||
await bot.reply(message, _format_cve(cve_id, rec))
|
||||
return
|
||||
|
||||
# Direct CVE-ID lookup
|
||||
cve_id = arg.upper()
|
||||
if not _CVE_RE.match(cve_id):
|
||||
# Maybe it's a search term without "search" prefix
|
||||
_refresh_if_stale()
|
||||
if not _index:
|
||||
await bot.reply(message, "No CVE data loaded (run !cve update)")
|
||||
return
|
||||
term_lower = arg.lower()
|
||||
rest = parts[2].strip() if len(parts) > 2 else ""
|
||||
if rest:
|
||||
term_lower = f"{term_lower} {rest.lower()}"
|
||||
matches = []
|
||||
for cid, rec in _index.items():
|
||||
if term_lower in rec["description"].lower():
|
||||
matches.append((cid, rec))
|
||||
if len(matches) >= _MAX_RESULTS:
|
||||
break
|
||||
if not matches:
|
||||
await bot.reply(message, f"No CVEs matching '{arg}'")
|
||||
else:
|
||||
for cid, rec in matches:
|
||||
await bot.reply(message, _format_cve(cid, rec))
|
||||
return
|
||||
|
||||
_refresh_if_stale()
|
||||
if not _index:
|
||||
await bot.reply(message, "No CVE data loaded (run !cve update)")
|
||||
return
|
||||
|
||||
rec = _index.get(cve_id)
|
||||
if not rec:
|
||||
await bot.reply(message, f"{cve_id}: not found in local index")
|
||||
return
|
||||
|
||||
await bot.reply(message, _format_cve(cve_id, rec))
|
||||
98
plugins/geoip.py
Normal file
98
plugins/geoip.py
Normal file
@@ -0,0 +1,98 @@
|
||||
"""Plugin: GeoIP lookup using MaxMind GeoLite2-City mmdb."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from derp.plugin import command
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_DB_PATHS = [
|
||||
Path("data/GeoLite2-City.mmdb"),
|
||||
Path("/usr/share/GeoIP/GeoLite2-City.mmdb"),
|
||||
Path.home() / ".local" / "share" / "GeoIP" / "GeoLite2-City.mmdb",
|
||||
]
|
||||
|
||||
_reader = None
|
||||
|
||||
|
||||
def _get_reader():
|
||||
"""Lazy-load the mmdb reader."""
|
||||
global _reader
|
||||
if _reader is not None:
|
||||
return _reader
|
||||
try:
|
||||
import maxminddb
|
||||
except ImportError:
|
||||
log.error("maxminddb package not installed")
|
||||
return None
|
||||
for path in _DB_PATHS:
|
||||
if path.is_file():
|
||||
_reader = maxminddb.open_database(str(path))
|
||||
log.info("geoip: loaded %s", path)
|
||||
return _reader
|
||||
log.warning("geoip: no GeoLite2-City.mmdb found")
|
||||
return None
|
||||
|
||||
|
||||
@command("geoip", help="GeoIP lookup: !geoip <ip>")
|
||||
async def cmd_geoip(bot, message):
|
||||
"""Look up geographic location for an IP address.
|
||||
|
||||
Usage:
|
||||
!geoip 8.8.8.8
|
||||
"""
|
||||
parts = message.text.split(None, 2)
|
||||
if len(parts) < 2:
|
||||
await bot.reply(message, "Usage: !geoip <ip>")
|
||||
return
|
||||
|
||||
addr = parts[1]
|
||||
try:
|
||||
ip = ipaddress.ip_address(addr)
|
||||
except ValueError:
|
||||
await bot.reply(message, f"Invalid IP address: {addr}")
|
||||
return
|
||||
|
||||
if ip.is_private or ip.is_loopback:
|
||||
await bot.reply(message, f"{addr}: private/loopback address")
|
||||
return
|
||||
|
||||
reader = _get_reader()
|
||||
if reader is None:
|
||||
await bot.reply(message, "GeoIP database not available (run update-data)")
|
||||
return
|
||||
|
||||
try:
|
||||
rec = reader.get(str(ip))
|
||||
except Exception as exc:
|
||||
await bot.reply(message, f"Lookup error: {exc}")
|
||||
return
|
||||
|
||||
if not rec:
|
||||
await bot.reply(message, f"{addr}: no GeoIP data")
|
||||
return
|
||||
|
||||
country = rec.get("country", {}).get("names", {}).get("en", "")
|
||||
iso = rec.get("country", {}).get("iso_code", "")
|
||||
city = rec.get("city", {}).get("names", {}).get("en", "")
|
||||
loc = rec.get("location", {})
|
||||
lat = loc.get("latitude")
|
||||
lon = loc.get("longitude")
|
||||
tz = loc.get("time_zone", "")
|
||||
|
||||
info = []
|
||||
if city and country:
|
||||
info.append(f"{city}, {country} ({iso})")
|
||||
elif country:
|
||||
info.append(f"{country} ({iso})")
|
||||
if lat is not None and lon is not None:
|
||||
info.append(f"{lat:.4f}, {lon:.4f}")
|
||||
if tz:
|
||||
info.append(tz)
|
||||
|
||||
result = f"{addr}: {' | '.join(info)}" if info else f"{addr}: no location data"
|
||||
await bot.reply(message, result)
|
||||
180
plugins/iprep.py
Normal file
180
plugins/iprep.py
Normal file
@@ -0,0 +1,180 @@
|
||||
"""Plugin: IP reputation check against Firehol blocklist feeds."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import logging
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from derp.plugin import command
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_DATA_DIR = Path("data/iprep")
|
||||
|
||||
# Firehol feeds: (filename, url, description)
|
||||
_FEEDS = [
|
||||
("firehol_level1.netset",
|
||||
"https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset",
|
||||
"Firehol L1"),
|
||||
("firehol_level2.netset",
|
||||
"https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level2.netset",
|
||||
"Firehol L2"),
|
||||
("et_compromised.ipset",
|
||||
"https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/et_compromised.ipset",
|
||||
"ET Compromised"),
|
||||
("bruteforcelogin.ipset",
|
||||
"https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/bruteforcelogin.ipset",
|
||||
"BruteForce"),
|
||||
("bi_any_2_30d.ipset",
|
||||
"https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/bi_any_2_30d.ipset",
|
||||
"Badips 30d"),
|
||||
]
|
||||
|
||||
_MAX_AGE = 86400 # Refresh cache after 24h
|
||||
|
||||
# Cache: feed_name -> (set of IPs/networks, load_time)
|
||||
_cache: dict[str, tuple[set[str], list, float]] = {}
|
||||
|
||||
|
||||
def _parse_feed(path: Path) -> tuple[set[str], list]:
|
||||
"""Parse a feed file into sets of IPs and CIDR networks."""
|
||||
ips: set[str] = set()
|
||||
nets: list = []
|
||||
try:
|
||||
for line in path.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if "/" in line:
|
||||
try:
|
||||
nets.append(ipaddress.ip_network(line, strict=False))
|
||||
except ValueError:
|
||||
continue
|
||||
else:
|
||||
try:
|
||||
ipaddress.ip_address(line)
|
||||
ips.add(line)
|
||||
except ValueError:
|
||||
continue
|
||||
except OSError:
|
||||
pass
|
||||
return ips, nets
|
||||
|
||||
|
||||
def _load_feed(name: str) -> tuple[set[str], list]:
|
||||
"""Load a feed from local cache, refreshing if stale."""
|
||||
now = time.monotonic()
|
||||
if name in _cache:
|
||||
ips, nets, loaded = _cache[name]
|
||||
if (now - loaded) < _MAX_AGE:
|
||||
return ips, nets
|
||||
|
||||
path = _DATA_DIR / name
|
||||
if not path.is_file():
|
||||
return set(), []
|
||||
|
||||
ips, nets = _parse_feed(path)
|
||||
_cache[name] = (ips, nets, now)
|
||||
return ips, nets
|
||||
|
||||
|
||||
def _check_ip(addr: str) -> list[str]:
|
||||
"""Check an IP against all loaded feeds. Returns list of matching feed names."""
|
||||
try:
|
||||
ip_obj = ipaddress.ip_address(addr)
|
||||
except ValueError:
|
||||
return []
|
||||
|
||||
hits = []
|
||||
for filename, _url, label in _FEEDS:
|
||||
ips, nets = _load_feed(filename)
|
||||
if addr in ips:
|
||||
hits.append(label)
|
||||
continue
|
||||
for net in nets:
|
||||
if ip_obj in net:
|
||||
hits.append(label)
|
||||
break
|
||||
return hits
|
||||
|
||||
|
||||
async def _download_feeds() -> tuple[int, int]:
|
||||
"""Download all feeds. Returns (success_count, fail_count)."""
|
||||
import asyncio
|
||||
import urllib.request
|
||||
|
||||
_DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
async def _fetch_one(filename: str, url: str) -> bool:
|
||||
def _do():
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "derp-bot"})
|
||||
with urllib.request.urlopen(req, timeout=30) as resp: # noqa: S310
|
||||
return resp.read()
|
||||
|
||||
try:
|
||||
data = await loop.run_in_executor(None, _do)
|
||||
(_DATA_DIR / filename).write_bytes(data)
|
||||
return True
|
||||
except Exception as exc:
|
||||
log.error("iprep: failed to fetch %s: %s", filename, exc)
|
||||
return False
|
||||
|
||||
tasks = [_fetch_one(fn, url) for fn, url, _ in _FEEDS]
|
||||
results = await asyncio.gather(*tasks)
|
||||
|
||||
# Clear cache to force reload
|
||||
_cache.clear()
|
||||
|
||||
ok = sum(1 for r in results if r)
|
||||
return ok, len(results) - ok
|
||||
|
||||
|
||||
@command("iprep", help="IP reputation: !iprep <ip|update>")
|
||||
async def cmd_iprep(bot, message):
|
||||
"""Check IP against Firehol/ET blocklist feeds.
|
||||
|
||||
Usage:
|
||||
!iprep 1.2.3.4 Check IP reputation
|
||||
!iprep update Download latest feeds
|
||||
"""
|
||||
parts = message.text.split(None, 2)
|
||||
if len(parts) < 2:
|
||||
await bot.reply(message, "Usage: !iprep <ip|update>")
|
||||
return
|
||||
|
||||
arg = parts[1].strip()
|
||||
|
||||
if arg == "update":
|
||||
await bot.reply(message, f"Downloading {len(_FEEDS)} feeds...")
|
||||
ok, fail = await _download_feeds()
|
||||
msg = f"Updated: {ok}/{len(_FEEDS)} feeds"
|
||||
if fail:
|
||||
msg += f" ({fail} failed)"
|
||||
await bot.reply(message, msg)
|
||||
return
|
||||
|
||||
try:
|
||||
ip = ipaddress.ip_address(arg)
|
||||
except ValueError:
|
||||
await bot.reply(message, f"Invalid IP address: {arg}")
|
||||
return
|
||||
|
||||
if ip.is_private or ip.is_loopback:
|
||||
await bot.reply(message, f"{arg}: private/loopback address")
|
||||
return
|
||||
|
||||
# Check if any feeds are loaded
|
||||
has_data = any((_DATA_DIR / fn).is_file() for fn, _, _ in _FEEDS)
|
||||
if not has_data:
|
||||
await bot.reply(message, "No feeds loaded (run !iprep update)")
|
||||
return
|
||||
|
||||
hits = _check_ip(str(ip))
|
||||
if hits:
|
||||
await bot.reply(message, f"{arg}: LISTED on {', '.join(hits)} "
|
||||
f"({len(hits)}/{len(_FEEDS)} feeds)")
|
||||
else:
|
||||
await bot.reply(message, f"{arg}: clean ({len(_FEEDS)} feeds checked)")
|
||||
145
plugins/torcheck.py
Normal file
145
plugins/torcheck.py
Normal file
@@ -0,0 +1,145 @@
|
||||
"""Plugin: check IPs against local Tor exit node list."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import logging
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from derp.plugin import command
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_EXIT_LIST_PATHS = [
|
||||
Path("data/tor-exit-nodes.txt"),
|
||||
Path("/var/lib/tor/exit-nodes.txt"),
|
||||
]
|
||||
|
||||
_MAX_AGE = 86400 # Refresh if older than 24h
|
||||
_TOR_EXIT_URL = "https://check.torproject.org/torbulkexitlist"
|
||||
|
||||
_exits: set[str] = set()
|
||||
_loaded_at: float = 0
|
||||
|
||||
|
||||
def _load_exits() -> set[str]:
|
||||
"""Load exit node IPs from local file."""
|
||||
for path in _EXIT_LIST_PATHS:
|
||||
if path.is_file():
|
||||
try:
|
||||
nodes = set()
|
||||
for line in path.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
try:
|
||||
ipaddress.ip_address(line)
|
||||
nodes.add(line)
|
||||
except ValueError:
|
||||
continue
|
||||
log.info("torcheck: loaded %d exit nodes from %s", len(nodes), path)
|
||||
return nodes
|
||||
except OSError:
|
||||
continue
|
||||
return set()
|
||||
|
||||
|
||||
def _refresh_if_stale() -> None:
|
||||
"""Reload the exit list if it hasn't been loaded or is stale."""
|
||||
global _exits, _loaded_at
|
||||
now = time.monotonic()
|
||||
if _exits and (now - _loaded_at) < _MAX_AGE:
|
||||
return
|
||||
nodes = _load_exits()
|
||||
if nodes:
|
||||
_exits = nodes
|
||||
_loaded_at = now
|
||||
|
||||
|
||||
async def _download_exits() -> int:
|
||||
"""Download the Tor bulk exit list. Returns count of nodes fetched."""
|
||||
import asyncio
|
||||
import urllib.request
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def _fetch():
|
||||
req = urllib.request.Request(_TOR_EXIT_URL, headers={"User-Agent": "derp-bot"})
|
||||
with urllib.request.urlopen(req, timeout=30) as resp: # noqa: S310
|
||||
return resp.read().decode("utf-8", errors="replace")
|
||||
|
||||
try:
|
||||
text = await loop.run_in_executor(None, _fetch)
|
||||
except Exception as exc:
|
||||
log.error("torcheck: download failed: %s", exc)
|
||||
return -1
|
||||
|
||||
nodes = set()
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
try:
|
||||
ipaddress.ip_address(line)
|
||||
nodes.add(line)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
if not nodes:
|
||||
return 0
|
||||
|
||||
# Write to first candidate path
|
||||
dest = _EXIT_LIST_PATHS[0]
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_text("\n".join(sorted(nodes)) + "\n")
|
||||
log.info("torcheck: saved %d exit nodes to %s", len(nodes), dest)
|
||||
|
||||
global _exits, _loaded_at
|
||||
_exits = nodes
|
||||
_loaded_at = time.monotonic()
|
||||
return len(nodes)
|
||||
|
||||
|
||||
@command("tor", help="Tor exit check: !tor <ip|update>")
|
||||
async def cmd_tor(bot, message):
|
||||
"""Check if an IP is a known Tor exit node.
|
||||
|
||||
Usage:
|
||||
!tor 1.2.3.4 Check IP against exit list
|
||||
!tor update Download latest exit node list
|
||||
"""
|
||||
parts = message.text.split(None, 2)
|
||||
if len(parts) < 2:
|
||||
await bot.reply(message, "Usage: !tor <ip|update>")
|
||||
return
|
||||
|
||||
arg = parts[1].strip()
|
||||
|
||||
if arg == "update":
|
||||
await bot.reply(message, "Downloading Tor exit list...")
|
||||
count = await _download_exits()
|
||||
if count < 0:
|
||||
await bot.reply(message, "Failed to download exit list")
|
||||
elif count == 0:
|
||||
await bot.reply(message, "Downloaded empty list")
|
||||
else:
|
||||
await bot.reply(message, f"Updated: {count} exit nodes")
|
||||
return
|
||||
|
||||
try:
|
||||
ip = ipaddress.ip_address(arg)
|
||||
except ValueError:
|
||||
await bot.reply(message, f"Invalid IP address: {arg}")
|
||||
return
|
||||
|
||||
_refresh_if_stale()
|
||||
if not _exits:
|
||||
await bot.reply(message, "No exit list loaded (run !tor update)")
|
||||
return
|
||||
|
||||
addr = str(ip)
|
||||
if addr in _exits:
|
||||
await bot.reply(message, f"{addr}: Tor exit node ({len(_exits)} nodes in list)")
|
||||
else:
|
||||
await bot.reply(message, f"{addr}: not a known Tor exit ({len(_exits)} nodes in list)")
|
||||
@@ -8,6 +8,9 @@ version = "0.1.0"
|
||||
description = "Asyncio IRC bot with plugin system"
|
||||
requires-python = ">=3.11"
|
||||
license = "MIT"
|
||||
dependencies = [
|
||||
"maxminddb>=2.0",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
derp = "derp.cli:main"
|
||||
|
||||
125
scripts/update-data.sh
Executable file
125
scripts/update-data.sh
Executable file
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env bash
|
||||
# Update local data files for derp wave 3 plugins.
|
||||
# Run from the project root: ./scripts/update-data.sh
|
||||
# Cron-friendly: exits 0 on success, 1 on any failure.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
DATA_DIR="$PROJECT_DIR/data"
|
||||
|
||||
# Colors (suppressed if NO_COLOR is set or stdout isn't a tty)
|
||||
if [[ -z "${NO_COLOR:-}" ]] && [[ -t 1 ]]; then
|
||||
GRN='\e[38;5;108m'
|
||||
RED='\e[38;5;131m'
|
||||
DIM='\e[2m'
|
||||
RST='\e[0m'
|
||||
else
|
||||
GRN='' RED='' DIM='' RST=''
|
||||
fi
|
||||
|
||||
info() { printf "${GRN}%s${RST} %s\n" "✓" "$*"; }
|
||||
err() { printf "${RED}%s${RST} %s\n" "✗" "$*" >&2; }
|
||||
dim() { printf "${DIM} %s${RST}\n" "$*"; }
|
||||
|
||||
FAILURES=0
|
||||
|
||||
# -- Tor exit nodes -----------------------------------------------------------
|
||||
update_tor() {
|
||||
local dest="$DATA_DIR/tor-exit-nodes.txt"
|
||||
local url="https://check.torproject.org/torbulkexitlist"
|
||||
mkdir -p "$DATA_DIR"
|
||||
dim "Downloading Tor exit list..."
|
||||
if curl -sS -fL --max-time 30 -o "$dest.tmp" "$url"; then
|
||||
local count
|
||||
count=$(grep -cE '^[0-9]' "$dest.tmp" || true)
|
||||
mv "$dest.tmp" "$dest"
|
||||
info "Tor exit nodes: $count IPs"
|
||||
else
|
||||
rm -f "$dest.tmp"
|
||||
err "Failed to download Tor exit list"
|
||||
((FAILURES++))
|
||||
fi
|
||||
}
|
||||
|
||||
# -- Firehol/ET feeds ---------------------------------------------------------
|
||||
update_iprep() {
|
||||
local dest_dir="$DATA_DIR/iprep"
|
||||
mkdir -p "$dest_dir"
|
||||
|
||||
local feeds=(
|
||||
"firehol_level1.netset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset"
|
||||
"firehol_level2.netset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level2.netset"
|
||||
"et_compromised.ipset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/et_compromised.ipset"
|
||||
"bruteforcelogin.ipset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/bruteforcelogin.ipset"
|
||||
"bi_any_2_30d.ipset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/bi_any_2_30d.ipset"
|
||||
)
|
||||
|
||||
local ok=0 fail=0
|
||||
for entry in "${feeds[@]}"; do
|
||||
local name="${entry%%:*}"
|
||||
local url="${entry#*:}"
|
||||
dim "Fetching $name..."
|
||||
if curl -sS -fL --max-time 30 -o "$dest_dir/$name.tmp" "$url"; then
|
||||
mv "$dest_dir/$name.tmp" "$dest_dir/$name"
|
||||
((ok++))
|
||||
else
|
||||
rm -f "$dest_dir/$name.tmp"
|
||||
((fail++))
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ $fail -gt 0 ]]; then
|
||||
err "IP rep feeds: $ok/${#feeds[@]} ($fail failed)"
|
||||
((FAILURES++))
|
||||
else
|
||||
info "IP rep feeds: $ok/${#feeds[@]}"
|
||||
fi
|
||||
}
|
||||
|
||||
# -- GeoLite2 databases -------------------------------------------------------
|
||||
update_geolite2() {
|
||||
# Requires MAXMIND_LICENSE_KEY env var
|
||||
if [[ -z "${MAXMIND_LICENSE_KEY:-}" ]]; then
|
||||
dim "Skipping GeoLite2 (set MAXMIND_LICENSE_KEY to enable)"
|
||||
return
|
||||
fi
|
||||
|
||||
local base="https://download.maxmind.com/app/geoip_download"
|
||||
mkdir -p "$DATA_DIR"
|
||||
|
||||
for edition in GeoLite2-City GeoLite2-ASN; do
|
||||
dim "Downloading $edition..."
|
||||
local url="${base}?edition_id=${edition}&license_key=${MAXMIND_LICENSE_KEY}&suffix=tar.gz"
|
||||
if curl -sS -fL --max-time 120 -o "$DATA_DIR/$edition.tar.gz" "$url"; then
|
||||
# Extract mmdb from tarball
|
||||
tar -xzf "$DATA_DIR/$edition.tar.gz" -C "$DATA_DIR" --strip-components=1 \
|
||||
--wildcards "*/$edition.mmdb"
|
||||
rm -f "$DATA_DIR/$edition.tar.gz"
|
||||
info "$edition.mmdb updated"
|
||||
else
|
||||
rm -f "$DATA_DIR/$edition.tar.gz"
|
||||
err "Failed to download $edition"
|
||||
((FAILURES++))
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# -- Main ---------------------------------------------------------------------
|
||||
printf "${DIM}derp data update${RST}\n"
|
||||
printf "${DIM}%s${RST}\n" "$(date -u '+%Y-%m-%d %H:%M UTC')"
|
||||
echo
|
||||
|
||||
update_tor
|
||||
update_iprep
|
||||
update_geolite2
|
||||
|
||||
echo
|
||||
if [[ $FAILURES -gt 0 ]]; then
|
||||
err "$FAILURES update(s) failed"
|
||||
exit 1
|
||||
else
|
||||
info "All updates complete"
|
||||
exit 0
|
||||
fi
|
||||
Reference in New Issue
Block a user