feat: add wave 3 local database plugins
GeoIP and ASN lookup via MaxMind GeoLite2 mmdb, Tor exit node check against local bulk exit list, IP reputation via Firehol/ET blocklist feeds, and CVE lookup against local NVD JSON mirror. Includes cron-friendly update script (scripts/update-data.sh) for all data sources and make update-data target. GeoLite2 requires a free MaxMind license key; all other sources are freely downloadable. Plugins: geoip, asn, torcheck, iprep, cve Commands: !geoip, !asn, !tor, !iprep, !cve Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
125
scripts/update-data.sh
Executable file
125
scripts/update-data.sh
Executable file
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env bash
|
||||
# Update local data files for derp wave 3 plugins.
|
||||
# Run from the project root: ./scripts/update-data.sh
|
||||
# Cron-friendly: exits 0 on success, 1 on any failure.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
DATA_DIR="$PROJECT_DIR/data"
|
||||
|
||||
# Colors (suppressed if NO_COLOR is set or stdout isn't a tty)
|
||||
if [[ -z "${NO_COLOR:-}" ]] && [[ -t 1 ]]; then
|
||||
GRN='\e[38;5;108m'
|
||||
RED='\e[38;5;131m'
|
||||
DIM='\e[2m'
|
||||
RST='\e[0m'
|
||||
else
|
||||
GRN='' RED='' DIM='' RST=''
|
||||
fi
|
||||
|
||||
info() { printf "${GRN}%s${RST} %s\n" "✓" "$*"; }
|
||||
err() { printf "${RED}%s${RST} %s\n" "✗" "$*" >&2; }
|
||||
dim() { printf "${DIM} %s${RST}\n" "$*"; }
|
||||
|
||||
FAILURES=0
|
||||
|
||||
# -- Tor exit nodes -----------------------------------------------------------
|
||||
update_tor() {
|
||||
local dest="$DATA_DIR/tor-exit-nodes.txt"
|
||||
local url="https://check.torproject.org/torbulkexitlist"
|
||||
mkdir -p "$DATA_DIR"
|
||||
dim "Downloading Tor exit list..."
|
||||
if curl -sS -fL --max-time 30 -o "$dest.tmp" "$url"; then
|
||||
local count
|
||||
count=$(grep -cE '^[0-9]' "$dest.tmp" || true)
|
||||
mv "$dest.tmp" "$dest"
|
||||
info "Tor exit nodes: $count IPs"
|
||||
else
|
||||
rm -f "$dest.tmp"
|
||||
err "Failed to download Tor exit list"
|
||||
((FAILURES++))
|
||||
fi
|
||||
}
|
||||
|
||||
# -- Firehol/ET feeds ---------------------------------------------------------
|
||||
update_iprep() {
|
||||
local dest_dir="$DATA_DIR/iprep"
|
||||
mkdir -p "$dest_dir"
|
||||
|
||||
local feeds=(
|
||||
"firehol_level1.netset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset"
|
||||
"firehol_level2.netset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level2.netset"
|
||||
"et_compromised.ipset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/et_compromised.ipset"
|
||||
"bruteforcelogin.ipset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/bruteforcelogin.ipset"
|
||||
"bi_any_2_30d.ipset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/bi_any_2_30d.ipset"
|
||||
)
|
||||
|
||||
local ok=0 fail=0
|
||||
for entry in "${feeds[@]}"; do
|
||||
local name="${entry%%:*}"
|
||||
local url="${entry#*:}"
|
||||
dim "Fetching $name..."
|
||||
if curl -sS -fL --max-time 30 -o "$dest_dir/$name.tmp" "$url"; then
|
||||
mv "$dest_dir/$name.tmp" "$dest_dir/$name"
|
||||
((ok++))
|
||||
else
|
||||
rm -f "$dest_dir/$name.tmp"
|
||||
((fail++))
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ $fail -gt 0 ]]; then
|
||||
err "IP rep feeds: $ok/${#feeds[@]} ($fail failed)"
|
||||
((FAILURES++))
|
||||
else
|
||||
info "IP rep feeds: $ok/${#feeds[@]}"
|
||||
fi
|
||||
}
|
||||
|
||||
# -- GeoLite2 databases -------------------------------------------------------
|
||||
update_geolite2() {
|
||||
# Requires MAXMIND_LICENSE_KEY env var
|
||||
if [[ -z "${MAXMIND_LICENSE_KEY:-}" ]]; then
|
||||
dim "Skipping GeoLite2 (set MAXMIND_LICENSE_KEY to enable)"
|
||||
return
|
||||
fi
|
||||
|
||||
local base="https://download.maxmind.com/app/geoip_download"
|
||||
mkdir -p "$DATA_DIR"
|
||||
|
||||
for edition in GeoLite2-City GeoLite2-ASN; do
|
||||
dim "Downloading $edition..."
|
||||
local url="${base}?edition_id=${edition}&license_key=${MAXMIND_LICENSE_KEY}&suffix=tar.gz"
|
||||
if curl -sS -fL --max-time 120 -o "$DATA_DIR/$edition.tar.gz" "$url"; then
|
||||
# Extract mmdb from tarball
|
||||
tar -xzf "$DATA_DIR/$edition.tar.gz" -C "$DATA_DIR" --strip-components=1 \
|
||||
--wildcards "*/$edition.mmdb"
|
||||
rm -f "$DATA_DIR/$edition.tar.gz"
|
||||
info "$edition.mmdb updated"
|
||||
else
|
||||
rm -f "$DATA_DIR/$edition.tar.gz"
|
||||
err "Failed to download $edition"
|
||||
((FAILURES++))
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# -- Main ---------------------------------------------------------------------
|
||||
printf "${DIM}derp data update${RST}\n"
|
||||
printf "${DIM}%s${RST}\n" "$(date -u '+%Y-%m-%d %H:%M UTC')"
|
||||
echo
|
||||
|
||||
update_tor
|
||||
update_iprep
|
||||
update_geolite2
|
||||
|
||||
echo
|
||||
if [[ $FAILURES -gt 0 ]]; then
|
||||
err "$FAILURES update(s) failed"
|
||||
exit 1
|
||||
else
|
||||
info "All updates complete"
|
||||
exit 0
|
||||
fi
|
||||
Reference in New Issue
Block a user