23b4d6f2a4
GeoIP and ASN lookup via MaxMind GeoLite2 mmdb, Tor exit node check against local bulk exit list, IP reputation via Firehol/ET blocklist feeds, and CVE lookup against local NVD JSON mirror. Includes cron-friendly update script (scripts/update-data.sh) for all data sources and make update-data target. GeoLite2 requires a free MaxMind license key; all other sources are freely downloadable. Plugins: geoip, asn, torcheck, iprep, cve Commands: !geoip, !asn, !tor, !iprep, !cve Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
126 lines
4.0 KiB
Bash
Executable File
126 lines
4.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Update local data files for derp wave 3 plugins.
|
|
# Run from the project root: ./scripts/update-data.sh
|
|
# Cron-friendly: exits 0 on success, 1 on any failure.
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
DATA_DIR="$PROJECT_DIR/data"
|
|
|
|
# Colors (suppressed if NO_COLOR is set or stdout isn't a tty)
|
|
if [[ -z "${NO_COLOR:-}" ]] && [[ -t 1 ]]; then
|
|
GRN='\e[38;5;108m'
|
|
RED='\e[38;5;131m'
|
|
DIM='\e[2m'
|
|
RST='\e[0m'
|
|
else
|
|
GRN='' RED='' DIM='' RST=''
|
|
fi
|
|
|
|
info() { printf "${GRN}%s${RST} %s\n" "✓" "$*"; }
|
|
err() { printf "${RED}%s${RST} %s\n" "✗" "$*" >&2; }
|
|
dim() { printf "${DIM} %s${RST}\n" "$*"; }
|
|
|
|
FAILURES=0
|
|
|
|
# -- Tor exit nodes -----------------------------------------------------------
|
|
update_tor() {
|
|
local dest="$DATA_DIR/tor-exit-nodes.txt"
|
|
local url="https://check.torproject.org/torbulkexitlist"
|
|
mkdir -p "$DATA_DIR"
|
|
dim "Downloading Tor exit list..."
|
|
if curl -sS -fL --max-time 30 -o "$dest.tmp" "$url"; then
|
|
local count
|
|
count=$(grep -cE '^[0-9]' "$dest.tmp" || true)
|
|
mv "$dest.tmp" "$dest"
|
|
info "Tor exit nodes: $count IPs"
|
|
else
|
|
rm -f "$dest.tmp"
|
|
err "Failed to download Tor exit list"
|
|
((FAILURES++))
|
|
fi
|
|
}
|
|
|
|
# -- Firehol/ET feeds ---------------------------------------------------------
|
|
update_iprep() {
|
|
local dest_dir="$DATA_DIR/iprep"
|
|
mkdir -p "$dest_dir"
|
|
|
|
local feeds=(
|
|
"firehol_level1.netset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset"
|
|
"firehol_level2.netset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level2.netset"
|
|
"et_compromised.ipset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/et_compromised.ipset"
|
|
"bruteforcelogin.ipset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/bruteforcelogin.ipset"
|
|
"bi_any_2_30d.ipset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/bi_any_2_30d.ipset"
|
|
)
|
|
|
|
local ok=0 fail=0
|
|
for entry in "${feeds[@]}"; do
|
|
local name="${entry%%:*}"
|
|
local url="${entry#*:}"
|
|
dim "Fetching $name..."
|
|
if curl -sS -fL --max-time 30 -o "$dest_dir/$name.tmp" "$url"; then
|
|
mv "$dest_dir/$name.tmp" "$dest_dir/$name"
|
|
((ok++))
|
|
else
|
|
rm -f "$dest_dir/$name.tmp"
|
|
((fail++))
|
|
fi
|
|
done
|
|
|
|
if [[ $fail -gt 0 ]]; then
|
|
err "IP rep feeds: $ok/${#feeds[@]} ($fail failed)"
|
|
((FAILURES++))
|
|
else
|
|
info "IP rep feeds: $ok/${#feeds[@]}"
|
|
fi
|
|
}
|
|
|
|
# -- GeoLite2 databases -------------------------------------------------------
|
|
update_geolite2() {
|
|
# Requires MAXMIND_LICENSE_KEY env var
|
|
if [[ -z "${MAXMIND_LICENSE_KEY:-}" ]]; then
|
|
dim "Skipping GeoLite2 (set MAXMIND_LICENSE_KEY to enable)"
|
|
return
|
|
fi
|
|
|
|
local base="https://download.maxmind.com/app/geoip_download"
|
|
mkdir -p "$DATA_DIR"
|
|
|
|
for edition in GeoLite2-City GeoLite2-ASN; do
|
|
dim "Downloading $edition..."
|
|
local url="${base}?edition_id=${edition}&license_key=${MAXMIND_LICENSE_KEY}&suffix=tar.gz"
|
|
if curl -sS -fL --max-time 120 -o "$DATA_DIR/$edition.tar.gz" "$url"; then
|
|
# Extract mmdb from tarball
|
|
tar -xzf "$DATA_DIR/$edition.tar.gz" -C "$DATA_DIR" --strip-components=1 \
|
|
--wildcards "*/$edition.mmdb"
|
|
rm -f "$DATA_DIR/$edition.tar.gz"
|
|
info "$edition.mmdb updated"
|
|
else
|
|
rm -f "$DATA_DIR/$edition.tar.gz"
|
|
err "Failed to download $edition"
|
|
((FAILURES++))
|
|
fi
|
|
done
|
|
}
|
|
|
|
# -- Main ---------------------------------------------------------------------
|
|
printf "${DIM}derp data update${RST}\n"
|
|
printf "${DIM}%s${RST}\n" "$(date -u '+%Y-%m-%d %H:%M UTC')"
|
|
echo
|
|
|
|
update_tor
|
|
update_iprep
|
|
update_geolite2
|
|
|
|
echo
|
|
if [[ $FAILURES -gt 0 ]]; then
|
|
err "$FAILURES update(s) failed"
|
|
exit 1
|
|
else
|
|
info "All updates complete"
|
|
exit 0
|
|
fi
|