#!/usr/bin/env bash # Update local data files for derp wave 3 plugins. # Run from the project root: ./scripts/update-data.sh # Cron-friendly: exits 0 on success, 1 on any failure. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" DATA_DIR="$PROJECT_DIR/data" # Colors (suppressed if NO_COLOR is set or stdout isn't a tty) if [[ -z "${NO_COLOR:-}" ]] && [[ -t 1 ]]; then GRN='\e[38;5;108m' RED='\e[38;5;131m' DIM='\e[2m' RST='\e[0m' else GRN='' RED='' DIM='' RST='' fi info() { printf "${GRN}%s${RST} %s\n" "✓" "$*"; } err() { printf "${RED}%s${RST} %s\n" "✗" "$*" >&2; } dim() { printf "${DIM} %s${RST}\n" "$*"; } FAILURES=0 # -- Tor exit nodes ----------------------------------------------------------- update_tor() { local dest="$DATA_DIR/tor-exit-nodes.txt" local url="https://check.torproject.org/torbulkexitlist" mkdir -p "$DATA_DIR" dim "Downloading Tor exit list..." if curl -sS -fL --max-time 30 -o "$dest.tmp" "$url"; then local count count=$(grep -cE '^[0-9]' "$dest.tmp" || true) mv "$dest.tmp" "$dest" info "Tor exit nodes: $count IPs" else rm -f "$dest.tmp" err "Failed to download Tor exit list" ((FAILURES++)) || true fi } # -- Firehol/ET feeds --------------------------------------------------------- update_iprep() { local dest_dir="$DATA_DIR/iprep" mkdir -p "$dest_dir" local feeds=( "firehol_level1.netset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset" "firehol_level2.netset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level2.netset" "et_compromised.ipset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/et_compromised.ipset" "dshield.netset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/dshield.netset" "spamhaus_drop.netset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/spamhaus_drop.netset" ) local ok=0 fail=0 for entry in "${feeds[@]}"; do local name="${entry%%:*}" local url="${entry#*:}" dim "Fetching $name..." if curl -sS -fL --max-time 30 -o "$dest_dir/$name.tmp" "$url"; then mv "$dest_dir/$name.tmp" "$dest_dir/$name" ((ok++)) || true else rm -f "$dest_dir/$name.tmp" ((fail++)) || true fi done if [[ $fail -gt 0 ]]; then err "IP rep feeds: $ok/${#feeds[@]} ($fail failed)" ((FAILURES++)) || true else info "IP rep feeds: $ok/${#feeds[@]}" fi } # -- IEEE OUI database -------------------------------------------------------- update_oui() { local dest="$DATA_DIR/oui.txt" local url="https://standards-oui.ieee.org/oui/oui.txt" mkdir -p "$DATA_DIR" dim "Downloading IEEE OUI database..." if curl -sS -fL --max-time 60 -o "$dest.tmp" "$url"; then local count count=$(grep -cE '^[0-9A-F]{2}-' "$dest.tmp" || true) mv "$dest.tmp" "$dest" info "OUI database: $count vendors" else rm -f "$dest.tmp" err "Failed to download OUI database" ((FAILURES++)) || true fi } # -- GeoLite2 databases ------------------------------------------------------- update_geolite2() { # Requires MAXMIND_LICENSE_KEY env var if [[ -z "${MAXMIND_LICENSE_KEY:-}" ]]; then dim "Skipping GeoLite2 (set MAXMIND_LICENSE_KEY to enable)" return fi local base="https://download.maxmind.com/app/geoip_download" mkdir -p "$DATA_DIR" for edition in GeoLite2-City GeoLite2-ASN; do dim "Downloading $edition..." local url="${base}?edition_id=${edition}&license_key=${MAXMIND_LICENSE_KEY}&suffix=tar.gz" if curl -sS -fL --max-time 120 -o "$DATA_DIR/$edition.tar.gz" "$url"; then # Extract mmdb from tarball tar -xzf "$DATA_DIR/$edition.tar.gz" -C "$DATA_DIR" --strip-components=1 \ --wildcards "*/$edition.mmdb" rm -f "$DATA_DIR/$edition.tar.gz" info "$edition.mmdb updated" else rm -f "$DATA_DIR/$edition.tar.gz" err "Failed to download $edition" ((FAILURES++)) || true fi done } # -- iptoasn ASN database ----------------------------------------------------- update_asn() { local dest="$DATA_DIR/ip2asn-v4.tsv" local url="https://iptoasn.com/data/ip2asn-v4.tsv.gz" mkdir -p "$DATA_DIR" dim "Downloading iptoasn database..." if curl -sS -fL --max-time 60 -o "$dest.gz" "$url" || curl -sS -fL --socks5-hostname 127.0.0.1:1080 --max-time 60 \ -o "$dest.gz" "$url"; then gunzip -f "$dest.gz" local count count=$(wc -l < "$dest") info "iptoasn: $count ranges" else rm -f "$dest.gz" err "Failed to download iptoasn database" ((FAILURES++)) || true fi } # -- Exploit-DB CSV ----------------------------------------------------------- update_exploitdb() { local dest_dir="$DATA_DIR/exploitdb" local dest="$dest_dir/files_exploits.csv" local url="https://gitlab.com/exploit-database/exploitdb/-/raw/main/files_exploits.csv" mkdir -p "$dest_dir" dim "Downloading exploit-db CSV..." if curl -sS -fL --max-time 60 -o "$dest.tmp" "$url"; then local count count=$(wc -l < "$dest.tmp") mv "$dest.tmp" "$dest" info "Exploit-DB: $count entries" else rm -f "$dest.tmp" err "Failed to download exploit-db CSV" ((FAILURES++)) || true fi } # -- Main --------------------------------------------------------------------- printf "${DIM}derp data update${RST}\n" printf "${DIM}%s${RST}\n" "$(date -u '+%Y-%m-%d %H:%M UTC')" echo update_tor update_iprep update_oui update_asn update_exploitdb update_geolite2 echo if [[ $FAILURES -gt 0 ]]; then err "$FAILURES update(s) failed" exit 1 else info "All updates complete" exit 0 fi