derp/scripts/update-data.sh

#!/usr/bin/env bash
# Update local data files for derp wave 3 plugins.
# Run from the project root: ./scripts/update-data.sh
# Cron-friendly: exits 0 on success, 1 on any failure.

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
DATA_DIR="$PROJECT_DIR/data"

# Colors (suppressed if NO_COLOR is set or stdout isn't a tty)
if [[ -z "${NO_COLOR:-}" ]] && [[ -t 1 ]]; then
    GRN='\e[38;5;108m'
    RED='\e[38;5;131m'
    DIM='\e[2m'
    RST='\e[0m'
else
    GRN='' RED='' DIM='' RST=''
fi

info()  { printf "${GRN}%s${RST} %s\n" "✓" "$*"; }
err()   { printf "${RED}%s${RST} %s\n" "✗" "$*" >&2; }
dim()   { printf "${DIM}  %s${RST}\n" "$*"; }

FAILURES=0

# -- Tor exit nodes -----------------------------------------------------------
update_tor() {
    local dest="$DATA_DIR/tor-exit-nodes.txt"
    local url="https://check.torproject.org/torbulkexitlist"
    mkdir -p "$DATA_DIR"
    dim "Downloading Tor exit list..."
    if curl -sS -fL --max-time 30 -o "$dest.tmp" "$url"; then
        local count
        count=$(grep -cE '^[0-9]' "$dest.tmp" || true)
        mv "$dest.tmp" "$dest"
        info "Tor exit nodes: $count IPs"
    else
        rm -f "$dest.tmp"
        err "Failed to download Tor exit list"
        ((FAILURES++)) || true
    fi
}

# -- Firehol/ET feeds ---------------------------------------------------------
update_iprep() {
    local dest_dir="$DATA_DIR/iprep"
    mkdir -p "$dest_dir"

    local feeds=(
        "firehol_level1.netset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset"
        "firehol_level2.netset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level2.netset"
        "et_compromised.ipset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/et_compromised.ipset"
        "dshield.netset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/dshield.netset"
        "spamhaus_drop.netset:https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/spamhaus_drop.netset"
    )

    local ok=0 fail=0
    for entry in "${feeds[@]}"; do
        local name="${entry%%:*}"
        local url="${entry#*:}"
        dim "Fetching $name..."
        if curl -sS -fL --max-time 30 -o "$dest_dir/$name.tmp" "$url"; then
            mv "$dest_dir/$name.tmp" "$dest_dir/$name"
            ((ok++)) || true
        else
            rm -f "$dest_dir/$name.tmp"
            ((fail++)) || true
        fi
    done

    if [[ $fail -gt 0 ]]; then
        err "IP rep feeds: $ok/${#feeds[@]} ($fail failed)"
        ((FAILURES++)) || true
    else
        info "IP rep feeds: $ok/${#feeds[@]}"
    fi
}

# -- IEEE OUI database --------------------------------------------------------
update_oui() {
    local dest="$DATA_DIR/oui.txt"
    local url="https://standards-oui.ieee.org/oui/oui.txt"
    mkdir -p "$DATA_DIR"
    dim "Downloading IEEE OUI database..."
    if curl -sS -fL --max-time 60 -o "$dest.tmp" "$url"; then
        local count
        count=$(grep -cE '^[0-9A-F]{2}-' "$dest.tmp" || true)
        mv "$dest.tmp" "$dest"
        info "OUI database: $count vendors"
    else
        rm -f "$dest.tmp"
        err "Failed to download OUI database"
        ((FAILURES++)) || true
    fi
}

# -- GeoLite2 databases -------------------------------------------------------
update_geolite2() {
    # Requires MAXMIND_LICENSE_KEY env var
    if [[ -z "${MAXMIND_LICENSE_KEY:-}" ]]; then
        dim "Skipping GeoLite2 (set MAXMIND_LICENSE_KEY to enable)"
        return
    fi

    local base="https://download.maxmind.com/app/geoip_download"
    mkdir -p "$DATA_DIR"

    for edition in GeoLite2-City GeoLite2-ASN; do
        dim "Downloading $edition..."
        local url="${base}?edition_id=${edition}&license_key=${MAXMIND_LICENSE_KEY}&suffix=tar.gz"
        if curl -sS -fL --max-time 120 -o "$DATA_DIR/$edition.tar.gz" "$url"; then
            # Extract mmdb from tarball
            tar -xzf "$DATA_DIR/$edition.tar.gz" -C "$DATA_DIR" --strip-components=1 \
                --wildcards "*/$edition.mmdb"
            rm -f "$DATA_DIR/$edition.tar.gz"
            info "$edition.mmdb updated"
        else
            rm -f "$DATA_DIR/$edition.tar.gz"
            err "Failed to download $edition"
            ((FAILURES++)) || true
        fi
    done
}

# -- iptoasn ASN database -----------------------------------------------------
update_asn() {
    local dest="$DATA_DIR/ip2asn-v4.tsv"
    local url="https://iptoasn.com/data/ip2asn-v4.tsv.gz"
    mkdir -p "$DATA_DIR"
    dim "Downloading iptoasn database..."
    if curl -sS -fL --max-time 60 -o "$dest.gz" "$url" ||
       curl -sS -fL --socks5-hostname 127.0.0.1:1080 --max-time 60 \
            -o "$dest.gz" "$url"; then
        gunzip -f "$dest.gz"
        local count
        count=$(wc -l < "$dest")
        info "iptoasn: $count ranges"
    else
        rm -f "$dest.gz"
        err "Failed to download iptoasn database"
        ((FAILURES++)) || true
    fi
}

# -- Exploit-DB CSV -----------------------------------------------------------
update_exploitdb() {
    local dest_dir="$DATA_DIR/exploitdb"
    local dest="$dest_dir/files_exploits.csv"
    local url="https://gitlab.com/exploit-database/exploitdb/-/raw/main/files_exploits.csv"
    mkdir -p "$dest_dir"
    dim "Downloading exploit-db CSV..."
    if curl -sS -fL --max-time 60 -o "$dest.tmp" "$url"; then
        local count
        count=$(wc -l < "$dest.tmp")
        mv "$dest.tmp" "$dest"
        info "Exploit-DB: $count entries"
    else
        rm -f "$dest.tmp"
        err "Failed to download exploit-db CSV"
        ((FAILURES++)) || true
    fi
}

# -- Main ---------------------------------------------------------------------
printf "${DIM}derp data update${RST}\n"
printf "${DIM}%s${RST}\n" "$(date -u '+%Y-%m-%d %H:%M UTC')"
echo

update_tor
update_iprep
update_oui
update_asn
update_exploitdb
update_geolite2

echo
if [[ $FAILURES -gt 0 ]]; then
    err "$FAILURES update(s) failed"
    exit 1
else
    info "All updates complete"
    exit 0
fi