fireclaw/tests/test-suite.sh

#!/bin/bash
# Fireclaw regression test suite
# Requires: overseer running, no agents active
# Usage: ./tests/test-suite.sh

set -uo pipefail

PASS=0
FAIL=0
SKIP=0

irc_cmd() {
  local wait=${1:-3}
  shift
  local cmds=""
  for cmd in "$@"; do
    cmds+="PRIVMSG #agents :${cmd}\r\n"
    cmds+="$(printf 'SLEEP %s' "$wait")\r\n"
  done
  {
    echo -e "NICK fctest\r\nUSER fctest 0 * :test\r\n"
    sleep 2
    echo -e "JOIN #agents\r\nJOIN #dev\r\n"
    sleep 1
    for cmd in "$@"; do
      echo -e "PRIVMSG #agents :${cmd}\r\n"
      sleep "$wait"
    done
    echo -e "QUIT\r\n"
  } | nc -q 2 127.0.0.1 6667 2>&1
}

assert_contains() {
  local output="$1"
  local expected="$2"
  local test_name="$3"
  if echo "$output" | grep -q "$expected"; then
    echo "  PASS: $test_name"
    ((PASS++))
  else
    echo "  FAIL: $test_name (expected: $expected)"
    ((FAIL++))
  fi
}

assert_not_contains() {
  local output="$1"
  local expected="$2"
  local test_name="$3"
  if echo "$output" | grep -q "$expected"; then
    echo "  FAIL: $test_name (unexpected: $expected)"
    ((FAIL++))
  else
    echo "  PASS: $test_name"
    ((PASS++))
  fi
}

cleanup_agent() {
  local name="$1"
  fireclaw agent stop "$name" 2>/dev/null || true
  sleep 1
  # Clean any leaked taps
  for tap in $(ip link show 2>/dev/null | grep -oP 'fctap\d+' | sort -u); do
    if [ "$tap" != "fctap200" ]; then
      # Check if tap is used by a running agent
      if ! fireclaw agent list 2>/dev/null | grep -q "$tap"; then
        sudo ip tuntap del "$tap" mode tap 2>/dev/null || true
      fi
    fi
  done
}

echo "========================================="
echo "Fireclaw Regression Test Suite"
echo "========================================="
echo ""

# Precondition: check overseer is running
echo "[Pre] Checking overseer..."
OUT=$(irc_cmd 2 "!help")
if echo "$OUT" | grep -q "overseer.*PRIVMSG.*Commands:"; then
  echo "  OK: Overseer is running"
else
  echo "  ERROR: Overseer not running. Start with: fireclaw overseer"
  exit 1
fi

# Clean any leftover agents
echo "[Pre] Cleaning leftover agents..."
for agent in $(fireclaw agent list 2>/dev/null | awk '{print $1}'); do
  cleanup_agent "$agent"
done
echo ""

# ==========================================
echo "--- Test 1: Overseer help command ---"
OUT=$(irc_cmd 2 "!help")
assert_contains "$OUT" "Commands:" "!help returns command list"

echo "--- Test 2: Overseer templates ---"
OUT=$(irc_cmd 2 "!templates")
assert_contains "$OUT" "worker" "templates includes worker"
assert_contains "$OUT" "coder" "templates includes coder"
assert_contains "$OUT" "researcher" "templates includes researcher"

echo "--- Test 3: Empty agent list ---"
OUT=$(irc_cmd 2 "!list")
assert_contains "$OUT" "No agents running" "no agents when clean"

echo "--- Test 4: Invalid template ---"
OUT=$(irc_cmd 2 "!invoke faketype")
assert_contains "$OUT" "not found" "invalid template rejected"

echo "--- Test 5: Missing arguments ---"
OUT=$(irc_cmd 2 "!invoke" "!destroy" "!model")
assert_contains "$OUT" "Usage:" "!invoke usage shown"

echo "--- Test 6: Destroy nonexistent ---"
OUT=$(irc_cmd 2 "!destroy ghost")
assert_contains "$OUT" "not running" "destroy nonexistent handled"

echo ""
echo "--- Test 7: Spawn worker agent ---"
OUT=$(irc_cmd 8 "!invoke worker")
assert_contains "$OUT" "Agent \"worker\" started" "worker spawned"

# Verify agent is running
AGENTS=$(fireclaw agent list 2>&1)
assert_contains "$AGENTS" "worker" "worker in agent list"

echo "--- Test 8: Worker appears in IRC ---"
OUT=$({
  echo -e "NICK fctest2\r\nUSER fctest2 0 * :t\r\n"
  sleep 2
  echo -e "JOIN #agents\r\n"
  sleep 1
  echo -e "NAMES #agents\r\n"
  sleep 1
  echo -e "QUIT\r\n"
} | nc -q 2 127.0.0.1 6667 2>&1)
assert_contains "$OUT" "worker" "worker in NAMES list"

echo "--- Test 9: Worker responds to mention ---"
OUT=$({
  echo -e "NICK fctest3\r\nUSER fctest3 0 * :t\r\n"
  sleep 2
  echo -e "JOIN #agents\r\n"
  sleep 1
  echo -e "PRIVMSG #agents :worker: say pong\r\n"
  sleep 25
  echo -e "QUIT\r\n"
} | nc -q 2 127.0.0.1 6667 2>&1)
assert_contains "$OUT" ":worker.*PRIVMSG" "worker responded"

echo "--- Test 10: Worker ignores non-mention ---"
OUT=$({
  echo -e "NICK fctest4\r\nUSER fctest4 0 * :t\r\n"
  sleep 2
  echo -e "JOIN #agents\r\n"
  sleep 1
  echo -e "PRIVMSG #agents :hello everyone\r\n"
  sleep 10
  echo -e "QUIT\r\n"
} | nc -q 2 127.0.0.1 6667 2>&1)
assert_not_contains "$OUT" ":worker.*PRIVMSG" "worker stays quiet on non-mention"

echo "--- Test 11: Duplicate prevention ---"
OUT=$(irc_cmd 3 "!invoke worker")
assert_contains "$OUT" "already running" "duplicate rejected"

echo "--- Test 12: Named agent from template ---"
OUT=$(irc_cmd 8 "!invoke worker helper2")
assert_contains "$OUT" 'Agent "helper2" started' "named agent spawned"
sleep 2

echo "--- Test 13: Multiple agents listed ---"
OUT=$(irc_cmd 3 "!list")
assert_contains "$OUT" "worker" "worker in list"
assert_contains "$OUT" "helper2" "helper2 in list"

echo "--- Test 14: Destroy specific agent ---"
OUT=$(irc_cmd 3 "!destroy helper2")
assert_contains "$OUT" 'Agent "helper2" destroyed' "helper2 destroyed"

# Verify only worker remains
sleep 2
OUT=$({
  echo -e "NICK fcverify\r\nUSER fcverify 0 * :t\r\n"
  sleep 2
  echo -e "JOIN #agents\r\n"
  sleep 1
  echo -e "PRIVMSG #agents :!list\r\n"
  sleep 3
  echo -e "QUIT\r\n"
} | nc -q 2 127.0.0.1 6667 2>&1 | grep ":overseer.*PRIVMSG.*#agents")
assert_contains "$OUT" "worker" "worker still running"
# Check that helper2 doesn't appear in the list output (exclude destroy confirmation)
LIST_LINE=$(echo "$OUT" | grep -v "destroyed" | grep -v "Invoking" || true)
assert_not_contains "$LIST_LINE" "helper2" "helper2 removed from list"
cleanup_agent "helper2"

echo "--- Test 15: Destroy worker ---"
OUT=$(irc_cmd 3 "!destroy worker")
assert_contains "$OUT" 'Agent "worker" destroyed' "worker destroyed"
cleanup_agent "worker"

echo "--- Test 16: Clean after destroy ---"
OUT=$(irc_cmd 2 "!list")
assert_contains "$OUT" "No agents running" "all agents cleaned"

echo ""
echo "--- Test 17: CLI agent start/list/stop ---"
fireclaw agent start worker 2>&1 | grep -q "started" && echo "  PASS: CLI agent start" && ((PASS++)) || { echo "  FAIL: CLI agent start"; ((FAIL++)); }
sleep 3
fireclaw agent list 2>&1 | grep -q "worker" && echo "  PASS: CLI agent list" && ((PASS++)) || { echo "  FAIL: CLI agent list"; ((FAIL++)); }
fireclaw agent stop worker 2>&1 | grep -q "stopped" && echo "  PASS: CLI agent stop" && ((PASS++)) || { echo "  FAIL: CLI agent stop"; ((FAIL++)); }
cleanup_agent "worker"

echo "--- Test 18: Ephemeral run still works ---"
OUT=$(fireclaw run "echo ephemeral-test" 2>&1)
assert_contains "$OUT" "ephemeral-test" "fireclaw run works"

echo ""
echo "--- Test 19: Overseer crash recovery ---"
# Spawn a worker
fireclaw agent start worker 2>&1 | grep -q "started" && echo "  PASS: worker started for crash test" && ((PASS++)) || { echo "  FAIL: start worker for crash test"; ((FAIL++)); }
sleep 5
WORKER_PID=$(python3 -c "import json; d=json.load(open('$HOME/.fireclaw/agents.json')); print(d.get('worker',{}).get('pid',''))" 2>/dev/null)
# SIGKILL the overseer (simulates crash, KillMode=process keeps worker alive)
OVERSEER_PID=$(systemctl show fireclaw-overseer -p MainPID --value 2>/dev/null)
if [ -n "$OVERSEER_PID" ] && [ "$OVERSEER_PID" != "0" ]; then
  sudo kill -9 "$OVERSEER_PID" 2>/dev/null
  sleep 8  # Wait for systemd to auto-restart (RestartSec=5 + buffer)
  # Check worker survived
  if kill -0 "$WORKER_PID" 2>/dev/null; then
    echo "  PASS: worker survived overseer crash" && ((PASS++))
  else
    echo "  FAIL: worker died with overseer" && ((FAIL++))
  fi
  # Check overseer restarted and adopted
  NEW_PID=$(systemctl show fireclaw-overseer -p MainPID --value 2>/dev/null)
  if [ -n "$NEW_PID" ] && [ "$NEW_PID" != "0" ] && [ "$NEW_PID" != "$OVERSEER_PID" ]; then
    echo "  PASS: overseer auto-restarted" && ((PASS++))
  else
    echo "  FAIL: overseer did not restart" && ((FAIL++))
  fi
  # Check adopted via !list
  sleep 2
  OUT=$({
    echo -e "NICK fcrecov\r\nUSER fcrecov 0 * :t\r\n"
    sleep 2
    echo -e "JOIN #agents\r\n"
    sleep 1
    echo -e "PRIVMSG #agents :!list\r\n"
    sleep 3
    echo -e "QUIT\r\n"
  } | nc -q 2 127.0.0.1 6667 2>&1)
  assert_contains "$OUT" "worker" "overseer adopted worker after crash"
  # Cleanup
  {
    echo -e "NICK fcrecov2\r\nUSER fcrecov2 0 * :t\r\n"
    sleep 2
    echo -e "JOIN #agents\r\n"
    sleep 1
    echo -e "PRIVMSG #agents :!destroy worker\r\n"
    sleep 5
    echo -e "QUIT\r\n"
  } | nc -q 2 127.0.0.1 6667 2>&1
else
  echo "  SKIP: overseer not running via systemd, skipping crash test" && ((SKIP++))
  ((SKIP++))
  ((SKIP++))
  ((SKIP++))
  cleanup_agent "worker"
fi

echo ""
echo "--- Test 20: Graceful agent shutdown (IRC QUIT) ---"
# Spawn and destroy, check for clean QUIT
{
  echo -e "NICK fcquit\r\nUSER fcquit 0 * :t\r\n"
  sleep 2
  echo -e "JOIN #agents\r\n"
  sleep 1
  echo -e "PRIVMSG #agents :!invoke worker\r\n"
  sleep 8
  echo -e "PRIVMSG #agents :!destroy worker\r\n"
  sleep 5
  echo -e "QUIT\r\n"
} | nc -q 2 127.0.0.1 6667 > /tmp/fc-quit-test.txt 2>&1
if grep -q "QUIT.*shutting down" /tmp/fc-quit-test.txt; then
  echo "  PASS: agent sent IRC QUIT on destroy" && ((PASS++))
else
  echo "  FAIL: agent did not send IRC QUIT" && ((FAIL++))
fi
rm -f /tmp/fc-quit-test.txt
cleanup_agent "worker"

echo ""
echo "========================================="
echo "Results: $PASS passed, $FAIL failed, $SKIP skipped"
echo "========================================="

[ "$FAIL" -eq 0 ] && exit 0 || exit 1