feat: add multi-server support

Connect to multiple IRC servers concurrently from a single config file.
Plugins are loaded once and shared; per-server state is isolated via
separate SQLite databases and per-bot runtime state (bot._pstate).

- Add build_server_configs() for [servers.*] config layout
- Bot.__init__ gains name parameter, _pstate dict for plugin isolation
- cli.py runs multiple bots via asyncio.gather
- 9 stateful plugins migrated from module-level dicts to _ps(bot) pattern
- Backward compatible: legacy [server] config works unchanged

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
user
2026-02-21 19:04:20 +01:00
parent e9528bd879
commit 073659607e
27 changed files with 987 additions and 735 deletions

View File

@@ -77,12 +77,19 @@ _DEVTO_API = "https://dev.to/api/articles"
_MEDIUM_FEED_URL = "https://medium.com/feed/tag"
_HUGGINGFACE_API = "https://huggingface.co/api/models"
# -- Module-level tracking ---------------------------------------------------
# -- Per-bot plugin runtime state --------------------------------------------
_pollers: dict[str, asyncio.Task] = {}
_subscriptions: dict[str, dict] = {}
_errors: dict[str, dict[str, int]] = {}
_poll_count: dict[str, int] = {}
def _ps(bot):
"""Per-bot plugin runtime state."""
return bot._pstate.setdefault("alert", {
"pollers": {},
"subs": {},
"errors": {},
"poll_count": {},
"db_conn": None,
"db_path": "data/alert_history.db",
})
# -- Concurrent fetch helper -------------------------------------------------
@@ -121,18 +128,16 @@ def _fetch_many(targets, *, build_req, timeout, parse):
# -- History database --------------------------------------------------------
_DB_PATH = Path("data/alert_history.db")
_conn: sqlite3.Connection | None = None
def _db() -> sqlite3.Connection:
def _db(bot) -> sqlite3.Connection:
"""Lazy-init the history database connection and schema."""
global _conn
if _conn is not None:
return _conn
_DB_PATH.parent.mkdir(parents=True, exist_ok=True)
_conn = sqlite3.connect(str(_DB_PATH))
_conn.execute("""
ps = _ps(bot)
if ps["db_conn"] is not None:
return ps["db_conn"]
db_path = Path(ps.get("db_path", "data/alert_history.db"))
db_path.parent.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(str(db_path))
conn.execute("""
CREATE TABLE IF NOT EXISTS results (
id INTEGER PRIMARY KEY AUTOINCREMENT,
channel TEXT NOT NULL,
@@ -152,34 +157,35 @@ def _db() -> sqlite3.Connection:
("extra", "''"),
]:
try:
_conn.execute(
conn.execute(
f"ALTER TABLE results ADD COLUMN {col} TEXT NOT NULL DEFAULT {default}"
)
except sqlite3.OperationalError:
pass # column already exists
_conn.execute(
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_results_alert ON results(channel, alert)"
)
_conn.execute(
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_results_short_id ON results(short_id)"
)
# Backfill short_id for rows that predate the column
for row_id, backend, item_id in _conn.execute(
for row_id, backend, item_id in conn.execute(
"SELECT id, backend, item_id FROM results WHERE short_id = ''"
).fetchall():
_conn.execute(
conn.execute(
"UPDATE results SET short_id = ? WHERE id = ?",
(_make_short_id(backend, item_id), row_id),
)
_conn.commit()
return _conn
conn.commit()
ps["db_conn"] = conn
return conn
def _save_result(channel: str, alert: str, backend: str, item: dict,
def _save_result(bot, channel: str, alert: str, backend: str, item: dict,
short_url: str = "") -> str:
"""Persist a matched result to the history database. Returns short_id."""
short_id = _make_short_id(backend, item.get("id", ""))
db = _db()
db = _db(bot)
db.execute(
"INSERT INTO results"
" (channel, alert, backend, item_id, title, url, date, found_at,"
@@ -1814,19 +1820,20 @@ def _delete(bot, key: str) -> None:
async def _poll_once(bot, key: str, announce: bool = True) -> None:
"""Single poll cycle for one alert subscription (all backends)."""
data = _subscriptions.get(key)
ps = _ps(bot)
data = ps["subs"].get(key)
if data is None:
data = _load(bot, key)
if data is None:
return
_subscriptions[key] = data
ps["subs"][key] = data
keyword = data["keyword"]
now = datetime.now(timezone.utc).isoformat()
data["last_poll"] = now
cycle = _poll_count[key] = _poll_count.get(key, 0) + 1
tag_errors = _errors.setdefault(key, {})
cycle = ps["poll_count"][key] = ps["poll_count"].get(key, 0) + 1
tag_errors = ps["errors"].setdefault(key, {})
loop = asyncio.get_running_loop()
for tag, backend in _BACKENDS.items():
@@ -1917,7 +1924,7 @@ async def _poll_once(bot, key: str, announce: bool = True) -> None:
except Exception:
pass
short_id = _save_result(
channel, name, tag, item, short_url=short_url,
bot, channel, name, tag, item, short_url=short_url,
)
title = item["title"] or "(no title)"
extra = item.get("extra", "")
@@ -1938,7 +1945,7 @@ async def _poll_once(bot, key: str, announce: bool = True) -> None:
seen_list = seen_list[-_MAX_SEEN:]
data.setdefault("seen", {})[tag] = seen_list
_subscriptions[key] = data
ps["subs"][key] = data
_save(bot, key, data)
@@ -1946,7 +1953,7 @@ async def _poll_loop(bot, key: str) -> None:
"""Infinite poll loop for one alert subscription."""
try:
while True:
data = _subscriptions.get(key) or _load(bot, key)
data = _ps(bot)["subs"].get(key) or _load(bot, key)
if data is None:
return
interval = data.get("interval", _DEFAULT_INTERVAL)
@@ -1958,35 +1965,38 @@ async def _poll_loop(bot, key: str) -> None:
def _start_poller(bot, key: str) -> None:
"""Create and track a poller task."""
existing = _pollers.get(key)
ps = _ps(bot)
existing = ps["pollers"].get(key)
if existing and not existing.done():
return
task = asyncio.create_task(_poll_loop(bot, key))
_pollers[key] = task
ps["pollers"][key] = task
def _stop_poller(key: str) -> None:
def _stop_poller(bot, key: str) -> None:
"""Cancel and remove a poller task."""
task = _pollers.pop(key, None)
ps = _ps(bot)
task = ps["pollers"].pop(key, None)
if task and not task.done():
task.cancel()
_subscriptions.pop(key, None)
_errors.pop(key, None)
_poll_count.pop(key, None)
ps["subs"].pop(key, None)
ps["errors"].pop(key, None)
ps["poll_count"].pop(key, None)
# -- Restore on connect -----------------------------------------------------
def _restore(bot) -> None:
"""Rebuild pollers from persisted state."""
ps = _ps(bot)
for key in bot.state.keys("alert"):
existing = _pollers.get(key)
existing = ps["pollers"].get(key)
if existing and not existing.done():
continue
data = _load(bot, key)
if data is None:
continue
_subscriptions[key] = data
ps["subs"][key] = data
_start_poller(bot, key)
@@ -2056,9 +2066,9 @@ async def cmd_alert(bot, message):
if data is None:
await bot.reply(message, f"No alert '{name}' in this channel")
return
_subscriptions[key] = data
_ps(bot)["subs"][key] = data
await _poll_once(bot, key, announce=True)
data = _subscriptions.get(key, data)
data = _ps(bot)["subs"].get(key, data)
errs = data.get("last_errors", {})
if errs:
tags = ", ".join(sorted(errs))
@@ -2087,7 +2097,7 @@ async def cmd_alert(bot, message):
limit = max(1, min(int(parts[3]), 20))
except ValueError:
limit = 5
db = _db()
db = _db(bot)
rows = db.execute(
"SELECT id, backend, title, url, date, found_at, short_id,"
" short_url, extra FROM results"
@@ -2141,7 +2151,7 @@ async def cmd_alert(bot, message):
return
short_id = parts[2].lower()
channel = message.target
db = _db()
db = _db(bot)
row = db.execute(
"SELECT alert, backend, title, url, date, found_at, short_id,"
" extra"
@@ -2216,7 +2226,7 @@ async def cmd_alert(bot, message):
"seen": {},
}
_save(bot, key, data)
_subscriptions[key] = data
_ps(bot)["subs"][key] = data
# Seed seen IDs in background (silent poll), then start the poller
async def _seed():
@@ -2251,7 +2261,7 @@ async def cmd_alert(bot, message):
await bot.reply(message, f"No alert '{name}' in this channel")
return
_stop_poller(key)
_stop_poller(bot, key)
_delete(bot, key)
await bot.reply(message, f"Removed '{name}'")
return

View File

@@ -58,7 +58,7 @@ async def cmd_help(bot, message):
@command("version", help="Show bot version")
async def cmd_version(bot, message):
"""Report the running version."""
await bot.reply(message, f"derp {__version__}")
await bot.reply(message, f"derp {__version__} ({bot.name})")
@command("uptime", help="Show how long the bot has been running")

View File

@@ -18,10 +18,15 @@ _MIN_INTERVAL = 60
_MAX_INTERVAL = 604800 # 7 days
_MAX_JOBS = 20
# -- Module-level tracking ---------------------------------------------------
# -- Per-bot plugin runtime state --------------------------------------------
_jobs: dict[str, dict] = {}
_tasks: dict[str, asyncio.Task] = {}
def _ps(bot):
"""Per-bot plugin runtime state."""
return bot._pstate.setdefault("cron", {
"jobs": {},
"tasks": {},
})
# -- Pure helpers ------------------------------------------------------------
@@ -101,7 +106,7 @@ async def _cron_loop(bot, key: str) -> None:
"""Repeating loop: sleep, then dispatch the stored command."""
try:
while True:
data = _jobs.get(key)
data = _ps(bot)["jobs"].get(key)
if not data:
return
await asyncio.sleep(data["interval"])
@@ -118,33 +123,36 @@ async def _cron_loop(bot, key: str) -> None:
def _start_job(bot, key: str) -> None:
"""Create and track a cron task."""
existing = _tasks.get(key)
ps = _ps(bot)
existing = ps["tasks"].get(key)
if existing and not existing.done():
return
task = asyncio.create_task(_cron_loop(bot, key))
_tasks[key] = task
ps["tasks"][key] = task
def _stop_job(key: str) -> None:
def _stop_job(bot, key: str) -> None:
"""Cancel and remove a cron task."""
task = _tasks.pop(key, None)
ps = _ps(bot)
task = ps["tasks"].pop(key, None)
if task and not task.done():
task.cancel()
_jobs.pop(key, None)
ps["jobs"].pop(key, None)
# -- Restore on connect -----------------------------------------------------
def _restore(bot) -> None:
"""Rebuild cron tasks from persisted state."""
ps = _ps(bot)
for key in bot.state.keys("cron"):
existing = _tasks.get(key)
existing = ps["tasks"].get(key)
if existing and not existing.done():
continue
data = _load(bot, key)
if data is None:
continue
_jobs[key] = data
ps["jobs"][key] = data
_start_job(bot, key)
@@ -211,7 +219,7 @@ async def cmd_cron(bot, message):
if not found_key:
await bot.reply(message, f"No cron job #{cron_id}")
return
_stop_job(found_key)
_stop_job(bot, found_key)
_delete(bot, found_key)
await bot.reply(message, f"Removed cron #{cron_id}")
return
@@ -275,7 +283,7 @@ async def cmd_cron(bot, message):
"added_by": message.nick,
}
_save(bot, key, data)
_jobs[key] = data
_ps(bot)["jobs"][key] = data
_start_job(bot, key)
fmt_interval = _format_duration(interval)

View File

@@ -28,11 +28,15 @@ _MAX_MONITORS = 20
_MAX_SNIPPET_LEN = 80
_MAX_TITLE_LEN = 60
# -- Module-level tracking ---------------------------------------------------
# -- Per-bot runtime state ---------------------------------------------------
_pollers: dict[str, asyncio.Task] = {}
_monitors: dict[str, dict] = {}
_errors: dict[str, int] = {}
def _ps(bot):
"""Per-bot plugin runtime state."""
return bot._pstate.setdefault("pastemoni", {
"pollers": {},
"monitors": {},
"errors": {},
})
# -- Pure helpers ------------------------------------------------------------
@@ -239,12 +243,13 @@ _BACKENDS: dict[str, callable] = {
async def _poll_once(bot, key: str, announce: bool = True) -> None:
"""Single poll cycle for one monitor (all backends)."""
data = _monitors.get(key)
ps = _ps(bot)
data = ps["monitors"].get(key)
if data is None:
data = _load(bot, key)
if data is None:
return
_monitors[key] = data
ps["monitors"][key] = data
keyword = data["keyword"]
now = datetime.now(timezone.utc).isoformat()
@@ -294,11 +299,11 @@ async def _poll_once(bot, key: str, announce: bool = True) -> None:
data.setdefault("seen", {})[tag] = seen_list
if had_success:
_errors[key] = 0
ps["errors"][key] = 0
else:
_errors[key] = _errors.get(key, 0) + 1
ps["errors"][key] = ps["errors"].get(key, 0) + 1
_monitors[key] = data
ps["monitors"][key] = data
_save(bot, key, data)
@@ -306,11 +311,12 @@ async def _poll_loop(bot, key: str) -> None:
"""Infinite poll loop for one monitor."""
try:
while True:
data = _monitors.get(key) or _load(bot, key)
ps = _ps(bot)
data = ps["monitors"].get(key) or _load(bot, key)
if data is None:
return
interval = data.get("interval", _DEFAULT_INTERVAL)
errs = _errors.get(key, 0)
errs = ps["errors"].get(key, 0)
if errs >= 5:
interval = min(interval * 2, _MAX_INTERVAL)
await asyncio.sleep(interval)
@@ -321,34 +327,37 @@ async def _poll_loop(bot, key: str) -> None:
def _start_poller(bot, key: str) -> None:
"""Create and track a poller task."""
existing = _pollers.get(key)
ps = _ps(bot)
existing = ps["pollers"].get(key)
if existing and not existing.done():
return
task = asyncio.create_task(_poll_loop(bot, key))
_pollers[key] = task
ps["pollers"][key] = task
def _stop_poller(key: str) -> None:
def _stop_poller(bot, key: str) -> None:
"""Cancel and remove a poller task."""
task = _pollers.pop(key, None)
ps = _ps(bot)
task = ps["pollers"].pop(key, None)
if task and not task.done():
task.cancel()
_monitors.pop(key, None)
_errors.pop(key, 0)
ps["monitors"].pop(key, None)
ps["errors"].pop(key, 0)
# -- Restore on connect -----------------------------------------------------
def _restore(bot) -> None:
"""Rebuild pollers from persisted state."""
ps = _ps(bot)
for key in bot.state.keys("pastemoni"):
existing = _pollers.get(key)
existing = ps["pollers"].get(key)
if existing and not existing.done():
continue
data = _load(bot, key)
if data is None:
continue
_monitors[key] = data
ps["monitors"][key] = data
_start_poller(bot, key)
@@ -417,9 +426,9 @@ async def cmd_pastemoni(bot, message):
if data is None:
await bot.reply(message, f"No monitor '{name}' in this channel")
return
_monitors[key] = data
_ps(bot)["monitors"][key] = data
await _poll_once(bot, key, announce=True)
data = _monitors.get(key, data)
data = _ps(bot)["monitors"].get(key, data)
errs = data.get("last_errors", {})
if errs:
tags = ", ".join(sorted(errs))
@@ -480,7 +489,7 @@ async def cmd_pastemoni(bot, message):
"seen": {},
}
_save(bot, key, data)
_monitors[key] = data
_ps(bot)["monitors"][key] = data
async def _seed():
await _poll_once(bot, key, announce=False)
@@ -514,7 +523,7 @@ async def cmd_pastemoni(bot, message):
await bot.reply(message, f"No monitor '{name}' in this channel")
return
_stop_poller(key)
_stop_poller(bot, key)
_delete(bot, key)
await bot.reply(message, f"Removed '{name}'")
return

View File

@@ -118,32 +118,35 @@ def _delete_saved(bot, rid: str) -> None:
bot.state.delete("remind", rid)
# ---- In-memory tracking -----------------------------------------------------
# ---- Per-bot runtime state --------------------------------------------------
# {rid: (task, target, nick, label, created, repeating)}
_reminders: dict[str, tuple[asyncio.Task, str, str, str, str, bool]] = {}
# Reverse lookup: (target, nick) -> [rid, ...]
_by_user: dict[tuple[str, str], list[str]] = {}
# Calendar-based rids (persisted)
_calendar: set[str] = set()
def _ps(bot):
"""Per-bot plugin runtime state."""
return bot._pstate.setdefault("remind", {
"reminders": {},
"by_user": {},
"calendar": set(),
})
def _cleanup(rid: str, target: str, nick: str) -> None:
def _cleanup(bot, rid: str, target: str, nick: str) -> None:
"""Remove a reminder from tracking structures."""
_reminders.pop(rid, None)
_calendar.discard(rid)
ps = _ps(bot)
ps["reminders"].pop(rid, None)
ps["calendar"].discard(rid)
ukey = (target, nick)
if ukey in _by_user:
_by_user[ukey] = [r for r in _by_user[ukey] if r != rid]
if not _by_user[ukey]:
del _by_user[ukey]
if ukey in ps["by_user"]:
ps["by_user"][ukey] = [r for r in ps["by_user"][ukey] if r != rid]
if not ps["by_user"][ukey]:
del ps["by_user"][ukey]
def _track(rid: str, task: asyncio.Task, target: str, nick: str,
def _track(bot, rid: str, task: asyncio.Task, target: str, nick: str,
label: str, created: str, repeating: bool) -> None:
"""Add a reminder to in-memory tracking."""
_reminders[rid] = (task, target, nick, label, created, repeating)
_by_user.setdefault((target, nick), []).append(rid)
ps = _ps(bot)
ps["reminders"][rid] = (task, target, nick, label, created, repeating)
ps["by_user"].setdefault((target, nick), []).append(rid)
# ---- Coroutines -------------------------------------------------------------
@@ -159,7 +162,7 @@ async def _remind_once(bot, rid: str, target: str, nick: str, label: str,
except asyncio.CancelledError:
pass
finally:
_cleanup(rid, target, nick)
_cleanup(bot, rid, target, nick)
async def _remind_repeat(bot, rid: str, target: str, nick: str, label: str,
@@ -174,7 +177,7 @@ async def _remind_repeat(bot, rid: str, target: str, nick: str, label: str,
except asyncio.CancelledError:
pass
finally:
_cleanup(rid, target, nick)
_cleanup(bot, rid, target, nick)
async def _schedule_at(bot, rid: str, target: str, nick: str, label: str,
@@ -191,7 +194,7 @@ async def _schedule_at(bot, rid: str, target: str, nick: str, label: str,
except asyncio.CancelledError:
pass
finally:
_cleanup(rid, target, nick)
_cleanup(bot, rid, target, nick)
async def _schedule_yearly(bot, rid: str, target: str, nick: str,
@@ -219,16 +222,17 @@ async def _schedule_yearly(bot, rid: str, target: str, nick: str,
except asyncio.CancelledError:
pass
finally:
_cleanup(rid, target, nick)
_cleanup(bot, rid, target, nick)
# ---- Restore on connect -----------------------------------------------------
def _restore(bot) -> None:
"""Restore persisted calendar reminders from bot.state."""
ps = _ps(bot)
for rid in bot.state.keys("remind"):
# Skip if already active
entry = _reminders.get(rid)
entry = ps["reminders"].get(rid)
if entry and not entry[0].done():
continue
raw = bot.state.get("remind", rid)
@@ -272,8 +276,8 @@ def _restore(bot) -> None:
else:
continue
_calendar.add(rid)
_track(rid, task, target, nick, label, created, rtype == "yearly")
ps["calendar"].add(rid)
_track(bot, rid, task, target, nick, label, created, rtype == "yearly")
@event("001")
@@ -311,12 +315,13 @@ async def cmd_remind(bot, message):
# ---- List ----------------------------------------------------------------
if sub == "list":
rids = _by_user.get(ukey, [])
ps = _ps(bot)
rids = ps["by_user"].get(ukey, [])
active = []
for rid in rids:
entry = _reminders.get(rid)
entry = ps["reminders"].get(rid)
if entry and not entry[0].done():
if rid in _calendar:
if rid in ps["calendar"]:
# Show next fire time
raw = bot.state.get("remind", rid)
if raw:
@@ -347,7 +352,7 @@ async def cmd_remind(bot, message):
if not rid:
await bot.reply(message, "Usage: !remind cancel <id>")
return
entry = _reminders.get(rid)
entry = _ps(bot)["reminders"].get(rid)
if entry and not entry[0].done() and entry[2] == nick:
entry[0].cancel()
_delete_saved(bot, rid)
@@ -397,11 +402,11 @@ async def cmd_remind(bot, message):
"created": created,
}
_save(bot, rid, data)
_calendar.add(rid)
_ps(bot)["calendar"].add(rid)
task = asyncio.create_task(
_schedule_at(bot, rid, target, nick, label, fire_utc, created),
)
_track(rid, task, target, nick, label, created, False)
_track(bot, rid, task, target, nick, label, created, False)
local_str = fire_dt.strftime("%Y-%m-%d %H:%M")
await bot.reply(message, f"Reminder #{rid} set (at {local_str})")
return
@@ -459,12 +464,12 @@ async def cmd_remind(bot, message):
"created": created,
}
_save(bot, rid, data)
_calendar.add(rid)
_ps(bot)["calendar"].add(rid)
task = asyncio.create_task(
_schedule_yearly(bot, rid, target, nick, label, fire_utc,
month, day_raw, hour, minute, tz, created),
)
_track(rid, task, target, nick, label, created, True)
_track(bot, rid, task, target, nick, label, created, True)
local_str = fire_dt.strftime("%Y-%m-%d %H:%M")
await bot.reply(message, f"Reminder #{rid} set (yearly {month_day}, next {local_str})")
return
@@ -501,7 +506,7 @@ async def cmd_remind(bot, message):
_remind_once(bot, rid, target, nick, label, duration, created),
)
_track(rid, task, target, nick, label, created, repeating)
_track(bot, rid, task, target, nick, label, created, repeating)
kind = f"every {_format_duration(duration)}" if repeating else _format_duration(duration)
await bot.reply(message, f"Reminder #{rid} set ({kind})")

View File

@@ -27,11 +27,15 @@ _MAX_FEEDS = 20
_ATOM_NS = "{http://www.w3.org/2005/Atom}"
_DC_NS = "{http://purl.org/dc/elements/1.1/}"
# -- Module-level tracking ---------------------------------------------------
# -- Per-bot runtime state ---------------------------------------------------
_pollers: dict[str, asyncio.Task] = {}
_feeds: dict[str, dict] = {}
_errors: dict[str, int] = {}
def _ps(bot):
"""Per-bot plugin runtime state."""
return bot._pstate.setdefault("rss", {
"pollers": {},
"feeds": {},
"errors": {},
})
# -- Pure helpers ------------------------------------------------------------
@@ -209,12 +213,13 @@ def _parse_feed(body: bytes) -> tuple[str, list[dict]]:
async def _poll_once(bot, key: str, announce: bool = True) -> None:
"""Single poll cycle for one feed."""
data = _feeds.get(key)
ps = _ps(bot)
data = ps["feeds"].get(key)
if data is None:
data = _load(bot, key)
if data is None:
return
_feeds[key] = data
ps["feeds"][key] = data
url = data["url"]
etag = data.get("etag", "")
@@ -230,16 +235,16 @@ async def _poll_once(bot, key: str, announce: bool = True) -> None:
if result["error"]:
data["last_error"] = result["error"]
_errors[key] = _errors.get(key, 0) + 1
_feeds[key] = data
ps["errors"][key] = ps["errors"].get(key, 0) + 1
ps["feeds"][key] = data
_save(bot, key, data)
return
# HTTP 304 -- not modified
if result["status"] == 304:
data["last_error"] = ""
_errors[key] = 0
_feeds[key] = data
ps["errors"][key] = 0
ps["feeds"][key] = data
_save(bot, key, data)
return
@@ -247,14 +252,14 @@ async def _poll_once(bot, key: str, announce: bool = True) -> None:
data["etag"] = result["etag"]
data["last_modified"] = result["last_modified"]
data["last_error"] = ""
_errors[key] = 0
ps["errors"][key] = 0
try:
feed_title, items = _parse_feed(result["body"])
except Exception as exc:
data["last_error"] = f"Parse error: {exc}"
_errors[key] = _errors.get(key, 0) + 1
_feeds[key] = data
ps["errors"][key] = ps["errors"].get(key, 0) + 1
ps["feeds"][key] = data
_save(bot, key, data)
return
@@ -292,7 +297,7 @@ async def _poll_once(bot, key: str, announce: bool = True) -> None:
seen_list = seen_list[-_MAX_SEEN:]
data["seen"] = seen_list
_feeds[key] = data
ps["feeds"][key] = data
_save(bot, key, data)
@@ -300,12 +305,13 @@ async def _poll_loop(bot, key: str) -> None:
"""Infinite poll loop for one feed."""
try:
while True:
data = _feeds.get(key) or _load(bot, key)
ps = _ps(bot)
data = ps["feeds"].get(key) or _load(bot, key)
if data is None:
return
interval = data.get("interval", _DEFAULT_INTERVAL)
# Back off on consecutive errors
errs = _errors.get(key, 0)
errs = ps["errors"].get(key, 0)
if errs >= 5:
interval = min(interval * 2, _MAX_INTERVAL)
await asyncio.sleep(interval)
@@ -316,34 +322,37 @@ async def _poll_loop(bot, key: str) -> None:
def _start_poller(bot, key: str) -> None:
"""Create and track a poller task."""
existing = _pollers.get(key)
ps = _ps(bot)
existing = ps["pollers"].get(key)
if existing and not existing.done():
return
task = asyncio.create_task(_poll_loop(bot, key))
_pollers[key] = task
ps["pollers"][key] = task
def _stop_poller(key: str) -> None:
def _stop_poller(bot, key: str) -> None:
"""Cancel and remove a poller task."""
task = _pollers.pop(key, None)
ps = _ps(bot)
task = ps["pollers"].pop(key, None)
if task and not task.done():
task.cancel()
_feeds.pop(key, None)
_errors.pop(key, 0)
ps["feeds"].pop(key, None)
ps["errors"].pop(key, 0)
# -- Restore on connect -----------------------------------------------------
def _restore(bot) -> None:
"""Rebuild pollers from persisted state."""
ps = _ps(bot)
for key in bot.state.keys("rss"):
existing = _pollers.get(key)
existing = ps["pollers"].get(key)
if existing and not existing.done():
continue
data = _load(bot, key)
if data is None:
continue
_feeds[key] = data
ps["feeds"][key] = data
_start_poller(bot, key)
@@ -411,9 +420,10 @@ async def cmd_rss(bot, message):
if data is None:
await bot.reply(message, f"No feed '{name}' in this channel")
return
_feeds[key] = data
ps = _ps(bot)
ps["feeds"][key] = data
await _poll_once(bot, key, announce=True)
data = _feeds.get(key, data)
data = ps["feeds"].get(key, data)
if data.get("last_error"):
await bot.reply(message, f"{name}: error -- {data['last_error']}")
else:
@@ -494,7 +504,7 @@ async def cmd_rss(bot, message):
"title": feed_title,
}
_save(bot, key, data)
_feeds[key] = data
_ps(bot)["feeds"][key] = data
_start_poller(bot, key)
display = feed_title or name
@@ -525,7 +535,7 @@ async def cmd_rss(bot, message):
await bot.reply(message, f"No feed '{name}' in this channel")
return
_stop_poller(key)
_stop_poller(bot, key)
_delete(bot, key)
await bot.reply(message, f"Unsubscribed '{name}'")
return

View File

@@ -23,11 +23,15 @@ _FETCH_TIMEOUT = 10
_MAX_TITLE_LEN = 80
_MAX_STREAMERS = 20
# -- Module-level tracking ---------------------------------------------------
# -- Per-bot runtime state ---------------------------------------------------
_pollers: dict[str, asyncio.Task] = {}
_streamers: dict[str, dict] = {}
_errors: dict[str, int] = {}
def _ps(bot):
"""Per-bot plugin runtime state."""
return bot._pstate.setdefault("twitch", {
"pollers": {},
"streamers": {},
"errors": {},
})
# -- Pure helpers ------------------------------------------------------------
@@ -149,12 +153,13 @@ def _delete(bot, key: str) -> None:
async def _poll_once(bot, key: str, announce: bool = True) -> None:
"""Single poll cycle for one Twitch streamer."""
data = _streamers.get(key)
ps = _ps(bot)
data = ps["streamers"].get(key)
if data is None:
data = _load(bot, key)
if data is None:
return
_streamers[key] = data
ps["streamers"][key] = data
login = data["login"]
@@ -166,13 +171,13 @@ async def _poll_once(bot, key: str, announce: bool = True) -> None:
if result["error"]:
data["last_error"] = result["error"]
_errors[key] = _errors.get(key, 0) + 1
_streamers[key] = data
ps["errors"][key] = ps["errors"].get(key, 0) + 1
ps["streamers"][key] = data
_save(bot, key, data)
return
data["last_error"] = ""
_errors[key] = 0
ps["errors"][key] = 0
was_live = data.get("was_live", False)
old_stream_id = data.get("stream_id", "")
@@ -202,7 +207,7 @@ async def _poll_once(bot, key: str, announce: bool = True) -> None:
else:
data["was_live"] = False
_streamers[key] = data
ps["streamers"][key] = data
_save(bot, key, data)
@@ -210,11 +215,12 @@ async def _poll_loop(bot, key: str) -> None:
"""Infinite poll loop for one Twitch streamer."""
try:
while True:
data = _streamers.get(key) or _load(bot, key)
ps = _ps(bot)
data = ps["streamers"].get(key) or _load(bot, key)
if data is None:
return
interval = data.get("interval", _DEFAULT_INTERVAL)
errs = _errors.get(key, 0)
errs = ps["errors"].get(key, 0)
if errs >= 5:
interval = min(interval * 2, _MAX_INTERVAL)
await asyncio.sleep(interval)
@@ -225,34 +231,37 @@ async def _poll_loop(bot, key: str) -> None:
def _start_poller(bot, key: str) -> None:
"""Create and track a poller task."""
existing = _pollers.get(key)
ps = _ps(bot)
existing = ps["pollers"].get(key)
if existing and not existing.done():
return
task = asyncio.create_task(_poll_loop(bot, key))
_pollers[key] = task
ps["pollers"][key] = task
def _stop_poller(key: str) -> None:
def _stop_poller(bot, key: str) -> None:
"""Cancel and remove a poller task."""
task = _pollers.pop(key, None)
ps = _ps(bot)
task = ps["pollers"].pop(key, None)
if task and not task.done():
task.cancel()
_streamers.pop(key, None)
_errors.pop(key, 0)
ps["streamers"].pop(key, None)
ps["errors"].pop(key, 0)
# -- Restore on connect -----------------------------------------------------
def _restore(bot) -> None:
"""Rebuild pollers from persisted state."""
ps = _ps(bot)
for key in bot.state.keys("twitch"):
existing = _pollers.get(key)
existing = ps["pollers"].get(key)
if existing and not existing.done():
continue
data = _load(bot, key)
if data is None:
continue
_streamers[key] = data
ps["streamers"][key] = data
_start_poller(bot, key)
@@ -329,9 +338,10 @@ async def cmd_twitch(bot, message):
if data is None:
await bot.reply(message, f"No streamer '{name}' in this channel")
return
_streamers[key] = data
ps = _ps(bot)
ps["streamers"][key] = data
await _poll_once(bot, key, announce=True)
data = _streamers.get(key, data)
data = ps["streamers"].get(key, data)
if data.get("last_error"):
await bot.reply(message, f"{name}: error -- {data['last_error']}")
elif data.get("was_live"):
@@ -417,7 +427,7 @@ async def cmd_twitch(bot, message):
"last_error": "",
}
_save(bot, key, data)
_streamers[key] = data
_ps(bot)["streamers"][key] = data
_start_poller(bot, key)
reply = f"Following '{name}' ({display_name})"
@@ -446,7 +456,7 @@ async def cmd_twitch(bot, message):
await bot.reply(message, f"No streamer '{name}' in this channel")
return
_stop_poller(key)
_stop_poller(bot, key)
_delete(bot, key)
await bot.reply(message, f"Unfollowed '{name}'")
return

View File

@@ -38,9 +38,14 @@ _SKIP_EXTS = frozenset({
# Trailing punctuation to strip, but preserve balanced parens
_TRAIL_CHARS = set(".,;:!?)>]")
# -- Module-level state ------------------------------------------------------
# -- Per-bot state -----------------------------------------------------------
_seen: dict[str, float] = {}
def _ps(bot):
"""Per-bot plugin runtime state."""
return bot._pstate.setdefault("urltitle", {
"seen": {},
})
# -- HTML parser -------------------------------------------------------------
@@ -202,21 +207,22 @@ def _fetch_title(url: str) -> tuple[str, str]:
# -- Cooldown ----------------------------------------------------------------
def _check_cooldown(url: str, cooldown: int) -> bool:
def _check_cooldown(bot, url: str, cooldown: int) -> bool:
"""Return True if the URL is within the cooldown window."""
seen = _ps(bot)["seen"]
now = time.monotonic()
last = _seen.get(url)
last = seen.get(url)
if last is not None and (now - last) < cooldown:
return True
# Prune if cache is too large
if len(_seen) >= _CACHE_MAX:
if len(seen) >= _CACHE_MAX:
cutoff = now - cooldown
stale = [k for k, v in _seen.items() if v < cutoff]
stale = [k for k, v in seen.items() if v < cutoff]
for k in stale:
del _seen[k]
del seen[k]
_seen[url] = now
seen[url] = now
return False
@@ -261,7 +267,7 @@ async def on_privmsg(bot, message):
for url in urls:
if _is_ignored_url(url, ignore_hosts):
continue
if _check_cooldown(url, cooldown):
if _check_cooldown(bot, url, cooldown):
continue
title, desc = await loop.run_in_executor(None, _fetch_title, url)

View File

@@ -14,9 +14,15 @@ from derp.plugin import command, event
log = logging.getLogger(__name__)
_MAX_BODY = 65536 # 64 KB
_server: asyncio.Server | None = None
_request_count: int = 0
_started: float = 0.0
def _ps(bot):
"""Per-bot plugin runtime state."""
return bot._pstate.setdefault("webhook", {
"server": None,
"request_count": 0,
"started": 0.0,
})
def _verify_signature(secret: str, body: bytes, signature: str) -> bool:
@@ -47,7 +53,7 @@ async def _handle_request(reader: asyncio.StreamReader,
writer: asyncio.StreamWriter,
bot, secret: str) -> None:
"""Parse one HTTP request and dispatch to IRC."""
global _request_count
ps = _ps(bot)
try:
# Read request line
@@ -117,7 +123,7 @@ async def _handle_request(reader: asyncio.StreamReader,
else:
await bot.send(channel, text)
_request_count += 1
ps["request_count"] += 1
writer.write(_http_response(200, "OK", "sent"))
log.info("webhook: relayed to %s (%d bytes)", channel, len(text))
@@ -140,9 +146,9 @@ async def _handle_request(reader: asyncio.StreamReader,
@event("001")
async def on_connect(bot, message):
"""Start the webhook HTTP server on connect (if enabled)."""
global _server, _started, _request_count
ps = _ps(bot)
if _server is not None:
if ps["server"] is not None:
return # already running
cfg = bot.config.get("webhook", {})
@@ -157,9 +163,9 @@ async def on_connect(bot, message):
await _handle_request(reader, writer, bot, secret)
try:
_server = await asyncio.start_server(handler, host, port)
_started = time.monotonic()
_request_count = 0
ps["server"] = await asyncio.start_server(handler, host, port)
ps["started"] = time.monotonic()
ps["request_count"] = 0
log.info("webhook: listening on %s:%d", host, port)
except OSError as exc:
log.error("webhook: failed to bind %s:%d: %s", host, port, exc)
@@ -168,18 +174,20 @@ async def on_connect(bot, message):
@command("webhook", help="Show webhook listener status", admin=True)
async def cmd_webhook(bot, message):
"""Display webhook server status."""
if _server is None:
ps = _ps(bot)
if ps["server"] is None:
await bot.reply(message, "Webhook: not running")
return
socks = _server.sockets
socks = ps["server"].sockets
if socks:
addr = socks[0].getsockname()
address = f"{addr[0]}:{addr[1]}"
else:
address = "unknown"
elapsed = int(time.monotonic() - _started)
elapsed = int(time.monotonic() - ps["started"])
hours, rem = divmod(elapsed, 3600)
minutes, secs = divmod(rem, 60)
parts = []
@@ -192,5 +200,5 @@ async def cmd_webhook(bot, message):
await bot.reply(
message,
f"Webhook: {address} | {_request_count} requests | up {uptime}",
f"Webhook: {address} | {ps['request_count']} requests | up {uptime}",
)

View File

@@ -40,11 +40,15 @@ _BROWSER_UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"
_MAX_TITLE_LEN = 80
_MAX_CHANNELS = 20
# -- Module-level tracking ---------------------------------------------------
# -- Per-bot runtime state ---------------------------------------------------
_pollers: dict[str, asyncio.Task] = {}
_channels: dict[str, dict] = {}
_errors: dict[str, int] = {}
def _ps(bot):
"""Per-bot plugin runtime state."""
return bot._pstate.setdefault("yt", {
"pollers": {},
"channels": {},
"errors": {},
})
# -- Pure helpers ------------------------------------------------------------
@@ -317,12 +321,13 @@ def _delete(bot, key: str) -> None:
async def _poll_once(bot, key: str, announce: bool = True) -> None:
"""Single poll cycle for one YouTube channel."""
data = _channels.get(key)
ps = _ps(bot)
data = ps["channels"].get(key)
if data is None:
data = _load(bot, key)
if data is None:
return
_channels[key] = data
ps["channels"][key] = data
url = data["feed_url"]
etag = data.get("etag", "")
@@ -338,16 +343,16 @@ async def _poll_once(bot, key: str, announce: bool = True) -> None:
if result["error"]:
data["last_error"] = result["error"]
_errors[key] = _errors.get(key, 0) + 1
_channels[key] = data
ps["errors"][key] = ps["errors"].get(key, 0) + 1
ps["channels"][key] = data
_save(bot, key, data)
return
# HTTP 304 -- not modified
if result["status"] == 304:
data["last_error"] = ""
_errors[key] = 0
_channels[key] = data
ps["errors"][key] = 0
ps["channels"][key] = data
_save(bot, key, data)
return
@@ -355,14 +360,14 @@ async def _poll_once(bot, key: str, announce: bool = True) -> None:
data["etag"] = result["etag"]
data["last_modified"] = result["last_modified"]
data["last_error"] = ""
_errors[key] = 0
ps["errors"][key] = 0
try:
_, items = _parse_feed(result["body"])
except Exception as exc:
data["last_error"] = f"Parse error: {exc}"
_errors[key] = _errors.get(key, 0) + 1
_channels[key] = data
ps["errors"][key] = ps["errors"].get(key, 0) + 1
ps["channels"][key] = data
_save(bot, key, data)
return
@@ -429,7 +434,7 @@ async def _poll_once(bot, key: str, announce: bool = True) -> None:
seen_list = seen_list[-_MAX_SEEN:]
data["seen"] = seen_list
_channels[key] = data
ps["channels"][key] = data
_save(bot, key, data)
@@ -437,12 +442,13 @@ async def _poll_loop(bot, key: str) -> None:
"""Infinite poll loop for one YouTube channel."""
try:
while True:
data = _channels.get(key) or _load(bot, key)
ps = _ps(bot)
data = ps["channels"].get(key) or _load(bot, key)
if data is None:
return
interval = data.get("interval", _DEFAULT_INTERVAL)
# Back off on consecutive errors
errs = _errors.get(key, 0)
errs = ps["errors"].get(key, 0)
if errs >= 5:
interval = min(interval * 2, _MAX_INTERVAL)
await asyncio.sleep(interval)
@@ -453,34 +459,37 @@ async def _poll_loop(bot, key: str) -> None:
def _start_poller(bot, key: str) -> None:
"""Create and track a poller task."""
existing = _pollers.get(key)
ps = _ps(bot)
existing = ps["pollers"].get(key)
if existing and not existing.done():
return
task = asyncio.create_task(_poll_loop(bot, key))
_pollers[key] = task
ps["pollers"][key] = task
def _stop_poller(key: str) -> None:
def _stop_poller(bot, key: str) -> None:
"""Cancel and remove a poller task."""
task = _pollers.pop(key, None)
ps = _ps(bot)
task = ps["pollers"].pop(key, None)
if task and not task.done():
task.cancel()
_channels.pop(key, None)
_errors.pop(key, 0)
ps["channels"].pop(key, None)
ps["errors"].pop(key, 0)
# -- Restore on connect -----------------------------------------------------
def _restore(bot) -> None:
"""Rebuild pollers from persisted state."""
ps = _ps(bot)
for key in bot.state.keys("yt"):
existing = _pollers.get(key)
existing = ps["pollers"].get(key)
if existing and not existing.done():
continue
data = _load(bot, key)
if data is None:
continue
_channels[key] = data
ps["channels"][key] = data
_start_poller(bot, key)
@@ -548,9 +557,10 @@ async def cmd_yt(bot, message):
if data is None:
await bot.reply(message, f"No channel '{name}' in this channel")
return
_channels[key] = data
ps = _ps(bot)
ps["channels"][key] = data
await _poll_once(bot, key, announce=True)
data = _channels.get(key, data)
data = ps["channels"].get(key, data)
if data.get("last_error"):
await bot.reply(message, f"{name}: error -- {data['last_error']}")
else:
@@ -652,7 +662,7 @@ async def cmd_yt(bot, message):
"title": channel_title,
}
_save(bot, key, data)
_channels[key] = data
_ps(bot)["channels"][key] = data
_start_poller(bot, key)
display = channel_title or name
@@ -683,7 +693,7 @@ async def cmd_yt(bot, message):
await bot.reply(message, f"No channel '{name}' in this channel")
return
_stop_poller(key)
_stop_poller(bot, key)
_delete(bot, key)
await bot.reply(message, f"Unfollowed '{name}'")
return