diff --git a/cache.ts b/cache.ts index 405dfc8..44c5037 100644 --- a/cache.ts +++ b/cache.ts @@ -140,15 +140,13 @@ export function createCache(cacheTtlMs: number, cacheWindowMs: number): Cache { hits++; } - // If `after` points to a message older than the cache window, return [] + // Clamp: if `after` is older than the cache window, use the window + // start so we return all cached messages instead of an empty result const afterTs = snowflakeToTimestamp(after); const windowStart = Date.now() - cacheWindowMs; - if (afterTs < windowStart) { - return { data: [], cachedAt: entry.cachedAt }; - } - - // Filter to only messages with snowflake ID > after - const afterBigInt = BigInt(after); + const afterBigInt = afterTs < windowStart + ? ((BigInt(windowStart) - DISCORD_EPOCH) << 22n) - 1n + : BigInt(after); let filtered = entry.data.filter((msg) => BigInt(msg.id) > afterBigInt); // Apply limit (return newest N, matching Discord API behavior) diff --git a/index.ts b/index.ts index 0480930..42a7198 100644 --- a/index.ts +++ b/index.ts @@ -3,7 +3,7 @@ import { createDiscordClient } from "./discord"; import type { DiscordClient, DiscordMessage } from "./discord"; import { createCache } from "./cache"; import type { Cache } from "./cache"; -import { initialPoll, startPollingLoop, createLogger } from "./poller"; +import { initialPoll, startPollingLoop, createLogger, createChannelHealth } from "./poller"; const VERSION = "0.2.0"; const startTime = Date.now(); @@ -159,6 +159,7 @@ if (import.meta.main) { const client = createDiscordClient(config.discordBotToken); const cache = createCache(config.cacheTtlMs, config.cacheWindowMs); const logger = createLogger(config.logLevel); + const health = createChannelHealth(config.pollIntervalMs); logger.info(`scream-hole v${VERSION} starting...`); @@ -168,6 +169,7 @@ if (import.meta.main) { cache, config.discordGuildId, logger, + health, ); const handler = createHandler(cache, config.discordGuildId, config.cacheTtlMs, client); @@ -178,7 +180,7 @@ if (import.meta.main) { }); // Start the continuous polling loop - const poller = startPollingLoop(client, cache, config); + const poller = startPollingLoop(client, cache, config, health); const intervalSec = (config.pollIntervalMs / 1000).toFixed(1); logger.info( diff --git a/poller.ts b/poller.ts index c73a47f..8877d94 100644 --- a/poller.ts +++ b/poller.ts @@ -16,6 +16,7 @@ const GUILD_TEXT_CHANNEL = 0; const PER_CHANNEL_TIMEOUT_MS = 10_000; const INITIAL_POLL_TIMEOUT_MS = 30_000; +const MAX_BACKOFF_EXPONENT = 4; // max multiplier: 2^4 = 16× poll interval export interface Logger { debug(msg: string): void; @@ -80,6 +81,7 @@ async function pollCycle( guildId: string, logger: Logger, perChannelTimeout: number, + health: ChannelHealth, ): Promise { // Fetch channels const channels = await withTimeout( @@ -97,6 +99,10 @@ async function pollCycle( // Fetch messages for each text channel for (const channel of textChannels) { + if (health.shouldSkip(channel.id)) { + logger.debug(`Skipping #${channel.name ?? channel.id} (in backoff)`); + continue; + } try { const messages = await withTimeout( client.fetchMessages(channel.id), @@ -104,14 +110,20 @@ async function pollCycle( `fetchMessages(${channel.id})`, ); cache.setMessages(channel.id, messages); + if (health.recordSuccess(channel.id)) { + logger.info(`Channel #${channel.name ?? channel.id} recovered`); + } logger.debug( `Cached ${messages.length} messages for #${channel.name ?? channel.id}`, ); } catch (err) { + const { failures, backoffMs } = health.recordFailure(channel.id); logger.error( `Failed to fetch messages for channel ${channel.id}: ${err instanceof Error ? err.message : String(err)}`, ); - // Continue to next channel — don't crash + logger.warn( + `Channel #${channel.name ?? channel.id} failed ${failures}x consecutively, backing off ${Math.round(backoffMs / 1000)}s`, + ); } } @@ -130,10 +142,11 @@ export async function initialPoll( cache: Cache, guildId: string, logger: Logger, + health: ChannelHealth, ): Promise { try { const channelCount = await withTimeout( - pollCycle(client, cache, guildId, logger, PER_CHANNEL_TIMEOUT_MS), + pollCycle(client, cache, guildId, logger, PER_CHANNEL_TIMEOUT_MS, health), INITIAL_POLL_TIMEOUT_MS, "initial poll", ); @@ -155,6 +168,7 @@ export function startPollingLoop( client: DiscordClient, cache: Cache, config: Config, + health: ChannelHealth, ): { stop: () => void; logger: Logger } { const logger = createLogger(config.logLevel); let timer: ReturnType | null = null; @@ -167,6 +181,7 @@ export function startPollingLoop( config.discordGuildId, logger, PER_CHANNEL_TIMEOUT_MS, + health, ); logger.debug("Poll cycle complete"); } catch (err) { @@ -187,4 +202,44 @@ export function startPollingLoop( }; } +export interface ChannelHealth { + /** Returns true if the channel is in backoff and should be skipped this cycle. */ + shouldSkip(channelId: string): boolean; + /** Record a successful fetch. Resets backoff. Returns true if the channel was previously failing. */ + recordSuccess(channelId: string): boolean; + /** Record a failed fetch. Returns consecutive failure count and backoff duration. */ + recordFailure(channelId: string): { failures: number; backoffMs: number }; +} + +export function createChannelHealth(pollIntervalMs: number): ChannelHealth { + const entries = new Map(); + + return { + shouldSkip(channelId: string): boolean { + const entry = entries.get(channelId); + if (!entry) return false; + return Date.now() < entry.backoffUntil; + }, + + recordSuccess(channelId: string): boolean { + const entry = entries.get(channelId); + const wasInBackoff = !!entry && Date.now() < entry.backoffUntil; + entries.delete(channelId); + return wasInBackoff; + }, + + recordFailure(channelId: string): { failures: number; backoffMs: number } { + const existing = entries.get(channelId); + const failures = existing ? existing.failures + 1 : 1; + const exponent = Math.min(failures - 1, MAX_BACKOFF_EXPONENT); + const backoffMs = pollIntervalMs * Math.pow(2, exponent); + entries.set(channelId, { + failures, + backoffUntil: Date.now() + backoffMs, + }); + return { failures, backoffMs }; + }, + }; +} + export { createLogger, withTimeout }; diff --git a/tests/cache.test.ts b/tests/cache.test.ts index ce37a3f..678bc72 100644 --- a/tests/cache.test.ts +++ b/tests/cache.test.ts @@ -100,7 +100,7 @@ describe("cache messages", () => { expect(cache.getMessages("unknown-ch", after)).toBeUndefined(); }); - test("returns empty array when `after` is older than cache window", () => { + test("clamps `after` to window start when older than cache window", () => { const cache = createCache(60_000, FOUR_HOURS); const now = Date.now(); @@ -108,10 +108,29 @@ describe("cache messages", () => { cache.setMessages("ch-1", [makeMessage(recentId, "ch-1", "recent")]); // `after` is 5 hours ago — outside the 4-hour window + // Should clamp to window start and return all messages within the window const oldId = timestampToSnowflake(now - 5 * 60 * 60 * 1000); const result = cache.getMessages("ch-1", oldId); expect(result).toBeDefined(); - expect(result!.data).toEqual([]); + expect(result!.data).toHaveLength(1); + expect(result!.data[0].content).toBe("recent"); + }); + + test("returns all cached messages when after=0", () => { + const cache = createCache(60_000, FOUR_HOURS); + const now = Date.now(); + + const id1 = timestampToSnowflake(now - 60_000); + const id2 = timestampToSnowflake(now - 30_000); + cache.setMessages("ch-1", [ + makeMessage(id1, "ch-1", "msg-1"), + makeMessage(id2, "ch-1", "msg-2"), + ]); + + // after=0 is the "give me everything" pattern — should return all + const result = cache.getMessages("ch-1", "0"); + expect(result).toBeDefined(); + expect(result!.data).toHaveLength(2); }); test("evicts messages older than cache window", () => { diff --git a/tests/index.test.ts b/tests/index.test.ts index c5d0b77..f307f60 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -182,7 +182,7 @@ describe("GET /api/v10/channels/{channelId}/messages", () => { expect(body[1].content).toBe("new message"); }); - test("returns empty array when `after` is older than cache window", async () => { + test("clamps `after` to window start when older than cache window", async () => { const cache = createCache(TEST_TTL, TEST_WINDOW); const now = Date.now(); @@ -193,6 +193,7 @@ describe("GET /api/v10/channels/{channelId}/messages", () => { ]); // `after` pointing to 5 hours ago — outside 4-hour window + // Should clamp and return all messages within the window const oldId = timestampToSnowflake(now - 5 * 60 * 60 * 1000); const handler = createHandler(cache, "guild-1"); @@ -202,8 +203,9 @@ describe("GET /api/v10/channels/{channelId}/messages", () => { const res = await handler(req); expect(res.status).toBe(200); - const body = await res.json(); - expect(body).toEqual([]); + const body = (await res.json()) as { content: string }[]; + expect(body).toHaveLength(1); + expect(body[0].content).toBe("recent"); }); test("returns 404 for unknown channel", async () => { diff --git a/tests/poller.test.ts b/tests/poller.test.ts index ae971ca..8f46539 100644 --- a/tests/poller.test.ts +++ b/tests/poller.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test"; -import { initialPoll, createLogger, withTimeout } from "../poller"; +import { initialPoll, createLogger, withTimeout, createChannelHealth } from "../poller"; +import type { ChannelHealth } from "../poller"; import { createCache } from "../cache"; import type { DiscordClient, DiscordChannel, DiscordMessage } from "../discord"; @@ -49,8 +50,9 @@ describe("initialPoll", () => { const client = makeMockClient(channels, { "ch-1": messages }); const cache = createCache(60_000, 4 * 60 * 60 * 1000); const logger = createLogger("error"); // suppress output in tests + const health = createChannelHealth(15_000); - const count = await initialPoll(client, cache, "guild-1", logger); + const count = await initialPoll(client, cache, "guild-1", logger, health); expect(count).toBe(1); // only 1 text channel expect(cache.getChannels("guild-1")).toBeDefined(); @@ -77,8 +79,9 @@ describe("initialPoll", () => { const cache = createCache(60_000, 4 * 60 * 60 * 1000); const logger = createLogger("error"); + const health = createChannelHealth(15_000); - const count = await initialPoll(client, cache, "guild-1", logger); + const count = await initialPoll(client, cache, "guild-1", logger, health); expect(count).toBe(0); }); @@ -116,8 +119,9 @@ describe("initialPoll", () => { const cache = createCache(60_000, 4 * 60 * 60 * 1000); const logger = createLogger("error"); + const health = createChannelHealth(15_000); - const count = await initialPoll(client, cache, "guild-1", logger); + const count = await initialPoll(client, cache, "guild-1", logger, health); expect(count).toBe(2); // both text channels counted // ch-1 failed — should have no messages @@ -151,6 +155,164 @@ describe("withTimeout", () => { }); }); +describe("createChannelHealth", () => { + test("shouldSkip returns false for unknown channels", () => { + const health = createChannelHealth(15_000); + expect(health.shouldSkip("unknown")).toBe(false); + }); + + test("shouldSkip returns true after failure", () => { + const health = createChannelHealth(15_000); + health.recordFailure("ch-1"); + expect(health.shouldSkip("ch-1")).toBe(true); + }); + + test("recordSuccess resets backoff", () => { + const health = createChannelHealth(15_000); + health.recordFailure("ch-1"); + expect(health.shouldSkip("ch-1")).toBe(true); + + const recovered = health.recordSuccess("ch-1"); + expect(recovered).toBe(true); + expect(health.shouldSkip("ch-1")).toBe(false); + }); + + test("recordSuccess returns false for channels not previously failing", () => { + const health = createChannelHealth(15_000); + expect(health.recordSuccess("ch-1")).toBe(false); + }); + + test("consecutive failures increase backoff duration", () => { + const health = createChannelHealth(1_000); + + const r1 = health.recordFailure("ch-1"); + expect(r1.failures).toBe(1); + expect(r1.backoffMs).toBe(1_000); // 1s * 2^0 + + const r2 = health.recordFailure("ch-1"); + expect(r2.failures).toBe(2); + expect(r2.backoffMs).toBe(2_000); // 1s * 2^1 + + const r3 = health.recordFailure("ch-1"); + expect(r3.failures).toBe(3); + expect(r3.backoffMs).toBe(4_000); // 1s * 2^2 + }); + + test("backoff caps at 16x poll interval", () => { + const health = createChannelHealth(1_000); + + // Ramp up to the cap (failure 5 = exponent min(4,4) = 16x) + for (let i = 0; i < 4; i++) { + health.recordFailure("ch-1"); + } + + // Failure 5: cap first applies + const atCap = health.recordFailure("ch-1"); + expect(atCap.failures).toBe(5); + expect(atCap.backoffMs).toBe(16_000); // 1000 * 2^4 + + // Failure 6: stays capped, doesn't grow + const pastCap = health.recordFailure("ch-1"); + expect(pastCap.failures).toBe(6); + expect(pastCap.backoffMs).toBe(16_000); + }); +}); + +describe("channel backoff in polling", () => { + test("skips channels in backoff on subsequent polls", async () => { + const now = Date.now(); + const channels: DiscordChannel[] = [ + { id: "ch-ok", type: 0, name: "general" }, + { id: "ch-fail", type: 0, name: "restricted" }, + ]; + + const id1 = timestampToSnowflake(now - 60_000); + const fetchCounts = new Map(); + + const client: DiscordClient = { + async fetchChannels(): Promise { + return channels; + }, + async fetchMessages(channelId: string): Promise { + fetchCounts.set(channelId, (fetchCounts.get(channelId) ?? 0) + 1); + if (channelId === "ch-fail") { + throw new Error("403 Forbidden"); + } + return [ + { + id: id1, + channel_id: channelId, + content: "hello", + timestamp: new Date().toISOString(), + author: { id: "u-1", username: "user1" }, + }, + ]; + }, + async sendMessage() { + return { ok: true, status: 200, headers: new Headers(), body: {} }; + }, + }; + + const cache = createCache(60_000, 4 * 60 * 60 * 1000); + const logger = createLogger("error"); + const health = createChannelHealth(15_000); + + // First poll — both channels attempted + await initialPoll(client, cache, "guild-1", logger, health); + expect(fetchCounts.get("ch-ok")).toBe(1); + expect(fetchCounts.get("ch-fail")).toBe(1); + + // Second poll immediately — ch-fail should be skipped (in backoff) + await initialPoll(client, cache, "guild-1", logger, health); + expect(fetchCounts.get("ch-ok")).toBe(2); + expect(fetchCounts.get("ch-fail")).toBe(1); // NOT incremented + }); + + test("channel recovers after successful fetch", async () => { + const now = Date.now(); + const channels: DiscordChannel[] = [ + { id: "ch-flaky", type: 0, name: "flaky" }, + ]; + + const id1 = timestampToSnowflake(now - 60_000); + const health = createChannelHealth(1); // 1ms backoff so it expires instantly + + // Seed a failure + health.recordFailure("ch-flaky"); + + // Wait for backoff to expire (1ms base * 2^0 = 1ms) + await new Promise((r) => setTimeout(r, 5)); + + const client: DiscordClient = { + async fetchChannels(): Promise { + return channels; + }, + async fetchMessages(): Promise { + return [ + { + id: id1, + channel_id: "ch-flaky", + content: "back online", + timestamp: new Date().toISOString(), + author: { id: "u-1", username: "user1" }, + }, + ]; + }, + async sendMessage() { + return { ok: true, status: 200, headers: new Headers(), body: {} }; + }, + }; + + const cache = createCache(60_000, 4 * 60 * 60 * 1000); + const logger = createLogger("error"); + + await initialPoll(client, cache, "guild-1", logger, health); + + // Channel should no longer be in backoff + expect(health.shouldSkip("ch-flaky")).toBe(false); + }); +}); + describe("createLogger", () => { test("creates a logger that does not throw", () => { const logger = createLogger("debug");