feat(queue): async delivery via Cloudflare Queues with retry backoff and DLQ

server/lib/queue/consumer.ts

@@ -0,0 +1,90 @@
+import type { Env, WebhookEvent, WebhookProvider } from "../types";
+import { dispatchEvent } from "../core/dispatch";
+import { loadConfig } from "../config";
+import { loadGroups } from "../web/groups";
+import { log } from "../lib/log";
+import {
+  DELIVERY_DLQ,
+  type DeliveryMessage,
+  classifyDelivery,
+  deliveryStateKey,
+  discardPayload,
+  getDeliveryState,
+  resolvePayload,
+  retryDelay,
+  setDeliveryState,
+} from "./delivery";
+
+export async function handleQueueBatch(
+  batch: MessageBatch<DeliveryMessage>,
+  env: Env,
+): Promise<void> {
+  for (const message of batch.messages) {
+    const body = message.body;
+    if (batch.queue === DELIVERY_DLQ) {
+      await markDead(env, body);
+      message.ack();
+      continue;
+    }
+    await processMessage(env, body, message);
+  }
+}
+
+async function processMessage(
+  env: Env,
+  body: DeliveryMessage,
+  message: Message<DeliveryMessage>,
+): Promise<void> {
+  const key = deliveryStateKey(body.provider, body.groupId, body.deliveryId);
+  const prior = await getDeliveryState(env, key);
+  if (prior === "delivered" || prior === "dead") {
+    message.ack();
+    return;
+  }
+
+  await setDeliveryState(env, key, "processing");
+
+  const payload = await resolvePayload(env, body);
+  const event: WebhookEvent = {
+    event: body.event,
+    payload,
+    deliveryId: body.deliveryId,
+    provider: body.provider as WebhookProvider,
+    installationId: body.installationId,
+  };
+
+  try {
+    const config = await loadConfig(env);
+    if (body.groupId) {
+      config.routes = config.routes.filter((r) => r.groupId === body.groupId);
+    }
+    const groups = await loadGroups(env.KV);
+    const summary = await dispatchEvent(config, event, env, groups);
+    const { failed, retryable } = classifyDelivery(summary);
+
+    if (!failed) {
+      await setDeliveryState(env, key, "delivered");
+      await discardPayload(env, body);
+      message.ack();
+      return;
+    }
+    if (retryable) {
+      await setDeliveryState(env, key, "retrying");
+      message.retry({ delaySeconds: retryDelay(message.attempts) });
+      return;
+    }
+    await setDeliveryState(env, key, "failed");
+    await discardPayload(env, body);
+    message.ack();
+  } catch (err) {
+    log.error({ deliveryId: body.deliveryId, err }, "Queue delivery failed");
+    await setDeliveryState(env, key, "retrying");
+    message.retry({ delaySeconds: retryDelay(message.attempts) });
+  }
+}
+
+async function markDead(env: Env, body: DeliveryMessage): Promise<void> {
+  const key = deliveryStateKey(body.provider, body.groupId, body.deliveryId);
+  await setDeliveryState(env, key, "dead");
+  await discardPayload(env, body);
+}

server/lib/queue/delivery.ts

@@ -0,0 +1,133 @@
+import type { Env } from "../types";
+
+export type DeliveryStatus =
+  "pending" | "processing" | "delivered" | "retrying" | "failed" | "dead";
+
+export interface DeliveryMessage {
+  deliveryId: string;
+  groupId?: string;
+  provider: string;
+  event: string;
+  payload?: Record<string, unknown>;
+  payloadRef?: string;
+  installationId?: number;
+  receivedAt: number;
+  requestId?: string;
+}
+
+export interface DispatchFailure {
+  target: string;
+  error?: string;
+  errorCode?: string;
+  status?: number;
+}
+
+export interface DispatchSummary {
+  attempts: number;
+  failures: DispatchFailure[];
+}
+
+export const DELIVERY_QUEUE = "webhooker-delivery";
+export const DELIVERY_DLQ = "webhooker-delivery-dlq";
+
+const MAX_QUEUE_MESSAGE_BYTES = 100_000;
+const PAYLOAD_KV_TTL_SECONDS = 60 * 60 * 24;
+const STATE_KV_TTL_SECONDS = 60 * 60 * 24;
+
+const RETRYABLE_ERROR_CODES = new Set(["DISCORD_5XX", "TELEGRAM_5XX", "NETWORK", "RETRIES"]);
+
+const RETRY_DELAYS_SECONDS = [5, 30, 120, 600];
+
+export function isRetryableError(code?: string): boolean {
+  return code == null || RETRYABLE_ERROR_CODES.has(code);
+}
+
+export function classifyDelivery(summary: DispatchSummary): {
+  failed: boolean;
+  retryable: boolean;
+} {
+  if (summary.failures.length === 0) return { failed: false, retryable: false };
+  const retryable = summary.failures.every((f) => isRetryableError(f.errorCode));
+  const permanent = summary.failures.some((f) => !isRetryableError(f.errorCode));
+  return { failed: true, retryable: retryable && !permanent };
+}
+
+export function retryDelay(attempt: number): number {
+  if (attempt < 1) return RETRY_DELAYS_SECONDS[0];
+  const idx = Math.min(attempt - 1, RETRY_DELAYS_SECONDS.length - 1);
+  return RETRY_DELAYS_SECONDS[idx];
+}
+
+function scopeKey(provider: string, groupId: string | undefined, deliveryId: string): string {
+  return `${provider}:${groupId ?? "global"}:${deliveryId}`;
+}
+
+export function deliveryStateKey(
+  provider: string,
+  groupId: string | undefined,
+  deliveryId: string,
+): string {
+  return `delivery-state:${scopeKey(provider, groupId, deliveryId)}`;
+}
+
+function payloadKey(provider: string, groupId: string | undefined, deliveryId: string): string {
+  return `queue:payload:${scopeKey(provider, groupId, deliveryId)}`;
+}
+
+export async function setDeliveryState(
+  env: Env,
+  key: string,
+  status: DeliveryStatus,
+): Promise<void> {
+  await env.KV.put(key, JSON.stringify({ status, at: Date.now() }), {
+    expirationTtl: STATE_KV_TTL_SECONDS,
+  });
+}
+
+export async function getDeliveryState(env: Env, key: string): Promise<DeliveryStatus | null> {
+  const raw = await env.KV.get(key);
+  if (!raw) return null;
+  try {
+    return (JSON.parse(raw) as { status?: DeliveryStatus }).status ?? null;
+  } catch {
+    return null;
+  }
+}
+
+export async function enqueueWebhook(env: Env, message: DeliveryMessage): Promise<void> {
+  const queue = env.QUEUE;
+  if (!queue) throw new Error("QUEUE binding is not configured");
+  const { payload, ...rest } = message;
+  const direct: DeliveryMessage = { ...rest, payload };
+  if (JSON.stringify(direct).length <= MAX_QUEUE_MESSAGE_BYTES) {
+    await queue.send(direct);
+    return;
+  }
+  const payloadRef = payloadKey(message.provider, message.groupId, message.deliveryId);
+  await env.KV.put(payloadRef, JSON.stringify(payload ?? {}), {
+    expirationTtl: PAYLOAD_KV_TTL_SECONDS,
+  });
+  await queue.send({ ...rest, payloadRef });
+}
+
+export async function resolvePayload(
+  env: Env,
+  message: DeliveryMessage,
+): Promise<Record<string, unknown>> {
+  if (message.payload) return message.payload;
+  if (message.payloadRef) {
+    const raw = await env.KV.get(message.payloadRef);
+    if (raw) {
+      try {
+        return JSON.parse(raw) as Record<string, unknown>;
+      } catch {
+        return {};
+      }
+    }
+  }
+  return {};
+}
+
+export async function discardPayload(env: Env, message: DeliveryMessage): Promise<void> {
+  if (message.payloadRef) await env.KV.delete(message.payloadRef);
+}