feat(queue): async delivery via Cloudflare Queues with retry backoff and DLQ

server/lib/core/dispatch.ts

@@ -13,6 +13,7 @@ import {
 } from "../web/groups";
 import { getDriver } from "../drivers";
 import type { SendResult } from "../drivers/types";
+import type { DispatchFailure, DispatchSummary } from "../queue/delivery";
 
 /** One dispatch attempt (route × target), collected for the group webhook log. */
 interface DispatchAttempt {
@@ -22,6 +23,8 @@ interface DispatchAttempt {
   target: string;
   ok: boolean;
   error?: string;
+  errorCode?: string;
+  status?: number;
 }
 
 export async function dispatchEvent(
@@ -29,7 +32,7 @@ export async function dispatchEvent(
   event: WebhookEvent,
   env: Env,
   groups?: Group[],
-): Promise<void> {
+): Promise<DispatchSummary> {
   const loadedGroups = groups ?? (await loadGroups(env.KV));
   const groupById = new Map(loadedGroups.map((g) => [g.id, g]));
 
@@ -76,6 +79,16 @@ export async function dispatchEvent(
 
   await sendGroupLogs(attempts);
 
+  const failures: DispatchFailure[] = attempts
+    .filter((a) => !a.ok)
+    .map((a) => ({
+      target: a.target,
+      error: a.error,
+      errorCode: a.errorCode,
+      status: a.status,
+    }));
+  return { attempts: attempts.length, failures };
+
   async function sendGroupLogs(list: DispatchAttempt[]): Promise<void> {
     const byGroup = new Map<string, DispatchAttempt[]>();
     for (const a of list) {
@@ -309,6 +322,8 @@ export async function dispatchEvent(
             target: targetStr,
             ok: false,
             error,
+            errorCode: result.errorCode,
+            status: result.status,
           });
           await recordSend(env.DB, {
             ...base,

server/lib/queue/consumer.ts

@@ -0,0 +1,90 @@
+import type { Env, WebhookEvent, WebhookProvider } from "../types";
+import { dispatchEvent } from "../core/dispatch";
+import { loadConfig } from "../config";
+import { loadGroups } from "../web/groups";
+import { log } from "../lib/log";
+import {
+  DELIVERY_DLQ,
+  type DeliveryMessage,
+  classifyDelivery,
+  deliveryStateKey,
+  discardPayload,
+  getDeliveryState,
+  resolvePayload,
+  retryDelay,
+  setDeliveryState,
+} from "./delivery";
+
+export async function handleQueueBatch(
+  batch: MessageBatch<DeliveryMessage>,
+  env: Env,
+): Promise<void> {
+  for (const message of batch.messages) {
+    const body = message.body;
+    if (batch.queue === DELIVERY_DLQ) {
+      await markDead(env, body);
+      message.ack();
+      continue;
+    }
+    await processMessage(env, body, message);
+  }
+}
+
+async function processMessage(
+  env: Env,
+  body: DeliveryMessage,
+  message: Message<DeliveryMessage>,
+): Promise<void> {
+  const key = deliveryStateKey(body.provider, body.groupId, body.deliveryId);
+  const prior = await getDeliveryState(env, key);
+  if (prior === "delivered" || prior === "dead") {
+    message.ack();
+    return;
+  }
+
+  await setDeliveryState(env, key, "processing");
+
+  const payload = await resolvePayload(env, body);
+  const event: WebhookEvent = {
+    event: body.event,
+    payload,
+    deliveryId: body.deliveryId,
+    provider: body.provider as WebhookProvider,
+    installationId: body.installationId,
+  };
+
+  try {
+    const config = await loadConfig(env);
+    if (body.groupId) {
+      config.routes = config.routes.filter((r) => r.groupId === body.groupId);
+    }
+    const groups = await loadGroups(env.KV);
+    const summary = await dispatchEvent(config, event, env, groups);
+    const { failed, retryable } = classifyDelivery(summary);
+
+    if (!failed) {
+      await setDeliveryState(env, key, "delivered");
+      await discardPayload(env, body);
+      message.ack();
+      return;
+    }
+    if (retryable) {
+      await setDeliveryState(env, key, "retrying");
+      message.retry({ delaySeconds: retryDelay(message.attempts) });
+      return;
+    }
+    await setDeliveryState(env, key, "failed");
+    await discardPayload(env, body);
+    message.ack();
+  } catch (err) {
+    log.error({ deliveryId: body.deliveryId, err }, "Queue delivery failed");
+    await setDeliveryState(env, key, "retrying");
+    message.retry({ delaySeconds: retryDelay(message.attempts) });
+  }
+}
+
+async function markDead(env: Env, body: DeliveryMessage): Promise<void> {
+  const key = deliveryStateKey(body.provider, body.groupId, body.deliveryId);
+  await setDeliveryState(env, key, "dead");
+  await discardPayload(env, body);
+}

server/lib/queue/delivery.ts

@@ -0,0 +1,133 @@
+import type { Env } from "../types";
+
+export type DeliveryStatus =
+  "pending" | "processing" | "delivered" | "retrying" | "failed" | "dead";
+
+export interface DeliveryMessage {
+  deliveryId: string;
+  groupId?: string;
+  provider: string;
+  event: string;
+  payload?: Record<string, unknown>;
+  payloadRef?: string;
+  installationId?: number;
+  receivedAt: number;
+  requestId?: string;
+}
+
+export interface DispatchFailure {
+  target: string;
+  error?: string;
+  errorCode?: string;
+  status?: number;
+}
+
+export interface DispatchSummary {
+  attempts: number;
+  failures: DispatchFailure[];
+}
+
+export const DELIVERY_QUEUE = "webhooker-delivery";
+export const DELIVERY_DLQ = "webhooker-delivery-dlq";
+
+const MAX_QUEUE_MESSAGE_BYTES = 100_000;
+const PAYLOAD_KV_TTL_SECONDS = 60 * 60 * 24;
+const STATE_KV_TTL_SECONDS = 60 * 60 * 24;
+
+const RETRYABLE_ERROR_CODES = new Set(["DISCORD_5XX", "TELEGRAM_5XX", "NETWORK", "RETRIES"]);
+
+const RETRY_DELAYS_SECONDS = [5, 30, 120, 600];
+
+export function isRetryableError(code?: string): boolean {
+  return code == null || RETRYABLE_ERROR_CODES.has(code);
+}
+
+export function classifyDelivery(summary: DispatchSummary): {
+  failed: boolean;
+  retryable: boolean;
+} {
+  if (summary.failures.length === 0) return { failed: false, retryable: false };
+  const retryable = summary.failures.every((f) => isRetryableError(f.errorCode));
+  const permanent = summary.failures.some((f) => !isRetryableError(f.errorCode));
+  return { failed: true, retryable: retryable && !permanent };
+}
+
+export function retryDelay(attempt: number): number {
+  if (attempt < 1) return RETRY_DELAYS_SECONDS[0];
+  const idx = Math.min(attempt - 1, RETRY_DELAYS_SECONDS.length - 1);
+  return RETRY_DELAYS_SECONDS[idx];
+}
+
+function scopeKey(provider: string, groupId: string | undefined, deliveryId: string): string {
+  return `${provider}:${groupId ?? "global"}:${deliveryId}`;
+}
+
+export function deliveryStateKey(
+  provider: string,
+  groupId: string | undefined,
+  deliveryId: string,
+): string {
+  return `delivery-state:${scopeKey(provider, groupId, deliveryId)}`;
+}
+
+function payloadKey(provider: string, groupId: string | undefined, deliveryId: string): string {
+  return `queue:payload:${scopeKey(provider, groupId, deliveryId)}`;
+}
+
+export async function setDeliveryState(
+  env: Env,
+  key: string,
+  status: DeliveryStatus,
+): Promise<void> {
+  await env.KV.put(key, JSON.stringify({ status, at: Date.now() }), {
+    expirationTtl: STATE_KV_TTL_SECONDS,
+  });
+}
+
+export async function getDeliveryState(env: Env, key: string): Promise<DeliveryStatus | null> {
+  const raw = await env.KV.get(key);
+  if (!raw) return null;
+  try {
+    return (JSON.parse(raw) as { status?: DeliveryStatus }).status ?? null;
+  } catch {
+    return null;
+  }
+}
+
+export async function enqueueWebhook(env: Env, message: DeliveryMessage): Promise<void> {
+  const queue = env.QUEUE;
+  if (!queue) throw new Error("QUEUE binding is not configured");
+  const { payload, ...rest } = message;
+  const direct: DeliveryMessage = { ...rest, payload };
+  if (JSON.stringify(direct).length <= MAX_QUEUE_MESSAGE_BYTES) {
+    await queue.send(direct);
+    return;
+  }
+  const payloadRef = payloadKey(message.provider, message.groupId, message.deliveryId);
+  await env.KV.put(payloadRef, JSON.stringify(payload ?? {}), {
+    expirationTtl: PAYLOAD_KV_TTL_SECONDS,
+  });
+  await queue.send({ ...rest, payloadRef });
+}
+
+export async function resolvePayload(
+  env: Env,
+  message: DeliveryMessage,
+): Promise<Record<string, unknown>> {
+  if (message.payload) return message.payload;
+  if (message.payloadRef) {
+    const raw = await env.KV.get(message.payloadRef);
+    if (raw) {
+      try {
+        return JSON.parse(raw) as Record<string, unknown>;
+      } catch {
+        return {};
+      }
+    }
+  }
+  return {};
+}
+
+export async function discardPayload(env: Env, message: DeliveryMessage): Promise<void> {
+  if (message.payloadRef) await env.KV.delete(message.payloadRef);
+}

server/lib/types.ts

@@ -27,6 +27,7 @@ export interface Env {
   ASSETS?: Fetcher;
   KV: KVNamespace;
   DB: D1Database;
+  QUEUE?: Queue;
 }
 
 export interface Config {

server/lib/webhook.ts

@@ -11,6 +11,7 @@ import { cfEnv, cfWaitUntil, headersFrom } from "./cf";
 import { log } from "./lib/log";
 import { deliveryKey, kvIdempotencyStore } from "./lib/idempotency";
 import { newCorrelationId } from "./lib/correlation";
+import { enqueueWebhook, type DeliveryMessage } from "./queue/delivery";
 
 const MAX_BODY_SIZE = 1024 * 1024;
 
@@ -135,6 +136,24 @@ export async function processWebhook(
     config.routes = config.routes.filter((r) => r.groupId === tenantId);
   }
 
+  if (env.QUEUE) {
+    const message: DeliveryMessage = {
+      deliveryId: event.deliveryId ?? requestId,
+      groupId: tenantId,
+      provider: provider.id,
+      event: event.event,
+      payload: event.payload,
+      installationId: event.installationId,
+      receivedAt: Date.now(),
+      requestId,
+    };
+    const enqueue = enqueueWebhook(env, message).catch((err) =>
+      log.error({ requestId, err }, "Failed to enqueue webhook"),
+    );
+    waitUntil(enqueue);
+    return { status: 200, body: { ok: true, requestId } };
+  }
+
   const dispatch = dispatchEvent(config, event, env, groups).catch((err) =>
     log.error({ requestId, err }, "Dispatch failed"),
   );

server/plugins/queue-consumer.ts

@@ -0,0 +1,8 @@
+import type { Env } from "../lib/types";
+import { handleQueueBatch } from "../lib/queue/consumer";
+
+export default defineNitroPlugin((nitroApp) => {
+  nitroApp.hooks.hook("cloudflare:queue", async ({ batch, env }) => {
+    await handleQueueBatch(batch, env as Env);
+  });
+});