diff --git a/server/lib/queue/consumer.ts b/server/lib/queue/consumer.ts index 0cd24b9..7006748 100644 --- a/server/lib/queue/consumer.ts +++ b/server/lib/queue/consumer.ts @@ -1,103 +1,412 @@ import type { Config, Env, Group, WebhookEvent, WebhookProvider } from "../types"; -import { dispatchEvent } from "../core/dispatch"; -import { loadConfig } from "../config"; -import { loadGroups } from "../web/groups"; +import { dispatchEvent, type DispatchOptions } from "../core/dispatch"; +import { loadConfigSnapshot, loadGroupFresh } from "../config"; +import { groupAliasChain } from "../web/group-lifecycle"; import { log } from "../lib/log"; +import { recordSendBatch } from "../lib/send-log-batch"; import { DELIVERY_DLQ, + PayloadUnavailableError, type DeliveryMessage, type DispatchSummary, + acquireDelivery, classifyDelivery, + finishLease, + getDeliveryState, + renewLease, + type DeliveryStatus, deliveryStateKey, discardPayload, - getDeliveryState, + hadPermanentFailure, + isRetryableFailure, + markDeliveryDead, + markTargetsDone, resolvePayload, retryDelay, - setDeliveryState, + targetDoneChecker, } from "./delivery"; +type Dispatch = ( + config: Config, + event: WebhookEvent, + env: Env, + groups?: Group[], + options?: DispatchOptions, +) => Promise; + export async function handleQueueBatch( batch: MessageBatch, env: Env, - dispatch: ( - config: Config, - event: WebhookEvent, - env: Env, - groups?: Group[], - ) => Promise = dispatchEvent, + dispatch: Dispatch = dispatchEvent, + budgetMs = BATCH_BUDGET_MS, ): Promise { + // Shared by the whole batch: the limits it guards (D1 queries and + // subrequests) are per invocation. + const budget: BatchBudget = { deadline: Date.now() + budgetMs, sendsLeft: BATCH_SEND_LIMIT }; for (const message of batch.messages) { const body = message.body; - if (batch.queue === DELIVERY_DLQ) { - await markDead(env, body); - message.ack(); - continue; + try { + if (batch.queue === DELIVERY_DLQ) { + if (await markDead(env, body)) message.ack(); + // Another copy holds a live lease and may still need the payload. + else message.retry({ delaySeconds: BUSY_RETRY_SECONDS }); + continue; + } + // Out of time or sends before it started (earlier messages took them): + // back to the queue for a later invocation — a retry (one more queue + // read), not a copy (a write and a delete more). Waiting is not a + // failure, though: a delivery never started whose retry would use up + // its last one (sending it to the DLQ untried) is copied instead. One + // that was started has real failures among its attempts; a copy would + // start their count over, so it keeps the retry. + if (spent(budget)) { + if (message.attempts > QUEUE_MAX_RETRIES && env.QUEUE && (await neverStarted(env, body))) { + try { + await env.QUEUE.send(body, { delaySeconds: CONTINUE_DELAY_SECONDS }); + message.ack(); + continue; + } catch (err) { + log.warn({ deliveryId: body.deliveryId, err }, "Failed to queue a delivery for later"); + } + } + message.retry({ delaySeconds: CONTINUE_DELAY_SECONDS }); + continue; + } + await processMessage(env, body, message, dispatch, budget); + } catch (err) { + // One message's storage failure must not fail (and replay) the batch. + log.error({ deliveryId: body.deliveryId, err }, "Queue message handling failed"); + message.retry({ delaySeconds: retryDelay(message.attempts) }); } - await processMessage(env, body, message, dispatch); } } +/** Renew a held lease when it is older than this (the lease lasts five minutes). */ +const LEASE_RENEW_MS = 60_000; +const FINAL: ReadonlySet = new Set(["delivered", "failed", "dead"]); + +/** Wait for an unexpired lease of another consumer: its full length plus a margin. */ +const BUSY_RETRY_SECONDS = 310; +/** + * Sending time of one batch: a consumer invocation may run 15 minutes, and a + * target takes under a minute at worst (bounded platform calls), so sending + * stops in time to record what was done. + */ +const BATCH_BUDGET_MS = 12 * 60_000; +/** + * Targets one batch sends at most: each costs D1 queries (its ledger record, + * an update chain's lock and tracking) and subrequests, and one invocation + * may make only 50 of each on Workers Free (with ~7 D1 queries per delivery + * besides; batches are small, see wrangler.jsonc). A message left unstarted + * goes back to the queue (a retry: one queue read); a message cut short is + * continued by a new one only when it made progress. + */ +const BATCH_SEND_LIMIT = 20; + +/** What a batch may still spend: its time and its sends. */ +interface BatchBudget { + deadline: number; + sendsLeft: number; +} + +function spent(budget: BatchBudget): boolean { + return Date.now() >= budget.deadline || budget.sendsLeft <= 0; +} +const CONTINUE_DELAY_SECONDS = 5; +/** The delivery queue's `max_retries` (`wrangler.jsonc`). */ +const QUEUE_MAX_RETRIES = 5; + +/** No attempt has taken the delivery's lease yet (its attempts were all waits). */ +async function neverStarted(env: Env, body: DeliveryMessage): Promise { + const key = deliveryStateKey(body.provider, body.groupId, body.deliveryId, body.groupInstance); + const state = await getDeliveryState(env, key); + return state == null || state === "pending"; +} + +/** Clean up after a terminal state; a failure must not reopen the delivery. */ +async function discardQuietly(env: Env, body: DeliveryMessage): Promise { + await discardPayload(env, body).catch((err) => + log.warn({ deliveryId: body.deliveryId, err }, "Failed to discard a parked payload"), + ); +} + async function processMessage( env: Env, body: DeliveryMessage, message: Message, - dispatch: ( - config: Config, - event: WebhookEvent, - env: Env, - groups?: Group[], - ) => Promise, + dispatch: Dispatch, + budget: BatchBudget, ): Promise { - const key = deliveryStateKey(body.provider, body.groupId, body.deliveryId); - const prior = await getDeliveryState(env, key); - if (prior === "delivered" || prior === "dead") { + const key = deliveryStateKey(body.provider, body.groupId, body.deliveryId, body.groupInstance); + if (await finishedUnderLegacyKey(env, body)) { message.ack(); return; } + const lease = await acquireDelivery(env, key); + if (lease.status === "finished") { + message.ack(); + return; + } + if (lease.status === "busy") { + // Another consumer holds this delivery (or a holder vanished): one retry + // after the lease surely expired, instead of spending the retry budget + // polling a lease that is still valid. + message.retry({ delaySeconds: BUSY_RETRY_SECONDS }); + return; + } - await setDeliveryState(env, key, "processing"); - - const payload = await resolvePayload(env, body); - const event: WebhookEvent = { - event: body.event, - payload, - deliveryId: body.deliveryId, - provider: body.provider as WebhookProvider, - installationId: body.installationId, - }; - - try { - const config = await loadConfig(env); - if (body.groupId) { - config.routes = config.routes.filter((r) => r.groupId === body.groupId); - } - const groups = await loadGroups(env.KV); - const summary = await dispatch(config, event, env, groups); - const { failed, retryable } = classifyDelivery(summary); - - if (!failed) { - await setDeliveryState(env, key, "delivered"); - await discardPayload(env, body); - message.ack(); - return; - } - if (retryable) { - await setDeliveryState(env, key, "retrying"); + let token = lease.token; + let renewedAt = Date.now(); + let leaseLost = false; + /** A renewal failed with an unknown outcome: stop sending, but never assume a new holder. */ + let leaseUncertain = false; + let renewing: Promise | null = null; + /** The batch's budget ran out: the rest is left to a continuation message. */ + let outOfTime = false; + /** End this attempt with `status`; acks instead when the lease was lost. */ + const finish = async (status: DeliveryStatus, then: "ack" | "retry"): Promise => { + const owned = !leaseLost && (await finishLease(env, key, token, status)); + if (owned && then === "retry") { message.retry({ delaySeconds: retryDelay(message.attempts) }); return; } - await setDeliveryState(env, key, "failed"); - await discardPayload(env, body); + if (!owned && !leaseLost) { + // The outcome could not be written, yet no new holder is known (our own + // renewal may have gone through unseen, or the lease lapsed unnoticed): + // come back once any lease surely expired; the state then decides. + message.retry({ delaySeconds: BUSY_RETRY_SECONDS }); + return; + } + // Lost lease: the new holder owns the delivery (and its outcome). message.ack(); + if (owned && FINAL.has(status)) await discardQuietly(env, body); + }; + + let summary: DispatchSummary; + // Targets recorded as sent the moment they were sent (see `afterTarget`). + const recorded = new Set(); + try { + const payload = await resolvePayload(env, body); + const event: WebhookEvent = { + event: body.event, + payload, + deliveryId: body.deliveryId, + provider: body.provider as WebhookProvider, + installationId: body.installationId, + }; + // Read fresh, not from this isolate's cache: a group (or route) created + // moments ago elsewhere must not be taken for missing — for a tenant + // delivery that verdict is final. + const { config, groups } = await loadConfigSnapshot(env, { fresh: true }); + // A tenant delivery belongs to the group instance that received it — under + // whatever id it has now — and never to another group that took over the + // id since. A message from before instances belongs to the group that had + // its id back then (`legacyId`), also when that group was renamed before + // instances existed (followed through the rename aliases) — never to a + // group created since or renamed to the id. + let tenantId = body.groupId; + if (tenantId && body.groupInstance) { + tenantId = groups.find((g) => g.instance === body.groupInstance)?.id; + } else if (tenantId) { + const owner = (id: string): string | undefined => groups.find((g) => g.legacyId === id)?.id; + let found = owner(tenantId); + for (const id of found ? [] : (await groupAliasChain(env, tenantId)).slice(1)) { + found ??= owner(id); + } + tenantId = found; + } + if (body.groupId && (!tenantId || !groups.some((g) => g.id === tenantId))) { + // A group row that cannot be read may be the tenant: retry until it is + // repaired rather than give the delivery up as one of a deleted group. + if (config.unreadableGroups?.length) { + log.error( + { deliveryId: body.deliveryId, unreadable: config.unreadableGroups }, + "Tenant unresolved while some group configuration is unreadable", + ); + await finish("retrying", "retry"); + return; + } + // The group was deleted: nothing can receive the event any more. + log.warn({ deliveryId: body.deliveryId, groupId: body.groupId }, "Tenant group is gone"); + await finish("failed", "ack"); + return; + } + const options: DispatchOptions = { + tenantId, + // Always consult the ledger: another queue message of the same delivery + // (a sender redelivery) may already have sent some targets. + skipTarget: targetDoneChecker(env, key), + // Keep the lease alive during a long dispatch; once it is lost, stop + // sending (the new holder takes over the remaining targets). + beforeTarget: async () => { + if (leaseLost || leaseUncertain || outOfTime) return false; + if (spent(budget)) { + outOfTime = true; + return false; + } + budget.sendsLeft--; + if (Date.now() - renewedAt < LEASE_RENEW_MS) return true; + // Routes run in parallel: they share one renewal. Two renewals with + // the same token would race, and the loser would take its own + // successful renewal for a lost lease. + renewing ??= (async (): Promise => { + try { + token = await renewLease(env, key, token); + renewedAt = Date.now(); + leaseLost = token == null && lease.token != null; + return !leaseLost; + } catch (err) { + log.warn({ deliveryId: body.deliveryId, err }, "Lease renewal failed"); + leaseUncertain = true; + return false; + } finally { + renewing = null; + } + })(); + return renewing; + }, + afterTarget: async (targetKey) => { + await markTargetsDone(env, key, [targetKey]); + recorded.add(targetKey); + }, + }; + const scoped = tenantId + ? { ...config, routes: config.routes.filter((r) => r.groupId === tenantId) } + : config; + summary = await dispatch(scoped, event, env, groups, options); } catch (err) { + if (err instanceof PayloadUnavailableError) { + // Nothing left to dispatch in this message: drop it, loudly, without + // retrying forever or pretending the delivery succeeded. The delivery + // stays open (not final): the sender's redelivery, with its own payload, + // can still deliver it. + log.error({ deliveryId: body.deliveryId, err: err.message }, "Queued payload unavailable"); + // On record where admins look (the group's send log and failure + // metrics): the event needs a redelivery from its sender. + await recordSendBatch(env.DB, [ + { + ts: Date.now(), + routeId: "", + groupId: body.groupId, + groupInstance: body.groupInstance, + event: body.event, + target: "", + ok: false, + error: `${err.message}; redeliver the webhook to send it`, + errorCode: "PAYLOAD_MISSING", + deliveryId: body.deliveryId, + }, + ]); + await finish("retrying", "ack"); + return; + } log.error({ deliveryId: body.deliveryId, err }, "Queue delivery failed"); - await setDeliveryState(env, key, "retrying"); - message.retry({ delaySeconds: retryDelay(message.attempts) }); + await finish("retrying", "retry"); + return; } + + const { failed, retryable } = classifyDelivery(summary); + // Record finished targets first, whatever comes next: if a later write + // fails and the message is retried, these targets are never sent again. + const permanent = summary.failures.filter((f) => !isRetryableFailure(f)); + try { + await markTargetsDone( + env, + key, + [...(summary.succeeded ?? []), ...permanent.flatMap((f) => f.key ?? [])].filter( + (k) => !recorded.has(k), + ), + { permanentFailure: permanent.length > 0 }, + ); + } catch (err) { + // The outcome is not on record: retry (giving the lease back now) rather + // than finish the delivery with a forgotten permanent failure. + log.error({ deliveryId: body.deliveryId, err }, "Failed to record delivery targets"); + await finish("retrying", "retry"); + return; + } + if (failed && retryable) { + // Stopped for time, not for a failure: continued by a new message (sent + // before the lease is given back; a lease still held only delays it) — + // only when this attempt finished targets. One that made no progress + // (its targets kept failing until time ran out) is an ordinary retry and + // counts against the retry limit, so it cannot loop forever. + const progressed = (summary.succeeded?.length ?? 0) > 0 || permanent.length > 0; + if (outOfTime && progressed && !leaseLost && env.QUEUE) { + try { + await env.QUEUE.send(body, { delaySeconds: CONTINUE_DELAY_SECONDS }); + await finish("retrying", "ack"); + return; + } catch (err) { + log.warn({ deliveryId: body.deliveryId, err }, "Failed to queue the rest of a delivery"); + } + } + await finish("retrying", "retry"); + return; + } + // Targets that failed for good in an earlier attempt still count. + const anyPermanent = failed || (await hadPermanentFailure(env, key)); + await finish(anyPermanent ? "failed" : "delivered", "ack"); } -async function markDead(env: Env, body: DeliveryMessage): Promise { - const key = deliveryStateKey(body.provider, body.groupId, body.deliveryId); - await setDeliveryState(env, key, "dead"); +/** + * The delivery already finished under the state key used before deliveries + * were scoped by `@global` / group instance (`delivery-state:{provider}: + * {groupId | "global"}:{id}`, kept 7 days): a sender's redelivery of an event + * handled before the upgrade is then not sent again. The old key is only + * trusted where its owner is certain — for a tenant, the group that had the + * id back then (its `legacyId`); for the global endpoint, while no group named + * "global" (which used the same key) exists or was deleted or renamed away + * recently. + */ +async function finishedUnderLegacyKey(env: Env, body: DeliveryMessage): Promise { + // The global endpoint's old key was shared with any tenant once named + // "global" — even one deleted before the upgrade, which nothing records: + // its owner is never certain, so it is never trusted (a redelivery is sent + // again rather than an event dropped). Nothing to read for it, then. + if (!body.groupId || body.groupId === "global") return false; + // The store's one-time setup (seeding from KV, which also turns on reading + // older KV state) must have run before state is read: one group row reads + // it in, not the whole configuration (read below only when it matters). + const { group } = await loadGroupFresh(env, body.groupId); + // The message's own group, still under this id, with no older id of the + // same name (a group created since): nothing it could have finished under + // the old key, so that key is not read. + if ( + body.groupInstance && + group?.instance === body.groupInstance && + group.legacyId !== body.groupId + ) { + return false; + } + // Custom delivery ids got a namespace since (`d:`; `n:` for nonce + // identities, which did not exist before) — in messages queued since. A + // message queued before carries the id as the sender sent it, prefix-like + // text included. + const namespaced = body.provider === "custom" && (body.stateVersion ?? 0) >= 2; + if (namespaced && body.deliveryId.startsWith("n:")) return false; + const id = + namespaced && body.deliveryId.startsWith("d:") ? body.deliveryId.slice(2) : body.deliveryId; + const legacy = `delivery-state:${body.provider}:${body.groupId ?? "global"}:${id}`; + const state = await getDeliveryState(env, legacy); + if (!state || !FINAL.has(state)) return false; + // (A tenant named "global" shared this very key with the global endpoint: + // refused above.) + const { groups } = await loadConfigSnapshot(env, { fresh: true }); + const owner = body.groupInstance + ? groups.find((g) => g.instance === body.groupInstance) + : groups.find((g) => g.legacyId === body.groupId); + return owner?.legacyId === body.groupId; +} + +/** + * Settle a message that left the DLQ; false while another copy of the + * delivery holds a live lease (its state and payload are left alone then). + */ +async function markDead(env: Env, body: DeliveryMessage): Promise { + const key = deliveryStateKey(body.provider, body.groupId, body.deliveryId, body.groupInstance); + // A delivery another message of it finished (or is sending) keeps its state. + if ((await markDeliveryDead(env, key)) === "busy") return false; await discardPayload(env, body); + return true; } diff --git a/tests/queue-consumer.test.ts b/tests/queue-consumer.test.ts index 9199be5..bc160c9 100644 --- a/tests/queue-consumer.test.ts +++ b/tests/queue-consumer.test.ts @@ -4,12 +4,16 @@ import { invalidateConfigCache } from "../server/lib/config"; import { invalidateGroupsCache } from "../server/lib/web/groups"; import type { Env } from "../server/lib/types"; import type { DeliveryMessage, DispatchSummary } from "../server/lib/queue/delivery"; +import { createMemoryKV, createSqliteD1 } from "./d1-sqlite"; +import type { DispatchOptions } from "../server/lib/core/dispatch"; let summary: DispatchSummary; let dispatchCalls: number; +let lastArgs: unknown[]; -async function fakeDispatch(..._args: unknown[]): Promise { +async function fakeDispatch(...args: unknown[]): Promise { dispatchCalls += 1; + lastArgs = args; return summary; } @@ -136,4 +140,443 @@ describe("handleQueueBatch", () => { expect(msg.acked).toBe(true); expect(dispatchCalls).toBe(0); }); + + it("never records a lost parked payload as delivered, nor closes the delivery", async () => { + const { kv, store } = createMockKV(); + const msg = makeMessage( + body({ + payload: undefined, + payloadRef: "queue:payload:gone", + receivedAt: Date.now() - 3600_000, + }), + ); + await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), createEnv(kv), fakeDispatch); + expect(dispatchCalls).toBe(0); + expect(msg.acked).toBe(true); + // Left open: a redelivery carrying its own payload can still deliver it. + expect(JSON.parse(store.get(STATE_KEY)!)).toMatchObject({ status: "retrying" }); + const redelivery = makeMessage(body()); + await handleQueueBatch( + makeBatch("webhooker-delivery", [redelivery]), + createEnv(kv), + fakeDispatch, + ); + expect(dispatchCalls).toBe(1); + }); + + it("retries only the transient targets of a partially failed delivery", async () => { + const { kv, store } = createMockKV(); + const env = createEnv(kv); + summary = { + attempts: 3, + succeeded: ["g:r:discord:ok"], + failures: [ + { key: "g:r:discord:flaky", target: "flaky", errorCode: "DISCORD_5XX" }, + { key: "g:r:discord:bad", target: "bad", errorCode: "DISCORD_ERROR" }, + ], + }; + const first = makeMessage(body(), 1); + await handleQueueBatch(makeBatch("webhooker-delivery", [first]), env, fakeDispatch); + expect(first.retried).toEqual({ delaySeconds: 5 }); + // Every attempt consults the ledger (another message may have sent some). + expect(typeof (lastArgs[4] as { skipTarget?: unknown }).skipTarget).toBe("function"); + + const second = makeMessage(body(), 2); + summary = { attempts: 1, failures: [], succeeded: ["g:r:discord:flaky"] }; + await handleQueueBatch(makeBatch("webhooker-delivery", [second]), env, fakeDispatch); + const skip = (lastArgs[4] as { skipTarget: (k: string) => Promise }).skipTarget; + expect(await skip("g:r:discord:ok")).toBe(true); + expect(await skip("g:r:discord:bad")).toBe(true); + // Recorded too, now that the retry delivered it. + expect(await skip("g:r:discord:flaky")).toBe(true); + expect(second.acked).toBe(true); + // The 403 target never got the message: the delivery ends as failed. + expect(JSON.parse(store.get(STATE_KEY)!)).toMatchObject({ status: "failed" }); + }); + + it("follows a tenant group renamed while the delivery was queued", async () => { + const { kv, store } = createMemoryKV(); + store.set("config:groups", JSON.stringify([{ id: "new-id", name: "G", adminIds: [] }])); + store.set( + "config:routes", + JSON.stringify([ + { id: "r1", name: "R", enabled: true, filters: [], targets: [], groupId: "new-id" }, + ]), + ); + store.set("group:renamed:old-id", "new-id"); + const msg = makeMessage(body({ groupId: "old-id" })); + await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), createEnv(kv), fakeDispatch); + const config = lastArgs[0] as { routes: Array<{ id: string }> }; + expect(config.routes.map((r) => r.id)).toEqual(["r1"]); + expect((lastArgs[4] as { tenantId?: string }).tenantId).toBe("new-id"); + }); +}); + +describe("handleQueueBatch edge cases", () => { + beforeEach(() => { + summary = { attempts: 0, failures: [] }; + dispatchCalls = 0; + }); + + it("waits out a busy lease with a single long retry", async () => { + const { kv, store } = createMockKV(); + store.set(STATE_KEY, JSON.stringify({ status: "processing", at: Date.now() })); + // KV leases are best effort: emulate the D1 answer with a fake. + const env = { + ...createEnv(kv), + DB: { + prepare: (sql: string): unknown => { + const stmt = { + run: async (): Promise => ({ meta: { changes: 0 } }), + // Delivery state answers "processing"; the configuration is empty. + first: async (): Promise => + sql.includes("delivery_state") ? { status: "processing" } : null, + all: async (): Promise => ({ results: [] }), + }; + return { ...stmt, bind: (): unknown => stmt }; + }, + batch: async (stmts: unknown[]) => stmts.map(() => ({ results: [], meta: { changes: 0 } })), + } as unknown as D1Database, + }; + const msg = makeMessage(body()); + await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, fakeDispatch); + expect(dispatchCalls).toBe(0); + expect(msg.retried).toEqual({ delaySeconds: 310 }); + }); + + it("does not reopen a delivered delivery when payload cleanup fails", async () => { + const { kv, store } = createMockKV(); + const failingDelete = { + ...kv, + delete: async (): Promise => Promise.reject(new Error("KV down")), + }; + store.set("queue:payload:x", JSON.stringify({ a: 1 })); + summary = { attempts: 1, failures: [] }; + const msg = makeMessage( + body({ payload: undefined, payloadRef: "queue:payload:x", payloadRefType: "kv" }), + ); + await handleQueueBatch( + makeBatch("webhooker-delivery", [msg]), + createEnv(failingDelete as unknown as KVNamespace), + fakeDispatch, + ); + expect(msg.acked).toBe(true); + expect(msg.retried).toBeNull(); + expect(JSON.parse(store.get(STATE_KEY)!)).toMatchObject({ status: "delivered" }); + }); + + it("marks a delivery for a deleted tenant group as failed", async () => { + const { kv, store } = createMemoryKV(); + store.set("config:groups", JSON.stringify([])); + const msg = makeMessage(body({ groupId: "gone" })); + await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), createEnv(kv), fakeDispatch); + expect(dispatchCalls).toBe(0); + expect(msg.acked).toBe(true); + expect(JSON.parse(store.get("delivery-state:github:gone:d1")!)).toMatchObject({ + status: "failed", + }); + }); +}); + +describe("tenant resolution by group instance", () => { + beforeEach(() => { + summary = { attempts: 1, failures: [] }; + dispatchCalls = 0; + }); + + it("follows the instance across renames and ignores a re-created id", async () => { + const { kv, store } = createMemoryKV(); + // Identities as the server assigned them (after the one-time migration). + store.set("config:identity-migrated", "1"); + store.set( + "config:groups", + JSON.stringify([ + { id: "team", name: "T", adminIds: [], instance: "new-one" }, + { id: "moved", name: "M", adminIds: [], instance: "orig" }, + ]), + ); + store.set( + "config:routes", + JSON.stringify([ + { id: "r1", name: "R", enabled: true, filters: [], targets: [], groupId: "moved" }, + { id: "r2", name: "R", enabled: true, filters: [], targets: [], groupId: "team" }, + ]), + ); + const msg = makeMessage(body({ groupId: "team", groupInstance: "orig" })); + await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), createEnv(kv), fakeDispatch); + const config = lastArgs[0] as { routes: Array<{ id: string }> }; + // Delivered to the renamed original group, not to the new "team". + expect(config.routes.map((r) => r.id)).toEqual(["r1"]); + }); + + it("shares one lease renewal between routes running in parallel", async () => { + const { db } = createSqliteD1(); + const { kv } = createMemoryKV(); + const env: Env = { GITHUB_WEBHOOK_SECRET: "s", KV: kv, DB: db }; + let answers: boolean[] = []; + const dispatch = async (...args: unknown[]): Promise => { + const opts = args[4] as DispatchOptions; + const realNow = Date.now; + // Both routes reach their next target once a renewal is due. + Date.now = (): number => realNow() + 90_000; + try { + answers = await Promise.all([opts.beforeTarget!(), opts.beforeTarget!()]); + } finally { + Date.now = realNow; + } + return { attempts: 0, failures: [] }; + }; + const msg = makeMessage(body()); + await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, dispatch); + // Neither route takes the shared renewal for a lost lease. + expect(answers).toEqual([true, true]); + expect(msg.acked).toBe(true); + const row = await db + .prepare("SELECT status FROM delivery_state WHERE key = ?") + .bind("delivery-state:github:@global:d1") + .first<{ status: string }>(); + expect(row?.status).toBe("delivered"); + }); + + it("never acks when its own renewal went through unseen", async () => { + const { db: real } = createSqliteD1(); + // The renewal commits, but its answer is lost. + const db = { + ...real, + batch: real.batch.bind(real), + prepare: (sql: string): D1PreparedStatement => { + const stmt = real.prepare(sql); + if (!sql.includes("UPDATE delivery_state SET updated_at = ?")) return stmt; + return { + ...stmt, + bind: (...args: unknown[]) => { + const bound = stmt.bind(...args); + return { + ...bound, + run: async () => { + await bound.run(); + throw new Error("network"); + }, + } as D1PreparedStatement; + }, + } as D1PreparedStatement; + }, + } as D1Database; + const env: Env = { GITHUB_WEBHOOK_SECRET: "s", KV: createMemoryKV().kv, DB: db }; + const dispatch = async (...args: unknown[]): Promise => { + const opts = args[4] as DispatchOptions; + const realNow = Date.now; + Date.now = (): number => realNow() + 90_000; + try { + expect(await opts.beforeTarget!()).toBe(false); + } finally { + Date.now = realNow; + } + return { + attempts: 1, + failures: [{ key: "k", target: "t", error: "Delivery lease lost; left to its new holder" }], + }; + }; + const msg = makeMessage(body()); + await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, dispatch); + // No new holder exists: the message comes back once the lease surely expired. + expect(msg.acked).toBe(false); + expect(msg.retried).toEqual({ delaySeconds: 310 }); + }); + + it("never hands a pre-instance message to a group renamed onto its id", async () => { + const { kv, store } = createMemoryKV(); + // Identities as the server assigned them (after the one-time migration). + store.set("config:identity-migrated", "1"); + // "team" is held by a pre-instance group that had another id back then. + store.set( + "config:groups", + JSON.stringify([ + { + id: "team", + name: "T", + adminIds: [], + instance: "b", + instanceBackfilled: true, + legacyId: "other", + }, + ]), + ); + store.set("config:routes", JSON.stringify([])); + const msg = makeMessage(body({ groupId: "team" })); + await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), createEnv(kv), fakeDispatch); + expect(dispatchCalls).toBe(0); + expect(msg.acked).toBe(true); + }); + + it("never hands a pre-instance message to a group created since", async () => { + const { kv, store } = createMemoryKV(); + // Identities as the server assigned them (after the one-time migration). + store.set("config:identity-migrated", "1"); + // "team" now belongs to a group created after instances existed. + store.set( + "config:groups", + JSON.stringify([{ id: "team", name: "T", adminIds: [], instance: "fresh" }]), + ); + store.set( + "config:routes", + JSON.stringify([ + { id: "r2", name: "R", enabled: true, filters: [], targets: [], groupId: "team" }, + ]), + ); + const msg = makeMessage(body({ groupId: "team" })); + await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), createEnv(kv), fakeDispatch); + expect(dispatchCalls).toBe(0); + expect(msg.acked).toBe(true); + expect(JSON.parse(store.get("delivery-state:github:team:d1")!)).toMatchObject({ + status: "failed", + }); + }); +}); + +describe("consumer time budget", () => { + beforeEach(() => { + invalidateConfigCache(); + invalidateGroupsCache(); + dispatchCalls = 0; + }); + + it("gives a message it has no time left for back to the queue, uncopied", async () => { + const { kv } = createMockKV(); + const sent: unknown[] = []; + const env: Env = { + ...createEnv(kv), + QUEUE: { send: async (b: unknown) => void sent.push(b) } as unknown as Queue, + }; + const msg = makeMessage(body()); + await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, fakeDispatch, 0); + expect(dispatchCalls).toBe(0); + // Back to the queue as it is (one queue read), not copied (a write and a + // delete more). + expect(sent).toHaveLength(0); + expect(msg.retried).not.toBeNull(); + }); + + it("never spends a waiting message's last retry: it is copied instead", async () => { + const { kv } = createMockKV(); + const sent: unknown[] = []; + const env: Env = { + ...createEnv(kv), + QUEUE: { send: async (b: unknown) => void sent.push(b) } as unknown as Queue, + }; + // Left for later on every earlier attempt (max_retries is 5). + const msg = makeMessage(body(), 6); + await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, fakeDispatch, 0); + expect(dispatchCalls).toBe(0); + expect(sent).toHaveLength(1); + expect(msg.acked).toBe(true); + expect(msg.retried).toBeNull(); + }); + + it("keeps the retry count of a started delivery that has to wait", async () => { + const { kv, store } = createMockKV(); + // Taken and failed before: its attempts were real failures. + store.set(STATE_KEY, JSON.stringify({ status: "retrying", at: Date.now() })); + const sent: unknown[] = []; + const env: Env = { + ...createEnv(kv), + QUEUE: { send: async (b: unknown) => void sent.push(b) } as unknown as Queue, + }; + const msg = makeMessage(body(), 6); + await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, fakeDispatch, 0); + // Not copied (a copy would start its failure count over): retried. + expect(sent).toHaveLength(0); + expect(msg.retried).not.toBeNull(); + }); + + it("continues a delivery that ran out of time in a new message", async () => { + const { kv } = createMockKV(); + const sent: unknown[] = []; + const env: Env = { + ...createEnv(kv), + QUEUE: { send: async (b: unknown) => void sent.push(b) } as unknown as Queue, + }; + const slowDispatch = async (...args: unknown[]): Promise => { + const options = args[4] as DispatchOptions; + await new Promise((r) => setTimeout(r, 60)); + expect(await options.beforeTarget!()).toBe(false); + return { + attempts: 2, + succeeded: ["k0"], + failures: [{ key: "k", target: "t", error: "paused" }], + }; + }; + const msg = makeMessage(body()); + await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, slowDispatch, 30); + expect(sent).toHaveLength(1); + expect(msg.acked).toBe(true); + expect(msg.retried).toBeNull(); + }); +}); + +describe("consumer time budget without progress", () => { + it("retries (and counts) a delivery that ran out of time without finishing a target", async () => { + invalidateConfigCache(); + invalidateGroupsCache(); + const { kv } = createMockKV(); + const sent: unknown[] = []; + const env: Env = { + ...createEnv(kv), + QUEUE: { send: async (b: unknown) => void sent.push(b) } as unknown as Queue, + }; + const stuck = async (...args: unknown[]): Promise => { + const options = args[4] as DispatchOptions; + await new Promise((r) => setTimeout(r, 60)); + await options.beforeTarget!(); + return { attempts: 1, failures: [{ key: "k", target: "t", error: "timeout" }] }; + }; + const msg = makeMessage(body()); + await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, stuck, 30); + expect(sent).toHaveLength(0); + expect(msg.retried).not.toBeNull(); + }); + + it("reads a pre-upgrade custom delivery id as sent, prefix-like text included", async () => { + invalidateConfigCache(); + invalidateGroupsCache(); + const { db } = createSqliteD1(); + const { kv } = createMemoryKV(); + const env: Env = { GITHUB_WEBHOOK_SECRET: "s", KV: kv, DB: db }; + // The delivery "42" finished before the upgrade; "d:42" is another one. + const { setDeliveryState } = await import("../server/lib/queue/delivery"); + await setDeliveryState(env, "delivery-state:custom:global:42", "delivered"); + const msg = makeMessage(body({ provider: "custom", deliveryId: "d:42" })); + dispatchCalls = 0; + summary = { attempts: 0, failures: [] }; + await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, fakeDispatch); + expect(dispatchCalls).toBe(1); + }); +}); + +describe("consumer send budget", () => { + it("stops a batch after its send limit and continues the rest in a new message", async () => { + invalidateConfigCache(); + invalidateGroupsCache(); + const { kv } = createMockKV(); + const sent: unknown[] = []; + const env: Env = { + ...createEnv(kv), + QUEUE: { send: async (b: unknown) => void sent.push(b) } as unknown as Queue, + }; + let allowed = 0; + const wide = async (...args: unknown[]): Promise => { + const options = args[4] as DispatchOptions; + for (let i = 0; i < 40; i++) if (await options.beforeTarget!()) allowed++; + return { + attempts: 40, + succeeded: Array.from({ length: allowed }, (_, i) => `k${i}`), + failures: [{ key: "rest", target: "t", error: "paused" }], + }; + }; + const msg = makeMessage(body()); + await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, wide); + expect(allowed).toBe(20); + expect(sent).toHaveLength(1); + expect(msg.acked).toBe(true); + }); }); diff --git a/wrangler.jsonc b/wrangler.jsonc index 27f2a2d..e2dce82 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -51,7 +51,9 @@ "consumers": [ { "queue": "webhooker-delivery", - "max_batch_size": 10, + // Small batches: one invocation may make 50 D1 queries and 50 + // subrequests on Workers Free, and a delivery takes ~7 + its sends. + "max_batch_size": 3, "max_batch_timeout": 30, "max_retries": 5, "dead_letter_queue": "webhooker-delivery-dlq"