fix(queue): bound batches and resume deliveries

A queue invocation has shared query and subrequest limits.
Use a small batch and shared send budget, renew fenced leases and
resume only unfinished work without exhausting unstarted messages.
This commit is contained in:
翅膀 committed 2026-10-23 23:23:35 +08:00
1 parent 4c03543790
commit 4fc784ab9f
3 files changed
+808 -54

No files matched your search

+361 -52
View File
@@ -1,62 +1,191 @@
import type { Config, Env, Group, WebhookEvent, WebhookProvider } from "../types"; import type { Config, Env, Group, WebhookEvent, WebhookProvider } from "../types";
import { dispatchEvent } from "../core/dispatch"; import { dispatchEvent, type DispatchOptions } from "../core/dispatch";
import { loadConfig } from "../config"; import { loadConfigSnapshot, loadGroupFresh } from "../config";
import { loadGroups } from "../web/groups"; import { groupAliasChain } from "../web/group-lifecycle";
import { log } from "../lib/log"; import { log } from "../lib/log";
import { recordSendBatch } from "../lib/send-log-batch";
import { import {
DELIVERY_DLQ, DELIVERY_DLQ,
PayloadUnavailableError,
type DeliveryMessage, type DeliveryMessage,
type DispatchSummary, type DispatchSummary,
acquireDelivery,
classifyDelivery, classifyDelivery,
finishLease,
getDeliveryState,
renewLease,
type DeliveryStatus,
deliveryStateKey, deliveryStateKey,
discardPayload, discardPayload,
getDeliveryState, hadPermanentFailure,
isRetryableFailure,
markDeliveryDead,
markTargetsDone,
resolvePayload, resolvePayload,
retryDelay, retryDelay,
setDeliveryState, targetDoneChecker,
} from "./delivery"; } from "./delivery";
export async function handleQueueBatch( type Dispatch = (
batch: MessageBatch<DeliveryMessage>,
env: Env,
dispatch: (
config: Config, config: Config,
event: WebhookEvent, event: WebhookEvent,
env: Env, env: Env,
groups?: Group[], groups?: Group[],
) => Promise<DispatchSummary> = dispatchEvent, options?: DispatchOptions,
) => Promise<DispatchSummary>;
export async function handleQueueBatch(
batch: MessageBatch<DeliveryMessage>,
env: Env,
dispatch: Dispatch = dispatchEvent,
budgetMs = BATCH_BUDGET_MS,
): Promise<void> { ): Promise<void> {
// Shared by the whole batch: the limits it guards (D1 queries and
// subrequests) are per invocation.
const budget: BatchBudget = { deadline: Date.now() + budgetMs, sendsLeft: BATCH_SEND_LIMIT };
for (const message of batch.messages) { for (const message of batch.messages) {
const body = message.body; const body = message.body;
try {
if (batch.queue === DELIVERY_DLQ) { if (batch.queue === DELIVERY_DLQ) {
await markDead(env, body); if (await markDead(env, body)) message.ack();
message.ack(); // Another copy holds a live lease and may still need the payload.
else message.retry({ delaySeconds: BUSY_RETRY_SECONDS });
continue; continue;
} }
await processMessage(env, body, message, dispatch); // Out of time or sends before it started (earlier messages took them):
// back to the queue for a later invocation — a retry (one more queue
// read), not a copy (a write and a delete more). Waiting is not a
// failure, though: a delivery never started whose retry would use up
// its last one (sending it to the DLQ untried) is copied instead. One
// that was started has real failures among its attempts; a copy would
// start their count over, so it keeps the retry.
if (spent(budget)) {
if (message.attempts > QUEUE_MAX_RETRIES && env.QUEUE && (await neverStarted(env, body))) {
try {
await env.QUEUE.send(body, { delaySeconds: CONTINUE_DELAY_SECONDS });
message.ack();
continue;
} catch (err) {
log.warn({ deliveryId: body.deliveryId, err }, "Failed to queue a delivery for later");
} }
} }
message.retry({ delaySeconds: CONTINUE_DELAY_SECONDS });
continue;
}
await processMessage(env, body, message, dispatch, budget);
} catch (err) {
// One message's storage failure must not fail (and replay) the batch.
log.error({ deliveryId: body.deliveryId, err }, "Queue message handling failed");
message.retry({ delaySeconds: retryDelay(message.attempts) });
}
}
}
/** Renew a held lease when it is older than this (the lease lasts five minutes). */
const LEASE_RENEW_MS = 60_000;
const FINAL: ReadonlySet<DeliveryStatus> = new Set(["delivered", "failed", "dead"]);
/** Wait for an unexpired lease of another consumer: its full length plus a margin. */
const BUSY_RETRY_SECONDS = 310;
/**
* Sending time of one batch: a consumer invocation may run 15 minutes, and a
* target takes under a minute at worst (bounded platform calls), so sending
* stops in time to record what was done.
*/
const BATCH_BUDGET_MS = 12 * 60_000;
/**
* Targets one batch sends at most: each costs D1 queries (its ledger record,
* an update chain's lock and tracking) and subrequests, and one invocation
* may make only 50 of each on Workers Free (with ~7 D1 queries per delivery
* besides; batches are small, see wrangler.jsonc). A message left unstarted
* goes back to the queue (a retry: one queue read); a message cut short is
* continued by a new one only when it made progress.
*/
const BATCH_SEND_LIMIT = 20;
/** What a batch may still spend: its time and its sends. */
interface BatchBudget {
deadline: number;
sendsLeft: number;
}
function spent(budget: BatchBudget): boolean {
return Date.now() >= budget.deadline || budget.sendsLeft <= 0;
}
const CONTINUE_DELAY_SECONDS = 5;
/** The delivery queue's `max_retries` (`wrangler.jsonc`). */
const QUEUE_MAX_RETRIES = 5;
/** No attempt has taken the delivery's lease yet (its attempts were all waits). */
async function neverStarted(env: Env, body: DeliveryMessage): Promise<boolean> {
const key = deliveryStateKey(body.provider, body.groupId, body.deliveryId, body.groupInstance);
const state = await getDeliveryState(env, key);
return state == null || state === "pending";
}
/** Clean up after a terminal state; a failure must not reopen the delivery. */
async function discardQuietly(env: Env, body: DeliveryMessage): Promise<void> {
await discardPayload(env, body).catch((err) =>
log.warn({ deliveryId: body.deliveryId, err }, "Failed to discard a parked payload"),
);
}
async function processMessage( async function processMessage(
env: Env, env: Env,
body: DeliveryMessage, body: DeliveryMessage,
message: Message<DeliveryMessage>, message: Message<DeliveryMessage>,
dispatch: ( dispatch: Dispatch,
config: Config, budget: BatchBudget,
event: WebhookEvent,
env: Env,
groups?: Group[],
) => Promise<DispatchSummary>,
): Promise<void> { ): Promise<void> {
const key = deliveryStateKey(body.provider, body.groupId, body.deliveryId); const key = deliveryStateKey(body.provider, body.groupId, body.deliveryId, body.groupInstance);
const prior = await getDeliveryState(env, key); if (await finishedUnderLegacyKey(env, body)) {
if (prior === "delivered" || prior === "dead") {
message.ack(); message.ack();
return; return;
} }
const lease = await acquireDelivery(env, key);
if (lease.status === "finished") {
message.ack();
return;
}
if (lease.status === "busy") {
// Another consumer holds this delivery (or a holder vanished): one retry
// after the lease surely expired, instead of spending the retry budget
// polling a lease that is still valid.
message.retry({ delaySeconds: BUSY_RETRY_SECONDS });
return;
}
await setDeliveryState(env, key, "processing"); let token = lease.token;
let renewedAt = Date.now();
let leaseLost = false;
/** A renewal failed with an unknown outcome: stop sending, but never assume a new holder. */
let leaseUncertain = false;
let renewing: Promise<boolean> | null = null;
/** The batch's budget ran out: the rest is left to a continuation message. */
let outOfTime = false;
/** End this attempt with `status`; acks instead when the lease was lost. */
const finish = async (status: DeliveryStatus, then: "ack" | "retry"): Promise<void> => {
const owned = !leaseLost && (await finishLease(env, key, token, status));
if (owned && then === "retry") {
message.retry({ delaySeconds: retryDelay(message.attempts) });
return;
}
if (!owned && !leaseLost) {
// The outcome could not be written, yet no new holder is known (our own
// renewal may have gone through unseen, or the lease lapsed unnoticed):
// come back once any lease surely expired; the state then decides.
message.retry({ delaySeconds: BUSY_RETRY_SECONDS });
return;
}
// Lost lease: the new holder owns the delivery (and its outcome).
message.ack();
if (owned && FINAL.has(status)) await discardQuietly(env, body);
};
let summary: DispatchSummary;
// Targets recorded as sent the moment they were sent (see `afterTarget`).
const recorded = new Set<string>();
try {
const payload = await resolvePayload(env, body); const payload = await resolvePayload(env, body);
const event: WebhookEvent = { const event: WebhookEvent = {
event: body.event, event: body.event,
@@ -65,39 +194,219 @@ async function processMessage(
provider: body.provider as WebhookProvider, provider: body.provider as WebhookProvider,
installationId: body.installationId, installationId: body.installationId,
}; };
// Read fresh, not from this isolate's cache: a group (or route) created
// moments ago elsewhere must not be taken for missing — for a tenant
// delivery that verdict is final.
const { config, groups } = await loadConfigSnapshot(env, { fresh: true });
// A tenant delivery belongs to the group instance that received it — under
// whatever id it has now — and never to another group that took over the
// id since. A message from before instances belongs to the group that had
// its id back then (`legacyId`), also when that group was renamed before
// instances existed (followed through the rename aliases) — never to a
// group created since or renamed to the id.
let tenantId = body.groupId;
if (tenantId && body.groupInstance) {
tenantId = groups.find((g) => g.instance === body.groupInstance)?.id;
} else if (tenantId) {
const owner = (id: string): string | undefined => groups.find((g) => g.legacyId === id)?.id;
let found = owner(tenantId);
for (const id of found ? [] : (await groupAliasChain(env, tenantId)).slice(1)) {
found ??= owner(id);
}
tenantId = found;
}
if (body.groupId && (!tenantId || !groups.some((g) => g.id === tenantId))) {
// A group row that cannot be read may be the tenant: retry until it is
// repaired rather than give the delivery up as one of a deleted group.
if (config.unreadableGroups?.length) {
log.error(
{ deliveryId: body.deliveryId, unreadable: config.unreadableGroups },
"Tenant unresolved while some group configuration is unreadable",
);
await finish("retrying", "retry");
return;
}
// The group was deleted: nothing can receive the event any more.
log.warn({ deliveryId: body.deliveryId, groupId: body.groupId }, "Tenant group is gone");
await finish("failed", "ack");
return;
}
const options: DispatchOptions = {
tenantId,
// Always consult the ledger: another queue message of the same delivery
// (a sender redelivery) may already have sent some targets.
skipTarget: targetDoneChecker(env, key),
// Keep the lease alive during a long dispatch; once it is lost, stop
// sending (the new holder takes over the remaining targets).
beforeTarget: async () => {
if (leaseLost || leaseUncertain || outOfTime) return false;
if (spent(budget)) {
outOfTime = true;
return false;
}
budget.sendsLeft--;
if (Date.now() - renewedAt < LEASE_RENEW_MS) return true;
// Routes run in parallel: they share one renewal. Two renewals with
// the same token would race, and the loser would take its own
// successful renewal for a lost lease.
renewing ??= (async (): Promise<boolean> => {
try { try {
const config = await loadConfig(env); token = await renewLease(env, key, token);
if (body.groupId) { renewedAt = Date.now();
config.routes = config.routes.filter((r) => r.groupId === body.groupId); leaseLost = token == null && lease.token != null;
} return !leaseLost;
const groups = await loadGroups(env.KV);
const summary = await dispatch(config, event, env, groups);
const { failed, retryable } = classifyDelivery(summary);
if (!failed) {
await setDeliveryState(env, key, "delivered");
await discardPayload(env, body);
message.ack();
return;
}
if (retryable) {
await setDeliveryState(env, key, "retrying");
message.retry({ delaySeconds: retryDelay(message.attempts) });
return;
}
await setDeliveryState(env, key, "failed");
await discardPayload(env, body);
message.ack();
} catch (err) { } catch (err) {
log.error({ deliveryId: body.deliveryId, err }, "Queue delivery failed"); log.warn({ deliveryId: body.deliveryId, err }, "Lease renewal failed");
await setDeliveryState(env, key, "retrying"); leaseUncertain = true;
message.retry({ delaySeconds: retryDelay(message.attempts) }); return false;
} finally {
renewing = null;
} }
})();
return renewing;
},
afterTarget: async (targetKey) => {
await markTargetsDone(env, key, [targetKey]);
recorded.add(targetKey);
},
};
const scoped = tenantId
? { ...config, routes: config.routes.filter((r) => r.groupId === tenantId) }
: config;
summary = await dispatch(scoped, event, env, groups, options);
} catch (err) {
if (err instanceof PayloadUnavailableError) {
// Nothing left to dispatch in this message: drop it, loudly, without
// retrying forever or pretending the delivery succeeded. The delivery
// stays open (not final): the sender's redelivery, with its own payload,
// can still deliver it.
log.error({ deliveryId: body.deliveryId, err: err.message }, "Queued payload unavailable");
// On record where admins look (the group's send log and failure
// metrics): the event needs a redelivery from its sender.
await recordSendBatch(env.DB, [
{
ts: Date.now(),
routeId: "",
groupId: body.groupId,
groupInstance: body.groupInstance,
event: body.event,
target: "",
ok: false,
error: `${err.message}; redeliver the webhook to send it`,
errorCode: "PAYLOAD_MISSING",
deliveryId: body.deliveryId,
},
]);
await finish("retrying", "ack");
return;
}
log.error({ deliveryId: body.deliveryId, err }, "Queue delivery failed");
await finish("retrying", "retry");
return;
} }
async function markDead(env: Env, body: DeliveryMessage): Promise<void> { const { failed, retryable } = classifyDelivery(summary);
const key = deliveryStateKey(body.provider, body.groupId, body.deliveryId); // Record finished targets first, whatever comes next: if a later write
await setDeliveryState(env, key, "dead"); // fails and the message is retried, these targets are never sent again.
const permanent = summary.failures.filter((f) => !isRetryableFailure(f));
try {
await markTargetsDone(
env,
key,
[...(summary.succeeded ?? []), ...permanent.flatMap((f) => f.key ?? [])].filter(
(k) => !recorded.has(k),
),
{ permanentFailure: permanent.length > 0 },
);
} catch (err) {
// The outcome is not on record: retry (giving the lease back now) rather
// than finish the delivery with a forgotten permanent failure.
log.error({ deliveryId: body.deliveryId, err }, "Failed to record delivery targets");
await finish("retrying", "retry");
return;
}
if (failed && retryable) {
// Stopped for time, not for a failure: continued by a new message (sent
// before the lease is given back; a lease still held only delays it) —
// only when this attempt finished targets. One that made no progress
// (its targets kept failing until time ran out) is an ordinary retry and
// counts against the retry limit, so it cannot loop forever.
const progressed = (summary.succeeded?.length ?? 0) > 0 || permanent.length > 0;
if (outOfTime && progressed && !leaseLost && env.QUEUE) {
try {
await env.QUEUE.send(body, { delaySeconds: CONTINUE_DELAY_SECONDS });
await finish("retrying", "ack");
return;
} catch (err) {
log.warn({ deliveryId: body.deliveryId, err }, "Failed to queue the rest of a delivery");
}
}
await finish("retrying", "retry");
return;
}
// Targets that failed for good in an earlier attempt still count.
const anyPermanent = failed || (await hadPermanentFailure(env, key));
await finish(anyPermanent ? "failed" : "delivered", "ack");
}
/**
* The delivery already finished under the state key used before deliveries
* were scoped by `@global` / group instance (`delivery-state:{provider}:
* {groupId | "global"}:{id}`, kept 7 days): a sender's redelivery of an event
* handled before the upgrade is then not sent again. The old key is only
* trusted where its owner is certain — for a tenant, the group that had the
* id back then (its `legacyId`); for the global endpoint, while no group named
* "global" (which used the same key) exists or was deleted or renamed away
* recently.
*/
async function finishedUnderLegacyKey(env: Env, body: DeliveryMessage): Promise<boolean> {
// The global endpoint's old key was shared with any tenant once named
// "global" — even one deleted before the upgrade, which nothing records:
// its owner is never certain, so it is never trusted (a redelivery is sent
// again rather than an event dropped). Nothing to read for it, then.
if (!body.groupId || body.groupId === "global") return false;
// The store's one-time setup (seeding from KV, which also turns on reading
// older KV state) must have run before state is read: one group row reads
// it in, not the whole configuration (read below only when it matters).
const { group } = await loadGroupFresh(env, body.groupId);
// The message's own group, still under this id, with no older id of the
// same name (a group created since): nothing it could have finished under
// the old key, so that key is not read.
if (
body.groupInstance &&
group?.instance === body.groupInstance &&
group.legacyId !== body.groupId
) {
return false;
}
// Custom delivery ids got a namespace since (`d:`; `n:` for nonce
// identities, which did not exist before) — in messages queued since. A
// message queued before carries the id as the sender sent it, prefix-like
// text included.
const namespaced = body.provider === "custom" && (body.stateVersion ?? 0) >= 2;
if (namespaced && body.deliveryId.startsWith("n:")) return false;
const id =
namespaced && body.deliveryId.startsWith("d:") ? body.deliveryId.slice(2) : body.deliveryId;
const legacy = `delivery-state:${body.provider}:${body.groupId ?? "global"}:${id}`;
const state = await getDeliveryState(env, legacy);
if (!state || !FINAL.has(state)) return false;
// (A tenant named "global" shared this very key with the global endpoint:
// refused above.)
const { groups } = await loadConfigSnapshot(env, { fresh: true });
const owner = body.groupInstance
? groups.find((g) => g.instance === body.groupInstance)
: groups.find((g) => g.legacyId === body.groupId);
return owner?.legacyId === body.groupId;
}
/**
* Settle a message that left the DLQ; false while another copy of the
* delivery holds a live lease (its state and payload are left alone then).
*/
async function markDead(env: Env, body: DeliveryMessage): Promise<boolean> {
const key = deliveryStateKey(body.provider, body.groupId, body.deliveryId, body.groupInstance);
// A delivery another message of it finished (or is sending) keeps its state.
if ((await markDeliveryDead(env, key)) === "busy") return false;
await discardPayload(env, body); await discardPayload(env, body);
return true;
} }
+444 -1
View File
@@ -4,12 +4,16 @@ import { invalidateConfigCache } from "../server/lib/config";
import { invalidateGroupsCache } from "../server/lib/web/groups"; import { invalidateGroupsCache } from "../server/lib/web/groups";
import type { Env } from "../server/lib/types"; import type { Env } from "../server/lib/types";
import type { DeliveryMessage, DispatchSummary } from "../server/lib/queue/delivery"; import type { DeliveryMessage, DispatchSummary } from "../server/lib/queue/delivery";
import { createMemoryKV, createSqliteD1 } from "./d1-sqlite";
import type { DispatchOptions } from "../server/lib/core/dispatch";
let summary: DispatchSummary; let summary: DispatchSummary;
let dispatchCalls: number; let dispatchCalls: number;
let lastArgs: unknown[];
async function fakeDispatch(..._args: unknown[]): Promise<DispatchSummary> { async function fakeDispatch(...args: unknown[]): Promise<DispatchSummary> {
dispatchCalls += 1; dispatchCalls += 1;
lastArgs = args;
return summary; return summary;
} }
@@ -136,4 +140,443 @@ describe("handleQueueBatch", () => {
expect(msg.acked).toBe(true); expect(msg.acked).toBe(true);
expect(dispatchCalls).toBe(0); expect(dispatchCalls).toBe(0);
}); });
it("never records a lost parked payload as delivered, nor closes the delivery", async () => {
const { kv, store } = createMockKV();
const msg = makeMessage(
body({
payload: undefined,
payloadRef: "queue:payload:gone",
receivedAt: Date.now() - 3600_000,
}),
);
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), createEnv(kv), fakeDispatch);
expect(dispatchCalls).toBe(0);
expect(msg.acked).toBe(true);
// Left open: a redelivery carrying its own payload can still deliver it.
expect(JSON.parse(store.get(STATE_KEY)!)).toMatchObject({ status: "retrying" });
const redelivery = makeMessage(body());
await handleQueueBatch(
makeBatch("webhooker-delivery", [redelivery]),
createEnv(kv),
fakeDispatch,
);
expect(dispatchCalls).toBe(1);
});
it("retries only the transient targets of a partially failed delivery", async () => {
const { kv, store } = createMockKV();
const env = createEnv(kv);
summary = {
attempts: 3,
succeeded: ["g:r:discord:ok"],
failures: [
{ key: "g:r:discord:flaky", target: "flaky", errorCode: "DISCORD_5XX" },
{ key: "g:r:discord:bad", target: "bad", errorCode: "DISCORD_ERROR" },
],
};
const first = makeMessage(body(), 1);
await handleQueueBatch(makeBatch("webhooker-delivery", [first]), env, fakeDispatch);
expect(first.retried).toEqual({ delaySeconds: 5 });
// Every attempt consults the ledger (another message may have sent some).
expect(typeof (lastArgs[4] as { skipTarget?: unknown }).skipTarget).toBe("function");
const second = makeMessage(body(), 2);
summary = { attempts: 1, failures: [], succeeded: ["g:r:discord:flaky"] };
await handleQueueBatch(makeBatch("webhooker-delivery", [second]), env, fakeDispatch);
const skip = (lastArgs[4] as { skipTarget: (k: string) => Promise<boolean> }).skipTarget;
expect(await skip("g:r:discord:ok")).toBe(true);
expect(await skip("g:r:discord:bad")).toBe(true);
// Recorded too, now that the retry delivered it.
expect(await skip("g:r:discord:flaky")).toBe(true);
expect(second.acked).toBe(true);
// The 403 target never got the message: the delivery ends as failed.
expect(JSON.parse(store.get(STATE_KEY)!)).toMatchObject({ status: "failed" });
});
it("follows a tenant group renamed while the delivery was queued", async () => {
const { kv, store } = createMemoryKV();
store.set("config:groups", JSON.stringify([{ id: "new-id", name: "G", adminIds: [] }]));
store.set(
"config:routes",
JSON.stringify([
{ id: "r1", name: "R", enabled: true, filters: [], targets: [], groupId: "new-id" },
]),
);
store.set("group:renamed:old-id", "new-id");
const msg = makeMessage(body({ groupId: "old-id" }));
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), createEnv(kv), fakeDispatch);
const config = lastArgs[0] as { routes: Array<{ id: string }> };
expect(config.routes.map((r) => r.id)).toEqual(["r1"]);
expect((lastArgs[4] as { tenantId?: string }).tenantId).toBe("new-id");
});
});
describe("handleQueueBatch edge cases", () => {
beforeEach(() => {
summary = { attempts: 0, failures: [] };
dispatchCalls = 0;
});
it("waits out a busy lease with a single long retry", async () => {
const { kv, store } = createMockKV();
store.set(STATE_KEY, JSON.stringify({ status: "processing", at: Date.now() }));
// KV leases are best effort: emulate the D1 answer with a fake.
const env = {
...createEnv(kv),
DB: {
prepare: (sql: string): unknown => {
const stmt = {
run: async (): Promise<unknown> => ({ meta: { changes: 0 } }),
// Delivery state answers "processing"; the configuration is empty.
first: async (): Promise<unknown> =>
sql.includes("delivery_state") ? { status: "processing" } : null,
all: async (): Promise<unknown> => ({ results: [] }),
};
return { ...stmt, bind: (): unknown => stmt };
},
batch: async (stmts: unknown[]) => stmts.map(() => ({ results: [], meta: { changes: 0 } })),
} as unknown as D1Database,
};
const msg = makeMessage(body());
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, fakeDispatch);
expect(dispatchCalls).toBe(0);
expect(msg.retried).toEqual({ delaySeconds: 310 });
});
it("does not reopen a delivered delivery when payload cleanup fails", async () => {
const { kv, store } = createMockKV();
const failingDelete = {
...kv,
delete: async (): Promise<never> => Promise.reject(new Error("KV down")),
};
store.set("queue:payload:x", JSON.stringify({ a: 1 }));
summary = { attempts: 1, failures: [] };
const msg = makeMessage(
body({ payload: undefined, payloadRef: "queue:payload:x", payloadRefType: "kv" }),
);
await handleQueueBatch(
makeBatch("webhooker-delivery", [msg]),
createEnv(failingDelete as unknown as KVNamespace),
fakeDispatch,
);
expect(msg.acked).toBe(true);
expect(msg.retried).toBeNull();
expect(JSON.parse(store.get(STATE_KEY)!)).toMatchObject({ status: "delivered" });
});
it("marks a delivery for a deleted tenant group as failed", async () => {
const { kv, store } = createMemoryKV();
store.set("config:groups", JSON.stringify([]));
const msg = makeMessage(body({ groupId: "gone" }));
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), createEnv(kv), fakeDispatch);
expect(dispatchCalls).toBe(0);
expect(msg.acked).toBe(true);
expect(JSON.parse(store.get("delivery-state:github:gone:d1")!)).toMatchObject({
status: "failed",
});
});
});
describe("tenant resolution by group instance", () => {
beforeEach(() => {
summary = { attempts: 1, failures: [] };
dispatchCalls = 0;
});
it("follows the instance across renames and ignores a re-created id", async () => {
const { kv, store } = createMemoryKV();
// Identities as the server assigned them (after the one-time migration).
store.set("config:identity-migrated", "1");
store.set(
"config:groups",
JSON.stringify([
{ id: "team", name: "T", adminIds: [], instance: "new-one" },
{ id: "moved", name: "M", adminIds: [], instance: "orig" },
]),
);
store.set(
"config:routes",
JSON.stringify([
{ id: "r1", name: "R", enabled: true, filters: [], targets: [], groupId: "moved" },
{ id: "r2", name: "R", enabled: true, filters: [], targets: [], groupId: "team" },
]),
);
const msg = makeMessage(body({ groupId: "team", groupInstance: "orig" }));
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), createEnv(kv), fakeDispatch);
const config = lastArgs[0] as { routes: Array<{ id: string }> };
// Delivered to the renamed original group, not to the new "team".
expect(config.routes.map((r) => r.id)).toEqual(["r1"]);
});
it("shares one lease renewal between routes running in parallel", async () => {
const { db } = createSqliteD1();
const { kv } = createMemoryKV();
const env: Env = { GITHUB_WEBHOOK_SECRET: "s", KV: kv, DB: db };
let answers: boolean[] = [];
const dispatch = async (...args: unknown[]): Promise<DispatchSummary> => {
const opts = args[4] as DispatchOptions;
const realNow = Date.now;
// Both routes reach their next target once a renewal is due.
Date.now = (): number => realNow() + 90_000;
try {
answers = await Promise.all([opts.beforeTarget!(), opts.beforeTarget!()]);
} finally {
Date.now = realNow;
}
return { attempts: 0, failures: [] };
};
const msg = makeMessage(body());
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, dispatch);
// Neither route takes the shared renewal for a lost lease.
expect(answers).toEqual([true, true]);
expect(msg.acked).toBe(true);
const row = await db
.prepare("SELECT status FROM delivery_state WHERE key = ?")
.bind("delivery-state:github:@global:d1")
.first<{ status: string }>();
expect(row?.status).toBe("delivered");
});
it("never acks when its own renewal went through unseen", async () => {
const { db: real } = createSqliteD1();
// The renewal commits, but its answer is lost.
const db = {
...real,
batch: real.batch.bind(real),
prepare: (sql: string): D1PreparedStatement => {
const stmt = real.prepare(sql);
if (!sql.includes("UPDATE delivery_state SET updated_at = ?")) return stmt;
return {
...stmt,
bind: (...args: unknown[]) => {
const bound = stmt.bind(...args);
return {
...bound,
run: async () => {
await bound.run();
throw new Error("network");
},
} as D1PreparedStatement;
},
} as D1PreparedStatement;
},
} as D1Database;
const env: Env = { GITHUB_WEBHOOK_SECRET: "s", KV: createMemoryKV().kv, DB: db };
const dispatch = async (...args: unknown[]): Promise<DispatchSummary> => {
const opts = args[4] as DispatchOptions;
const realNow = Date.now;
Date.now = (): number => realNow() + 90_000;
try {
expect(await opts.beforeTarget!()).toBe(false);
} finally {
Date.now = realNow;
}
return {
attempts: 1,
failures: [{ key: "k", target: "t", error: "Delivery lease lost; left to its new holder" }],
};
};
const msg = makeMessage(body());
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, dispatch);
// No new holder exists: the message comes back once the lease surely expired.
expect(msg.acked).toBe(false);
expect(msg.retried).toEqual({ delaySeconds: 310 });
});
it("never hands a pre-instance message to a group renamed onto its id", async () => {
const { kv, store } = createMemoryKV();
// Identities as the server assigned them (after the one-time migration).
store.set("config:identity-migrated", "1");
// "team" is held by a pre-instance group that had another id back then.
store.set(
"config:groups",
JSON.stringify([
{
id: "team",
name: "T",
adminIds: [],
instance: "b",
instanceBackfilled: true,
legacyId: "other",
},
]),
);
store.set("config:routes", JSON.stringify([]));
const msg = makeMessage(body({ groupId: "team" }));
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), createEnv(kv), fakeDispatch);
expect(dispatchCalls).toBe(0);
expect(msg.acked).toBe(true);
});
it("never hands a pre-instance message to a group created since", async () => {
const { kv, store } = createMemoryKV();
// Identities as the server assigned them (after the one-time migration).
store.set("config:identity-migrated", "1");
// "team" now belongs to a group created after instances existed.
store.set(
"config:groups",
JSON.stringify([{ id: "team", name: "T", adminIds: [], instance: "fresh" }]),
);
store.set(
"config:routes",
JSON.stringify([
{ id: "r2", name: "R", enabled: true, filters: [], targets: [], groupId: "team" },
]),
);
const msg = makeMessage(body({ groupId: "team" }));
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), createEnv(kv), fakeDispatch);
expect(dispatchCalls).toBe(0);
expect(msg.acked).toBe(true);
expect(JSON.parse(store.get("delivery-state:github:team:d1")!)).toMatchObject({
status: "failed",
});
});
});
describe("consumer time budget", () => {
beforeEach(() => {
invalidateConfigCache();
invalidateGroupsCache();
dispatchCalls = 0;
});
it("gives a message it has no time left for back to the queue, uncopied", async () => {
const { kv } = createMockKV();
const sent: unknown[] = [];
const env: Env = {
...createEnv(kv),
QUEUE: { send: async (b: unknown) => void sent.push(b) } as unknown as Queue,
};
const msg = makeMessage(body());
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, fakeDispatch, 0);
expect(dispatchCalls).toBe(0);
// Back to the queue as it is (one queue read), not copied (a write and a
// delete more).
expect(sent).toHaveLength(0);
expect(msg.retried).not.toBeNull();
});
it("never spends a waiting message's last retry: it is copied instead", async () => {
const { kv } = createMockKV();
const sent: unknown[] = [];
const env: Env = {
...createEnv(kv),
QUEUE: { send: async (b: unknown) => void sent.push(b) } as unknown as Queue,
};
// Left for later on every earlier attempt (max_retries is 5).
const msg = makeMessage(body(), 6);
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, fakeDispatch, 0);
expect(dispatchCalls).toBe(0);
expect(sent).toHaveLength(1);
expect(msg.acked).toBe(true);
expect(msg.retried).toBeNull();
});
it("keeps the retry count of a started delivery that has to wait", async () => {
const { kv, store } = createMockKV();
// Taken and failed before: its attempts were real failures.
store.set(STATE_KEY, JSON.stringify({ status: "retrying", at: Date.now() }));
const sent: unknown[] = [];
const env: Env = {
...createEnv(kv),
QUEUE: { send: async (b: unknown) => void sent.push(b) } as unknown as Queue,
};
const msg = makeMessage(body(), 6);
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, fakeDispatch, 0);
// Not copied (a copy would start its failure count over): retried.
expect(sent).toHaveLength(0);
expect(msg.retried).not.toBeNull();
});
it("continues a delivery that ran out of time in a new message", async () => {
const { kv } = createMockKV();
const sent: unknown[] = [];
const env: Env = {
...createEnv(kv),
QUEUE: { send: async (b: unknown) => void sent.push(b) } as unknown as Queue,
};
const slowDispatch = async (...args: unknown[]): Promise<DispatchSummary> => {
const options = args[4] as DispatchOptions;
await new Promise((r) => setTimeout(r, 60));
expect(await options.beforeTarget!()).toBe(false);
return {
attempts: 2,
succeeded: ["k0"],
failures: [{ key: "k", target: "t", error: "paused" }],
};
};
const msg = makeMessage(body());
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, slowDispatch, 30);
expect(sent).toHaveLength(1);
expect(msg.acked).toBe(true);
expect(msg.retried).toBeNull();
});
});
describe("consumer time budget without progress", () => {
it("retries (and counts) a delivery that ran out of time without finishing a target", async () => {
invalidateConfigCache();
invalidateGroupsCache();
const { kv } = createMockKV();
const sent: unknown[] = [];
const env: Env = {
...createEnv(kv),
QUEUE: { send: async (b: unknown) => void sent.push(b) } as unknown as Queue,
};
const stuck = async (...args: unknown[]): Promise<DispatchSummary> => {
const options = args[4] as DispatchOptions;
await new Promise((r) => setTimeout(r, 60));
await options.beforeTarget!();
return { attempts: 1, failures: [{ key: "k", target: "t", error: "timeout" }] };
};
const msg = makeMessage(body());
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, stuck, 30);
expect(sent).toHaveLength(0);
expect(msg.retried).not.toBeNull();
});
it("reads a pre-upgrade custom delivery id as sent, prefix-like text included", async () => {
invalidateConfigCache();
invalidateGroupsCache();
const { db } = createSqliteD1();
const { kv } = createMemoryKV();
const env: Env = { GITHUB_WEBHOOK_SECRET: "s", KV: kv, DB: db };
// The delivery "42" finished before the upgrade; "d:42" is another one.
const { setDeliveryState } = await import("../server/lib/queue/delivery");
await setDeliveryState(env, "delivery-state:custom:global:42", "delivered");
const msg = makeMessage(body({ provider: "custom", deliveryId: "d:42" }));
dispatchCalls = 0;
summary = { attempts: 0, failures: [] };
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, fakeDispatch);
expect(dispatchCalls).toBe(1);
});
});
describe("consumer send budget", () => {
it("stops a batch after its send limit and continues the rest in a new message", async () => {
invalidateConfigCache();
invalidateGroupsCache();
const { kv } = createMockKV();
const sent: unknown[] = [];
const env: Env = {
...createEnv(kv),
QUEUE: { send: async (b: unknown) => void sent.push(b) } as unknown as Queue,
};
let allowed = 0;
const wide = async (...args: unknown[]): Promise<DispatchSummary> => {
const options = args[4] as DispatchOptions;
for (let i = 0; i < 40; i++) if (await options.beforeTarget!()) allowed++;
return {
attempts: 40,
succeeded: Array.from({ length: allowed }, (_, i) => `k${i}`),
failures: [{ key: "rest", target: "t", error: "paused" }],
};
};
const msg = makeMessage(body());
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, wide);
expect(allowed).toBe(20);
expect(sent).toHaveLength(1);
expect(msg.acked).toBe(true);
});
}); });
+3 -1
View File
@@ -51,7 +51,9 @@
"consumers": [ "consumers": [
{ {
"queue": "webhooker-delivery", "queue": "webhooker-delivery",
"max_batch_size": 10, // Small batches: one invocation may make 50 D1 queries and 50
// subrequests on Workers Free, and a delivery takes ~7 + its sends.
"max_batch_size": 3,
"max_batch_timeout": 30, "max_batch_timeout": 30,
"max_retries": 5, "max_retries": 5,
"dead_letter_queue": "webhooker-delivery-dlq" "dead_letter_queue": "webhooker-delivery-dlq"