mirror of
https://github.com/Elysola/WebHooker.git
synced 2026-11-21 01:23:40 +00:00
fix(queue): bound batches and resume deliveries
A queue invocation has shared query and subrequest limits. Use a small batch and shared send budget, renew fenced leases and resume only unfinished work without exhausting unstarted messages.
This commit is contained in:
1 parent
4c03543790
commit
4fc784ab9f
3 files changed
+808
-54
No files matched your search
+361
-52
@@ -1,62 +1,191 @@
|
|||||||
import type { Config, Env, Group, WebhookEvent, WebhookProvider } from "../types";
|
import type { Config, Env, Group, WebhookEvent, WebhookProvider } from "../types";
|
||||||
import { dispatchEvent } from "../core/dispatch";
|
import { dispatchEvent, type DispatchOptions } from "../core/dispatch";
|
||||||
import { loadConfig } from "../config";
|
import { loadConfigSnapshot, loadGroupFresh } from "../config";
|
||||||
import { loadGroups } from "../web/groups";
|
import { groupAliasChain } from "../web/group-lifecycle";
|
||||||
import { log } from "../lib/log";
|
import { log } from "../lib/log";
|
||||||
|
import { recordSendBatch } from "../lib/send-log-batch";
|
||||||
import {
|
import {
|
||||||
DELIVERY_DLQ,
|
DELIVERY_DLQ,
|
||||||
|
PayloadUnavailableError,
|
||||||
type DeliveryMessage,
|
type DeliveryMessage,
|
||||||
type DispatchSummary,
|
type DispatchSummary,
|
||||||
|
acquireDelivery,
|
||||||
classifyDelivery,
|
classifyDelivery,
|
||||||
|
finishLease,
|
||||||
|
getDeliveryState,
|
||||||
|
renewLease,
|
||||||
|
type DeliveryStatus,
|
||||||
deliveryStateKey,
|
deliveryStateKey,
|
||||||
discardPayload,
|
discardPayload,
|
||||||
getDeliveryState,
|
hadPermanentFailure,
|
||||||
|
isRetryableFailure,
|
||||||
|
markDeliveryDead,
|
||||||
|
markTargetsDone,
|
||||||
resolvePayload,
|
resolvePayload,
|
||||||
retryDelay,
|
retryDelay,
|
||||||
setDeliveryState,
|
targetDoneChecker,
|
||||||
} from "./delivery";
|
} from "./delivery";
|
||||||
|
|
||||||
export async function handleQueueBatch(
|
type Dispatch = (
|
||||||
batch: MessageBatch<DeliveryMessage>,
|
|
||||||
env: Env,
|
|
||||||
dispatch: (
|
|
||||||
config: Config,
|
config: Config,
|
||||||
event: WebhookEvent,
|
event: WebhookEvent,
|
||||||
env: Env,
|
env: Env,
|
||||||
groups?: Group[],
|
groups?: Group[],
|
||||||
) => Promise<DispatchSummary> = dispatchEvent,
|
options?: DispatchOptions,
|
||||||
|
) => Promise<DispatchSummary>;
|
||||||
|
|
||||||
|
export async function handleQueueBatch(
|
||||||
|
batch: MessageBatch<DeliveryMessage>,
|
||||||
|
env: Env,
|
||||||
|
dispatch: Dispatch = dispatchEvent,
|
||||||
|
budgetMs = BATCH_BUDGET_MS,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
|
// Shared by the whole batch: the limits it guards (D1 queries and
|
||||||
|
// subrequests) are per invocation.
|
||||||
|
const budget: BatchBudget = { deadline: Date.now() + budgetMs, sendsLeft: BATCH_SEND_LIMIT };
|
||||||
for (const message of batch.messages) {
|
for (const message of batch.messages) {
|
||||||
const body = message.body;
|
const body = message.body;
|
||||||
|
try {
|
||||||
if (batch.queue === DELIVERY_DLQ) {
|
if (batch.queue === DELIVERY_DLQ) {
|
||||||
await markDead(env, body);
|
if (await markDead(env, body)) message.ack();
|
||||||
message.ack();
|
// Another copy holds a live lease and may still need the payload.
|
||||||
|
else message.retry({ delaySeconds: BUSY_RETRY_SECONDS });
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
await processMessage(env, body, message, dispatch);
|
// Out of time or sends before it started (earlier messages took them):
|
||||||
|
// back to the queue for a later invocation — a retry (one more queue
|
||||||
|
// read), not a copy (a write and a delete more). Waiting is not a
|
||||||
|
// failure, though: a delivery never started whose retry would use up
|
||||||
|
// its last one (sending it to the DLQ untried) is copied instead. One
|
||||||
|
// that was started has real failures among its attempts; a copy would
|
||||||
|
// start their count over, so it keeps the retry.
|
||||||
|
if (spent(budget)) {
|
||||||
|
if (message.attempts > QUEUE_MAX_RETRIES && env.QUEUE && (await neverStarted(env, body))) {
|
||||||
|
try {
|
||||||
|
await env.QUEUE.send(body, { delaySeconds: CONTINUE_DELAY_SECONDS });
|
||||||
|
message.ack();
|
||||||
|
continue;
|
||||||
|
} catch (err) {
|
||||||
|
log.warn({ deliveryId: body.deliveryId, err }, "Failed to queue a delivery for later");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
message.retry({ delaySeconds: CONTINUE_DELAY_SECONDS });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
await processMessage(env, body, message, dispatch, budget);
|
||||||
|
} catch (err) {
|
||||||
|
// One message's storage failure must not fail (and replay) the batch.
|
||||||
|
log.error({ deliveryId: body.deliveryId, err }, "Queue message handling failed");
|
||||||
|
message.retry({ delaySeconds: retryDelay(message.attempts) });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Renew a held lease when it is older than this (the lease lasts five minutes). */
|
||||||
|
const LEASE_RENEW_MS = 60_000;
|
||||||
|
const FINAL: ReadonlySet<DeliveryStatus> = new Set(["delivered", "failed", "dead"]);
|
||||||
|
|
||||||
|
/** Wait for an unexpired lease of another consumer: its full length plus a margin. */
|
||||||
|
const BUSY_RETRY_SECONDS = 310;
|
||||||
|
/**
|
||||||
|
* Sending time of one batch: a consumer invocation may run 15 minutes, and a
|
||||||
|
* target takes under a minute at worst (bounded platform calls), so sending
|
||||||
|
* stops in time to record what was done.
|
||||||
|
*/
|
||||||
|
const BATCH_BUDGET_MS = 12 * 60_000;
|
||||||
|
/**
|
||||||
|
* Targets one batch sends at most: each costs D1 queries (its ledger record,
|
||||||
|
* an update chain's lock and tracking) and subrequests, and one invocation
|
||||||
|
* may make only 50 of each on Workers Free (with ~7 D1 queries per delivery
|
||||||
|
* besides; batches are small, see wrangler.jsonc). A message left unstarted
|
||||||
|
* goes back to the queue (a retry: one queue read); a message cut short is
|
||||||
|
* continued by a new one only when it made progress.
|
||||||
|
*/
|
||||||
|
const BATCH_SEND_LIMIT = 20;
|
||||||
|
|
||||||
|
/** What a batch may still spend: its time and its sends. */
|
||||||
|
interface BatchBudget {
|
||||||
|
deadline: number;
|
||||||
|
sendsLeft: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function spent(budget: BatchBudget): boolean {
|
||||||
|
return Date.now() >= budget.deadline || budget.sendsLeft <= 0;
|
||||||
|
}
|
||||||
|
const CONTINUE_DELAY_SECONDS = 5;
|
||||||
|
/** The delivery queue's `max_retries` (`wrangler.jsonc`). */
|
||||||
|
const QUEUE_MAX_RETRIES = 5;
|
||||||
|
|
||||||
|
/** No attempt has taken the delivery's lease yet (its attempts were all waits). */
|
||||||
|
async function neverStarted(env: Env, body: DeliveryMessage): Promise<boolean> {
|
||||||
|
const key = deliveryStateKey(body.provider, body.groupId, body.deliveryId, body.groupInstance);
|
||||||
|
const state = await getDeliveryState(env, key);
|
||||||
|
return state == null || state === "pending";
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Clean up after a terminal state; a failure must not reopen the delivery. */
|
||||||
|
async function discardQuietly(env: Env, body: DeliveryMessage): Promise<void> {
|
||||||
|
await discardPayload(env, body).catch((err) =>
|
||||||
|
log.warn({ deliveryId: body.deliveryId, err }, "Failed to discard a parked payload"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
async function processMessage(
|
async function processMessage(
|
||||||
env: Env,
|
env: Env,
|
||||||
body: DeliveryMessage,
|
body: DeliveryMessage,
|
||||||
message: Message<DeliveryMessage>,
|
message: Message<DeliveryMessage>,
|
||||||
dispatch: (
|
dispatch: Dispatch,
|
||||||
config: Config,
|
budget: BatchBudget,
|
||||||
event: WebhookEvent,
|
|
||||||
env: Env,
|
|
||||||
groups?: Group[],
|
|
||||||
) => Promise<DispatchSummary>,
|
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const key = deliveryStateKey(body.provider, body.groupId, body.deliveryId);
|
const key = deliveryStateKey(body.provider, body.groupId, body.deliveryId, body.groupInstance);
|
||||||
const prior = await getDeliveryState(env, key);
|
if (await finishedUnderLegacyKey(env, body)) {
|
||||||
if (prior === "delivered" || prior === "dead") {
|
|
||||||
message.ack();
|
message.ack();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
const lease = await acquireDelivery(env, key);
|
||||||
|
if (lease.status === "finished") {
|
||||||
|
message.ack();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (lease.status === "busy") {
|
||||||
|
// Another consumer holds this delivery (or a holder vanished): one retry
|
||||||
|
// after the lease surely expired, instead of spending the retry budget
|
||||||
|
// polling a lease that is still valid.
|
||||||
|
message.retry({ delaySeconds: BUSY_RETRY_SECONDS });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
await setDeliveryState(env, key, "processing");
|
let token = lease.token;
|
||||||
|
let renewedAt = Date.now();
|
||||||
|
let leaseLost = false;
|
||||||
|
/** A renewal failed with an unknown outcome: stop sending, but never assume a new holder. */
|
||||||
|
let leaseUncertain = false;
|
||||||
|
let renewing: Promise<boolean> | null = null;
|
||||||
|
/** The batch's budget ran out: the rest is left to a continuation message. */
|
||||||
|
let outOfTime = false;
|
||||||
|
/** End this attempt with `status`; acks instead when the lease was lost. */
|
||||||
|
const finish = async (status: DeliveryStatus, then: "ack" | "retry"): Promise<void> => {
|
||||||
|
const owned = !leaseLost && (await finishLease(env, key, token, status));
|
||||||
|
if (owned && then === "retry") {
|
||||||
|
message.retry({ delaySeconds: retryDelay(message.attempts) });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!owned && !leaseLost) {
|
||||||
|
// The outcome could not be written, yet no new holder is known (our own
|
||||||
|
// renewal may have gone through unseen, or the lease lapsed unnoticed):
|
||||||
|
// come back once any lease surely expired; the state then decides.
|
||||||
|
message.retry({ delaySeconds: BUSY_RETRY_SECONDS });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Lost lease: the new holder owns the delivery (and its outcome).
|
||||||
|
message.ack();
|
||||||
|
if (owned && FINAL.has(status)) await discardQuietly(env, body);
|
||||||
|
};
|
||||||
|
|
||||||
|
let summary: DispatchSummary;
|
||||||
|
// Targets recorded as sent the moment they were sent (see `afterTarget`).
|
||||||
|
const recorded = new Set<string>();
|
||||||
|
try {
|
||||||
const payload = await resolvePayload(env, body);
|
const payload = await resolvePayload(env, body);
|
||||||
const event: WebhookEvent = {
|
const event: WebhookEvent = {
|
||||||
event: body.event,
|
event: body.event,
|
||||||
@@ -65,39 +194,219 @@ async function processMessage(
|
|||||||
provider: body.provider as WebhookProvider,
|
provider: body.provider as WebhookProvider,
|
||||||
installationId: body.installationId,
|
installationId: body.installationId,
|
||||||
};
|
};
|
||||||
|
// Read fresh, not from this isolate's cache: a group (or route) created
|
||||||
|
// moments ago elsewhere must not be taken for missing — for a tenant
|
||||||
|
// delivery that verdict is final.
|
||||||
|
const { config, groups } = await loadConfigSnapshot(env, { fresh: true });
|
||||||
|
// A tenant delivery belongs to the group instance that received it — under
|
||||||
|
// whatever id it has now — and never to another group that took over the
|
||||||
|
// id since. A message from before instances belongs to the group that had
|
||||||
|
// its id back then (`legacyId`), also when that group was renamed before
|
||||||
|
// instances existed (followed through the rename aliases) — never to a
|
||||||
|
// group created since or renamed to the id.
|
||||||
|
let tenantId = body.groupId;
|
||||||
|
if (tenantId && body.groupInstance) {
|
||||||
|
tenantId = groups.find((g) => g.instance === body.groupInstance)?.id;
|
||||||
|
} else if (tenantId) {
|
||||||
|
const owner = (id: string): string | undefined => groups.find((g) => g.legacyId === id)?.id;
|
||||||
|
let found = owner(tenantId);
|
||||||
|
for (const id of found ? [] : (await groupAliasChain(env, tenantId)).slice(1)) {
|
||||||
|
found ??= owner(id);
|
||||||
|
}
|
||||||
|
tenantId = found;
|
||||||
|
}
|
||||||
|
if (body.groupId && (!tenantId || !groups.some((g) => g.id === tenantId))) {
|
||||||
|
// A group row that cannot be read may be the tenant: retry until it is
|
||||||
|
// repaired rather than give the delivery up as one of a deleted group.
|
||||||
|
if (config.unreadableGroups?.length) {
|
||||||
|
log.error(
|
||||||
|
{ deliveryId: body.deliveryId, unreadable: config.unreadableGroups },
|
||||||
|
"Tenant unresolved while some group configuration is unreadable",
|
||||||
|
);
|
||||||
|
await finish("retrying", "retry");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// The group was deleted: nothing can receive the event any more.
|
||||||
|
log.warn({ deliveryId: body.deliveryId, groupId: body.groupId }, "Tenant group is gone");
|
||||||
|
await finish("failed", "ack");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const options: DispatchOptions = {
|
||||||
|
tenantId,
|
||||||
|
// Always consult the ledger: another queue message of the same delivery
|
||||||
|
// (a sender redelivery) may already have sent some targets.
|
||||||
|
skipTarget: targetDoneChecker(env, key),
|
||||||
|
// Keep the lease alive during a long dispatch; once it is lost, stop
|
||||||
|
// sending (the new holder takes over the remaining targets).
|
||||||
|
beforeTarget: async () => {
|
||||||
|
if (leaseLost || leaseUncertain || outOfTime) return false;
|
||||||
|
if (spent(budget)) {
|
||||||
|
outOfTime = true;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
budget.sendsLeft--;
|
||||||
|
if (Date.now() - renewedAt < LEASE_RENEW_MS) return true;
|
||||||
|
// Routes run in parallel: they share one renewal. Two renewals with
|
||||||
|
// the same token would race, and the loser would take its own
|
||||||
|
// successful renewal for a lost lease.
|
||||||
|
renewing ??= (async (): Promise<boolean> => {
|
||||||
try {
|
try {
|
||||||
const config = await loadConfig(env);
|
token = await renewLease(env, key, token);
|
||||||
if (body.groupId) {
|
renewedAt = Date.now();
|
||||||
config.routes = config.routes.filter((r) => r.groupId === body.groupId);
|
leaseLost = token == null && lease.token != null;
|
||||||
}
|
return !leaseLost;
|
||||||
const groups = await loadGroups(env.KV);
|
|
||||||
const summary = await dispatch(config, event, env, groups);
|
|
||||||
const { failed, retryable } = classifyDelivery(summary);
|
|
||||||
|
|
||||||
if (!failed) {
|
|
||||||
await setDeliveryState(env, key, "delivered");
|
|
||||||
await discardPayload(env, body);
|
|
||||||
message.ack();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (retryable) {
|
|
||||||
await setDeliveryState(env, key, "retrying");
|
|
||||||
message.retry({ delaySeconds: retryDelay(message.attempts) });
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
await setDeliveryState(env, key, "failed");
|
|
||||||
await discardPayload(env, body);
|
|
||||||
message.ack();
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
log.error({ deliveryId: body.deliveryId, err }, "Queue delivery failed");
|
log.warn({ deliveryId: body.deliveryId, err }, "Lease renewal failed");
|
||||||
await setDeliveryState(env, key, "retrying");
|
leaseUncertain = true;
|
||||||
message.retry({ delaySeconds: retryDelay(message.attempts) });
|
return false;
|
||||||
|
} finally {
|
||||||
|
renewing = null;
|
||||||
}
|
}
|
||||||
|
})();
|
||||||
|
return renewing;
|
||||||
|
},
|
||||||
|
afterTarget: async (targetKey) => {
|
||||||
|
await markTargetsDone(env, key, [targetKey]);
|
||||||
|
recorded.add(targetKey);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const scoped = tenantId
|
||||||
|
? { ...config, routes: config.routes.filter((r) => r.groupId === tenantId) }
|
||||||
|
: config;
|
||||||
|
summary = await dispatch(scoped, event, env, groups, options);
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof PayloadUnavailableError) {
|
||||||
|
// Nothing left to dispatch in this message: drop it, loudly, without
|
||||||
|
// retrying forever or pretending the delivery succeeded. The delivery
|
||||||
|
// stays open (not final): the sender's redelivery, with its own payload,
|
||||||
|
// can still deliver it.
|
||||||
|
log.error({ deliveryId: body.deliveryId, err: err.message }, "Queued payload unavailable");
|
||||||
|
// On record where admins look (the group's send log and failure
|
||||||
|
// metrics): the event needs a redelivery from its sender.
|
||||||
|
await recordSendBatch(env.DB, [
|
||||||
|
{
|
||||||
|
ts: Date.now(),
|
||||||
|
routeId: "",
|
||||||
|
groupId: body.groupId,
|
||||||
|
groupInstance: body.groupInstance,
|
||||||
|
event: body.event,
|
||||||
|
target: "",
|
||||||
|
ok: false,
|
||||||
|
error: `${err.message}; redeliver the webhook to send it`,
|
||||||
|
errorCode: "PAYLOAD_MISSING",
|
||||||
|
deliveryId: body.deliveryId,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
await finish("retrying", "ack");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
log.error({ deliveryId: body.deliveryId, err }, "Queue delivery failed");
|
||||||
|
await finish("retrying", "retry");
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function markDead(env: Env, body: DeliveryMessage): Promise<void> {
|
const { failed, retryable } = classifyDelivery(summary);
|
||||||
const key = deliveryStateKey(body.provider, body.groupId, body.deliveryId);
|
// Record finished targets first, whatever comes next: if a later write
|
||||||
await setDeliveryState(env, key, "dead");
|
// fails and the message is retried, these targets are never sent again.
|
||||||
|
const permanent = summary.failures.filter((f) => !isRetryableFailure(f));
|
||||||
|
try {
|
||||||
|
await markTargetsDone(
|
||||||
|
env,
|
||||||
|
key,
|
||||||
|
[...(summary.succeeded ?? []), ...permanent.flatMap((f) => f.key ?? [])].filter(
|
||||||
|
(k) => !recorded.has(k),
|
||||||
|
),
|
||||||
|
{ permanentFailure: permanent.length > 0 },
|
||||||
|
);
|
||||||
|
} catch (err) {
|
||||||
|
// The outcome is not on record: retry (giving the lease back now) rather
|
||||||
|
// than finish the delivery with a forgotten permanent failure.
|
||||||
|
log.error({ deliveryId: body.deliveryId, err }, "Failed to record delivery targets");
|
||||||
|
await finish("retrying", "retry");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (failed && retryable) {
|
||||||
|
// Stopped for time, not for a failure: continued by a new message (sent
|
||||||
|
// before the lease is given back; a lease still held only delays it) —
|
||||||
|
// only when this attempt finished targets. One that made no progress
|
||||||
|
// (its targets kept failing until time ran out) is an ordinary retry and
|
||||||
|
// counts against the retry limit, so it cannot loop forever.
|
||||||
|
const progressed = (summary.succeeded?.length ?? 0) > 0 || permanent.length > 0;
|
||||||
|
if (outOfTime && progressed && !leaseLost && env.QUEUE) {
|
||||||
|
try {
|
||||||
|
await env.QUEUE.send(body, { delaySeconds: CONTINUE_DELAY_SECONDS });
|
||||||
|
await finish("retrying", "ack");
|
||||||
|
return;
|
||||||
|
} catch (err) {
|
||||||
|
log.warn({ deliveryId: body.deliveryId, err }, "Failed to queue the rest of a delivery");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await finish("retrying", "retry");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Targets that failed for good in an earlier attempt still count.
|
||||||
|
const anyPermanent = failed || (await hadPermanentFailure(env, key));
|
||||||
|
await finish(anyPermanent ? "failed" : "delivered", "ack");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The delivery already finished under the state key used before deliveries
|
||||||
|
* were scoped by `@global` / group instance (`delivery-state:{provider}:
|
||||||
|
* {groupId | "global"}:{id}`, kept 7 days): a sender's redelivery of an event
|
||||||
|
* handled before the upgrade is then not sent again. The old key is only
|
||||||
|
* trusted where its owner is certain — for a tenant, the group that had the
|
||||||
|
* id back then (its `legacyId`); for the global endpoint, while no group named
|
||||||
|
* "global" (which used the same key) exists or was deleted or renamed away
|
||||||
|
* recently.
|
||||||
|
*/
|
||||||
|
async function finishedUnderLegacyKey(env: Env, body: DeliveryMessage): Promise<boolean> {
|
||||||
|
// The global endpoint's old key was shared with any tenant once named
|
||||||
|
// "global" — even one deleted before the upgrade, which nothing records:
|
||||||
|
// its owner is never certain, so it is never trusted (a redelivery is sent
|
||||||
|
// again rather than an event dropped). Nothing to read for it, then.
|
||||||
|
if (!body.groupId || body.groupId === "global") return false;
|
||||||
|
// The store's one-time setup (seeding from KV, which also turns on reading
|
||||||
|
// older KV state) must have run before state is read: one group row reads
|
||||||
|
// it in, not the whole configuration (read below only when it matters).
|
||||||
|
const { group } = await loadGroupFresh(env, body.groupId);
|
||||||
|
// The message's own group, still under this id, with no older id of the
|
||||||
|
// same name (a group created since): nothing it could have finished under
|
||||||
|
// the old key, so that key is not read.
|
||||||
|
if (
|
||||||
|
body.groupInstance &&
|
||||||
|
group?.instance === body.groupInstance &&
|
||||||
|
group.legacyId !== body.groupId
|
||||||
|
) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
// Custom delivery ids got a namespace since (`d:`; `n:` for nonce
|
||||||
|
// identities, which did not exist before) — in messages queued since. A
|
||||||
|
// message queued before carries the id as the sender sent it, prefix-like
|
||||||
|
// text included.
|
||||||
|
const namespaced = body.provider === "custom" && (body.stateVersion ?? 0) >= 2;
|
||||||
|
if (namespaced && body.deliveryId.startsWith("n:")) return false;
|
||||||
|
const id =
|
||||||
|
namespaced && body.deliveryId.startsWith("d:") ? body.deliveryId.slice(2) : body.deliveryId;
|
||||||
|
const legacy = `delivery-state:${body.provider}:${body.groupId ?? "global"}:${id}`;
|
||||||
|
const state = await getDeliveryState(env, legacy);
|
||||||
|
if (!state || !FINAL.has(state)) return false;
|
||||||
|
// (A tenant named "global" shared this very key with the global endpoint:
|
||||||
|
// refused above.)
|
||||||
|
const { groups } = await loadConfigSnapshot(env, { fresh: true });
|
||||||
|
const owner = body.groupInstance
|
||||||
|
? groups.find((g) => g.instance === body.groupInstance)
|
||||||
|
: groups.find((g) => g.legacyId === body.groupId);
|
||||||
|
return owner?.legacyId === body.groupId;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Settle a message that left the DLQ; false while another copy of the
|
||||||
|
* delivery holds a live lease (its state and payload are left alone then).
|
||||||
|
*/
|
||||||
|
async function markDead(env: Env, body: DeliveryMessage): Promise<boolean> {
|
||||||
|
const key = deliveryStateKey(body.provider, body.groupId, body.deliveryId, body.groupInstance);
|
||||||
|
// A delivery another message of it finished (or is sending) keeps its state.
|
||||||
|
if ((await markDeliveryDead(env, key)) === "busy") return false;
|
||||||
await discardPayload(env, body);
|
await discardPayload(env, body);
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
@@ -4,12 +4,16 @@ import { invalidateConfigCache } from "../server/lib/config";
|
|||||||
import { invalidateGroupsCache } from "../server/lib/web/groups";
|
import { invalidateGroupsCache } from "../server/lib/web/groups";
|
||||||
import type { Env } from "../server/lib/types";
|
import type { Env } from "../server/lib/types";
|
||||||
import type { DeliveryMessage, DispatchSummary } from "../server/lib/queue/delivery";
|
import type { DeliveryMessage, DispatchSummary } from "../server/lib/queue/delivery";
|
||||||
|
import { createMemoryKV, createSqliteD1 } from "./d1-sqlite";
|
||||||
|
import type { DispatchOptions } from "../server/lib/core/dispatch";
|
||||||
|
|
||||||
let summary: DispatchSummary;
|
let summary: DispatchSummary;
|
||||||
let dispatchCalls: number;
|
let dispatchCalls: number;
|
||||||
|
let lastArgs: unknown[];
|
||||||
|
|
||||||
async function fakeDispatch(..._args: unknown[]): Promise<DispatchSummary> {
|
async function fakeDispatch(...args: unknown[]): Promise<DispatchSummary> {
|
||||||
dispatchCalls += 1;
|
dispatchCalls += 1;
|
||||||
|
lastArgs = args;
|
||||||
return summary;
|
return summary;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -136,4 +140,443 @@ describe("handleQueueBatch", () => {
|
|||||||
expect(msg.acked).toBe(true);
|
expect(msg.acked).toBe(true);
|
||||||
expect(dispatchCalls).toBe(0);
|
expect(dispatchCalls).toBe(0);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("never records a lost parked payload as delivered, nor closes the delivery", async () => {
|
||||||
|
const { kv, store } = createMockKV();
|
||||||
|
const msg = makeMessage(
|
||||||
|
body({
|
||||||
|
payload: undefined,
|
||||||
|
payloadRef: "queue:payload:gone",
|
||||||
|
receivedAt: Date.now() - 3600_000,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), createEnv(kv), fakeDispatch);
|
||||||
|
expect(dispatchCalls).toBe(0);
|
||||||
|
expect(msg.acked).toBe(true);
|
||||||
|
// Left open: a redelivery carrying its own payload can still deliver it.
|
||||||
|
expect(JSON.parse(store.get(STATE_KEY)!)).toMatchObject({ status: "retrying" });
|
||||||
|
const redelivery = makeMessage(body());
|
||||||
|
await handleQueueBatch(
|
||||||
|
makeBatch("webhooker-delivery", [redelivery]),
|
||||||
|
createEnv(kv),
|
||||||
|
fakeDispatch,
|
||||||
|
);
|
||||||
|
expect(dispatchCalls).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("retries only the transient targets of a partially failed delivery", async () => {
|
||||||
|
const { kv, store } = createMockKV();
|
||||||
|
const env = createEnv(kv);
|
||||||
|
summary = {
|
||||||
|
attempts: 3,
|
||||||
|
succeeded: ["g:r:discord:ok"],
|
||||||
|
failures: [
|
||||||
|
{ key: "g:r:discord:flaky", target: "flaky", errorCode: "DISCORD_5XX" },
|
||||||
|
{ key: "g:r:discord:bad", target: "bad", errorCode: "DISCORD_ERROR" },
|
||||||
|
],
|
||||||
|
};
|
||||||
|
const first = makeMessage(body(), 1);
|
||||||
|
await handleQueueBatch(makeBatch("webhooker-delivery", [first]), env, fakeDispatch);
|
||||||
|
expect(first.retried).toEqual({ delaySeconds: 5 });
|
||||||
|
// Every attempt consults the ledger (another message may have sent some).
|
||||||
|
expect(typeof (lastArgs[4] as { skipTarget?: unknown }).skipTarget).toBe("function");
|
||||||
|
|
||||||
|
const second = makeMessage(body(), 2);
|
||||||
|
summary = { attempts: 1, failures: [], succeeded: ["g:r:discord:flaky"] };
|
||||||
|
await handleQueueBatch(makeBatch("webhooker-delivery", [second]), env, fakeDispatch);
|
||||||
|
const skip = (lastArgs[4] as { skipTarget: (k: string) => Promise<boolean> }).skipTarget;
|
||||||
|
expect(await skip("g:r:discord:ok")).toBe(true);
|
||||||
|
expect(await skip("g:r:discord:bad")).toBe(true);
|
||||||
|
// Recorded too, now that the retry delivered it.
|
||||||
|
expect(await skip("g:r:discord:flaky")).toBe(true);
|
||||||
|
expect(second.acked).toBe(true);
|
||||||
|
// The 403 target never got the message: the delivery ends as failed.
|
||||||
|
expect(JSON.parse(store.get(STATE_KEY)!)).toMatchObject({ status: "failed" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("follows a tenant group renamed while the delivery was queued", async () => {
|
||||||
|
const { kv, store } = createMemoryKV();
|
||||||
|
store.set("config:groups", JSON.stringify([{ id: "new-id", name: "G", adminIds: [] }]));
|
||||||
|
store.set(
|
||||||
|
"config:routes",
|
||||||
|
JSON.stringify([
|
||||||
|
{ id: "r1", name: "R", enabled: true, filters: [], targets: [], groupId: "new-id" },
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
store.set("group:renamed:old-id", "new-id");
|
||||||
|
const msg = makeMessage(body({ groupId: "old-id" }));
|
||||||
|
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), createEnv(kv), fakeDispatch);
|
||||||
|
const config = lastArgs[0] as { routes: Array<{ id: string }> };
|
||||||
|
expect(config.routes.map((r) => r.id)).toEqual(["r1"]);
|
||||||
|
expect((lastArgs[4] as { tenantId?: string }).tenantId).toBe("new-id");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("handleQueueBatch edge cases", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
summary = { attempts: 0, failures: [] };
|
||||||
|
dispatchCalls = 0;
|
||||||
|
});
|
||||||
|
|
||||||
|
it("waits out a busy lease with a single long retry", async () => {
|
||||||
|
const { kv, store } = createMockKV();
|
||||||
|
store.set(STATE_KEY, JSON.stringify({ status: "processing", at: Date.now() }));
|
||||||
|
// KV leases are best effort: emulate the D1 answer with a fake.
|
||||||
|
const env = {
|
||||||
|
...createEnv(kv),
|
||||||
|
DB: {
|
||||||
|
prepare: (sql: string): unknown => {
|
||||||
|
const stmt = {
|
||||||
|
run: async (): Promise<unknown> => ({ meta: { changes: 0 } }),
|
||||||
|
// Delivery state answers "processing"; the configuration is empty.
|
||||||
|
first: async (): Promise<unknown> =>
|
||||||
|
sql.includes("delivery_state") ? { status: "processing" } : null,
|
||||||
|
all: async (): Promise<unknown> => ({ results: [] }),
|
||||||
|
};
|
||||||
|
return { ...stmt, bind: (): unknown => stmt };
|
||||||
|
},
|
||||||
|
batch: async (stmts: unknown[]) => stmts.map(() => ({ results: [], meta: { changes: 0 } })),
|
||||||
|
} as unknown as D1Database,
|
||||||
|
};
|
||||||
|
const msg = makeMessage(body());
|
||||||
|
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, fakeDispatch);
|
||||||
|
expect(dispatchCalls).toBe(0);
|
||||||
|
expect(msg.retried).toEqual({ delaySeconds: 310 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not reopen a delivered delivery when payload cleanup fails", async () => {
|
||||||
|
const { kv, store } = createMockKV();
|
||||||
|
const failingDelete = {
|
||||||
|
...kv,
|
||||||
|
delete: async (): Promise<never> => Promise.reject(new Error("KV down")),
|
||||||
|
};
|
||||||
|
store.set("queue:payload:x", JSON.stringify({ a: 1 }));
|
||||||
|
summary = { attempts: 1, failures: [] };
|
||||||
|
const msg = makeMessage(
|
||||||
|
body({ payload: undefined, payloadRef: "queue:payload:x", payloadRefType: "kv" }),
|
||||||
|
);
|
||||||
|
await handleQueueBatch(
|
||||||
|
makeBatch("webhooker-delivery", [msg]),
|
||||||
|
createEnv(failingDelete as unknown as KVNamespace),
|
||||||
|
fakeDispatch,
|
||||||
|
);
|
||||||
|
expect(msg.acked).toBe(true);
|
||||||
|
expect(msg.retried).toBeNull();
|
||||||
|
expect(JSON.parse(store.get(STATE_KEY)!)).toMatchObject({ status: "delivered" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("marks a delivery for a deleted tenant group as failed", async () => {
|
||||||
|
const { kv, store } = createMemoryKV();
|
||||||
|
store.set("config:groups", JSON.stringify([]));
|
||||||
|
const msg = makeMessage(body({ groupId: "gone" }));
|
||||||
|
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), createEnv(kv), fakeDispatch);
|
||||||
|
expect(dispatchCalls).toBe(0);
|
||||||
|
expect(msg.acked).toBe(true);
|
||||||
|
expect(JSON.parse(store.get("delivery-state:github:gone:d1")!)).toMatchObject({
|
||||||
|
status: "failed",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("tenant resolution by group instance", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
summary = { attempts: 1, failures: [] };
|
||||||
|
dispatchCalls = 0;
|
||||||
|
});
|
||||||
|
|
||||||
|
it("follows the instance across renames and ignores a re-created id", async () => {
|
||||||
|
const { kv, store } = createMemoryKV();
|
||||||
|
// Identities as the server assigned them (after the one-time migration).
|
||||||
|
store.set("config:identity-migrated", "1");
|
||||||
|
store.set(
|
||||||
|
"config:groups",
|
||||||
|
JSON.stringify([
|
||||||
|
{ id: "team", name: "T", adminIds: [], instance: "new-one" },
|
||||||
|
{ id: "moved", name: "M", adminIds: [], instance: "orig" },
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
store.set(
|
||||||
|
"config:routes",
|
||||||
|
JSON.stringify([
|
||||||
|
{ id: "r1", name: "R", enabled: true, filters: [], targets: [], groupId: "moved" },
|
||||||
|
{ id: "r2", name: "R", enabled: true, filters: [], targets: [], groupId: "team" },
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
const msg = makeMessage(body({ groupId: "team", groupInstance: "orig" }));
|
||||||
|
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), createEnv(kv), fakeDispatch);
|
||||||
|
const config = lastArgs[0] as { routes: Array<{ id: string }> };
|
||||||
|
// Delivered to the renamed original group, not to the new "team".
|
||||||
|
expect(config.routes.map((r) => r.id)).toEqual(["r1"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("shares one lease renewal between routes running in parallel", async () => {
|
||||||
|
const { db } = createSqliteD1();
|
||||||
|
const { kv } = createMemoryKV();
|
||||||
|
const env: Env = { GITHUB_WEBHOOK_SECRET: "s", KV: kv, DB: db };
|
||||||
|
let answers: boolean[] = [];
|
||||||
|
const dispatch = async (...args: unknown[]): Promise<DispatchSummary> => {
|
||||||
|
const opts = args[4] as DispatchOptions;
|
||||||
|
const realNow = Date.now;
|
||||||
|
// Both routes reach their next target once a renewal is due.
|
||||||
|
Date.now = (): number => realNow() + 90_000;
|
||||||
|
try {
|
||||||
|
answers = await Promise.all([opts.beforeTarget!(), opts.beforeTarget!()]);
|
||||||
|
} finally {
|
||||||
|
Date.now = realNow;
|
||||||
|
}
|
||||||
|
return { attempts: 0, failures: [] };
|
||||||
|
};
|
||||||
|
const msg = makeMessage(body());
|
||||||
|
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, dispatch);
|
||||||
|
// Neither route takes the shared renewal for a lost lease.
|
||||||
|
expect(answers).toEqual([true, true]);
|
||||||
|
expect(msg.acked).toBe(true);
|
||||||
|
const row = await db
|
||||||
|
.prepare("SELECT status FROM delivery_state WHERE key = ?")
|
||||||
|
.bind("delivery-state:github:@global:d1")
|
||||||
|
.first<{ status: string }>();
|
||||||
|
expect(row?.status).toBe("delivered");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("never acks when its own renewal went through unseen", async () => {
|
||||||
|
const { db: real } = createSqliteD1();
|
||||||
|
// The renewal commits, but its answer is lost.
|
||||||
|
const db = {
|
||||||
|
...real,
|
||||||
|
batch: real.batch.bind(real),
|
||||||
|
prepare: (sql: string): D1PreparedStatement => {
|
||||||
|
const stmt = real.prepare(sql);
|
||||||
|
if (!sql.includes("UPDATE delivery_state SET updated_at = ?")) return stmt;
|
||||||
|
return {
|
||||||
|
...stmt,
|
||||||
|
bind: (...args: unknown[]) => {
|
||||||
|
const bound = stmt.bind(...args);
|
||||||
|
return {
|
||||||
|
...bound,
|
||||||
|
run: async () => {
|
||||||
|
await bound.run();
|
||||||
|
throw new Error("network");
|
||||||
|
},
|
||||||
|
} as D1PreparedStatement;
|
||||||
|
},
|
||||||
|
} as D1PreparedStatement;
|
||||||
|
},
|
||||||
|
} as D1Database;
|
||||||
|
const env: Env = { GITHUB_WEBHOOK_SECRET: "s", KV: createMemoryKV().kv, DB: db };
|
||||||
|
const dispatch = async (...args: unknown[]): Promise<DispatchSummary> => {
|
||||||
|
const opts = args[4] as DispatchOptions;
|
||||||
|
const realNow = Date.now;
|
||||||
|
Date.now = (): number => realNow() + 90_000;
|
||||||
|
try {
|
||||||
|
expect(await opts.beforeTarget!()).toBe(false);
|
||||||
|
} finally {
|
||||||
|
Date.now = realNow;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
attempts: 1,
|
||||||
|
failures: [{ key: "k", target: "t", error: "Delivery lease lost; left to its new holder" }],
|
||||||
|
};
|
||||||
|
};
|
||||||
|
const msg = makeMessage(body());
|
||||||
|
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, dispatch);
|
||||||
|
// No new holder exists: the message comes back once the lease surely expired.
|
||||||
|
expect(msg.acked).toBe(false);
|
||||||
|
expect(msg.retried).toEqual({ delaySeconds: 310 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("never hands a pre-instance message to a group renamed onto its id", async () => {
|
||||||
|
const { kv, store } = createMemoryKV();
|
||||||
|
// Identities as the server assigned them (after the one-time migration).
|
||||||
|
store.set("config:identity-migrated", "1");
|
||||||
|
// "team" is held by a pre-instance group that had another id back then.
|
||||||
|
store.set(
|
||||||
|
"config:groups",
|
||||||
|
JSON.stringify([
|
||||||
|
{
|
||||||
|
id: "team",
|
||||||
|
name: "T",
|
||||||
|
adminIds: [],
|
||||||
|
instance: "b",
|
||||||
|
instanceBackfilled: true,
|
||||||
|
legacyId: "other",
|
||||||
|
},
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
store.set("config:routes", JSON.stringify([]));
|
||||||
|
const msg = makeMessage(body({ groupId: "team" }));
|
||||||
|
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), createEnv(kv), fakeDispatch);
|
||||||
|
expect(dispatchCalls).toBe(0);
|
||||||
|
expect(msg.acked).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("never hands a pre-instance message to a group created since", async () => {
|
||||||
|
const { kv, store } = createMemoryKV();
|
||||||
|
// Identities as the server assigned them (after the one-time migration).
|
||||||
|
store.set("config:identity-migrated", "1");
|
||||||
|
// "team" now belongs to a group created after instances existed.
|
||||||
|
store.set(
|
||||||
|
"config:groups",
|
||||||
|
JSON.stringify([{ id: "team", name: "T", adminIds: [], instance: "fresh" }]),
|
||||||
|
);
|
||||||
|
store.set(
|
||||||
|
"config:routes",
|
||||||
|
JSON.stringify([
|
||||||
|
{ id: "r2", name: "R", enabled: true, filters: [], targets: [], groupId: "team" },
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
const msg = makeMessage(body({ groupId: "team" }));
|
||||||
|
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), createEnv(kv), fakeDispatch);
|
||||||
|
expect(dispatchCalls).toBe(0);
|
||||||
|
expect(msg.acked).toBe(true);
|
||||||
|
expect(JSON.parse(store.get("delivery-state:github:team:d1")!)).toMatchObject({
|
||||||
|
status: "failed",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("consumer time budget", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
invalidateConfigCache();
|
||||||
|
invalidateGroupsCache();
|
||||||
|
dispatchCalls = 0;
|
||||||
|
});
|
||||||
|
|
||||||
|
it("gives a message it has no time left for back to the queue, uncopied", async () => {
|
||||||
|
const { kv } = createMockKV();
|
||||||
|
const sent: unknown[] = [];
|
||||||
|
const env: Env = {
|
||||||
|
...createEnv(kv),
|
||||||
|
QUEUE: { send: async (b: unknown) => void sent.push(b) } as unknown as Queue,
|
||||||
|
};
|
||||||
|
const msg = makeMessage(body());
|
||||||
|
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, fakeDispatch, 0);
|
||||||
|
expect(dispatchCalls).toBe(0);
|
||||||
|
// Back to the queue as it is (one queue read), not copied (a write and a
|
||||||
|
// delete more).
|
||||||
|
expect(sent).toHaveLength(0);
|
||||||
|
expect(msg.retried).not.toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("never spends a waiting message's last retry: it is copied instead", async () => {
|
||||||
|
const { kv } = createMockKV();
|
||||||
|
const sent: unknown[] = [];
|
||||||
|
const env: Env = {
|
||||||
|
...createEnv(kv),
|
||||||
|
QUEUE: { send: async (b: unknown) => void sent.push(b) } as unknown as Queue,
|
||||||
|
};
|
||||||
|
// Left for later on every earlier attempt (max_retries is 5).
|
||||||
|
const msg = makeMessage(body(), 6);
|
||||||
|
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, fakeDispatch, 0);
|
||||||
|
expect(dispatchCalls).toBe(0);
|
||||||
|
expect(sent).toHaveLength(1);
|
||||||
|
expect(msg.acked).toBe(true);
|
||||||
|
expect(msg.retried).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the retry count of a started delivery that has to wait", async () => {
|
||||||
|
const { kv, store } = createMockKV();
|
||||||
|
// Taken and failed before: its attempts were real failures.
|
||||||
|
store.set(STATE_KEY, JSON.stringify({ status: "retrying", at: Date.now() }));
|
||||||
|
const sent: unknown[] = [];
|
||||||
|
const env: Env = {
|
||||||
|
...createEnv(kv),
|
||||||
|
QUEUE: { send: async (b: unknown) => void sent.push(b) } as unknown as Queue,
|
||||||
|
};
|
||||||
|
const msg = makeMessage(body(), 6);
|
||||||
|
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, fakeDispatch, 0);
|
||||||
|
// Not copied (a copy would start its failure count over): retried.
|
||||||
|
expect(sent).toHaveLength(0);
|
||||||
|
expect(msg.retried).not.toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("continues a delivery that ran out of time in a new message", async () => {
|
||||||
|
const { kv } = createMockKV();
|
||||||
|
const sent: unknown[] = [];
|
||||||
|
const env: Env = {
|
||||||
|
...createEnv(kv),
|
||||||
|
QUEUE: { send: async (b: unknown) => void sent.push(b) } as unknown as Queue,
|
||||||
|
};
|
||||||
|
const slowDispatch = async (...args: unknown[]): Promise<DispatchSummary> => {
|
||||||
|
const options = args[4] as DispatchOptions;
|
||||||
|
await new Promise((r) => setTimeout(r, 60));
|
||||||
|
expect(await options.beforeTarget!()).toBe(false);
|
||||||
|
return {
|
||||||
|
attempts: 2,
|
||||||
|
succeeded: ["k0"],
|
||||||
|
failures: [{ key: "k", target: "t", error: "paused" }],
|
||||||
|
};
|
||||||
|
};
|
||||||
|
const msg = makeMessage(body());
|
||||||
|
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, slowDispatch, 30);
|
||||||
|
expect(sent).toHaveLength(1);
|
||||||
|
expect(msg.acked).toBe(true);
|
||||||
|
expect(msg.retried).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("consumer time budget without progress", () => {
|
||||||
|
it("retries (and counts) a delivery that ran out of time without finishing a target", async () => {
|
||||||
|
invalidateConfigCache();
|
||||||
|
invalidateGroupsCache();
|
||||||
|
const { kv } = createMockKV();
|
||||||
|
const sent: unknown[] = [];
|
||||||
|
const env: Env = {
|
||||||
|
...createEnv(kv),
|
||||||
|
QUEUE: { send: async (b: unknown) => void sent.push(b) } as unknown as Queue,
|
||||||
|
};
|
||||||
|
const stuck = async (...args: unknown[]): Promise<DispatchSummary> => {
|
||||||
|
const options = args[4] as DispatchOptions;
|
||||||
|
await new Promise((r) => setTimeout(r, 60));
|
||||||
|
await options.beforeTarget!();
|
||||||
|
return { attempts: 1, failures: [{ key: "k", target: "t", error: "timeout" }] };
|
||||||
|
};
|
||||||
|
const msg = makeMessage(body());
|
||||||
|
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, stuck, 30);
|
||||||
|
expect(sent).toHaveLength(0);
|
||||||
|
expect(msg.retried).not.toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reads a pre-upgrade custom delivery id as sent, prefix-like text included", async () => {
|
||||||
|
invalidateConfigCache();
|
||||||
|
invalidateGroupsCache();
|
||||||
|
const { db } = createSqliteD1();
|
||||||
|
const { kv } = createMemoryKV();
|
||||||
|
const env: Env = { GITHUB_WEBHOOK_SECRET: "s", KV: kv, DB: db };
|
||||||
|
// The delivery "42" finished before the upgrade; "d:42" is another one.
|
||||||
|
const { setDeliveryState } = await import("../server/lib/queue/delivery");
|
||||||
|
await setDeliveryState(env, "delivery-state:custom:global:42", "delivered");
|
||||||
|
const msg = makeMessage(body({ provider: "custom", deliveryId: "d:42" }));
|
||||||
|
dispatchCalls = 0;
|
||||||
|
summary = { attempts: 0, failures: [] };
|
||||||
|
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, fakeDispatch);
|
||||||
|
expect(dispatchCalls).toBe(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("consumer send budget", () => {
|
||||||
|
it("stops a batch after its send limit and continues the rest in a new message", async () => {
|
||||||
|
invalidateConfigCache();
|
||||||
|
invalidateGroupsCache();
|
||||||
|
const { kv } = createMockKV();
|
||||||
|
const sent: unknown[] = [];
|
||||||
|
const env: Env = {
|
||||||
|
...createEnv(kv),
|
||||||
|
QUEUE: { send: async (b: unknown) => void sent.push(b) } as unknown as Queue,
|
||||||
|
};
|
||||||
|
let allowed = 0;
|
||||||
|
const wide = async (...args: unknown[]): Promise<DispatchSummary> => {
|
||||||
|
const options = args[4] as DispatchOptions;
|
||||||
|
for (let i = 0; i < 40; i++) if (await options.beforeTarget!()) allowed++;
|
||||||
|
return {
|
||||||
|
attempts: 40,
|
||||||
|
succeeded: Array.from({ length: allowed }, (_, i) => `k${i}`),
|
||||||
|
failures: [{ key: "rest", target: "t", error: "paused" }],
|
||||||
|
};
|
||||||
|
};
|
||||||
|
const msg = makeMessage(body());
|
||||||
|
await handleQueueBatch(makeBatch("webhooker-delivery", [msg]), env, wide);
|
||||||
|
expect(allowed).toBe(20);
|
||||||
|
expect(sent).toHaveLength(1);
|
||||||
|
expect(msg.acked).toBe(true);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
+3
-1
@@ -51,7 +51,9 @@
|
|||||||
"consumers": [
|
"consumers": [
|
||||||
{
|
{
|
||||||
"queue": "webhooker-delivery",
|
"queue": "webhooker-delivery",
|
||||||
"max_batch_size": 10,
|
// Small batches: one invocation may make 50 D1 queries and 50
|
||||||
|
// subrequests on Workers Free, and a delivery takes ~7 + its sends.
|
||||||
|
"max_batch_size": 3,
|
||||||
"max_batch_timeout": 30,
|
"max_batch_timeout": 30,
|
||||||
"max_retries": 5,
|
"max_retries": 5,
|
||||||
"dead_letter_queue": "webhooker-delivery-dlq"
|
"dead_letter_queue": "webhooker-delivery-dlq"
|
||||||
|
|||||||
Reference in new issue
Block a user