Files
WebHooker/server/lib/queue/delivery.ts
T
翅膀喵andRhenCloud 978c53eeb3 fix(security): harden webhook delivery pipeline and group storage (#11)
* fix(log): preserve structured error details

Error objects lose their useful details when serialized as plain JSON.
Keep the message, stack and cause in structured log output.

* fix(runtime): bound shared in-flight waits

A canceled request can leave its shared promise pending forever.
Bound joins and coordinate a single replacement so later requests
make progress without multiplying upstream work.

* fix(bot): reject oversized callback bodies

Reject declared oversized bot callbacks before reading their bodies.
This keeps unauthenticated requests from wasting worker resources.

* test(d1): exercise real sqlite transactions

Exercise real foreign keys, transactions and conditional statements
instead of mocks that cannot reproduce lost updates or cascades.

* fix(storage): preserve claims across kv fallback

Only missing tables justify KV fallback; other failures must surface.
Remember fallback writes so migration to D1 cannot forget claims
and replay completed work. Share marker checks without extra writes.

* fix(filter): use isolated linear-time patterns

Backtracking patterns can exhaust worker CPU on untrusted payloads.
Use linear-time matching and separate bounded cache namespaces so
glob and regex entries cannot corrupt each other.

* fix(filter): read each event branch correctly

Deployment and check branches live in event-specific nested fields.
Read those fields and remove only the actual leading ref prefix.

* fix(filter): read only own payload properties

Inherited object properties are not webhook payload data.
Read own properties only so JSONPath cannot traverse prototypes.

* fix(filter): validate executable condition trees

Malformed trees and unsupported patterns must not silently turn an
exclude rule into a pass. Validate the shape and only the patterns
the evaluator actually executes.

* fix(i18n): bound and cache translation loads

Repeated misses and simultaneous override loads spend KV reads for
the same language. Cache confirmed misses and share bounded loads
without caching read failures as missing translations.

* i18n(bot): explain command and account failures

Explain account linking, command failures and skipped delivery
outcomes consistently in both supported languages.

* fix(github): bound and reuse app token fetches

Repeated token exchanges waste requests and JWT work for every event.
Reuse valid installation tokens, share refreshes and back off briefly
after failures while keeping every shared wait bounded.

* fix(github): restrict enrichment request urls

Webhook-provided URLs must never receive an installation token on
another host. Restrict enrichment to GitHub repository API paths,
reject redirects and bound optional requests.

* fix(formatter): preserve invalid code points

Out-of-range numeric entities can throw while formatting a delivery.
Keep invalid code points verbatim instead of aborting the message.

* fix(formatter): preserve bounded markdown text

Code spans, fences, indentation and autolinks are Markdown content,
not HTML tags to strip. Preserve them while bounding cleanup input
and reusing results across routes for the same event.

* fix(formatter): strip only leading ref prefixes

A short branch or tag may itself contain refs/heads or refs/tags.
Strip one leading full-ref prefix at the caller and keep short names
unchanged when generating labels and links.

* fix(gitea): link push authors to their forge

A Gitea author is not the same-named GitHub account.
Build author profile links from the originating forge.

* fix(formatter): render deployment state values

The deployment status object stringifies as [object Object].
Render its actual state in the status field.

* fix(formatter): read security severity levels

The scanner severity is not the security severity of an alert.
Prefer the security level so critical and high alerts get the right
label and color.

* fix(formatter): hide actions on other forges

Action buttons call the GitHub API as the linked user.
Hide them for other forges to avoid operating on a same-named
GitHub repository.

* fix(check): isolate updates by check suite

Same-named checks from different suites must not edit one message.
Include the suite or app identity and retain the former key for
controlled adoption of existing tracked messages.

* fix(logs): scope queries before limiting results

Filter authorized groups in SQL before LIMIT and aggregate only
visible rows. Preserve current group attribution for late writes,
surface read failures and combine related totals in one scan.

* perf(d1): index pruning and failed send queries

Pruning and failure views should not scan all historical rows.
Index the high-value access paths without adding a write-heavy
index for infrequent delivery-detail lookups.

* fix(discord): bound individual api requests

An unbounded API request can outlive an update-chain lock.
Bound each request so another holder cannot race a late edit.

* fix(discord): defer long rate-limit waits

Long rate-limit sleeps block delivery progress inside a worker.
Return a retryable 429 for the queue when the wait exceeds the bound.

* fix(discord): encode channel path segments

Stored channel and message ids are path segments, not URL paths.
Encode them so aliases cannot address another Discord resource.

* fix(discord): reuse nonces after lost responses

A successful create may lose its response. Reuse a stable nonce
with enforce_nonce so retries recover that message instead of
posting duplicates or accepting an unreadable success.

* fix(discord): constrain embeds and role mentions

Respect combined embed and component limits as well as field caps.
Restrict allowed mentions to validated route roles so payload text
cannot notify everyone.

* fix(discord): resume bounded command syncs

Application changes, command rollbacks and large guild lists can
leave commands stale indefinitely. Track the current hash per app
and resume bounded paginated work without needless KV writes.

* fix(github): verify installation management

An installation id is user-controlled, and repository access is not
installation management. Verify the owning account or an active
organization admin using the user token and read:org scope.

* fix(github): resolve immutable member ids

A login can be renamed and reclaimed before its first sign-in.
Resolve the current immutable user id when membership is granted
so a future holder of the login cannot inherit access.

* fix(auth): revoke stale sessions and tokens

Revoked or rotated credentials must stop working despite stale KV
replicas. Track the live token and revocations in D1 and preserve
immutable identity bindings rather than trusting a recycled login.

* fix(auth): bind sessions and tokens to identity

Automatic retries can repeat non-idempotent user actions.
Disable those retries and expose a write hook so bot handlers can
record completion immediately before the GitHub write.

* fix(discord): defer and deduplicate interactions

Acknowledge slow actions before Discord times out and deduplicate
interaction writes. Preserve comment text and reject edits whose
source changed after the modal opened.

* fix(telegram): always verify webhook secrets

Webhook verification must not silently disable itself when the
optional secret is absent. Derive a stable secret from the bot
token and reject callbacks when verification is unavailable.

* fix(telegram): bound individual api requests

Bound individual platform calls so they cannot outlive the lock
protecting a message update or stall a whole delivery invocation.

* fix(telegram): honor nested rate-limit delays

Telegram places retry_after inside parameters, not at the top level.
Read that value and defer long waits to the delivery queue.

* fix(telegram): report unreadable send results

An unreadable successful response may already have sent the message.
Report an explicit uncertain result instead of retrying a create
without an idempotency key or pretending it succeeded.

* fix(telegram): keep editable messages as text

Guessing photo versus text from message content makes later edits
use the wrong API and a smaller caption budget. Always send text
and use the avatar as a link preview.

* fix(telegram): render hard breaks as newlines

Telegram HTML does not support br tags. Render Markdown hard breaks
as plain newlines so otherwise valid messages are accepted.

* fix(telegram): clamp visible html characters

Telegram counts visible characters, not HTML source bytes.
Truncate without splitting tags, entities or surrogate pairs.

* fix(telegram): honor the author emoji toggle

The author marker must obey the group emoji preference just like
formatter-added decoration.

* fix(telegram): keep completed commands claimed

Redelivery after a completed write must never post another comment.
Keep completion claims despite reply failures, scope updates per bot
and only issue account-login links in private chats.

* fix(feishu): bound individual api requests

Bound individual platform calls so they cannot outlive the lock
protecting a message update or stall a whole delivery invocation.

* fix(feishu): classify retryable api failures

Business error codes can be transient even in HTTP 200 responses.
Classify token and rate-limit errors for retry and defer long waits.

* fix(feishu): retry sends with stable uuids

An unreadable response does not prove a send failed.
Reuse the same uuid for retries so recovering the result cannot
create a second Feishu message.

* fix(feishu): refresh and evict tokens safely

Separate app token caches, stop reusing refused tokens and share
refreshes. Cache failures briefly and keep valid tokens in memory
when a best-effort KV cache write fails.

* fix(feishu): use the card update endpoint

Delayed card updates use the card update endpoint; the config
endpoint cannot update an interaction response.

* fix(feishu): neutralize untrusted mentions

Forge text is untrusted and can contain at-all tags.
Neutralize mentions in card bodies and footers before rendering.

* fix(feishu): authenticate and dedupe callbacks

Authenticate signed or token-verified callbacks before dispatching
commands. Decode encrypted envelopes and handle card actions using
the correct fields without claiming irrelevant chat events.

* fix(target): normalize actual destinations

Different target spellings can name the same actual destination.
Normalize channel, thread and topic identities for ownership checks
and per-route duplicate suppression.

* fix(config): transact per-group configuration

Whole-table writes and cached snapshots can overwrite other groups
or resurrect removed configuration. Use per-group guarded writes,
one-time identity migration and explicit unreadable-row failures.

* fix(group): preserve identity through lifecycle

Group names are mutable and reusable, so they cannot own durable
state. Preserve immutable identities across lifecycle operations and
commit invite use with membership under transaction-time guards.

* fix(oauth): confirm account links and installs

A login link or installation id is not proof of account ownership.
Verify installation management, bind browser state and require an
explicit account-link confirmation before granting chat access.

* fix(queue): fence leases and record target outcomes

Retries must remember completed targets and distinguish active
lease holders. Fence state updates, preserve permanent outcomes and
reject missing parked payloads instead of dispatching empty data.

* fix(dispatch): isolate and resume tenant sends

Check tenant sources before filters and never dispatch orphan routes.
Record each target outcome, isolate update chains by forge and group
identity, and keep stale events from overwriting newer messages.

* fix(queue): bound batches and resume deliveries

A queue invocation has shared query and subrequest limits.
Use a small batch and shared send budget, renew fenced leases and
resume only unfinished work without exhausting unstarted messages.

* fix(admin): guard revisions and tenant targets

Read-time authorization alone cannot protect concurrent edits.
Guard revisions, current roles and destination ownership in the
write transaction, and use per-group endpoints for isolated changes.

* fix(webhook): confirm only accepted deliveries

Acknowledging before queue acceptance permanently loses deliveries.
Confirm only accepted work, release failed claims, scope replay
identities and strip tenant-supplied installation authority.

* fix(storage): prune abandoned parked payloads

Oversized payloads can remain after interrupted enqueue attempts.
Prune objects beyond their retry lifetime without deleting payloads
that a current delivery may still need.

* fix(console): expose api conflict statuses

Callers need the HTTP status to distinguish stale edits from other
failures and reload current data after a conflict.

* refactor(console): model group revisions

Carry server revisions explicitly so console writes can reject
changes based on an outdated group snapshot.

* fix(console): send loaded route revisions

Send the revision actually loaded with the route list and adopt the
server response after a save. Reload on conflict instead of silently
overwriting another editor.

* fix(console): guard fragment saves by revision

Fragments share a group version with other configuration writes.
Submit the loaded revision and reload on conflict to preserve
concurrent edits.

* fix(console): save groups with guarded writes

Whole-list saves can overwrite unrelated groups. Use per-group
revision-aware writes, retain rename results and confirm deletion
on the exact group, routes and version shown to the user.

* fix(console): reject edits of stale routes

A conflict reload can leave an older route editor open.
Compare its original value before saving so it cannot overwrite
the newly loaded route.

* fix(console): show overview loading failures

Failed overview requests must not look like an empty dashboard or
a successful refresh. Show the reason while retaining prior data.

* fix(console): explain rejected install bindings

Explain why an installation binding was refused instead of silently
returning to the console with no visible outcome.

* fix(console): allow selecting a group named all

The all sentinel collides with a valid group id.
Use a value that cannot be a group id so that group stays selectable.

* fix(console): discard stale log detail responses

An older detail request can finish after a newer selection or close.
Discard it so the dialog never shows the previously clicked record.

* fix(console): retain legacy group owners

Groups predating member roles store owners in adminIds.
Load those as owners so adding a member never drops all old owners.

* fix(console): lock member edits during saves

Edits made during a save are overwritten by its result.
Disable member controls and guard handlers until the write settles.

* fix(console): show invite loading failures

Failed invite loads must not look like no pending invites.
Show the error and discard responses from a previously shown group.

* fix(console): retain legacy discord log targets

A stored log target without platform means Discord.
Keep that meaning in the editor instead of clearing the target.

* fix(console): retain undisplayed forge sources

An editor that only displays some providers must preserve the others.
Keep stored custom sources when saving unrelated group settings.

* fix(console): lock the group editor while saving

Prevent edits while a group save is in flight and make super-only
installation settings readonly instead of silently ignoring edits.

* fix(console): lock route edits and show failures

Lock the route editor while saving and show fragment load failures
with a retry action rather than presenting a false empty list.

* fix(config): expose the legal contact override

Declare the public runtime key so the documented environment
override reaches legal pages in deployed Nuxt builds.

* build(deploy): migrate d1 before deployment

Apply pending D1 migrations before deploying code that requires
them. Stop deployment when migration fails instead of running
against an incomplete schema.

* docs(worker): align guides with runtime behavior

Synchronize both language guides, API references and examples with
the implemented authorization, storage and delivery behavior.
Document migration requirements and accepted operational limits.

* fix(storage): repair orphaned KV routes

Repair orphaned KV routes before serving KV-only configuration.

* fix(security): fully escape forge HTML

---------

Co-authored-by: RhenCloud <i@rhen.cloud>
2026-10-24 16:19:06 +08:00

535 lines
19 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import type { Env } from "../types";
import { r2PayloadStore } from "../storage/payload";
import { canUseD1, isMissingTable, kvFallbackActive, noteKvFallback } from "../storage/d1";
import { idempotencyStore, kvIdempotencyStore, tenantScope } from "../lib/idempotency";
import { log } from "../lib/log";
export type DeliveryStatus =
"pending" | "processing" | "delivered" | "retrying" | "failed" | "dead";
export interface DeliveryMessage {
deliveryId: string;
/** Tenant group of a per-group ingress (`/webhook/{groupId}`); absent for the global endpoint. */
groupId?: string;
/**
* The tenant group's immutable instance at ingress. The consumer delivers
* only to the group holding it now (following renames), never to another
* group that later took over the id.
*/
groupInstance?: string;
provider: string;
event: string;
payload?: Record<string, unknown>;
/** R2 object key (new) or KV key (legacy `queue:payload:*`). */
payloadRef?: string;
/** Distinguishes R2 payload refs from legacy KV keys. */
payloadRefType?: "r2" | "kv";
installationId?: number;
receivedAt: number;
requestId?: string;
/**
* 2 for messages whose state keys use the current scopes (`@global`,
* instances). Absent on messages queued before: only those may have state
* under the old `...:global:` key.
*/
stateVersion?: number;
}
export interface DispatchFailure {
/** Stable identity of the route × target, used to skip it on a retry. */
key?: string;
target: string;
error?: string;
errorCode?: string;
status?: number;
}
export interface DispatchSummary {
attempts: number;
failures: DispatchFailure[];
/** Keys of the route × targets that were sent successfully. */
succeeded?: string[];
}
export const DELIVERY_QUEUE = "webhooker-delivery";
export const DELIVERY_DLQ = "webhooker-delivery-dlq";
/**
* Queue messages may be 128 KB (128 000 bytes, a little of it taken by the
* queue's own metadata); measured in UTF-8 bytes of the JSON. Close to the
* limit, so only payloads that cannot go in the message are parked — each
* parked one costs a KV write, read and delete (scarce on Workers Free).
*/
const MAX_QUEUE_MESSAGE_BYTES = 120_000;
/**
* Parked payloads must outlive every retry and the DLQ hand-off. Queue
* messages are retained up to four days, so the payload is too.
*/
export const PAYLOAD_TTL_SECONDS = 4 * 86400;
/**
* What a delivery finished (state, done targets, permanent failures) is kept
* as long as delivery state (pruned after 7 days): a redelivery within that
* window — queued or inline — never sends a finished target again.
*/
const LEDGER_TTL_SECONDS = 7 * 86400;
const STATE_KV_TTL_SECONDS = LEDGER_TTL_SECONDS;
/** How long a just-parked KV payload may still be invisible (eventual consistency). */
const KV_VISIBILITY_GRACE_MS = 10 * 60_000;
/** A consumer holds a delivery this long before another one may take over. */
const LEASE_MS = 5 * 60_000;
const RETRYABLE_ERROR_CODES = new Set([
"DISCORD_5XX",
"TELEGRAM_5XX",
"FEISHU_5XX",
"FEISHU_TOKEN",
"NETWORK",
"RETRIES",
"RATE_LIMITED",
]);
const RETRY_DELAYS_SECONDS = [5, 30, 120, 600];
/** The stored payload of a delivery is gone or unreadable: it cannot be dispatched. */
export class PayloadUnavailableError extends Error {
constructor(message: string) {
super(message);
this.name = "PayloadUnavailableError";
}
}
export function isRetryableError(code?: string): boolean {
return code == null || RETRYABLE_ERROR_CODES.has(code);
}
/** Transient by code, or by HTTP status (5xx, 429) whatever the platform's code. */
export function isRetryableFailure(failure: DispatchFailure): boolean {
if (failure.status != null && (failure.status >= 500 || failure.status === 429)) return true;
return isRetryableError(failure.errorCode);
}
/**
* A delivery is retried when any target failed transiently. Targets that
* succeeded or failed permanently are recorded first (see `markTargetsDone`)
* so the retry only re-sends the transient ones: one bad target never cancels
* the retries of the others.
*/
export function classifyDelivery(summary: DispatchSummary): {
failed: boolean;
retryable: boolean;
} {
if (summary.failures.length === 0) return { failed: false, retryable: false };
return { failed: true, retryable: summary.failures.some(isRetryableFailure) };
}
export function retryDelay(attempt: number): number {
if (attempt < 1) return RETRY_DELAYS_SECONDS[0] ?? 5;
const idx = Math.min(attempt - 1, RETRY_DELAYS_SECONDS.length - 1);
return RETRY_DELAYS_SECONDS[idx] ?? 600;
}
function scopeKey(
provider: string,
groupId: string | undefined,
deliveryId: string,
instance?: string,
): string {
return `${provider}:${tenantScope(groupId, instance)}:${deliveryId}`;
}
/** State key of a delivery; tenant deliveries are keyed by their group instance. */
export function deliveryStateKey(
provider: string,
groupId: string | undefined,
deliveryId: string,
instance?: string,
): string {
return `delivery-state:${scopeKey(provider, groupId, deliveryId, instance)}`;
}
/**
* KV key of a parked payload. Unique per enqueue: a redelivery of the same
* delivery parks its own copy, so cleaning up after its failed send can never
* remove the payload an earlier queue message still needs.
*/
function payloadKey(provider: string, groupId: string | undefined, deliveryId: string): string {
return `queue:payload:${scopeKey(provider, groupId, deliveryId)}:${crypto.randomUUID()}`;
}
export async function setDeliveryState(
env: Env,
key: string,
status: DeliveryStatus,
): Promise<void> {
const db = env.DB;
if (canUseD1(db)) {
try {
await db
.prepare(
`INSERT INTO delivery_state (key, status, updated_at)
VALUES (?, ?, ?)
ON CONFLICT(key) DO UPDATE SET status = excluded.status, updated_at = excluded.updated_at`,
)
.bind(key, status, Date.now())
.run();
return;
} catch (err) {
if (!isMissingTable(err)) throw err;
log.warn({ err, key }, "D1 delivery_state table missing, falling back to KV");
}
}
// Marked, so D1 (missing table now, or added to the deployment later)
// still honors this state.
await noteKvFallback(env.KV);
await env.KV.put(key, JSON.stringify({ status, at: Date.now() }), {
expirationTtl: STATE_KV_TTL_SECONDS,
});
}
export async function getDeliveryState(env: Env, key: string): Promise<DeliveryStatus | null> {
const db = env.DB;
if (canUseD1(db)) {
try {
const row = await db
.prepare("SELECT status FROM delivery_state WHERE key = ?")
.bind(key)
.first<{ status: DeliveryStatus }>();
if (row?.status || !(await kvFallbackActive(env.KV))) return row?.status ?? null;
} catch (err) {
if (!isMissingTable(err)) throw err;
log.warn({ err, key }, "D1 delivery_state table missing, falling back to KV");
await noteKvFallback(env.KV);
}
}
const raw = await env.KV.get(key);
if (!raw) return null;
try {
return (JSON.parse(raw) as { status?: DeliveryStatus }).status ?? null;
} catch {
return null;
}
}
export type LeaseResult = "acquired" | "finished" | "busy";
/**
* A delivery lease: its status and, when acquired, the fencing token (the
* `updated_at` the holder wrote). Renewals and terminal writes only apply
* while the row still carries the holder's token, so a consumer whose lease
* was taken over can neither overwrite the new holder's state nor go on as if
* it still owned the delivery. Null token: KV fallback (no fencing).
*/
export interface Lease {
status: LeaseResult;
token: number | null;
}
const FINISHED: ReadonlySet<DeliveryStatus | null> = new Set(["delivered", "dead", "failed"]);
/**
* Take a delivery for processing. With D1 this is one conditional UPSERT: it
* succeeds only for a new delivery, one waiting for a retry, or one whose
* previous holder's lease expired — so two consumers can never dispatch the
* same delivery at the same time. `finished` deliveries are done for good;
* `busy` ones are being processed elsewhere right now.
*/
export async function acquireDelivery(env: Env, key: string): Promise<Lease> {
const db = env.DB;
if (canUseD1(db)) {
try {
// Finished while the table was missing (state lives in KV then).
if (await kvFallbackActive(env.KV)) {
const kvState = await env.KV.get(key);
if (
kvState &&
FINISHED.has((JSON.parse(kvState) as { status?: DeliveryStatus }).status ?? null)
) {
return { status: "finished", token: null };
}
}
const now = Date.now();
const result = await db
.prepare(
`INSERT INTO delivery_state (key, status, updated_at) VALUES (?, 'processing', ?)
ON CONFLICT(key) DO UPDATE SET status = 'processing', updated_at = excluded.updated_at
WHERE delivery_state.status IN ('pending', 'retrying')
OR (delivery_state.status = 'processing' AND delivery_state.updated_at < ?)`,
)
.bind(key, now, now - LEASE_MS)
.run();
if ((result.meta?.changes ?? 0) > 0) return { status: "acquired", token: now };
const status = await getDeliveryState(env, key);
return { status: FINISHED.has(status) ? "finished" : "busy", token: null };
} catch (err) {
if (!isMissingTable(err)) throw err;
log.warn({ err, key }, "D1 delivery_state table missing, falling back to KV");
await noteKvFallback(env.KV);
}
}
// KV: best effort (no atomic compare-and-set).
const prior = await getDeliveryState(env, key);
if (FINISHED.has(prior)) return { status: "finished", token: null };
await setDeliveryState(env, key, "processing");
return { status: "acquired", token: null };
}
/**
* Mark a delivery dead (its message left the DLQ) — atomically, and only
* while it is neither finished nor held by a live lease: a copy of the same
* delivery that finished or is still sending keeps its state. Without D1:
* best effort.
*/
export async function markDeliveryDead(
env: Env,
key: string,
): Promise<"dead" | "finished" | "busy"> {
if (canUseD1(env.DB)) {
try {
const now = Date.now();
const result = await env.DB.prepare(
`INSERT INTO delivery_state (key, status, updated_at) VALUES (?, 'dead', ?)
ON CONFLICT(key) DO UPDATE SET status = 'dead', updated_at = excluded.updated_at
WHERE delivery_state.status IN ('pending', 'retrying')
OR (delivery_state.status = 'processing' AND delivery_state.updated_at < ?)`,
)
.bind(key, now, now - LEASE_MS)
.run();
if ((result.meta?.changes ?? 0) > 0) return "dead";
return FINISHED.has(await getDeliveryState(env, key)) ? "finished" : "busy";
} catch (err) {
if (!isMissingTable(err)) throw err;
}
}
if (FINISHED.has(await getDeliveryState(env, key))) return "finished";
await setDeliveryState(env, key, "dead");
return "dead";
}
/**
* Extend a held lease; the new token, or null when the lease was lost (taken
* over after expiry). Without D1 the lease cannot be fenced and stays valid.
*/
export async function renewLease(
env: Env,
key: string,
token: number | null,
): Promise<number | null> {
if (token == null || !canUseD1(env.DB)) return token;
const now = Math.max(Date.now(), token + 1);
const result = await env.DB.prepare(
`UPDATE delivery_state SET updated_at = ?
WHERE key = ? AND status = 'processing' AND updated_at = ?`,
)
.bind(now, key, token)
.run();
return (result.meta?.changes ?? 0) > 0 ? now : null;
}
/**
* Write the outcome of a held lease (`retrying` or a terminal status); false
* when the lease was lost meanwhile — the new holder's state then stands.
*/
export async function finishLease(
env: Env,
key: string,
token: number | null,
status: DeliveryStatus,
): Promise<boolean> {
if (token == null || !canUseD1(env.DB)) {
await setDeliveryState(env, key, status);
return true;
}
const result = await env.DB.prepare(
`UPDATE delivery_state SET status = ?, updated_at = ?
WHERE key = ? AND status = 'processing' AND updated_at = ?`,
)
.bind(status, Math.max(Date.now(), token + 1), key, token)
.run();
return (result.meta?.changes ?? 0) > 0;
}
function targetDoneKey(stateKey: string, targetKey: string): string {
return `delivery-target:${stateKey}:${targetKey}`;
}
/**
* Remember route × targets that need no further attempt for this delivery —
* one D1 transaction for all of them (KV claims without D1 or its table).
* With `permanentFailure`, the delivery's permanent-failure marker is written
* in the same transaction: a target is never skipped as finished while the
* failure that finished it is forgotten (which would let the delivery end
* up `delivered`).
*/
export async function markTargetsDone(
env: Env,
stateKey: string,
targetKeys: string[],
opts: { permanentFailure?: boolean } = {},
): Promise<void> {
const keys = targetKeys.map((t) => targetDoneKey(stateKey, t));
// The marker goes first: in the KV fallback, a write failing midway leaves
// at worst a marker without its target (the target is simply retried).
if (opts.permanentFailure) keys.unshift(permanentFailureKey(stateKey));
if (keys.length === 0) return;
if (canUseD1(env.DB)) {
const now = Date.now();
const expires = now + LEDGER_TTL_SECONDS * 1000;
try {
await env.DB.batch(
keys.map((key) =>
env.DB.prepare(
`INSERT INTO dedup_keys (key, claimed_at, expires_at) VALUES (?, ?, ?)
ON CONFLICT(key) DO UPDATE SET expires_at = excluded.expires_at`,
).bind(key, now, expires),
),
);
return;
} catch (err) {
if (!isMissingTable(err)) throw err;
}
}
await noteKvFallback(env.KV);
const store = kvIdempotencyStore(env.KV);
for (const key of keys) {
await store.claim(key, LEDGER_TTL_SECONDS);
}
}
/**
* Whether a route × target already finished in an earlier attempt of this
* delivery. The delivery's finished targets are read in one D1 query (on the
* first check) rather than one per target: a large fan-out would otherwise
* spend the invocation's D1 queries before sending anything.
*/
export function targetDoneChecker(
env: Env,
stateKey: string,
): (targetKey: string) => Promise<boolean> {
const store = idempotencyStore(env.DB, env.KV);
let finished: Promise<Set<string> | null> | null = null;
return async (targetKey) => {
const key = targetDoneKey(stateKey, targetKey);
finished ??= finishedTargets(env, stateKey);
const known = await finished;
if (!known) return store.has(key);
if (known.has(key)) return true;
// Copies written to KV while the table was missing: looked up only for
// the targets D1 does not know as finished.
return (await kvFallbackActive(env.KV)) ? kvIdempotencyStore(env.KV).has(key) : false;
};
}
/** The delivery's finished-target keys, or null where they are read one by one. */
async function finishedTargets(env: Env, stateKey: string): Promise<Set<string> | null> {
if (!canUseD1(env.DB)) return null;
const prefix = targetDoneKey(stateKey, "");
try {
const { results } = await env.DB.prepare(
"SELECT key FROM dedup_keys WHERE key >= ? AND key < ? AND expires_at > ?",
)
.bind(prefix, `${prefix}\uffff`, Date.now())
.all<{ key: string }>();
return new Set((results ?? []).map((r) => r.key));
} catch (err) {
if (isMissingTable(err)) return null;
throw err;
}
}
function permanentFailureKey(stateKey: string): string {
return `delivery-failed:${stateKey}`;
}
/** Whether an earlier attempt of this delivery left a target failed for good. */
export function hadPermanentFailure(env: Env, stateKey: string): Promise<boolean> {
return idempotencyStore(env.DB, env.KV).has(permanentFailureKey(stateKey));
}
function byteLength(text: string): number {
return new TextEncoder().encode(text).byteLength;
}
/**
* Enqueue a verified webhook. Payloads too large for a queue message are
* parked in R2 (or KV) first; when the send itself fails, the parked copy is
* removed again and the error propagates, so the caller can tell the sender
* to retry instead of acknowledging an event that was never queued.
*/
export async function enqueueWebhook(env: Env, message: DeliveryMessage): Promise<void> {
const queue = env.QUEUE;
if (!queue) throw new Error("QUEUE binding is not configured");
const { payload, ...rest } = message;
const direct: DeliveryMessage = { ...rest, payload };
if (byteLength(JSON.stringify(direct)) <= MAX_QUEUE_MESSAGE_BYTES) {
await queue.send(direct);
return;
}
const serialized = JSON.stringify(payload ?? {});
let parked: DeliveryMessage;
if (env.PAYLOAD) {
const key = await r2PayloadStore(env).put(serialized, PAYLOAD_TTL_SECONDS);
parked = { ...rest, payloadRef: key, payloadRefType: "r2" };
} else {
const payloadRef = payloadKey(message.provider, message.groupId, message.deliveryId);
await env.KV.put(payloadRef, serialized, { expirationTtl: PAYLOAD_TTL_SECONDS });
parked = { ...rest, payloadRef, payloadRefType: "kv" };
}
// A failed send leaves the parked copy in place: the queue may have taken
// the message after all (the error can be a lost response), and deleting
// the payload would doom it. An unused copy expires (KV TTL) or is pruned
// as an orphan (R2).
await queue.send(parked);
}
function parsePayload(raw: string, ref: string): Record<string, unknown> {
try {
const parsed = JSON.parse(raw) as unknown;
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
return parsed as Record<string, unknown>;
}
} catch {
// reported below
}
throw new PayloadUnavailableError(`Parked payload ${ref} is corrupt`);
}
/**
* The event payload of a queued delivery. A parked payload that is missing or
* corrupt is an explicit error — dispatching an empty payload instead would
* match nothing and record the delivery as delivered.
*/
export async function resolvePayload(
env: Env,
message: DeliveryMessage,
): Promise<Record<string, unknown>> {
if (message.payload) return message.payload;
if (!message.payloadRef) throw new PayloadUnavailableError("Delivery carries no payload");
if (message.payloadRefType === "r2") {
if (!env.PAYLOAD) throw new Error("R2 binding is not configured");
const raw = await r2PayloadStore(env).get(message.payloadRef);
if (raw == null) {
throw new PayloadUnavailableError(`Parked payload ${message.payloadRef} is missing`);
}
return parsePayload(raw, message.payloadRef);
}
const raw = await env.KV.get(message.payloadRef);
if (raw == null) {
// KV writes can take a while to become visible elsewhere: a payload
// parked minutes ago is retried, not declared lost.
if (Date.now() - message.receivedAt < KV_VISIBILITY_GRACE_MS) {
throw new Error(`Parked payload ${message.payloadRef} is not visible yet`);
}
throw new PayloadUnavailableError(`Parked payload ${message.payloadRef} is missing`);
}
return parsePayload(raw, message.payloadRef);
}
export async function discardPayload(env: Env, message: DeliveryMessage): Promise<void> {
if (!message.payloadRef) return;
if (message.payloadRefType === "r2") {
if (env.PAYLOAD) await r2PayloadStore(env).delete(message.payloadRef);
return;
}
await env.KV.delete(message.payloadRef);
}