feat(storage): Add delivery state store switch (D1 by default or KV) (#12)

This commit is contained in:
w9auto committed 2026-11-17 22:46:50 +08:00
1 parent fc4495dfb4
commit 0c126d4fee
21 files changed
+756 -184

No files matched your search

+6 -5
View File
@@ -16,7 +16,8 @@ import type { SendRecord } from "../lib/send-log";
import { recordSendBatch } from "../lib/send-log-batch";
import { explainRoute } from "../config/schema";
import { chainLock, messageTracker, type MessageTracker } from "../lib/message-tracker";
import { idempotencyStore, instanceScope, tenantScope } from "../lib/idempotency";
import { instanceScope, tenantScope } from "../lib/idempotency";
import { deliveryClaimStore, deliveryStateUsesD1 } from "../storage/state-store";
import {
loadGroups,
groupAcceptsOwners,
@@ -254,7 +255,7 @@ async function deliverUpdate(
const chainId = chain.id;
const trackTarget = chain.target;
const lockKey = `msg:lock:${chainId}:${trackTarget}`;
const lock = chainLock(env.DB, env.KV);
const lock = chainLock(env.DB, env.KV, deliveryStateUsesD1(env));
let token: string | null = null;
for (let attempt = 0; attempt < LOCK_ATTEMPTS && !token; attempt++) {
token = await lock.acquire(lockKey, LOCK_TTL_SECONDS);
@@ -394,7 +395,7 @@ export async function dispatchEvent(
const { tenantId, skipTarget, beforeTarget, afterTarget } = options;
const loadedGroups = groups ?? (await loadGroups(env.KV));
const groupById = new Map(loadedGroups.map((g) => [g.id, g]));
const tracker = messageTracker(env.DB, env.KV);
const tracker = messageTracker(env.DB, env.KV, deliveryStateUsesD1(env));
// Message language is configured per group (Group.lang), not per route;
// loaded when a message is built in it (a group nothing is sent to costs
@@ -677,7 +678,7 @@ export async function dispatchEvent(
)}|${event.deliveryId}|${group.instance ?? groupId}|${outcome}`,
)}`
: null;
if (summaryKey && !(await idempotencyStore(env.DB, env.KV).claim(summaryKey, 86400))) {
if (summaryKey && !(await deliveryClaimStore(env).claim(summaryKey, 86400))) {
continue;
}
claimed = summaryKey;
@@ -692,7 +693,7 @@ export async function dispatchEvent(
}
// Not posted (a failed or thrown send): a retry may try again.
if (claimed) {
await idempotencyStore(env.DB, env.KV)
await deliveryClaimStore(env)
.release(claimed)
.catch(() => undefined);
}
+8 -4
View File
@@ -118,10 +118,14 @@ function withKvFallback(
};
}
export function idempotencyStore(db: D1Database, kv: KVNamespace): IdempotencyStore {
if (canUseD1(db)) return withKvFallback(d1IdempotencyStore(db), kvIdempotencyStore(kv), kv);
// Without D1, claims live in KV: marked, so a D1 binding added later still
// honors the ones not expired yet.
/** D1 when the binding can take it, else KV (no binding, or a caller that wants KV). */
export function idempotencyStore(
db: D1Database | undefined | null,
kv: KVNamespace,
): IdempotencyStore {
if (canUseD1(db)) return withKvFallback(d1IdempotencyStore(db!), kvIdempotencyStore(kv), kv);
// Without D1, or with a caller that wants KV: claims live in KV, marked, so
// D1 still honors the ones not expired yet.
const store = kvIdempotencyStore(kv);
return {
...store,
+24 -7
View File
@@ -209,10 +209,19 @@ function withKvFallback(
};
}
export function messageTracker(db: D1Database, kv: KVNamespace): MessageTracker {
if (canUseD1(db)) return withKvFallback(d1MessageTracker(db), kvMessageTracker(kv), kv, db);
// Without D1, tracking lives in KV: marked, so a D1 binding added later
// still finds the messages to edit.
/**
* Message tracking of the delivery pipeline. `useD1` is the configured store
* (`DELIVERY_STATE_STORE`): KV when it is off, marked so a deployment switched
* back to D1 still finds the messages to edit.
*/
export function messageTracker(
db: D1Database,
kv: KVNamespace,
useD1: boolean = canUseD1(db),
): MessageTracker {
if (useD1) return withKvFallback(d1MessageTracker(db), kvMessageTracker(kv), kv, db);
// Without D1 — or with the delivery store set to KV — tracking lives in KV:
// marked, so D1 still finds the messages to edit.
const tracker = kvMessageTracker(kv);
return {
...tracker,
@@ -231,7 +240,8 @@ export function messageTracker(db: D1Database, kv: KVNamespace): MessageTracker
* Serializes the updates of one message chain. With D1 the lock is a
* `dedup_keys` row taken by an atomic conditional UPSERT (taken over only once
* expired) and released only by its holder, identified by the claim time it
* wrote. Without D1 it degrades to a best-effort KV marker.
* wrote. Without D1 — or with the delivery store set to KV — it degrades to a
* best-effort KV marker.
*/
export interface ChainLock {
/** A holder token, or null while someone else holds the lock. */
@@ -239,7 +249,11 @@ export interface ChainLock {
release(key: string, token: string): Promise<void>;
}
export function chainLock(db: D1Database, kv: KVNamespace): ChainLock {
export function chainLock(
db: D1Database,
kv: KVNamespace,
useD1: boolean = canUseD1(db),
): ChainLock {
const kvLock: ChainLock = {
async acquire(key, ttlSeconds) {
if (await kv.get(key)) return null;
@@ -251,9 +265,12 @@ export function chainLock(db: D1Database, kv: KVNamespace): ChainLock {
if ((await kv.get(key)) === token) await kv.delete(key);
},
};
if (!canUseD1(db)) return kvLock;
if (!useD1) return kvLock;
return {
async acquire(key, ttlSeconds): Promise<string | null> {
// A lock taken while delivering through KV (before switching back) is
// honored too: another holder would edit the same message chain.
if ((await kvFallbackActive(kv)) && (await kv.get(key))) return null;
const now = Date.now();
try {
const result = await db
+3 -2
View File
@@ -2,7 +2,8 @@ import type { Env, WebhookEvent } from "../../types";
import type { Provider } from "../types";
import { hmacSha256Hex, timingSafeEqual } from "../hmac";
import { verifySignature } from "../github/verify";
import { claimOnce, idempotencyStore } from "../../lib/idempotency";
import { claimOnce } from "../../lib/idempotency";
import { deliveryClaimStore } from "../../storage/state-store";
const REPLAY_WINDOW_SECONDS = 300;
/** How long an accepted nonce is remembered (longer than the signature window). */
@@ -63,7 +64,7 @@ async function verifyReplayProtected(
const key = await replayNonceKey(secret, nonce);
let state;
try {
state = await claimOnce(idempotencyStore(env.DB, env.KV), key, NONCE_CLAIM_SECONDS);
state = await claimOnce(deliveryClaimStore(env), key, NONCE_CLAIM_SECONDS);
} catch {
// A storage failure says nothing about the signature.
throw new ReplayPendingError("Replay protection is temporarily unavailable");
+73 -27
View File
@@ -1,7 +1,8 @@
import type { Env } from "../types";
import { r2PayloadStore } from "../storage/payload";
import { canUseD1, isMissingTable, kvFallbackActive, noteKvFallback } from "../storage/d1";
import { idempotencyStore, kvIdempotencyStore, tenantScope } from "../lib/idempotency";
import { isMissingTable, kvFallbackActive, noteKvFallback } from "../storage/d1";
import { deliveryClaimStore, deliveryStateUsesD1 } from "../storage/state-store";
import { kvIdempotencyStore, tenantScope } from "../lib/idempotency";
import { log } from "../lib/log";
export type DeliveryStatus =
@@ -162,7 +163,7 @@ export async function setDeliveryState(
status: DeliveryStatus,
): Promise<void> {
const db = env.DB;
if (canUseD1(db)) {
if (deliveryStateUsesD1(env)) {
try {
await db
.prepare(
@@ -188,7 +189,7 @@ export async function setDeliveryState(
export async function getDeliveryState(env: Env, key: string): Promise<DeliveryStatus | null> {
const db = env.DB;
if (canUseD1(db)) {
if (deliveryStateUsesD1(env)) {
try {
const row = await db
.prepare("SELECT status FROM delivery_state WHERE key = ?")
@@ -201,10 +202,23 @@ export async function getDeliveryState(env: Env, key: string): Promise<DeliveryS
await noteKvFallback(env.KV);
}
}
return (await readKvDeliveryState(env, key))?.status ?? null;
}
/**
* The KV-stored state of a delivery, with the time it was written (the lease
* stamp). Null when absent or unreadable.
*/
async function readKvDeliveryState(
env: Env,
key: string,
): Promise<{ status: DeliveryStatus; at: number } | null> {
const raw = await env.KV.get(key);
if (!raw) return null;
try {
return (JSON.parse(raw) as { status?: DeliveryStatus }).status ?? null;
const parsed = JSON.parse(raw) as { status?: DeliveryStatus; at?: unknown };
if (!parsed.status) return null;
return { status: parsed.status, at: Number(parsed.at) || 0 };
} catch {
return null;
}
@@ -217,7 +231,8 @@ export type LeaseResult = "acquired" | "finished" | "busy";
* `updated_at` the holder wrote). Renewals and terminal writes only apply
* while the row still carries the holder's token, so a consumer whose lease
* was taken over can neither overwrite the new holder's state nor go on as if
* it still owned the delivery. Null token: KV fallback (no fencing).
* it still owned the delivery. Null token: KV (configured store or fallback),
* where a fresh lease stamp plays the same role without fencing.
*/
export interface Lease {
status: LeaseResult;
@@ -231,21 +246,25 @@ const FINISHED: ReadonlySet<DeliveryStatus | null> = new Set(["delivered", "dead
* succeeds only for a new delivery, one waiting for a retry, or one whose
* previous holder's lease expired — so two consumers can never dispatch the
* same delivery at the same time. `finished` deliveries are done for good;
* `busy` ones are being processed elsewhere right now.
* `busy` ones are being processed elsewhere right now. With KV the same stamp
* keeps a second consumer off a live lease, but there is no fencing token: a
* holder whose lease expired cannot tell that another one took over.
*/
export async function acquireDelivery(env: Env, key: string): Promise<Lease> {
const db = env.DB;
if (canUseD1(db)) {
if (deliveryStateUsesD1(env)) {
try {
// Finished while the table was missing (state lives in KV then).
if (await kvFallbackActive(env.KV)) {
const kvState = await env.KV.get(key);
if (
kvState &&
FINISHED.has((JSON.parse(kvState) as { status?: DeliveryStatus }).status ?? null)
) {
const kvState = await readKvDeliveryState(env, key);
if (kvState && FINISHED.has(kvState.status)) {
return { status: "finished", token: null };
}
// A holder delivering through KV (before switching back) keeps its
// delivery for the rest of its lease.
if (kvState?.status === "processing" && kvState.at > Date.now() - LEASE_MS) {
return { status: "busy", token: null };
}
}
const now = Date.now();
const result = await db
@@ -266,9 +285,13 @@ export async function acquireDelivery(env: Env, key: string): Promise<Lease> {
await noteKvFallback(env.KV);
}
}
// KV: best effort (no atomic compare-and-set).
const prior = await getDeliveryState(env, key);
if (FINISHED.has(prior)) return { status: "finished", token: null };
// KV: no compare-and-set, but the lease stamp still keeps a second consumer
// off a delivery that is being sent right now.
const prior = await readKvDeliveryState(env, key);
if (prior && FINISHED.has(prior.status)) return { status: "finished", token: null };
if (prior?.status === "processing" && prior.at > Date.now() - LEASE_MS) {
return { status: "busy", token: null };
}
await setDeliveryState(env, key, "processing");
return { status: "acquired", token: null };
}
@@ -276,15 +299,21 @@ export async function acquireDelivery(env: Env, key: string): Promise<Lease> {
/**
* Mark a delivery dead (its message left the DLQ) — atomically, and only
* while it is neither finished nor held by a live lease: a copy of the same
* delivery that finished or is still sending keeps its state. Without D1:
* delivery that finished or is still sending keeps its state. With KV:
* best effort.
*/
export async function markDeliveryDead(
env: Env,
key: string,
): Promise<"dead" | "finished" | "busy"> {
if (canUseD1(env.DB)) {
if (deliveryStateUsesD1(env)) {
try {
// A holder delivering through KV (before switching back) keeps its
// delivery: the DLQ copy must not close it under that holder.
if (await kvFallbackActive(env.KV)) {
const kvState = await readKvDeliveryState(env, key);
if (kvState?.status === "processing" && kvState.at > Date.now() - LEASE_MS) return "busy";
}
const now = Date.now();
const result = await env.DB.prepare(
`INSERT INTO delivery_state (key, status, updated_at) VALUES (?, 'dead', ?)
@@ -300,21 +329,37 @@ export async function markDeliveryDead(
if (!isMissingTable(err)) throw err;
}
}
if (FINISHED.has(await getDeliveryState(env, key))) return "finished";
const prior = await readKvDeliveryState(env, key);
if (prior && FINISHED.has(prior.status)) return "finished";
// Same lease stamp as `acquireDelivery`: a live sender keeps its delivery.
if (prior?.status === "processing" && prior.at > Date.now() - LEASE_MS) return "busy";
await setDeliveryState(env, key, "dead");
return "dead";
}
/**
* Extend a held lease; the new token, or null when the lease was lost (taken
* over after expiry). Without D1 the lease cannot be fenced and stays valid.
* over after expiry). In KV there is no fencing token to renew: the lease
* stamp is refreshed instead, so another consumer does not take a delivery
* that is still being sent.
*/
export async function renewLease(
env: Env,
key: string,
token: number | null,
): Promise<number | null> {
if (token == null || !canUseD1(env.DB)) return token;
if (!deliveryStateUsesD1(env)) {
if (token == null) {
const prior = await readKvDeliveryState(env, key);
// At most every half lease: the stamp stays fresh with a read per
// renewal and a write per half lease, sparing the KV write quota.
if (prior?.status === "processing" && Date.now() - prior.at > LEASE_MS / 2) {
await setDeliveryState(env, key, "processing");
}
}
return token;
}
if (token == null) return token;
const now = Math.max(Date.now(), token + 1);
const result = await env.DB.prepare(
`UPDATE delivery_state SET updated_at = ?
@@ -335,7 +380,7 @@ export async function finishLease(
token: number | null,
status: DeliveryStatus,
): Promise<boolean> {
if (token == null || !canUseD1(env.DB)) {
if (token == null || !deliveryStateUsesD1(env)) {
await setDeliveryState(env, key, status);
return true;
}
@@ -354,7 +399,8 @@ function targetDoneKey(stateKey: string, targetKey: string): string {
/**
* Remember route × targets that need no further attempt for this delivery —
* one D1 transaction for all of them (KV claims without D1 or its table).
* one D1 transaction for all of them (KV claims when the store is KV or D1
* lacks the table).
* With `permanentFailure`, the delivery's permanent-failure marker is written
* in the same transaction: a target is never skipped as finished while the
* failure that finished it is forgotten (which would let the delivery end
@@ -371,7 +417,7 @@ export async function markTargetsDone(
// at worst a marker without its target (the target is simply retried).
if (opts.permanentFailure) keys.unshift(permanentFailureKey(stateKey));
if (keys.length === 0) return;
if (canUseD1(env.DB)) {
if (deliveryStateUsesD1(env)) {
const now = Date.now();
const expires = now + LEDGER_TTL_SECONDS * 1000;
try {
@@ -405,7 +451,7 @@ export function targetDoneChecker(
env: Env,
stateKey: string,
): (targetKey: string) => Promise<boolean> {
const store = idempotencyStore(env.DB, env.KV);
const store = deliveryClaimStore(env);
let finished: Promise<Set<string> | null> | null = null;
return async (targetKey) => {
const key = targetDoneKey(stateKey, targetKey);
@@ -421,7 +467,7 @@ export function targetDoneChecker(
/** The delivery's finished-target keys, or null where they are read one by one. */
async function finishedTargets(env: Env, stateKey: string): Promise<Set<string> | null> {
if (!canUseD1(env.DB)) return null;
if (!deliveryStateUsesD1(env)) return null;
const prefix = targetDoneKey(stateKey, "");
try {
const { results } = await env.DB.prepare(
@@ -442,7 +488,7 @@ function permanentFailureKey(stateKey: string): string {
/** Whether an earlier attempt of this delivery left a target failed for good. */
export function hadPermanentFailure(env: Env, stateKey: string): Promise<boolean> {
return idempotencyStore(env.DB, env.KV).has(permanentFailureKey(stateKey));
return deliveryClaimStore(env).has(permanentFailureKey(stateKey));
}
function byteLength(text: string): number {
+57
View File
@@ -0,0 +1,57 @@
import type { Env } from "../types";
import { idempotencyStore, type IdempotencyStore } from "../lib/idempotency";
import { log } from "../lib/log";
import { canUseD1 } from "./d1";
/**
* Where the state of a webhook delivery lives: its dedup claim and replay
* nonce at ingress, its lease and status, the finished-target ledger and the
* messages it edits. D1 is the default: Workers Free allows 100,000 row
* writes/day there, while each write is a round trip to the database. KV
* answers from the edge, but allows only ~1,000 writes/day on the free plan
* and has no compare-and-set, so a lease or a claim can be taken twice in a
* race — the deliberate trade for latency at low volume.
*/
export type DeliveryStateStore = "d1" | "kv";
/** Unknown values already reported (the binding is fixed per deployment). */
const warned = new Set<string>();
/** The configured store: `DELIVERY_STATE_STORE=kv` switches, anything else is D1. */
export function deliveryStateStore(env: Pick<Env, "DELIVERY_STATE_STORE">): DeliveryStateStore {
const value = (env.DELIVERY_STATE_STORE ?? "").trim().toLowerCase();
if (value === "kv") return "kv";
if (value !== "" && value !== "d1" && !warned.has(value)) {
warned.add(value);
log.warn({ value }, "Unknown DELIVERY_STATE_STORE, using d1");
}
return "d1";
}
/**
* Whether delivery state is written to D1: the configured store, when the
* binding can take it. State written to KV is marked, so a deployment
* switched back to D1 still honors its claims and leases. The other direction
* is not read: with `kv` set, the D1 rows of the past days — dedup claims and
* their confirmations, delivery states, the finished-target ledger, message
* tracking (30 days) — are invisible, so a redelivery of an event that already
* finished is sent again, targets already sent are sent again, and an
* in-progress `workflow_run`/`check_run` chain posts a new message. Draining
* the queue only keeps in-flight deliveries from overlapping; the D1 rows stay
* invisible either way, so carry the state over (or accept the repeats).
*/
export function deliveryStateUsesD1(env: Pick<Env, "DB" | "DELIVERY_STATE_STORE">): boolean {
return deliveryStateStore(env) === "d1" && canUseD1(env.DB);
}
/**
* The claim store of the delivery pipeline (ingress dedup, replay nonces, the
* delivery target ledger). Out of scope: claims tied to a D1 transaction of
* their own (invites, sessions, tokens, bot interactions, group ids), which
* stay in D1 whichever store deliveries use.
*/
export function deliveryClaimStore(
env: Pick<Env, "DB" | "KV" | "DELIVERY_STATE_STORE">,
): IdempotencyStore {
return idempotencyStore(deliveryStateUsesD1(env) ? env.DB : undefined, env.KV);
}
+6
View File
@@ -31,6 +31,12 @@ export interface Env {
ALLOW_SELF_SIGNUP?: string;
/** Audit log retention in days (default 90). */
AUDIT_RETENTION_DAYS?: string;
/**
* Where webhook delivery state is kept: `d1` (default) or `kv`. D1 allows
* far more free writes but each one is a database round trip; KV answers at
* the edge and is best effort (no compare-and-set).
*/
DELIVERY_STATE_STORE?: string;
ASSETS?: Fetcher;
KV: KVNamespace;
DB: D1Database;
+7 -4
View File
@@ -11,6 +11,7 @@ import { recordAudit } from "./lib/audit";
import { cfEnv, cfWaitUntil, headersFrom } from "./cf";
import { log } from "./lib/log";
import { claimOnce, confirmClaim, deliveryKey, idempotencyStore } from "./lib/idempotency";
import { deliveryClaimStore } from "./storage/state-store";
import { newCorrelationId } from "./lib/correlation";
import {
deliveryStateKey,
@@ -131,7 +132,7 @@ export async function processWebhook(
// otherwise they are given back, so the sender's retry is not refused. A
// claim that could not be given back is never mistaken for an accepted
// event: a retry finds it unconfirmed and is asked to come back later.
const claims = idempotencyStore(env.DB, env.KV);
const claims = deliveryClaimStore(env);
const held: { key: string; ttl: number }[] = [];
const nonce = headers["x-webhooker-nonce"];
if (provider.id === "custom" && nonce && headers["x-webhooker-timestamp"]) {
@@ -217,17 +218,19 @@ export async function processWebhook(
// Done once per installation, on record as long as delivery state is
// kept: a redelivery after the dedup claim lapsed (minutes) must not
// redo a binding an admin undid since. Recorded only once it succeeded,
// so a provisioning that failed midway is completed by the retry.
// so a provisioning that failed midway is completed by the retry. Kept
// in D1 with the bindings it guards, whichever store deliveries use.
const doneKey = `installation-provisioned:${event.installationId}`;
const records = idempotencyStore(env.DB, env.KV);
try {
if (!(await claims.has(doneKey))) {
if (!(await records.has(doneKey))) {
const group = await ensureInstallationGroup(
env.KV,
event.installationId,
account,
env.DB,
);
await claims.claim(doneKey, PROVISIONED_TTL_SECONDS);
await records.claim(doneKey, PROVISIONED_TTL_SECONDS);
if (group) {
await recordAudit(env.DB, {
ts: Date.now(),