fix(storage): preserve claims across kv fallback

Only missing tables justify KV fallback; other failures must surface.
Remember fallback writes so migration to D1 cannot forget claims
and replay completed work. Share marker checks without extra writes.
This commit is contained in:
翅膀 committed 2026-10-23 23:21:17 +08:00
1 parent 4d5c5e412d
commit 50966a4c50
3 files changed
+289 -13

No files matched your search

+197 -11
View File
@@ -1,15 +1,24 @@
import { canUseD1 } from "../storage/d1";
import { canUseD1, isMissingTable, kvFallbackActive, noteKvFallback } from "../storage/d1";
import { log } from "./log";
/**
* Delivery idempotency: a small, reusable abstraction over "has this key been
* seen / can I claim it once" so webhook dedup, nonce replay protection and
* future delivery retry all share one semantics. Backed by D1 when the binding
* is present (atomic `INSERT OR IGNORE`), falling back to a best-effort
* get-then-put KV claim otherwise.
* seen / can I claim it once" so webhook dedup, nonce replay protection,
* invite consumption and bot-command dedup all share one semantics. Backed by
* D1 when the binding is present (atomic UPSERT), falling back to a
* best-effort get-then-put KV claim otherwise — and also when the D1 table
* does not exist yet (an unmigrated database). Other D1 failures propagate.
*/
export interface IdempotencyStore {
has(key: string): Promise<boolean>;
claim(key: string, ttlSeconds: number): Promise<boolean>;
/** Give a claim back (the claimed work failed and may be retried). */
release(key: string): Promise<void>;
}
/** KV rejects TTLs under 60 seconds. */
function kvTtl(ttlSeconds: number): number {
return Math.max(60, Math.ceil(ttlSeconds));
}
export function kvIdempotencyStore(kv: KVNamespace): IdempotencyStore {
@@ -21,8 +30,15 @@ export function kvIdempotencyStore(kv: KVNamespace): IdempotencyStore {
async claim(key, ttlSeconds): Promise<boolean> {
const value = await kv.get(key);
if (value !== null && value !== undefined) return false;
await kv.put(key, "1", { expirationTtl: ttlSeconds });
return true;
// KV has no compare-and-set: write a unique value and read it back, so
// of two racing claimers the one whose write was overwritten backs off
// (best effort — D1 is the atomic store).
const mine = crypto.randomUUID();
await kv.put(key, mine, { expirationTtl: kvTtl(ttlSeconds) });
return (await kv.get(key)) === mine;
},
async release(key): Promise<void> {
await kv.delete(key);
},
};
}
@@ -49,21 +65,191 @@ export function d1IdempotencyStore(db: D1Database): IdempotencyStore {
.run();
return (result.meta?.changes ?? 0) > 0;
},
async release(key): Promise<void> {
await db.prepare("DELETE FROM dedup_keys WHERE key = ?").bind(key).run();
},
};
}
/** Run against D1, degrading to KV only while the D1 table is missing. */
function withKvFallback(
primary: IdempotencyStore,
fallback: IdempotencyStore,
kvNs: KVNamespace,
): IdempotencyStore {
async function run<T>(op: string, d1: () => Promise<T>, kv: () => Promise<T>): Promise<T> {
try {
return await d1();
} catch (err) {
if (!isMissingTable(err)) throw err;
log.warn({ err, op }, "D1 dedup table missing, falling back to KV");
await noteKvFallback(kvNs);
return kv();
}
}
return {
has: (key) =>
run(
"has",
async () =>
(await primary.has(key)) || ((await kvFallbackActive(kvNs)) && fallback.has(key)),
() => fallback.has(key),
),
claim: (key, ttl) =>
run(
"claim",
async () => {
// Claimed in KV while the table was missing: still taken.
if ((await kvFallbackActive(kvNs)) && (await fallback.has(key))) return false;
return primary.claim(key, ttl);
},
() => fallback.claim(key, ttl),
),
release: (key) =>
run(
"release",
async () => {
await primary.release(key);
// A copy claimed in KV while the table was missing must go too.
if (await kvFallbackActive(kvNs)) await fallback.release(key);
},
() => fallback.release(key),
),
};
}
export function idempotencyStore(db: D1Database, kv: KVNamespace): IdempotencyStore {
return canUseD1(db) ? d1IdempotencyStore(db) : kvIdempotencyStore(kv);
if (canUseD1(db)) return withKvFallback(d1IdempotencyStore(db), kvIdempotencyStore(kv), kv);
// Without D1, claims live in KV: marked, so a D1 binding added later still
// honors the ones not expired yet.
const store = kvIdempotencyStore(kv);
return {
...store,
async claim(key, ttl): Promise<boolean> {
await noteKvFallback(kv);
return store.claim(key, ttl);
},
};
}
/**
* Canonical dedup key for a webhook delivery: provider + tenant + delivery id.
* The legacy global endpoint has no tenant, so `groupId` is "global".
* Key segment naming a group instance. The `@` keeps it apart from every group
* id (ids are `[a-z0-9-]`): an id chosen to equal some instance string can
* never address that instance's keys, and no id-prefix scan reaches them.
*/
export function instanceScope(instance: string): string {
return `@i:${instance}`;
}
/**
* Scope segment of a tenant: the group's immutable instance when known (a
* group re-created under a former id never shares state with its
* predecessor, and a renamed group keeps its own), else the group id
* (messages from before instances), or a marker no group id can take for the
* legacy global endpoint.
*/
export function tenantScope(groupId: string | undefined, instance?: string): string {
if (groupId && instance) return instanceScope(instance);
return groupId ?? "@global";
}
/**
* Marker that the work a claim guarded was accepted for good. A claim alone
* may be left over from an attempt that failed and could not give it back;
* only a confirmed one proves a retry is a duplicate. The prefix keeps it
* apart from every claim key (none starts with it), whatever ids they embed.
*/
export function acceptedKey(key: string): string {
return `accepted:${key}`;
}
/**
* - `claimed`: this caller runs the work (and must confirm or release it);
* - `done`: an earlier run was confirmed — a duplicate;
* - `pending`: another run holds the claim unconfirmed (still running, or it
* failed and could not give the claim back): retry later, after `ttl` at most.
*/
export type ClaimState = "claimed" | "done" | "pending";
/**
* Claim one-time work: a short processing claim (`ttlSeconds`), confirmed
* with `confirmClaim` once the work is done for good. A failed run that also
* fails to give its claim back blocks retries only until the claim expires,
* and is never mistaken for a finished one.
*/
export async function claimOnce(
store: IdempotencyStore,
key: string,
ttlSeconds: number,
): Promise<ClaimState> {
if (await store.has(acceptedKey(key))) return "done";
if (await store.claim(key, ttlSeconds)) return "claimed";
return (await store.has(acceptedKey(key))) ? "done" : "pending";
}
const CONFIRM_ATTEMPTS = 3;
/**
* Record that claimed work was done; duplicates are recognized for
* `ttlSeconds`. Tried a few times: once the work happened, a lost
* confirmation leaves only the short processing claim between a late
* redelivery and running the work again.
*/
export async function confirmClaim(
store: IdempotencyStore,
key: string,
ttlSeconds: number,
): Promise<void> {
for (let attempt = 1; ; attempt++) {
try {
await store.claim(acceptedKey(key), ttlSeconds);
return;
} catch (err) {
if (attempt >= CONFIRM_ATTEMPTS) throw err;
await new Promise((r) => setTimeout(r, 200 * attempt));
}
}
}
/**
* Statement recording `key` (as `claim` would) in the same D1 transaction as
* other writes: the record and the writes commit together, so a failure
* whose answer was lost can be told apart afterwards (`has(key)`).
*/
export function recordStatement(
db: D1Database,
key: string,
ttlSeconds: number,
): D1PreparedStatement {
const now = Date.now();
return db
.prepare(
`INSERT INTO dedup_keys (key, claimed_at, expires_at) VALUES (?, ?, ?)
ON CONFLICT(key) DO UPDATE SET
claimed_at = excluded.claimed_at, expires_at = excluded.expires_at`,
)
.bind(key, now, now + ttlSeconds * 1000);
}
/**
* Statement aborting its D1 transaction while `key` is recorded (unexpired):
* a write done once can never be done again by a replay.
*/
export function unrecordedGuard(db: D1Database, key: string): D1PreparedStatement {
return db
.prepare(
`SELECT CASE WHEN EXISTS (SELECT 1 FROM dedup_keys WHERE key = ? AND expires_at > ?)
THEN json('already recorded') ELSE 1 END AS ok`,
)
.bind(key, Date.now());
}
/** Canonical dedup key for a webhook delivery: provider + tenant + delivery id. */
export function deliveryKey(
provider: string,
groupId: string | undefined,
deliveryId: string,
instance?: string,
): string {
return `delivery:${provider}:${groupId ?? "global"}:${deliveryId}`;
return `delivery:${provider}:${tenantScope(groupId, instance)}:${deliveryId}`;
}
+89
View File
@@ -1,3 +1,5 @@
import { sharedFlight, type Flight } from "../lib/shared-flight";
/**
* True when a D1Database binding looks like the real thing. Production D1
* always exposes both `prepare` and `batch`; test harnesses range from
@@ -9,3 +11,90 @@
export function canUseD1(db: D1Database | undefined | null): boolean {
return typeof db?.prepare === "function" && typeof (db as D1Database).batch === "function";
}
/**
* The D1 error says the table does not exist yet (migrations not applied).
* Only this case may degrade a D1-backed store to KV: on any other failure
* the two stores would each hold part of the state (a claim taken in D1 is
* invisible in KV), so the error must propagate and the caller retry.
*/
export function isMissingTable(err: unknown): boolean {
return /no such table/i.test(String(err));
}
/**
* Set whenever state is written to KV instead of D1: by a D1-backed store
* whose table was missing, and by every store of a deployment without D1
* (which may get a D1 binding later). Until it expires — longer than any state
* written that way lives, counted from the last such write — reads that find
* nothing in D1 also look in KV, so that state is not forgotten. Its value is
* its own expiry (ms), so a fresh enough marker is not rewritten.
*/
const KV_FALLBACK_MARKER = "storage:kv-fallback";
const KV_FALLBACK_TTL = 35 * 86400;
const MARKER_RECHECK_MS = 5 * 60_000;
const MARKER_ACTIVE_RECHECK_MS = 3600_000;
/** A fallback write refreshes the marker at most this often (per isolate). */
const MARKER_REFRESH_MS = 3600_000;
const fallbackMarkers = new WeakMap<
KVNamespace,
{ active: boolean; until: number; writtenAt?: number }
>();
/** Marker reads in flight: concurrent checks share one KV read. */
const markerReads = new WeakMap<KVNamespace, Flight<boolean>>();
/** Marker refreshes in flight: concurrent fallbacks share one read and write. */
const markerWrites = new WeakMap<KVNamespace, Flight<void>>();
/** How long another request's marker read or write is waited for. */
const MARKER_FLIGHT_MS = 10_000;
export function noteKvFallback(kv: KVNamespace): Promise<void> {
const cached = fallbackMarkers.get(kv);
if (cached?.writtenAt != null && Date.now() - cached.writtenAt < MARKER_REFRESH_MS) {
return Promise.resolve();
}
return sharedFlight(markerWrites, kv, () => writeFallbackMarker(kv), MARKER_FLIGHT_MS);
}
async function writeFallbackMarker(kv: KVNamespace): Promise<void> {
const stored = Number(await kv.get(KV_FALLBACK_MARKER));
// Rewritten at most about daily, whatever the number of isolates.
if (!(stored - Date.now() > (KV_FALLBACK_TTL - 86400) * 1000)) {
await kv.put(KV_FALLBACK_MARKER, String(Date.now() + KV_FALLBACK_TTL * 1000), {
expirationTtl: KV_FALLBACK_TTL,
});
}
// Remembered only once it is stored: a failed write is tried again by the
// next fallback instead of being taken for done.
fallbackMarkers.set(kv, {
active: true,
until: Date.now() + MARKER_ACTIVE_RECHECK_MS,
writtenAt: Date.now(),
});
}
export async function kvFallbackActive(kv: KVNamespace): Promise<boolean> {
// "No marker" is remembered for a few minutes too: this check runs on
// nearly every claim, and reading KV each time would spend the KV read
// quota (Workers Free) on a marker that is almost always absent. The cost
// is a short window after another isolate first sets it (accepted).
// A seen marker is read again hourly: once it has expired, the KV
// lookups it turns on stop too (a long-lived isolate would otherwise keep
// reading KV for state long gone).
const cached = fallbackMarkers.get(kv);
if (cached && Date.now() < cached.until) return cached.active;
return sharedFlight(
markerReads,
kv,
async () => {
const active = (await kv.get(KV_FALLBACK_MARKER)) != null;
fallbackMarkers.set(kv, {
active,
until: Date.now() + (active ? MARKER_ACTIVE_RECHECK_MS : MARKER_RECHECK_MS),
writtenAt: fallbackMarkers.get(kv)?.writtenAt,
});
return active;
},
MARKER_FLIGHT_MS,
);
}
+3 -2
View File
@@ -22,8 +22,9 @@ describe("deliveryKey", () => {
expect(deliveryKey("github", "team-a", "deliv-1")).toBe("delivery:github:team-a:deliv-1");
});
it("falls back to 'global' without a tenant", () => {
expect(deliveryKey("github", undefined, "deliv-1")).toBe("delivery:github:global:deliv-1");
it("uses a scope no group id can take without a tenant", () => {
expect(deliveryKey("github", undefined, "deliv-1")).toBe("delivery:github:@global:deliv-1");
expect(deliveryKey("github", "global", "deliv-1")).toBe("delivery:github:global:deliv-1");
});
});