diff --git a/server/lib/lib/idempotency.ts b/server/lib/lib/idempotency.ts index 46e847f..5eb351e 100644 --- a/server/lib/lib/idempotency.ts +++ b/server/lib/lib/idempotency.ts @@ -1,15 +1,24 @@ -import { canUseD1 } from "../storage/d1"; +import { canUseD1, isMissingTable, kvFallbackActive, noteKvFallback } from "../storage/d1"; +import { log } from "./log"; /** * Delivery idempotency: a small, reusable abstraction over "has this key been - * seen / can I claim it once" so webhook dedup, nonce replay protection and - * future delivery retry all share one semantics. Backed by D1 when the binding - * is present (atomic `INSERT OR IGNORE`), falling back to a best-effort - * get-then-put KV claim otherwise. + * seen / can I claim it once" so webhook dedup, nonce replay protection, + * invite consumption and bot-command dedup all share one semantics. Backed by + * D1 when the binding is present (atomic UPSERT), falling back to a + * best-effort get-then-put KV claim otherwise — and also when the D1 table + * does not exist yet (an unmigrated database). Other D1 failures propagate. */ export interface IdempotencyStore { has(key: string): Promise; claim(key: string, ttlSeconds: number): Promise; + /** Give a claim back (the claimed work failed and may be retried). */ + release(key: string): Promise; +} + +/** KV rejects TTLs under 60 seconds. */ +function kvTtl(ttlSeconds: number): number { + return Math.max(60, Math.ceil(ttlSeconds)); } export function kvIdempotencyStore(kv: KVNamespace): IdempotencyStore { @@ -21,8 +30,15 @@ export function kvIdempotencyStore(kv: KVNamespace): IdempotencyStore { async claim(key, ttlSeconds): Promise { const value = await kv.get(key); if (value !== null && value !== undefined) return false; - await kv.put(key, "1", { expirationTtl: ttlSeconds }); - return true; + // KV has no compare-and-set: write a unique value and read it back, so + // of two racing claimers the one whose write was overwritten backs off + // (best effort — D1 is the atomic store). + const mine = crypto.randomUUID(); + await kv.put(key, mine, { expirationTtl: kvTtl(ttlSeconds) }); + return (await kv.get(key)) === mine; + }, + async release(key): Promise { + await kv.delete(key); }, }; } @@ -49,21 +65,191 @@ export function d1IdempotencyStore(db: D1Database): IdempotencyStore { .run(); return (result.meta?.changes ?? 0) > 0; }, + async release(key): Promise { + await db.prepare("DELETE FROM dedup_keys WHERE key = ?").bind(key).run(); + }, + }; +} + +/** Run against D1, degrading to KV only while the D1 table is missing. */ +function withKvFallback( + primary: IdempotencyStore, + fallback: IdempotencyStore, + kvNs: KVNamespace, +): IdempotencyStore { + async function run(op: string, d1: () => Promise, kv: () => Promise): Promise { + try { + return await d1(); + } catch (err) { + if (!isMissingTable(err)) throw err; + log.warn({ err, op }, "D1 dedup table missing, falling back to KV"); + await noteKvFallback(kvNs); + return kv(); + } + } + return { + has: (key) => + run( + "has", + async () => + (await primary.has(key)) || ((await kvFallbackActive(kvNs)) && fallback.has(key)), + () => fallback.has(key), + ), + claim: (key, ttl) => + run( + "claim", + async () => { + // Claimed in KV while the table was missing: still taken. + if ((await kvFallbackActive(kvNs)) && (await fallback.has(key))) return false; + return primary.claim(key, ttl); + }, + () => fallback.claim(key, ttl), + ), + release: (key) => + run( + "release", + async () => { + await primary.release(key); + // A copy claimed in KV while the table was missing must go too. + if (await kvFallbackActive(kvNs)) await fallback.release(key); + }, + () => fallback.release(key), + ), }; } export function idempotencyStore(db: D1Database, kv: KVNamespace): IdempotencyStore { - return canUseD1(db) ? d1IdempotencyStore(db) : kvIdempotencyStore(kv); + if (canUseD1(db)) return withKvFallback(d1IdempotencyStore(db), kvIdempotencyStore(kv), kv); + // Without D1, claims live in KV: marked, so a D1 binding added later still + // honors the ones not expired yet. + const store = kvIdempotencyStore(kv); + return { + ...store, + async claim(key, ttl): Promise { + await noteKvFallback(kv); + return store.claim(key, ttl); + }, + }; } /** - * Canonical dedup key for a webhook delivery: provider + tenant + delivery id. - * The legacy global endpoint has no tenant, so `groupId` is "global". + * Key segment naming a group instance. The `@` keeps it apart from every group + * id (ids are `[a-z0-9-]`): an id chosen to equal some instance string can + * never address that instance's keys, and no id-prefix scan reaches them. */ +export function instanceScope(instance: string): string { + return `@i:${instance}`; +} + +/** + * Scope segment of a tenant: the group's immutable instance when known (a + * group re-created under a former id never shares state with its + * predecessor, and a renamed group keeps its own), else the group id + * (messages from before instances), or a marker no group id can take for the + * legacy global endpoint. + */ +export function tenantScope(groupId: string | undefined, instance?: string): string { + if (groupId && instance) return instanceScope(instance); + return groupId ?? "@global"; +} + +/** + * Marker that the work a claim guarded was accepted for good. A claim alone + * may be left over from an attempt that failed and could not give it back; + * only a confirmed one proves a retry is a duplicate. The prefix keeps it + * apart from every claim key (none starts with it), whatever ids they embed. + */ +export function acceptedKey(key: string): string { + return `accepted:${key}`; +} + +/** + * - `claimed`: this caller runs the work (and must confirm or release it); + * - `done`: an earlier run was confirmed — a duplicate; + * - `pending`: another run holds the claim unconfirmed (still running, or it + * failed and could not give the claim back): retry later, after `ttl` at most. + */ +export type ClaimState = "claimed" | "done" | "pending"; + +/** + * Claim one-time work: a short processing claim (`ttlSeconds`), confirmed + * with `confirmClaim` once the work is done for good. A failed run that also + * fails to give its claim back blocks retries only until the claim expires, + * and is never mistaken for a finished one. + */ +export async function claimOnce( + store: IdempotencyStore, + key: string, + ttlSeconds: number, +): Promise { + if (await store.has(acceptedKey(key))) return "done"; + if (await store.claim(key, ttlSeconds)) return "claimed"; + return (await store.has(acceptedKey(key))) ? "done" : "pending"; +} + +const CONFIRM_ATTEMPTS = 3; + +/** + * Record that claimed work was done; duplicates are recognized for + * `ttlSeconds`. Tried a few times: once the work happened, a lost + * confirmation leaves only the short processing claim between a late + * redelivery and running the work again. + */ +export async function confirmClaim( + store: IdempotencyStore, + key: string, + ttlSeconds: number, +): Promise { + for (let attempt = 1; ; attempt++) { + try { + await store.claim(acceptedKey(key), ttlSeconds); + return; + } catch (err) { + if (attempt >= CONFIRM_ATTEMPTS) throw err; + await new Promise((r) => setTimeout(r, 200 * attempt)); + } + } +} + +/** + * Statement recording `key` (as `claim` would) in the same D1 transaction as + * other writes: the record and the writes commit together, so a failure + * whose answer was lost can be told apart afterwards (`has(key)`). + */ +export function recordStatement( + db: D1Database, + key: string, + ttlSeconds: number, +): D1PreparedStatement { + const now = Date.now(); + return db + .prepare( + `INSERT INTO dedup_keys (key, claimed_at, expires_at) VALUES (?, ?, ?) + ON CONFLICT(key) DO UPDATE SET + claimed_at = excluded.claimed_at, expires_at = excluded.expires_at`, + ) + .bind(key, now, now + ttlSeconds * 1000); +} + +/** + * Statement aborting its D1 transaction while `key` is recorded (unexpired): + * a write done once can never be done again by a replay. + */ +export function unrecordedGuard(db: D1Database, key: string): D1PreparedStatement { + return db + .prepare( + `SELECT CASE WHEN EXISTS (SELECT 1 FROM dedup_keys WHERE key = ? AND expires_at > ?) + THEN json('already recorded') ELSE 1 END AS ok`, + ) + .bind(key, Date.now()); +} + +/** Canonical dedup key for a webhook delivery: provider + tenant + delivery id. */ export function deliveryKey( provider: string, groupId: string | undefined, deliveryId: string, + instance?: string, ): string { - return `delivery:${provider}:${groupId ?? "global"}:${deliveryId}`; + return `delivery:${provider}:${tenantScope(groupId, instance)}:${deliveryId}`; } diff --git a/server/lib/storage/d1.ts b/server/lib/storage/d1.ts index 6f60855..c385a4c 100644 --- a/server/lib/storage/d1.ts +++ b/server/lib/storage/d1.ts @@ -1,3 +1,5 @@ +import { sharedFlight, type Flight } from "../lib/shared-flight"; + /** * True when a D1Database binding looks like the real thing. Production D1 * always exposes both `prepare` and `batch`; test harnesses range from @@ -9,3 +11,90 @@ export function canUseD1(db: D1Database | undefined | null): boolean { return typeof db?.prepare === "function" && typeof (db as D1Database).batch === "function"; } + +/** + * The D1 error says the table does not exist yet (migrations not applied). + * Only this case may degrade a D1-backed store to KV: on any other failure + * the two stores would each hold part of the state (a claim taken in D1 is + * invisible in KV), so the error must propagate and the caller retry. + */ +export function isMissingTable(err: unknown): boolean { + return /no such table/i.test(String(err)); +} + +/** + * Set whenever state is written to KV instead of D1: by a D1-backed store + * whose table was missing, and by every store of a deployment without D1 + * (which may get a D1 binding later). Until it expires — longer than any state + * written that way lives, counted from the last such write — reads that find + * nothing in D1 also look in KV, so that state is not forgotten. Its value is + * its own expiry (ms), so a fresh enough marker is not rewritten. + */ +const KV_FALLBACK_MARKER = "storage:kv-fallback"; +const KV_FALLBACK_TTL = 35 * 86400; +const MARKER_RECHECK_MS = 5 * 60_000; +const MARKER_ACTIVE_RECHECK_MS = 3600_000; +/** A fallback write refreshes the marker at most this often (per isolate). */ +const MARKER_REFRESH_MS = 3600_000; +const fallbackMarkers = new WeakMap< + KVNamespace, + { active: boolean; until: number; writtenAt?: number } +>(); +/** Marker reads in flight: concurrent checks share one KV read. */ +const markerReads = new WeakMap>(); + +/** Marker refreshes in flight: concurrent fallbacks share one read and write. */ +const markerWrites = new WeakMap>(); +/** How long another request's marker read or write is waited for. */ +const MARKER_FLIGHT_MS = 10_000; + +export function noteKvFallback(kv: KVNamespace): Promise { + const cached = fallbackMarkers.get(kv); + if (cached?.writtenAt != null && Date.now() - cached.writtenAt < MARKER_REFRESH_MS) { + return Promise.resolve(); + } + return sharedFlight(markerWrites, kv, () => writeFallbackMarker(kv), MARKER_FLIGHT_MS); +} + +async function writeFallbackMarker(kv: KVNamespace): Promise { + const stored = Number(await kv.get(KV_FALLBACK_MARKER)); + // Rewritten at most about daily, whatever the number of isolates. + if (!(stored - Date.now() > (KV_FALLBACK_TTL - 86400) * 1000)) { + await kv.put(KV_FALLBACK_MARKER, String(Date.now() + KV_FALLBACK_TTL * 1000), { + expirationTtl: KV_FALLBACK_TTL, + }); + } + // Remembered only once it is stored: a failed write is tried again by the + // next fallback instead of being taken for done. + fallbackMarkers.set(kv, { + active: true, + until: Date.now() + MARKER_ACTIVE_RECHECK_MS, + writtenAt: Date.now(), + }); +} + +export async function kvFallbackActive(kv: KVNamespace): Promise { + // "No marker" is remembered for a few minutes too: this check runs on + // nearly every claim, and reading KV each time would spend the KV read + // quota (Workers Free) on a marker that is almost always absent. The cost + // is a short window after another isolate first sets it (accepted). + // A seen marker is read again hourly: once it has expired, the KV + // lookups it turns on stop too (a long-lived isolate would otherwise keep + // reading KV for state long gone). + const cached = fallbackMarkers.get(kv); + if (cached && Date.now() < cached.until) return cached.active; + return sharedFlight( + markerReads, + kv, + async () => { + const active = (await kv.get(KV_FALLBACK_MARKER)) != null; + fallbackMarkers.set(kv, { + active, + until: Date.now() + (active ? MARKER_ACTIVE_RECHECK_MS : MARKER_RECHECK_MS), + writtenAt: fallbackMarkers.get(kv)?.writtenAt, + }); + return active; + }, + MARKER_FLIGHT_MS, + ); +} diff --git a/tests/idempotency.test.ts b/tests/idempotency.test.ts index 618bfbe..6ae7f8c 100644 --- a/tests/idempotency.test.ts +++ b/tests/idempotency.test.ts @@ -22,8 +22,9 @@ describe("deliveryKey", () => { expect(deliveryKey("github", "team-a", "deliv-1")).toBe("delivery:github:team-a:deliv-1"); }); - it("falls back to 'global' without a tenant", () => { - expect(deliveryKey("github", undefined, "deliv-1")).toBe("delivery:github:global:deliv-1"); + it("uses a scope no group id can take without a tenant", () => { + expect(deliveryKey("github", undefined, "deliv-1")).toBe("delivery:github:@global:deliv-1"); + expect(deliveryKey("github", "global", "deliv-1")).toBe("delivery:github:global:deliv-1"); }); });