mirror of
https://github.com/Elysola/WebHooker.git
synced 2026-11-20 17:13:49 +00:00
fix(storage): preserve claims across kv fallback
Only missing tables justify KV fallback; other failures must surface. Remember fallback writes so migration to D1 cannot forget claims and replay completed work. Share marker checks without extra writes.
This commit is contained in:
1 parent
4d5c5e412d
commit
50966a4c50
3 files changed
+289
-13
No files matched your search
+197
-11
@@ -1,15 +1,24 @@
|
||||
import { canUseD1 } from "../storage/d1";
|
||||
import { canUseD1, isMissingTable, kvFallbackActive, noteKvFallback } from "../storage/d1";
|
||||
import { log } from "./log";
|
||||
|
||||
/**
|
||||
* Delivery idempotency: a small, reusable abstraction over "has this key been
|
||||
* seen / can I claim it once" so webhook dedup, nonce replay protection and
|
||||
* future delivery retry all share one semantics. Backed by D1 when the binding
|
||||
* is present (atomic `INSERT OR IGNORE`), falling back to a best-effort
|
||||
* get-then-put KV claim otherwise.
|
||||
* seen / can I claim it once" so webhook dedup, nonce replay protection,
|
||||
* invite consumption and bot-command dedup all share one semantics. Backed by
|
||||
* D1 when the binding is present (atomic UPSERT), falling back to a
|
||||
* best-effort get-then-put KV claim otherwise — and also when the D1 table
|
||||
* does not exist yet (an unmigrated database). Other D1 failures propagate.
|
||||
*/
|
||||
export interface IdempotencyStore {
|
||||
has(key: string): Promise<boolean>;
|
||||
claim(key: string, ttlSeconds: number): Promise<boolean>;
|
||||
/** Give a claim back (the claimed work failed and may be retried). */
|
||||
release(key: string): Promise<void>;
|
||||
}
|
||||
|
||||
/** KV rejects TTLs under 60 seconds. */
|
||||
function kvTtl(ttlSeconds: number): number {
|
||||
return Math.max(60, Math.ceil(ttlSeconds));
|
||||
}
|
||||
|
||||
export function kvIdempotencyStore(kv: KVNamespace): IdempotencyStore {
|
||||
@@ -21,8 +30,15 @@ export function kvIdempotencyStore(kv: KVNamespace): IdempotencyStore {
|
||||
async claim(key, ttlSeconds): Promise<boolean> {
|
||||
const value = await kv.get(key);
|
||||
if (value !== null && value !== undefined) return false;
|
||||
await kv.put(key, "1", { expirationTtl: ttlSeconds });
|
||||
return true;
|
||||
// KV has no compare-and-set: write a unique value and read it back, so
|
||||
// of two racing claimers the one whose write was overwritten backs off
|
||||
// (best effort — D1 is the atomic store).
|
||||
const mine = crypto.randomUUID();
|
||||
await kv.put(key, mine, { expirationTtl: kvTtl(ttlSeconds) });
|
||||
return (await kv.get(key)) === mine;
|
||||
},
|
||||
async release(key): Promise<void> {
|
||||
await kv.delete(key);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -49,21 +65,191 @@ export function d1IdempotencyStore(db: D1Database): IdempotencyStore {
|
||||
.run();
|
||||
return (result.meta?.changes ?? 0) > 0;
|
||||
},
|
||||
async release(key): Promise<void> {
|
||||
await db.prepare("DELETE FROM dedup_keys WHERE key = ?").bind(key).run();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Run against D1, degrading to KV only while the D1 table is missing. */
|
||||
function withKvFallback(
|
||||
primary: IdempotencyStore,
|
||||
fallback: IdempotencyStore,
|
||||
kvNs: KVNamespace,
|
||||
): IdempotencyStore {
|
||||
async function run<T>(op: string, d1: () => Promise<T>, kv: () => Promise<T>): Promise<T> {
|
||||
try {
|
||||
return await d1();
|
||||
} catch (err) {
|
||||
if (!isMissingTable(err)) throw err;
|
||||
log.warn({ err, op }, "D1 dedup table missing, falling back to KV");
|
||||
await noteKvFallback(kvNs);
|
||||
return kv();
|
||||
}
|
||||
}
|
||||
return {
|
||||
has: (key) =>
|
||||
run(
|
||||
"has",
|
||||
async () =>
|
||||
(await primary.has(key)) || ((await kvFallbackActive(kvNs)) && fallback.has(key)),
|
||||
() => fallback.has(key),
|
||||
),
|
||||
claim: (key, ttl) =>
|
||||
run(
|
||||
"claim",
|
||||
async () => {
|
||||
// Claimed in KV while the table was missing: still taken.
|
||||
if ((await kvFallbackActive(kvNs)) && (await fallback.has(key))) return false;
|
||||
return primary.claim(key, ttl);
|
||||
},
|
||||
() => fallback.claim(key, ttl),
|
||||
),
|
||||
release: (key) =>
|
||||
run(
|
||||
"release",
|
||||
async () => {
|
||||
await primary.release(key);
|
||||
// A copy claimed in KV while the table was missing must go too.
|
||||
if (await kvFallbackActive(kvNs)) await fallback.release(key);
|
||||
},
|
||||
() => fallback.release(key),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
export function idempotencyStore(db: D1Database, kv: KVNamespace): IdempotencyStore {
|
||||
return canUseD1(db) ? d1IdempotencyStore(db) : kvIdempotencyStore(kv);
|
||||
if (canUseD1(db)) return withKvFallback(d1IdempotencyStore(db), kvIdempotencyStore(kv), kv);
|
||||
// Without D1, claims live in KV: marked, so a D1 binding added later still
|
||||
// honors the ones not expired yet.
|
||||
const store = kvIdempotencyStore(kv);
|
||||
return {
|
||||
...store,
|
||||
async claim(key, ttl): Promise<boolean> {
|
||||
await noteKvFallback(kv);
|
||||
return store.claim(key, ttl);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Canonical dedup key for a webhook delivery: provider + tenant + delivery id.
|
||||
* The legacy global endpoint has no tenant, so `groupId` is "global".
|
||||
* Key segment naming a group instance. The `@` keeps it apart from every group
|
||||
* id (ids are `[a-z0-9-]`): an id chosen to equal some instance string can
|
||||
* never address that instance's keys, and no id-prefix scan reaches them.
|
||||
*/
|
||||
export function instanceScope(instance: string): string {
|
||||
return `@i:${instance}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Scope segment of a tenant: the group's immutable instance when known (a
|
||||
* group re-created under a former id never shares state with its
|
||||
* predecessor, and a renamed group keeps its own), else the group id
|
||||
* (messages from before instances), or a marker no group id can take for the
|
||||
* legacy global endpoint.
|
||||
*/
|
||||
export function tenantScope(groupId: string | undefined, instance?: string): string {
|
||||
if (groupId && instance) return instanceScope(instance);
|
||||
return groupId ?? "@global";
|
||||
}
|
||||
|
||||
/**
|
||||
* Marker that the work a claim guarded was accepted for good. A claim alone
|
||||
* may be left over from an attempt that failed and could not give it back;
|
||||
* only a confirmed one proves a retry is a duplicate. The prefix keeps it
|
||||
* apart from every claim key (none starts with it), whatever ids they embed.
|
||||
*/
|
||||
export function acceptedKey(key: string): string {
|
||||
return `accepted:${key}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* - `claimed`: this caller runs the work (and must confirm or release it);
|
||||
* - `done`: an earlier run was confirmed — a duplicate;
|
||||
* - `pending`: another run holds the claim unconfirmed (still running, or it
|
||||
* failed and could not give the claim back): retry later, after `ttl` at most.
|
||||
*/
|
||||
export type ClaimState = "claimed" | "done" | "pending";
|
||||
|
||||
/**
|
||||
* Claim one-time work: a short processing claim (`ttlSeconds`), confirmed
|
||||
* with `confirmClaim` once the work is done for good. A failed run that also
|
||||
* fails to give its claim back blocks retries only until the claim expires,
|
||||
* and is never mistaken for a finished one.
|
||||
*/
|
||||
export async function claimOnce(
|
||||
store: IdempotencyStore,
|
||||
key: string,
|
||||
ttlSeconds: number,
|
||||
): Promise<ClaimState> {
|
||||
if (await store.has(acceptedKey(key))) return "done";
|
||||
if (await store.claim(key, ttlSeconds)) return "claimed";
|
||||
return (await store.has(acceptedKey(key))) ? "done" : "pending";
|
||||
}
|
||||
|
||||
const CONFIRM_ATTEMPTS = 3;
|
||||
|
||||
/**
|
||||
* Record that claimed work was done; duplicates are recognized for
|
||||
* `ttlSeconds`. Tried a few times: once the work happened, a lost
|
||||
* confirmation leaves only the short processing claim between a late
|
||||
* redelivery and running the work again.
|
||||
*/
|
||||
export async function confirmClaim(
|
||||
store: IdempotencyStore,
|
||||
key: string,
|
||||
ttlSeconds: number,
|
||||
): Promise<void> {
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
try {
|
||||
await store.claim(acceptedKey(key), ttlSeconds);
|
||||
return;
|
||||
} catch (err) {
|
||||
if (attempt >= CONFIRM_ATTEMPTS) throw err;
|
||||
await new Promise((r) => setTimeout(r, 200 * attempt));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Statement recording `key` (as `claim` would) in the same D1 transaction as
|
||||
* other writes: the record and the writes commit together, so a failure
|
||||
* whose answer was lost can be told apart afterwards (`has(key)`).
|
||||
*/
|
||||
export function recordStatement(
|
||||
db: D1Database,
|
||||
key: string,
|
||||
ttlSeconds: number,
|
||||
): D1PreparedStatement {
|
||||
const now = Date.now();
|
||||
return db
|
||||
.prepare(
|
||||
`INSERT INTO dedup_keys (key, claimed_at, expires_at) VALUES (?, ?, ?)
|
||||
ON CONFLICT(key) DO UPDATE SET
|
||||
claimed_at = excluded.claimed_at, expires_at = excluded.expires_at`,
|
||||
)
|
||||
.bind(key, now, now + ttlSeconds * 1000);
|
||||
}
|
||||
|
||||
/**
|
||||
* Statement aborting its D1 transaction while `key` is recorded (unexpired):
|
||||
* a write done once can never be done again by a replay.
|
||||
*/
|
||||
export function unrecordedGuard(db: D1Database, key: string): D1PreparedStatement {
|
||||
return db
|
||||
.prepare(
|
||||
`SELECT CASE WHEN EXISTS (SELECT 1 FROM dedup_keys WHERE key = ? AND expires_at > ?)
|
||||
THEN json('already recorded') ELSE 1 END AS ok`,
|
||||
)
|
||||
.bind(key, Date.now());
|
||||
}
|
||||
|
||||
/** Canonical dedup key for a webhook delivery: provider + tenant + delivery id. */
|
||||
export function deliveryKey(
|
||||
provider: string,
|
||||
groupId: string | undefined,
|
||||
deliveryId: string,
|
||||
instance?: string,
|
||||
): string {
|
||||
return `delivery:${provider}:${groupId ?? "global"}:${deliveryId}`;
|
||||
return `delivery:${provider}:${tenantScope(groupId, instance)}:${deliveryId}`;
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
import { sharedFlight, type Flight } from "../lib/shared-flight";
|
||||
|
||||
/**
|
||||
* True when a D1Database binding looks like the real thing. Production D1
|
||||
* always exposes both `prepare` and `batch`; test harnesses range from
|
||||
@@ -9,3 +11,90 @@
|
||||
export function canUseD1(db: D1Database | undefined | null): boolean {
|
||||
return typeof db?.prepare === "function" && typeof (db as D1Database).batch === "function";
|
||||
}
|
||||
|
||||
/**
|
||||
* The D1 error says the table does not exist yet (migrations not applied).
|
||||
* Only this case may degrade a D1-backed store to KV: on any other failure
|
||||
* the two stores would each hold part of the state (a claim taken in D1 is
|
||||
* invisible in KV), so the error must propagate and the caller retry.
|
||||
*/
|
||||
export function isMissingTable(err: unknown): boolean {
|
||||
return /no such table/i.test(String(err));
|
||||
}
|
||||
|
||||
/**
|
||||
* Set whenever state is written to KV instead of D1: by a D1-backed store
|
||||
* whose table was missing, and by every store of a deployment without D1
|
||||
* (which may get a D1 binding later). Until it expires — longer than any state
|
||||
* written that way lives, counted from the last such write — reads that find
|
||||
* nothing in D1 also look in KV, so that state is not forgotten. Its value is
|
||||
* its own expiry (ms), so a fresh enough marker is not rewritten.
|
||||
*/
|
||||
const KV_FALLBACK_MARKER = "storage:kv-fallback";
|
||||
const KV_FALLBACK_TTL = 35 * 86400;
|
||||
const MARKER_RECHECK_MS = 5 * 60_000;
|
||||
const MARKER_ACTIVE_RECHECK_MS = 3600_000;
|
||||
/** A fallback write refreshes the marker at most this often (per isolate). */
|
||||
const MARKER_REFRESH_MS = 3600_000;
|
||||
const fallbackMarkers = new WeakMap<
|
||||
KVNamespace,
|
||||
{ active: boolean; until: number; writtenAt?: number }
|
||||
>();
|
||||
/** Marker reads in flight: concurrent checks share one KV read. */
|
||||
const markerReads = new WeakMap<KVNamespace, Flight<boolean>>();
|
||||
|
||||
/** Marker refreshes in flight: concurrent fallbacks share one read and write. */
|
||||
const markerWrites = new WeakMap<KVNamespace, Flight<void>>();
|
||||
/** How long another request's marker read or write is waited for. */
|
||||
const MARKER_FLIGHT_MS = 10_000;
|
||||
|
||||
export function noteKvFallback(kv: KVNamespace): Promise<void> {
|
||||
const cached = fallbackMarkers.get(kv);
|
||||
if (cached?.writtenAt != null && Date.now() - cached.writtenAt < MARKER_REFRESH_MS) {
|
||||
return Promise.resolve();
|
||||
}
|
||||
return sharedFlight(markerWrites, kv, () => writeFallbackMarker(kv), MARKER_FLIGHT_MS);
|
||||
}
|
||||
|
||||
async function writeFallbackMarker(kv: KVNamespace): Promise<void> {
|
||||
const stored = Number(await kv.get(KV_FALLBACK_MARKER));
|
||||
// Rewritten at most about daily, whatever the number of isolates.
|
||||
if (!(stored - Date.now() > (KV_FALLBACK_TTL - 86400) * 1000)) {
|
||||
await kv.put(KV_FALLBACK_MARKER, String(Date.now() + KV_FALLBACK_TTL * 1000), {
|
||||
expirationTtl: KV_FALLBACK_TTL,
|
||||
});
|
||||
}
|
||||
// Remembered only once it is stored: a failed write is tried again by the
|
||||
// next fallback instead of being taken for done.
|
||||
fallbackMarkers.set(kv, {
|
||||
active: true,
|
||||
until: Date.now() + MARKER_ACTIVE_RECHECK_MS,
|
||||
writtenAt: Date.now(),
|
||||
});
|
||||
}
|
||||
|
||||
export async function kvFallbackActive(kv: KVNamespace): Promise<boolean> {
|
||||
// "No marker" is remembered for a few minutes too: this check runs on
|
||||
// nearly every claim, and reading KV each time would spend the KV read
|
||||
// quota (Workers Free) on a marker that is almost always absent. The cost
|
||||
// is a short window after another isolate first sets it (accepted).
|
||||
// A seen marker is read again hourly: once it has expired, the KV
|
||||
// lookups it turns on stop too (a long-lived isolate would otherwise keep
|
||||
// reading KV for state long gone).
|
||||
const cached = fallbackMarkers.get(kv);
|
||||
if (cached && Date.now() < cached.until) return cached.active;
|
||||
return sharedFlight(
|
||||
markerReads,
|
||||
kv,
|
||||
async () => {
|
||||
const active = (await kv.get(KV_FALLBACK_MARKER)) != null;
|
||||
fallbackMarkers.set(kv, {
|
||||
active,
|
||||
until: Date.now() + (active ? MARKER_ACTIVE_RECHECK_MS : MARKER_RECHECK_MS),
|
||||
writtenAt: fallbackMarkers.get(kv)?.writtenAt,
|
||||
});
|
||||
return active;
|
||||
},
|
||||
MARKER_FLIGHT_MS,
|
||||
);
|
||||
}
|
||||
@@ -22,8 +22,9 @@ describe("deliveryKey", () => {
|
||||
expect(deliveryKey("github", "team-a", "deliv-1")).toBe("delivery:github:team-a:deliv-1");
|
||||
});
|
||||
|
||||
it("falls back to 'global' without a tenant", () => {
|
||||
expect(deliveryKey("github", undefined, "deliv-1")).toBe("delivery:github:global:deliv-1");
|
||||
it("uses a scope no group id can take without a tenant", () => {
|
||||
expect(deliveryKey("github", undefined, "deliv-1")).toBe("delivery:github:@global:deliv-1");
|
||||
expect(deliveryKey("github", "global", "deliv-1")).toBe("delivery:github:global:deliv-1");
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
Reference in new issue
Block a user