mirror of
https://github.com/Elysola/WebHooker.git
synced 2026-11-21 01:23:40 +00:00
fix(storage): preserve claims across kv fallback
Only missing tables justify KV fallback; other failures must surface. Remember fallback writes so migration to D1 cannot forget claims and replay completed work. Share marker checks without extra writes.
This commit is contained in:
1 parent
4d5c5e412d
commit
50966a4c50
3 files changed
+289
-13
No files matched your search
+197
-11
@@ -1,15 +1,24 @@
|
|||||||
import { canUseD1 } from "../storage/d1";
|
import { canUseD1, isMissingTable, kvFallbackActive, noteKvFallback } from "../storage/d1";
|
||||||
|
import { log } from "./log";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Delivery idempotency: a small, reusable abstraction over "has this key been
|
* Delivery idempotency: a small, reusable abstraction over "has this key been
|
||||||
* seen / can I claim it once" so webhook dedup, nonce replay protection and
|
* seen / can I claim it once" so webhook dedup, nonce replay protection,
|
||||||
* future delivery retry all share one semantics. Backed by D1 when the binding
|
* invite consumption and bot-command dedup all share one semantics. Backed by
|
||||||
* is present (atomic `INSERT OR IGNORE`), falling back to a best-effort
|
* D1 when the binding is present (atomic UPSERT), falling back to a
|
||||||
* get-then-put KV claim otherwise.
|
* best-effort get-then-put KV claim otherwise — and also when the D1 table
|
||||||
|
* does not exist yet (an unmigrated database). Other D1 failures propagate.
|
||||||
*/
|
*/
|
||||||
export interface IdempotencyStore {
|
export interface IdempotencyStore {
|
||||||
has(key: string): Promise<boolean>;
|
has(key: string): Promise<boolean>;
|
||||||
claim(key: string, ttlSeconds: number): Promise<boolean>;
|
claim(key: string, ttlSeconds: number): Promise<boolean>;
|
||||||
|
/** Give a claim back (the claimed work failed and may be retried). */
|
||||||
|
release(key: string): Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** KV rejects TTLs under 60 seconds. */
|
||||||
|
function kvTtl(ttlSeconds: number): number {
|
||||||
|
return Math.max(60, Math.ceil(ttlSeconds));
|
||||||
}
|
}
|
||||||
|
|
||||||
export function kvIdempotencyStore(kv: KVNamespace): IdempotencyStore {
|
export function kvIdempotencyStore(kv: KVNamespace): IdempotencyStore {
|
||||||
@@ -21,8 +30,15 @@ export function kvIdempotencyStore(kv: KVNamespace): IdempotencyStore {
|
|||||||
async claim(key, ttlSeconds): Promise<boolean> {
|
async claim(key, ttlSeconds): Promise<boolean> {
|
||||||
const value = await kv.get(key);
|
const value = await kv.get(key);
|
||||||
if (value !== null && value !== undefined) return false;
|
if (value !== null && value !== undefined) return false;
|
||||||
await kv.put(key, "1", { expirationTtl: ttlSeconds });
|
// KV has no compare-and-set: write a unique value and read it back, so
|
||||||
return true;
|
// of two racing claimers the one whose write was overwritten backs off
|
||||||
|
// (best effort — D1 is the atomic store).
|
||||||
|
const mine = crypto.randomUUID();
|
||||||
|
await kv.put(key, mine, { expirationTtl: kvTtl(ttlSeconds) });
|
||||||
|
return (await kv.get(key)) === mine;
|
||||||
|
},
|
||||||
|
async release(key): Promise<void> {
|
||||||
|
await kv.delete(key);
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -49,21 +65,191 @@ export function d1IdempotencyStore(db: D1Database): IdempotencyStore {
|
|||||||
.run();
|
.run();
|
||||||
return (result.meta?.changes ?? 0) > 0;
|
return (result.meta?.changes ?? 0) > 0;
|
||||||
},
|
},
|
||||||
|
async release(key): Promise<void> {
|
||||||
|
await db.prepare("DELETE FROM dedup_keys WHERE key = ?").bind(key).run();
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Run against D1, degrading to KV only while the D1 table is missing. */
|
||||||
|
function withKvFallback(
|
||||||
|
primary: IdempotencyStore,
|
||||||
|
fallback: IdempotencyStore,
|
||||||
|
kvNs: KVNamespace,
|
||||||
|
): IdempotencyStore {
|
||||||
|
async function run<T>(op: string, d1: () => Promise<T>, kv: () => Promise<T>): Promise<T> {
|
||||||
|
try {
|
||||||
|
return await d1();
|
||||||
|
} catch (err) {
|
||||||
|
if (!isMissingTable(err)) throw err;
|
||||||
|
log.warn({ err, op }, "D1 dedup table missing, falling back to KV");
|
||||||
|
await noteKvFallback(kvNs);
|
||||||
|
return kv();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
has: (key) =>
|
||||||
|
run(
|
||||||
|
"has",
|
||||||
|
async () =>
|
||||||
|
(await primary.has(key)) || ((await kvFallbackActive(kvNs)) && fallback.has(key)),
|
||||||
|
() => fallback.has(key),
|
||||||
|
),
|
||||||
|
claim: (key, ttl) =>
|
||||||
|
run(
|
||||||
|
"claim",
|
||||||
|
async () => {
|
||||||
|
// Claimed in KV while the table was missing: still taken.
|
||||||
|
if ((await kvFallbackActive(kvNs)) && (await fallback.has(key))) return false;
|
||||||
|
return primary.claim(key, ttl);
|
||||||
|
},
|
||||||
|
() => fallback.claim(key, ttl),
|
||||||
|
),
|
||||||
|
release: (key) =>
|
||||||
|
run(
|
||||||
|
"release",
|
||||||
|
async () => {
|
||||||
|
await primary.release(key);
|
||||||
|
// A copy claimed in KV while the table was missing must go too.
|
||||||
|
if (await kvFallbackActive(kvNs)) await fallback.release(key);
|
||||||
|
},
|
||||||
|
() => fallback.release(key),
|
||||||
|
),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export function idempotencyStore(db: D1Database, kv: KVNamespace): IdempotencyStore {
|
export function idempotencyStore(db: D1Database, kv: KVNamespace): IdempotencyStore {
|
||||||
return canUseD1(db) ? d1IdempotencyStore(db) : kvIdempotencyStore(kv);
|
if (canUseD1(db)) return withKvFallback(d1IdempotencyStore(db), kvIdempotencyStore(kv), kv);
|
||||||
|
// Without D1, claims live in KV: marked, so a D1 binding added later still
|
||||||
|
// honors the ones not expired yet.
|
||||||
|
const store = kvIdempotencyStore(kv);
|
||||||
|
return {
|
||||||
|
...store,
|
||||||
|
async claim(key, ttl): Promise<boolean> {
|
||||||
|
await noteKvFallback(kv);
|
||||||
|
return store.claim(key, ttl);
|
||||||
|
},
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Canonical dedup key for a webhook delivery: provider + tenant + delivery id.
|
* Key segment naming a group instance. The `@` keeps it apart from every group
|
||||||
* The legacy global endpoint has no tenant, so `groupId` is "global".
|
* id (ids are `[a-z0-9-]`): an id chosen to equal some instance string can
|
||||||
|
* never address that instance's keys, and no id-prefix scan reaches them.
|
||||||
*/
|
*/
|
||||||
|
export function instanceScope(instance: string): string {
|
||||||
|
return `@i:${instance}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Scope segment of a tenant: the group's immutable instance when known (a
|
||||||
|
* group re-created under a former id never shares state with its
|
||||||
|
* predecessor, and a renamed group keeps its own), else the group id
|
||||||
|
* (messages from before instances), or a marker no group id can take for the
|
||||||
|
* legacy global endpoint.
|
||||||
|
*/
|
||||||
|
export function tenantScope(groupId: string | undefined, instance?: string): string {
|
||||||
|
if (groupId && instance) return instanceScope(instance);
|
||||||
|
return groupId ?? "@global";
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Marker that the work a claim guarded was accepted for good. A claim alone
|
||||||
|
* may be left over from an attempt that failed and could not give it back;
|
||||||
|
* only a confirmed one proves a retry is a duplicate. The prefix keeps it
|
||||||
|
* apart from every claim key (none starts with it), whatever ids they embed.
|
||||||
|
*/
|
||||||
|
export function acceptedKey(key: string): string {
|
||||||
|
return `accepted:${key}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* - `claimed`: this caller runs the work (and must confirm or release it);
|
||||||
|
* - `done`: an earlier run was confirmed — a duplicate;
|
||||||
|
* - `pending`: another run holds the claim unconfirmed (still running, or it
|
||||||
|
* failed and could not give the claim back): retry later, after `ttl` at most.
|
||||||
|
*/
|
||||||
|
export type ClaimState = "claimed" | "done" | "pending";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Claim one-time work: a short processing claim (`ttlSeconds`), confirmed
|
||||||
|
* with `confirmClaim` once the work is done for good. A failed run that also
|
||||||
|
* fails to give its claim back blocks retries only until the claim expires,
|
||||||
|
* and is never mistaken for a finished one.
|
||||||
|
*/
|
||||||
|
export async function claimOnce(
|
||||||
|
store: IdempotencyStore,
|
||||||
|
key: string,
|
||||||
|
ttlSeconds: number,
|
||||||
|
): Promise<ClaimState> {
|
||||||
|
if (await store.has(acceptedKey(key))) return "done";
|
||||||
|
if (await store.claim(key, ttlSeconds)) return "claimed";
|
||||||
|
return (await store.has(acceptedKey(key))) ? "done" : "pending";
|
||||||
|
}
|
||||||
|
|
||||||
|
const CONFIRM_ATTEMPTS = 3;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record that claimed work was done; duplicates are recognized for
|
||||||
|
* `ttlSeconds`. Tried a few times: once the work happened, a lost
|
||||||
|
* confirmation leaves only the short processing claim between a late
|
||||||
|
* redelivery and running the work again.
|
||||||
|
*/
|
||||||
|
export async function confirmClaim(
|
||||||
|
store: IdempotencyStore,
|
||||||
|
key: string,
|
||||||
|
ttlSeconds: number,
|
||||||
|
): Promise<void> {
|
||||||
|
for (let attempt = 1; ; attempt++) {
|
||||||
|
try {
|
||||||
|
await store.claim(acceptedKey(key), ttlSeconds);
|
||||||
|
return;
|
||||||
|
} catch (err) {
|
||||||
|
if (attempt >= CONFIRM_ATTEMPTS) throw err;
|
||||||
|
await new Promise((r) => setTimeout(r, 200 * attempt));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Statement recording `key` (as `claim` would) in the same D1 transaction as
|
||||||
|
* other writes: the record and the writes commit together, so a failure
|
||||||
|
* whose answer was lost can be told apart afterwards (`has(key)`).
|
||||||
|
*/
|
||||||
|
export function recordStatement(
|
||||||
|
db: D1Database,
|
||||||
|
key: string,
|
||||||
|
ttlSeconds: number,
|
||||||
|
): D1PreparedStatement {
|
||||||
|
const now = Date.now();
|
||||||
|
return db
|
||||||
|
.prepare(
|
||||||
|
`INSERT INTO dedup_keys (key, claimed_at, expires_at) VALUES (?, ?, ?)
|
||||||
|
ON CONFLICT(key) DO UPDATE SET
|
||||||
|
claimed_at = excluded.claimed_at, expires_at = excluded.expires_at`,
|
||||||
|
)
|
||||||
|
.bind(key, now, now + ttlSeconds * 1000);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Statement aborting its D1 transaction while `key` is recorded (unexpired):
|
||||||
|
* a write done once can never be done again by a replay.
|
||||||
|
*/
|
||||||
|
export function unrecordedGuard(db: D1Database, key: string): D1PreparedStatement {
|
||||||
|
return db
|
||||||
|
.prepare(
|
||||||
|
`SELECT CASE WHEN EXISTS (SELECT 1 FROM dedup_keys WHERE key = ? AND expires_at > ?)
|
||||||
|
THEN json('already recorded') ELSE 1 END AS ok`,
|
||||||
|
)
|
||||||
|
.bind(key, Date.now());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Canonical dedup key for a webhook delivery: provider + tenant + delivery id. */
|
||||||
export function deliveryKey(
|
export function deliveryKey(
|
||||||
provider: string,
|
provider: string,
|
||||||
groupId: string | undefined,
|
groupId: string | undefined,
|
||||||
deliveryId: string,
|
deliveryId: string,
|
||||||
|
instance?: string,
|
||||||
): string {
|
): string {
|
||||||
return `delivery:${provider}:${groupId ?? "global"}:${deliveryId}`;
|
return `delivery:${provider}:${tenantScope(groupId, instance)}:${deliveryId}`;
|
||||||
}
|
}
|
||||||
@@ -1,3 +1,5 @@
|
|||||||
|
import { sharedFlight, type Flight } from "../lib/shared-flight";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* True when a D1Database binding looks like the real thing. Production D1
|
* True when a D1Database binding looks like the real thing. Production D1
|
||||||
* always exposes both `prepare` and `batch`; test harnesses range from
|
* always exposes both `prepare` and `batch`; test harnesses range from
|
||||||
@@ -9,3 +11,90 @@
|
|||||||
export function canUseD1(db: D1Database | undefined | null): boolean {
|
export function canUseD1(db: D1Database | undefined | null): boolean {
|
||||||
return typeof db?.prepare === "function" && typeof (db as D1Database).batch === "function";
|
return typeof db?.prepare === "function" && typeof (db as D1Database).batch === "function";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The D1 error says the table does not exist yet (migrations not applied).
|
||||||
|
* Only this case may degrade a D1-backed store to KV: on any other failure
|
||||||
|
* the two stores would each hold part of the state (a claim taken in D1 is
|
||||||
|
* invisible in KV), so the error must propagate and the caller retry.
|
||||||
|
*/
|
||||||
|
export function isMissingTable(err: unknown): boolean {
|
||||||
|
return /no such table/i.test(String(err));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set whenever state is written to KV instead of D1: by a D1-backed store
|
||||||
|
* whose table was missing, and by every store of a deployment without D1
|
||||||
|
* (which may get a D1 binding later). Until it expires — longer than any state
|
||||||
|
* written that way lives, counted from the last such write — reads that find
|
||||||
|
* nothing in D1 also look in KV, so that state is not forgotten. Its value is
|
||||||
|
* its own expiry (ms), so a fresh enough marker is not rewritten.
|
||||||
|
*/
|
||||||
|
const KV_FALLBACK_MARKER = "storage:kv-fallback";
|
||||||
|
const KV_FALLBACK_TTL = 35 * 86400;
|
||||||
|
const MARKER_RECHECK_MS = 5 * 60_000;
|
||||||
|
const MARKER_ACTIVE_RECHECK_MS = 3600_000;
|
||||||
|
/** A fallback write refreshes the marker at most this often (per isolate). */
|
||||||
|
const MARKER_REFRESH_MS = 3600_000;
|
||||||
|
const fallbackMarkers = new WeakMap<
|
||||||
|
KVNamespace,
|
||||||
|
{ active: boolean; until: number; writtenAt?: number }
|
||||||
|
>();
|
||||||
|
/** Marker reads in flight: concurrent checks share one KV read. */
|
||||||
|
const markerReads = new WeakMap<KVNamespace, Flight<boolean>>();
|
||||||
|
|
||||||
|
/** Marker refreshes in flight: concurrent fallbacks share one read and write. */
|
||||||
|
const markerWrites = new WeakMap<KVNamespace, Flight<void>>();
|
||||||
|
/** How long another request's marker read or write is waited for. */
|
||||||
|
const MARKER_FLIGHT_MS = 10_000;
|
||||||
|
|
||||||
|
export function noteKvFallback(kv: KVNamespace): Promise<void> {
|
||||||
|
const cached = fallbackMarkers.get(kv);
|
||||||
|
if (cached?.writtenAt != null && Date.now() - cached.writtenAt < MARKER_REFRESH_MS) {
|
||||||
|
return Promise.resolve();
|
||||||
|
}
|
||||||
|
return sharedFlight(markerWrites, kv, () => writeFallbackMarker(kv), MARKER_FLIGHT_MS);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function writeFallbackMarker(kv: KVNamespace): Promise<void> {
|
||||||
|
const stored = Number(await kv.get(KV_FALLBACK_MARKER));
|
||||||
|
// Rewritten at most about daily, whatever the number of isolates.
|
||||||
|
if (!(stored - Date.now() > (KV_FALLBACK_TTL - 86400) * 1000)) {
|
||||||
|
await kv.put(KV_FALLBACK_MARKER, String(Date.now() + KV_FALLBACK_TTL * 1000), {
|
||||||
|
expirationTtl: KV_FALLBACK_TTL,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
// Remembered only once it is stored: a failed write is tried again by the
|
||||||
|
// next fallback instead of being taken for done.
|
||||||
|
fallbackMarkers.set(kv, {
|
||||||
|
active: true,
|
||||||
|
until: Date.now() + MARKER_ACTIVE_RECHECK_MS,
|
||||||
|
writtenAt: Date.now(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function kvFallbackActive(kv: KVNamespace): Promise<boolean> {
|
||||||
|
// "No marker" is remembered for a few minutes too: this check runs on
|
||||||
|
// nearly every claim, and reading KV each time would spend the KV read
|
||||||
|
// quota (Workers Free) on a marker that is almost always absent. The cost
|
||||||
|
// is a short window after another isolate first sets it (accepted).
|
||||||
|
// A seen marker is read again hourly: once it has expired, the KV
|
||||||
|
// lookups it turns on stop too (a long-lived isolate would otherwise keep
|
||||||
|
// reading KV for state long gone).
|
||||||
|
const cached = fallbackMarkers.get(kv);
|
||||||
|
if (cached && Date.now() < cached.until) return cached.active;
|
||||||
|
return sharedFlight(
|
||||||
|
markerReads,
|
||||||
|
kv,
|
||||||
|
async () => {
|
||||||
|
const active = (await kv.get(KV_FALLBACK_MARKER)) != null;
|
||||||
|
fallbackMarkers.set(kv, {
|
||||||
|
active,
|
||||||
|
until: Date.now() + (active ? MARKER_ACTIVE_RECHECK_MS : MARKER_RECHECK_MS),
|
||||||
|
writtenAt: fallbackMarkers.get(kv)?.writtenAt,
|
||||||
|
});
|
||||||
|
return active;
|
||||||
|
},
|
||||||
|
MARKER_FLIGHT_MS,
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -22,8 +22,9 @@ describe("deliveryKey", () => {
|
|||||||
expect(deliveryKey("github", "team-a", "deliv-1")).toBe("delivery:github:team-a:deliv-1");
|
expect(deliveryKey("github", "team-a", "deliv-1")).toBe("delivery:github:team-a:deliv-1");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("falls back to 'global' without a tenant", () => {
|
it("uses a scope no group id can take without a tenant", () => {
|
||||||
expect(deliveryKey("github", undefined, "deliv-1")).toBe("delivery:github:global:deliv-1");
|
expect(deliveryKey("github", undefined, "deliv-1")).toBe("delivery:github:@global:deliv-1");
|
||||||
|
expect(deliveryKey("github", "global", "deliv-1")).toBe("delivery:github:global:deliv-1");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user