mirror of
https://github.com/Elysola/WebHooker.git
synced 2026-11-21 01:23:40 +00:00
feat(storage): migrate config, dedup and delivery state to D1
- Move oversized queue payloads from KV to R2 (PAYLOAD binding, webhooks/YYYY/MM/DD/*.json, KV queue:payload:* fallback)
- Persist routes/groups to D1 (d1_routes/d1_groups) with memory -> KV -> D1 three-tier cache, seeded from legacy KV config keys
- Move webhook dedup (dedup_keys), delivery state (delivery_state) and message tracking (message_tracking) to D1 via canUseD1 probe with automatic KV fallback
- Batch send_logs inserts (recordSendBatch) and add group_id/ts index
- Add storage-prune scheduled task for expired dedup/state/tracking rows
- Add TTL to invite:group:{id} index and audit all ephemeral KV keys
- Add D1 indexes for the new tables
- Sync AGENTS.md, README.md/zh and docs/ (en/zh) with the new storage layout
This commit is contained in:
46 files changed
+1450
-117
No files matched your search
@@ -1,10 +1,11 @@
|
||||
import { canUseD1 } from "../storage/d1";
|
||||
|
||||
/**
|
||||
* Delivery idempotency: a small, reusable abstraction over "has this key been
|
||||
* seen / can I claim it once" so webhook dedup, nonce replay protection and
|
||||
* future delivery retry all share one semantics. `claim` is best-effort atomic
|
||||
* on KV (get-then-put): under a concurrent double-send both callers may see
|
||||
* "unclaimed", but that matches the existing dedup behavior and is acceptable
|
||||
* because dispatch itself is idempotent per (provider, group, delivery).
|
||||
* future delivery retry all share one semantics. Backed by D1 when the binding
|
||||
* is present (atomic `INSERT OR IGNORE`), falling back to a best-effort
|
||||
* get-then-put KV claim otherwise.
|
||||
*/
|
||||
export interface IdempotencyStore {
|
||||
has(key: string): Promise<boolean>;
|
||||
@@ -26,6 +27,35 @@ export function kvIdempotencyStore(kv: KVNamespace): IdempotencyStore {
|
||||
};
|
||||
}
|
||||
|
||||
export function d1IdempotencyStore(db: D1Database): IdempotencyStore {
|
||||
return {
|
||||
async has(key): Promise<boolean> {
|
||||
const row = await db
|
||||
.prepare("SELECT 1 AS hit FROM dedup_keys WHERE key = ? AND expires_at > ?")
|
||||
.bind(key, Date.now())
|
||||
.first<{ hit: number }>();
|
||||
return row != null;
|
||||
},
|
||||
async claim(key, ttlSeconds): Promise<boolean> {
|
||||
const now = Date.now();
|
||||
const result = await db
|
||||
.prepare(
|
||||
`INSERT INTO dedup_keys (key, claimed_at, expires_at) VALUES (?, ?, ?)
|
||||
ON CONFLICT(key) DO UPDATE SET
|
||||
claimed_at = excluded.claimed_at, expires_at = excluded.expires_at
|
||||
WHERE dedup_keys.expires_at < excluded.expires_at`,
|
||||
)
|
||||
.bind(key, now, now + ttlSeconds * 1000)
|
||||
.run();
|
||||
return (result.meta?.changes ?? 0) > 0;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function idempotencyStore(db: D1Database, kv: KVNamespace): IdempotencyStore {
|
||||
return canUseD1(db) ? d1IdempotencyStore(db) : kvIdempotencyStore(kv);
|
||||
}
|
||||
|
||||
/**
|
||||
* Canonical dedup key for a webhook delivery: provider + tenant + delivery id.
|
||||
* The legacy global endpoint has no tenant, so `groupId` is "global".
|
||||
@@ -36,4 +66,4 @@ export function deliveryKey(
|
||||
deliveryId: string,
|
||||
): string {
|
||||
return `delivery:${provider}:${groupId ?? "global"}:${deliveryId}`;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user