feat(storage): migrate config, dedup and delivery state to D1

- Move oversized queue payloads from KV to R2 (PAYLOAD binding, webhooks/YYYY/MM/DD/*.json, KV queue:payload:* fallback)
- Persist routes/groups to D1 (d1_routes/d1_groups) with memory -> KV -> D1 three-tier cache, seeded from legacy KV config keys
- Move webhook dedup (dedup_keys), delivery state (delivery_state) and message tracking (message_tracking) to D1 via canUseD1 probe with automatic KV fallback
- Batch send_logs inserts (recordSendBatch) and add group_id/ts index
- Add storage-prune scheduled task for expired dedup/state/tracking rows
- Add TTL to invite:group:{id} index and audit all ephemeral KV keys
- Add D1 indexes for the new tables
- Sync AGENTS.md, README.md/zh and docs/ (en/zh) with the new storage layout
This commit is contained in:
RhenCloud committed 2026-08-17 15:01:20 +08:00
1 parent 2e1b0f022e
commit 25ebae4ae5
46 files changed
+1450 -117

No files matched your search

+35 -5
View File
@@ -1,10 +1,11 @@
import { canUseD1 } from "../storage/d1";
/**
* Delivery idempotency: a small, reusable abstraction over "has this key been
* seen / can I claim it once" so webhook dedup, nonce replay protection and
* future delivery retry all share one semantics. `claim` is best-effort atomic
* on KV (get-then-put): under a concurrent double-send both callers may see
* "unclaimed", but that matches the existing dedup behavior and is acceptable
* because dispatch itself is idempotent per (provider, group, delivery).
* future delivery retry all share one semantics. Backed by D1 when the binding
* is present (atomic `INSERT OR IGNORE`), falling back to a best-effort
* get-then-put KV claim otherwise.
*/
export interface IdempotencyStore {
has(key: string): Promise<boolean>;
@@ -26,6 +27,35 @@ export function kvIdempotencyStore(kv: KVNamespace): IdempotencyStore {
};
}
export function d1IdempotencyStore(db: D1Database): IdempotencyStore {
return {
async has(key): Promise<boolean> {
const row = await db
.prepare("SELECT 1 AS hit FROM dedup_keys WHERE key = ? AND expires_at > ?")
.bind(key, Date.now())
.first<{ hit: number }>();
return row != null;
},
async claim(key, ttlSeconds): Promise<boolean> {
const now = Date.now();
const result = await db
.prepare(
`INSERT INTO dedup_keys (key, claimed_at, expires_at) VALUES (?, ?, ?)
ON CONFLICT(key) DO UPDATE SET
claimed_at = excluded.claimed_at, expires_at = excluded.expires_at
WHERE dedup_keys.expires_at < excluded.expires_at`,
)
.bind(key, now, now + ttlSeconds * 1000)
.run();
return (result.meta?.changes ?? 0) > 0;
},
};
}
export function idempotencyStore(db: D1Database, kv: KVNamespace): IdempotencyStore {
return canUseD1(db) ? d1IdempotencyStore(db) : kvIdempotencyStore(kv);
}
/**
* Canonical dedup key for a webhook delivery: provider + tenant + delivery id.
* The legacy global endpoint has no tenant, so `groupId` is "global".
@@ -36,4 +66,4 @@ export function deliveryKey(
deliveryId: string,
): string {
return `delivery:${provider}:${groupId ?? "global"}:${deliveryId}`;
}
}