feat(storage): migrate config, dedup and delivery state to D1

- Move oversized queue payloads from KV to R2 (PAYLOAD binding, webhooks/YYYY/MM/DD/*.json, KV queue:payload:* fallback)
- Persist routes/groups to D1 (d1_routes/d1_groups) with memory -> KV -> D1 three-tier cache, seeded from legacy KV config keys
- Move webhook dedup (dedup_keys), delivery state (delivery_state) and message tracking (message_tracking) to D1 via canUseD1 probe with automatic KV fallback
- Batch send_logs inserts (recordSendBatch) and add group_id/ts index
- Add storage-prune scheduled task for expired dedup/state/tracking rows
- Add TTL to invite:group:{id} index and audit all ephemeral KV keys
- Add D1 indexes for the new tables
- Sync AGENTS.md, README.md/zh and docs/ (en/zh) with the new storage layout
This commit is contained in:
RhenCloud 2026-08-17 15:01:20 +08:00
parent 2e1b0f022e
commit 25ebae4ae5
No known key found for this signature in database
GPG key ID: A574A617378C4E0B
46 changed files with 1450 additions and 117 deletions

View file

@ -1,12 +1,41 @@
import type { Env, Config, Route } from "./types";
import { log } from "./lib/log";
import { migrateRoutes, validateRoutes } from "./config/schema";
import { d1ConfigStore, type ConfigStore } from "./storage/config-store";
const CONFIG_CACHE_TTL = 300_000;
const ROUTES_KEY = "config:routes";
let configCache: { config: Config; expiresAt: number } | null = null;
const configStores = new WeakMap<KVNamespace, ConfigStore>();
export function getConfigStore(kv: KVNamespace): ConfigStore | null {
return configStores.get(kv) ?? null;
}
export function initConfigStore(env: Env): ConfigStore {
return ensureStore(env);
}
export function resetConfigStore(kv?: KVNamespace): void {
if (kv) {
configStores.get(kv)?.invalidateCache();
configStores.delete(kv);
}
}
function ensureStore(env: Env): ConfigStore {
let store = configStores.get(env.KV);
if (!store) {
store = d1ConfigStore(env.DB, env.KV);
configStores.set(env.KV, store);
}
return store;
}
export async function loadRoutes(kv: KVNamespace): Promise<Route[]> {
const store = configStores.get(kv);
if (store) return store.loadRoutes();
try {
const stored = await kv.get<Route[]>(ROUTES_KEY, "json");
if (stored) return validateRoutes(migrateRoutes(stored));
@ -17,11 +46,16 @@ export async function loadRoutes(kv: KVNamespace): Promise<Route[]> {
}
export async function saveRoutes(kv: KVNamespace, routes: Route[]): Promise<void> {
const store = configStores.get(kv);
if (store) {
await store.saveRoutes(routes);
configCache = null;
return;
}
await kv.put(ROUTES_KEY, JSON.stringify(routes));
configCache = null;
}
/** Drop the in-memory route/config cache (used by the admin API and tests). */
export function invalidateConfigCache(): void {
configCache = null;
}
@ -31,6 +65,7 @@ export async function loadConfig(env: Env): Promise<Config> {
return configCache.config;
}
ensureStore(env);
const routes = await loadRoutes(env.KV);
const config: Config = {
@ -50,4 +85,4 @@ export async function loadConfig(env: Env): Promise<Config> {
configCache = { config, expiresAt: Date.now() + CONFIG_CACHE_TTL };
return config;
}
}