mirror of
https://github.com/ReCloudStudio/WebHooker.git
synced 2026-09-22 16:11:29 +00:00
feat(storage): migrate config, dedup and delivery state to D1
- Move oversized queue payloads from KV to R2 (PAYLOAD binding, webhooks/YYYY/MM/DD/*.json, KV queue:payload:* fallback)
- Persist routes/groups to D1 (d1_routes/d1_groups) with memory -> KV -> D1 three-tier cache, seeded from legacy KV config keys
- Move webhook dedup (dedup_keys), delivery state (delivery_state) and message tracking (message_tracking) to D1 via canUseD1 probe with automatic KV fallback
- Batch send_logs inserts (recordSendBatch) and add group_id/ts index
- Add storage-prune scheduled task for expired dedup/state/tracking rows
- Add TTL to invite:group:{id} index and audit all ephemeral KV keys
- Add D1 indexes for the new tables
- Sync AGENTS.md, README.md/zh and docs/ (en/zh) with the new storage layout
This commit is contained in:
parent
2e1b0f022e
commit
25ebae4ae5
46 changed files with 1450 additions and 117 deletions
242
server/lib/storage/config-store.ts
Normal file
242
server/lib/storage/config-store.ts
Normal file
|
|
@ -0,0 +1,242 @@
|
|||
import type { Group, Route } from "../types";
|
||||
import { log } from "../lib/log";
|
||||
|
||||
export interface ConfigStore {
|
||||
loadRoutes(): Promise<Route[]>;
|
||||
saveRoutes(routes: Route[]): Promise<void>;
|
||||
loadGroups(): Promise<Group[]>;
|
||||
saveGroups(groups: Group[]): Promise<void>;
|
||||
invalidateCache(): void;
|
||||
}
|
||||
|
||||
interface D1GroupRow {
|
||||
id: string;
|
||||
name: string;
|
||||
data: string;
|
||||
version: number;
|
||||
}
|
||||
|
||||
interface D1RouteRow {
|
||||
id: string;
|
||||
group_id: string;
|
||||
name: string;
|
||||
enabled: number;
|
||||
filters: string;
|
||||
targets: string;
|
||||
stop: number;
|
||||
fallback: number;
|
||||
discord_role_ids: string | null;
|
||||
ast: string | null;
|
||||
}
|
||||
|
||||
const CACHE_TTL = 300_000;
|
||||
const KV_ROUTES_KEY = "config:routes";
|
||||
const KV_GROUPS_KEY = "config:groups";
|
||||
|
||||
const KV_CACHE_TTL = 3600;
|
||||
|
||||
export function d1ConfigStore(db: D1Database, kv: KVNamespace): ConfigStore {
|
||||
let routesCache: { routes: Route[]; expiresAt: number } | null = null;
|
||||
let groupsCache: { groups: Group[]; expiresAt: number } | null = null;
|
||||
|
||||
async function loadRoutesFromD1(): Promise<Route[]> {
|
||||
const stmt = db.prepare(
|
||||
"SELECT id, group_id, name, enabled, filters, targets, stop, fallback, discord_role_ids, ast FROM d1_routes ORDER BY id",
|
||||
);
|
||||
if (typeof stmt.all !== "function") return [];
|
||||
const { results } = await stmt.all<D1RouteRow>();
|
||||
if (!results || results.length === 0) return [];
|
||||
return results.map((r) => ({
|
||||
id: r.id,
|
||||
groupId: r.group_id,
|
||||
name: r.name,
|
||||
enabled: r.enabled === 1,
|
||||
filters: JSON.parse(r.filters),
|
||||
targets: JSON.parse(r.targets),
|
||||
stop: r.stop === 1,
|
||||
fallback: r.fallback === 1,
|
||||
discordRoleIds: r.discord_role_ids ? JSON.parse(r.discord_role_ids) : undefined,
|
||||
ast: r.ast ? JSON.parse(r.ast) : undefined,
|
||||
}));
|
||||
}
|
||||
|
||||
async function loadGroupsFromD1(): Promise<Group[]> {
|
||||
const stmt = db.prepare("SELECT id, name, data, version FROM d1_groups ORDER BY id");
|
||||
if (typeof stmt.all !== "function") return [];
|
||||
const { results } = await stmt.all<D1GroupRow>();
|
||||
if (!results || results.length === 0) return [];
|
||||
return results.map((r) => JSON.parse(r.data) as Group);
|
||||
}
|
||||
|
||||
async function loadRoutesFromKV(): Promise<Route[]> {
|
||||
try {
|
||||
const raw = await kv.get<Route[]>(KV_ROUTES_KEY, "json");
|
||||
if (raw) return raw;
|
||||
} catch (err) {
|
||||
log.warn({ err }, "Failed to load routes from KV");
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
async function loadGroupsFromKV(): Promise<Group[]> {
|
||||
try {
|
||||
const raw = await kv.get<Group[]>(KV_GROUPS_KEY, "json");
|
||||
if (raw) return raw;
|
||||
} catch (err) {
|
||||
log.warn({ err }, "Failed to load groups from KV");
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
function routeStatements(routes: Route[]): D1PreparedStatement[] {
|
||||
const now = Date.now();
|
||||
return [
|
||||
db.prepare("DELETE FROM d1_routes"),
|
||||
...routes.map((r) =>
|
||||
db
|
||||
.prepare(
|
||||
`INSERT INTO d1_routes (id, group_id, name, enabled, filters, targets, stop, fallback, discord_role_ids, ast, version, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)`,
|
||||
)
|
||||
.bind(
|
||||
r.id,
|
||||
r.groupId ?? "",
|
||||
r.name,
|
||||
r.enabled ? 1 : 0,
|
||||
JSON.stringify(r.filters),
|
||||
JSON.stringify(r.targets),
|
||||
r.stop ? 1 : 0,
|
||||
r.fallback ? 1 : 0,
|
||||
r.discordRoleIds ? JSON.stringify(r.discordRoleIds) : null,
|
||||
r.ast ? JSON.stringify(r.ast) : null,
|
||||
now,
|
||||
now,
|
||||
),
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
function groupStatements(groups: Group[]): D1PreparedStatement[] {
|
||||
const now = Date.now();
|
||||
return [
|
||||
db.prepare("DELETE FROM d1_groups"),
|
||||
...groups.map((g) =>
|
||||
db
|
||||
.prepare(
|
||||
`INSERT INTO d1_groups (id, name, data, version, created_at, updated_at)
|
||||
VALUES (?, ?, ?, 1, ?, ?)`,
|
||||
)
|
||||
.bind(g.id, g.name, JSON.stringify(g), now, now),
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
async function seedRoutesToD1(routes: Route[]): Promise<void> {
|
||||
if (routes.length === 0) return;
|
||||
await db.batch(routeStatements(routes));
|
||||
}
|
||||
|
||||
async function seedGroupsToD1(groups: Group[]): Promise<void> {
|
||||
if (groups.length === 0) return;
|
||||
await db.batch(groupStatements(groups));
|
||||
}
|
||||
|
||||
async function syncRoutesToKV(routes: Route[], ttl: number): Promise<void> {
|
||||
try {
|
||||
await kv.put(KV_ROUTES_KEY, JSON.stringify(routes), { expirationTtl: ttl });
|
||||
} catch (err) {
|
||||
log.warn({ err }, "Failed to sync routes to KV cache");
|
||||
}
|
||||
}
|
||||
|
||||
async function syncGroupsToKV(groups: Group[], ttl: number): Promise<void> {
|
||||
try {
|
||||
await kv.put(KV_GROUPS_KEY, JSON.stringify(groups), { expirationTtl: ttl });
|
||||
} catch (err) {
|
||||
log.warn({ err }, "Failed to sync groups to KV cache");
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
async loadRoutes(): Promise<Route[]> {
|
||||
if (routesCache && Date.now() < routesCache.expiresAt) {
|
||||
return routesCache.routes;
|
||||
}
|
||||
|
||||
let routes: Route[] = [];
|
||||
try {
|
||||
routes = await loadRoutesFromD1();
|
||||
if (routes.length > 0) {
|
||||
syncRoutesToKV(routes, KV_CACHE_TTL).catch(() => undefined);
|
||||
} else {
|
||||
routes = await loadRoutesFromKV();
|
||||
if (routes.length > 0) {
|
||||
seedRoutesToD1(routes).catch((err) =>
|
||||
log.warn({ err }, "Failed to seed routes from KV to D1"),
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
log.warn({ err }, "D1 routes unavailable, falling back to KV");
|
||||
routes = await loadRoutesFromKV();
|
||||
}
|
||||
|
||||
routesCache = { routes, expiresAt: Date.now() + CACHE_TTL };
|
||||
return routes;
|
||||
},
|
||||
|
||||
async saveRoutes(routes: Route[]): Promise<void> {
|
||||
try {
|
||||
await db.batch(routeStatements(routes));
|
||||
await syncRoutesToKV(routes, KV_CACHE_TTL);
|
||||
} catch (err) {
|
||||
log.warn({ err }, "D1 routes unavailable, falling back to KV");
|
||||
await syncRoutesToKV(routes, 0);
|
||||
}
|
||||
routesCache = null;
|
||||
},
|
||||
|
||||
async loadGroups(): Promise<Group[]> {
|
||||
if (groupsCache && Date.now() < groupsCache.expiresAt) {
|
||||
return groupsCache.groups;
|
||||
}
|
||||
|
||||
let groups: Group[] = [];
|
||||
try {
|
||||
groups = await loadGroupsFromD1();
|
||||
if (groups.length > 0) {
|
||||
syncGroupsToKV(groups, KV_CACHE_TTL).catch(() => undefined);
|
||||
} else {
|
||||
groups = await loadGroupsFromKV();
|
||||
if (groups.length > 0) {
|
||||
seedGroupsToD1(groups).catch((err) =>
|
||||
log.warn({ err }, "Failed to seed groups from KV to D1"),
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
log.warn({ err }, "D1 groups unavailable, falling back to KV");
|
||||
groups = await loadGroupsFromKV();
|
||||
}
|
||||
|
||||
groupsCache = { groups, expiresAt: Date.now() + CACHE_TTL };
|
||||
return groups;
|
||||
},
|
||||
|
||||
async saveGroups(groups: Group[]): Promise<void> {
|
||||
try {
|
||||
await db.batch(groupStatements(groups));
|
||||
await syncGroupsToKV(groups, KV_CACHE_TTL);
|
||||
} catch (err) {
|
||||
log.warn({ err }, "D1 groups unavailable, falling back to KV");
|
||||
await syncGroupsToKV(groups, 0);
|
||||
}
|
||||
groupsCache = null;
|
||||
},
|
||||
|
||||
invalidateCache(): void {
|
||||
routesCache = null;
|
||||
groupsCache = null;
|
||||
},
|
||||
};
|
||||
}
|
||||
14
server/lib/storage/d1.ts
Normal file
14
server/lib/storage/d1.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
/**
|
||||
* True when a D1Database binding looks like the real thing. Production D1
|
||||
* always exposes both `prepare` and `batch`; test harnesses range from
|
||||
* `DB: {}` (no methods at all) to minimal mocks that only implement
|
||||
* `prepare`→`bind`→{`run`,`all`} without `batch`. Using the presence of
|
||||
* `batch` as the probe means every existing test keeps its KV fallback path
|
||||
* while production eagerly routes through D1.
|
||||
*/
|
||||
export function canUseD1(db: D1Database | undefined | null): boolean {
|
||||
return (
|
||||
typeof db?.prepare === "function" &&
|
||||
typeof (db as D1Database).batch === "function"
|
||||
);
|
||||
}
|
||||
64
server/lib/storage/payload.ts
Normal file
64
server/lib/storage/payload.ts
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
import type { Env } from "../types";
|
||||
|
||||
const PAYLOAD_PREFIX = "webhooks/";
|
||||
const DEFAULT_TTL_SECONDS = 3600;
|
||||
|
||||
export interface PayloadStore {
|
||||
put(payload: string, ttl?: number): Promise<string>;
|
||||
get(key: string): Promise<string | null>;
|
||||
delete(key: string): Promise<void>;
|
||||
}
|
||||
|
||||
function generateKey(): string {
|
||||
const bytes = new Uint8Array(16);
|
||||
crypto.getRandomValues(bytes);
|
||||
const hex = Array.from(bytes)
|
||||
.map((b) => b.toString(16).padStart(2, "0"))
|
||||
.join("");
|
||||
const now = new Date();
|
||||
const y = now.getUTCFullYear();
|
||||
const m = String(now.getUTCMonth() + 1).padStart(2, "0");
|
||||
const d = String(now.getUTCDate()).padStart(2, "0");
|
||||
return `${PAYLOAD_PREFIX}${y}/${m}/${d}/${hex}.json`;
|
||||
}
|
||||
|
||||
export function r2PayloadStore(env: Env): PayloadStore {
|
||||
const bucket = env.PAYLOAD;
|
||||
if (!bucket) {
|
||||
return {
|
||||
async put(_payload: string): Promise<string> {
|
||||
throw new Error("R2 binding is not configured");
|
||||
},
|
||||
async get(): Promise<null> {
|
||||
throw new Error("R2 binding is not configured");
|
||||
},
|
||||
async delete(): Promise<void> {
|
||||
throw new Error("R2 binding is not configured");
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
async put(payload: string, ttl?: number): Promise<string> {
|
||||
const key = generateKey();
|
||||
const expires = new Date(
|
||||
Date.now() + (ttl ?? DEFAULT_TTL_SECONDS) * 1000,
|
||||
);
|
||||
await bucket.put(key, payload, {
|
||||
httpMetadata: { contentType: "application/json" },
|
||||
customMetadata: { expires: expires.toISOString() },
|
||||
});
|
||||
return key;
|
||||
},
|
||||
|
||||
async get(key: string): Promise<string | null> {
|
||||
const obj = await bucket.get(key);
|
||||
if (!obj) return null;
|
||||
return obj.text();
|
||||
},
|
||||
|
||||
async delete(key: string): Promise<void> {
|
||||
await bucket.delete(key);
|
||||
},
|
||||
};
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue