feat(storage): migrate config, dedup and delivery state to D1

- Move oversized queue payloads from KV to R2 (PAYLOAD binding, webhooks/YYYY/MM/DD/*.json, KV queue:payload:* fallback)
- Persist routes/groups to D1 (d1_routes/d1_groups) with memory -> KV -> D1 three-tier cache, seeded from legacy KV config keys
- Move webhook dedup (dedup_keys), delivery state (delivery_state) and message tracking (message_tracking) to D1 via canUseD1 probe with automatic KV fallback
- Batch send_logs inserts (recordSendBatch) and add group_id/ts index
- Add storage-prune scheduled task for expired dedup/state/tracking rows
- Add TTL to invite:group:{id} index and audit all ephemeral KV keys
- Add D1 indexes for the new tables
- Sync AGENTS.md, README.md/zh and docs/ (en/zh) with the new storage layout
This commit is contained in:
RhenCloud 2026-08-17 15:01:20 +08:00
parent 2e1b0f022e
commit 25ebae4ae5
No known key found for this signature in database
GPG key ID: A574A617378C4E0B
46 changed files with 1450 additions and 117 deletions

View file

@ -0,0 +1,112 @@
import { describe, it, expect } from "bun:test";
import { recordSendBatch } from "../server/lib/lib/send-log-batch";
import type { SendRecord } from "../server/lib/lib/send-log";
interface BoundStmt {
sql: string;
args: unknown[];
}
function createMockDB(): { db: D1Database; rows: Array<Record<string, unknown>> } {
const rows: Array<Record<string, unknown>> = [];
const insertCols = [
"ts",
"route_id",
"group_id",
"event",
"repo",
"target",
"ok",
"error",
"status",
"message_id",
"delivery_id",
"platform",
"actor",
"action",
"duration_ms",
"error_code",
"attempts",
"detail",
];
const db = {
prepare: (sql: string) => ({
bind: (...args: unknown[]): BoundStmt => ({ sql, args }),
run: async (): Promise<{ success: boolean }> => ({ success: true }),
all: async (): Promise<{ results: Array<Record<string, unknown>> }> => ({
results: rows,
}),
}),
batch: async (stmts: BoundStmt[]): Promise<unknown[]> => {
for (const s of stmts) {
if (s.sql.startsWith("INSERT")) {
const row: Record<string, unknown> = {};
insertCols.forEach((col, i) => {
row[col] = s.args[i];
});
rows.push(row);
}
}
return [];
},
} as unknown as D1Database;
return { db, rows };
}
function record(overrides: Partial<SendRecord> = {}): SendRecord {
return {
ts: 1234,
routeId: "r1",
event: "push",
target: "111",
ok: true,
...overrides,
};
}
describe("recordSendBatch", () => {
it("is a no-op for an empty list", async () => {
const { db } = createMockDB();
await expect(recordSendBatch(db, [])).resolves.toBeUndefined();
});
it("batches multiple inserts in a single db.batch call", async () => {
const { db, rows } = createMockDB();
await recordSendBatch(db, [
record({ routeId: "a", ok: true }),
record({ routeId: "b", ok: false, error: "boom", groupId: "g1" }),
]);
expect(rows).toHaveLength(2);
expect(rows[0].route_id).toBe("a");
expect(rows[1].route_id).toBe("b");
expect(rows[1].group_id).toBe("g1");
expect(rows[1].ok).toBe(0);
expect(rows[1].error).toBe("boom");
});
it("serializes detail and encodes ok/error", async () => {
const { db, rows } = createMockDB();
await recordSendBatch(db, [
record({ detail: { a: 1 }, ok: false, error: "x", status: 500 }),
]);
expect(rows[0].detail).toBe('{"a":1}');
expect(rows[0].ok).toBe(0);
expect(rows[0].status).toBe(500);
});
it("swallows database errors", async () => {
const db = {
prepare: (): { bind: () => BoundStmt } => ({
bind: (): BoundStmt => {
throw new Error("nope");
},
}),
batch: async (): Promise<unknown[]> => {
throw new Error("batch failed");
},
} as unknown as D1Database;
await expect(
recordSendBatch(db, [record()]),
).resolves.toBeUndefined();
});
});