From 3a6a2cbbc536f6a2b3fee0b1acd2ab267fe5c626 Mon Sep 17 00:00:00 2001 From: RhenCloud Date: Mon, 3 Aug 2026 03:57:58 +0800 Subject: [PATCH] feat: migrate send logs and discord links to D1 --- AGENTS.md | 12 ++++-- docs/api/oauth.md | 2 +- docs/contributing.md | 4 +- docs/zh/api/oauth.md | 2 +- docs/zh/contributing.md | 4 +- migrations/0001_init.sql | 16 +++++++ src/__tests__/discord.test.ts | 14 +++++- src/__tests__/send-log.test.ts | 67 ++++++++++++++++------------- src/core/dispatch.ts | 4 +- src/drivers/discord/interactions.ts | 8 ++-- src/github/store.ts | 19 +++++--- src/lib/send-log.ts | 52 ++++++++++++---------- src/types.ts | 1 + src/web/admin-routes.ts | 4 +- src/web/oauth-routes.ts | 2 +- wrangler.jsonc | 7 +++ 16 files changed, 139 insertions(+), 79 deletions(-) create mode 100644 migrations/0001_init.sql diff --git a/AGENTS.md b/AGENTS.md index 04cbfb4..701a641 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,7 +11,7 @@ Core pipeline: GitHub Webhook → Worker (verify + filter + format) → Discord - Runtime: Cloudflare Workers - HTTP framework: Hono - Discord interactions: HTTPS Interactions Endpoint (`POST /discord/interactions`, Ed25519-signed) — no Discord Gateway / Durable Object; bot stays offline, messages always sent via REST -- Storage: Cloudflare KV (tokens, OAuth state, route config, admin sessions) +- Storage: Cloudflare KV (tokens, OAuth state, route config, admin sessions, delivery dedup) + D1 (send logs, discord-link mapping) - Signature verification: Web Crypto API (HMAC-SHA256 for GitHub, Ed25519 for Discord) - GitHub OAuth: octokit + jose (JWT) - Admin WebUI: `/admin` config console, OAuth-session protected via `ADMIN_USER_IDS` whitelist @@ -51,7 +51,7 @@ src/ │ └── index.ts # TelegramDriver stub (not implemented yet) ├── github/ │ ├── oauth.ts # OAuth URL, callback token exchange, getUserOctokit, comment/merge/close actions -│ └── store.ts # KV-based token CRUD + discord-link mapping (was token-store.ts) +│ └── store.ts # KV token CRUD + D1 discord-link mapping (was token-store.ts) ├── web/ # HTTP UI/API routes │ ├── oauth-routes.ts # GET /auth/github, callback (admin session / discord-link), DELETE /token/:userId │ ├── action-routes.ts # POST /api/comment|merge|react (Bearer token auth via KV lookup) @@ -62,7 +62,7 @@ src/ │ └── legal-routes.ts # legal / privacy / terms pages └── lib/ # shared infra ├── i18n.ts # loadTranslations, t() with param interpolation - ├── send-log.ts # SendRecord, recordSend, getSendLog + ├── send-log.ts # SendRecord, recordSend/getSendLog (D1 send_logs) ├── log.ts # JSON console logger (info/warn/error/fatal) └── locales/ # en.ts, zh.ts translation dictionaries ``` @@ -104,7 +104,8 @@ npm run lint # ESLint - **Local dev**: `.dev.vars` (wrangler reads this for env bindings) - **Production**: `wrangler secret put ` for each secret - **Routes**: KV key `config:routes` (JSON array, empty until configured) -- **KV namespace**: Required binding for token/state/config storage +- **KV namespace**: Required binding for token/state/config/session storage +- **D1 database**: Binding `DB` (database `webhooker`, id `214a0104-3235-47c0-b7bf-ddda95f3c8ac`) for `send_logs` + `discord_links` tables - **Discord**: `DISCORD_PUBLIC_KEY` (Interactions Endpoint signature verification, from Discord Developer Portal) and `DISCORD_APPLICATION_ID` (optional, auto-resolved via `GET /oauth2/applications/@me` when omitted) are required for interactions ## Deployment @@ -115,6 +116,9 @@ npx wrangler secret put DISCORD_TOKEN npx wrangler secret put DISCORD_PUBLIC_KEY npx wrangler kv namespace create KV # Update wrangler.jsonc with KV ID +npx wrangler d1 create webhooker +# Update wrangler.jsonc d1_databases with the database ID +npx wrangler d1 execute webhooker --remote --file ./migrations/0001_init.sql npx wrangler deploy ``` diff --git a/docs/api/oauth.md b/docs/api/oauth.md index b8244e9..154d6d1 100644 --- a/docs/api/oauth.md +++ b/docs/api/oauth.md @@ -77,7 +77,7 @@ Tokens are stored in KV with key pattern `token:{userId}`: } ``` -`expiresAt` is a Unix timestamp in milliseconds. KV entries expire at 90% of the token's lifetime (minimum 60 seconds). A reverse index `token-reverse:{sha256 of token}` maps the access token back to its user id so Bearer-authenticated endpoints can resolve the caller. Discord users linked to a GitHub account are stored under `discord-link:{discordUserId}`. +`expiresAt` is a Unix timestamp in milliseconds. KV entries expire at 90% of the token's lifetime (minimum 60 seconds). A reverse index `token-reverse:{sha256 of token}` maps the access token back to its user id so Bearer-authenticated endpoints can resolve the caller. Discord users linked to a GitHub account are stored in the D1 `discord_links` table. ## Using Tokens diff --git a/docs/contributing.md b/docs/contributing.md index 059b303..640ad25 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -37,7 +37,7 @@ src/ │ └── telegram/ # TelegramDriver stub (not implemented yet) ├── github/ # GitHub OAuth + as-user actions │ ├── oauth.ts # OAuth URL, callback token exchange, getUserOctokit, actions -│ └── store.ts # KV-based token CRUD + discord-link mapping +│ └── store.ts # KV token CRUD + D1 discord-link mapping ├── web/ # HTTP UI/API routes │ ├── oauth-routes.ts # GET /auth/github, callback, DELETE /token/:userId (KV state) │ ├── action-routes.ts # POST /api/comment|merge|close|react (Bearer token auth via KV lookup) @@ -48,7 +48,7 @@ src/ │ └── legal-routes.ts # Legal page routes └── lib/ # Shared infrastructure ├── i18n.ts # Message language overrides (en/zh) - ├── send-log.ts # Send logging (logs:send KV keys) + ├── send-log.ts # Send logging (D1 send_logs) ├── log.ts # JSON console logger (info/warn/error/fatal) └── locales/ # en.ts, zh.ts translation dictionaries ``` diff --git a/docs/zh/api/oauth.md b/docs/zh/api/oauth.md index e980785..f0c7307 100644 --- a/docs/zh/api/oauth.md +++ b/docs/zh/api/oauth.md @@ -77,7 +77,7 @@ Token 以键模式 `token:{userId}` 存储在 KV 中： } ``` -`expiresAt` 是毫秒级 Unix 时间戳。KV 条目在 Token 有效期的 90% 时过期（至少 60 秒）。反向索引 `token-reverse:{sha256 of token}` 将访问令牌映射回用户 id，使 Bearer 鉴权的端点能解析调用者。与 GitHub 账号绑定的 Discord 用户存储在 `discord-link:{discordUserId}` 下。 +`expiresAt` 是毫秒级 Unix 时间戳。KV 条目在 Token 有效期的 90% 时过期（至少 60 秒）。反向索引 `token-reverse:{sha256 of token}` 将访问令牌映射回用户 id，使 Bearer 鉴权的端点能解析调用者。与 GitHub 账号绑定的 Discord 用户存储在 D1 的 `discord_links` 表中。 ## 使用 Token diff --git a/docs/zh/contributing.md b/docs/zh/contributing.md index f30a2da..77eca18 100644 --- a/docs/zh/contributing.md +++ b/docs/zh/contributing.md @@ -37,7 +37,7 @@ src/ │ └── telegram/ # TelegramDriver 占位（未实现） ├── github/ # GitHub OAuth + 以用户身份操作 │ ├── oauth.ts # OAuth URL、回调 Token 交换、getUserOctokit、操作 -│ └── store.ts # 基于 KV 的 Token CRUD + discord-link 映射 +│ └── store.ts # KV Token CRUD + D1 discord-link 映射 ├── web/ # HTTP UI/API 路由 │ ├── oauth-routes.ts # GET /auth/github、回调、DELETE /token/:userId (KV 状态) │ ├── action-routes.ts # POST /api/comment|merge|close|react (通过 KV 查找进行 Bearer Token 鉴权) @@ -48,7 +48,7 @@ src/ │ └── legal-routes.ts # 法律页面路由 └── lib/ # 共享基础设施 ├── i18n.ts # 消息语言覆盖 (en/zh) - ├── send-log.ts # 发送日志 (logs:send KV 键) + ├── send-log.ts # 发送日志 (D1 send_logs) ├── log.ts # JSON 控制台日志 (info/warn/error/fatal) └── locales/ # en.ts、zh.ts 翻译字典 ``` diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql new file mode 100644 index 0000000..15015b8 --- /dev/null +++ b/migrations/0001_init.sql @@ -0,0 +1,16 @@ +CREATE TABLE IF NOT EXISTS send_logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts INTEGER NOT NULL, + route_id TEXT NOT NULL, + event TEXT NOT NULL, + repo TEXT, + target TEXT NOT NULL, + ok INTEGER NOT NULL, + error TEXT +); +CREATE INDEX IF NOT EXISTS idx_send_logs_ts ON send_logs (ts DESC); + +CREATE TABLE IF NOT EXISTS discord_links ( + discord_user_id TEXT PRIMARY KEY, + github_user_id TEXT NOT NULL +); diff --git a/src/__tests__/discord.test.ts b/src/__tests__/discord.test.ts index 4f7037d..aa6d3cb 100644 --- a/src/__tests__/discord.test.ts +++ b/src/__tests__/discord.test.ts @@ -12,6 +12,7 @@ function createEnv(overrides: Partial = {}): Env { return { GITHUB_WEBHOOK_SECRET: "secret", KV: {} as KVNamespace, + DB: {} as D1Database, ...overrides, }; } @@ -91,6 +92,17 @@ describe("dispatchEvent fallback routing", () => { } as unknown as KVNamespace; } + function createMockDB(): D1Database { + return { + prepare: () => ({ + bind: () => ({ + run: async () => ({ success: true }), + all: async () => ({ results: [] }), + }), + }), + } as unknown as D1Database; + } + const baseConfig = { baseUrl: "https://example.com", github: { @@ -110,7 +122,7 @@ describe("dispatchEvent fallback routing", () => { sent.push(url); return new Response("{}", { status: 200 }); }); - const env = createEnv({ KV: createMockKV() }); + const env = createEnv({ KV: createMockKV(), DB: createMockDB() }); const routes: Route[] = [ { id: "regular-push", diff --git a/src/__tests__/send-log.test.ts b/src/__tests__/send-log.test.ts index cddd9aa..fa3a859 100644 --- a/src/__tests__/send-log.test.ts +++ b/src/__tests__/send-log.test.ts @@ -1,39 +1,46 @@ import { describe, it, expect } from "bun:test"; import { recordSend, getSendLog } from "../lib/send-log"; -function createMockKV(): KVNamespace { - const store = new Map(); +function createMockDB(): D1Database { + const rows: Array> = []; return { - get: async (key: string, type?: string) => { - const entry = store.get(key); - if (!entry) return null; - if (entry.expiration && Date.now() / 1000 > entry.expiration) { - store.delete(key); - return null; - } - if (type === "json") return JSON.parse(entry.value); - return entry.value; - }, - put: async (key: string, value: string, opts?: { expirationTtl?: number }) => { - const expiration = opts?.expirationTtl ? Date.now() / 1000 + opts.expirationTtl : undefined; - store.set(key, { value, expiration }); - }, - delete: async (key: string) => { - store.delete(key); - }, - list: async () => ({ - keys: [...store.keys()].map((k) => ({ name: k })), - list_complete: true, - cacheStatus: null, + prepare: (sql: string) => ({ + bind: (..._args: unknown[]) => ({ + run: async (): Promise<{ success: boolean }> => { + if (sql.startsWith("INSERT")) { + const args = _args as unknown[]; + rows.push({ + ts: args[0], + route_id: args[1], + event: args[2], + repo: args[3], + target: args[4], + ok: args[5], + error: args[6], + }); + } + return { success: true }; + }, + all: async (): Promise<{ results: Array> }> => { + const args = _args as unknown[]; + const limit = (args[0] as number) ?? 50; + return { + results: rows + .slice() + .sort((a, b) => (b.ts as number) - (a.ts as number)) + .slice(0, limit), + }; + }, + }), }), - } as unknown as KVNamespace; + } as unknown as D1Database; } describe("send-log", () => { it("records and returns logs sorted newest first", async () => { - const kv = createMockKV(); - await recordSend(kv, { ts: 1000, routeId: "a", event: "push", target: "111", ok: true }); - await recordSend(kv, { + const db = createMockDB(); + await recordSend(db, { ts: 1000, routeId: "a", event: "push", target: "111", ok: true }); + await recordSend(db, { ts: 2000, routeId: "b", event: "issues", @@ -41,7 +48,7 @@ describe("send-log", () => { ok: false, error: "Missing Permissions", }); - const logs = await getSendLog(kv); + const logs = await getSendLog(db); expect(logs).toHaveLength(2); expect(logs[0]!.routeId).toBe("b"); expect(logs[0]!.ok).toBe(false); @@ -50,7 +57,7 @@ describe("send-log", () => { }); it("returns empty when no logs", async () => { - const kv = createMockKV(); - expect(await getSendLog(kv)).toEqual([]); + const db = createMockDB(); + expect(await getSendLog(db)).toEqual([]); }); }); diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index 650e30b..7849e27 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -48,7 +48,7 @@ export async function dispatchEvent(config: Config, event: WebhookEvent, env: En const message = formatEvent(route, event, tr, showEmoji); const result = await getDriver(route.target).send(message, route.target, env); if (!result.ok) throw new Error(result.error ?? "Send failed"); - await recordSend(env.KV, { + await recordSend(env.DB, { ts: Date.now(), routeId: route.id, event: event.event, @@ -57,7 +57,7 @@ export async function dispatchEvent(config: Config, event: WebhookEvent, env: En ok: true, }); } catch (err) { - await recordSend(env.KV, { + await recordSend(env.DB, { ts: Date.now(), routeId: route.id, event: event.event, diff --git a/src/drivers/discord/interactions.ts b/src/drivers/discord/interactions.ts index 61446ad..5b8f8d0 100644 --- a/src/drivers/discord/interactions.ts +++ b/src/drivers/discord/interactions.ts @@ -270,7 +270,7 @@ async function handleButton( customId: string | undefined, ): Promise { if (!userId) return respond(env, id, token, "无法识别你的 Discord 账号。"); - const githubUserId = await getDiscordLink(env.KV, userId); + const githubUserId = await getDiscordLink(env.DB, userId); if (!githubUserId) { return respond(env, id, token, "你还没有绑定 GitHub 账号，请先使用 `/gh login`。"); } @@ -348,7 +348,7 @@ async function cmdLogout( userId: string | null, ): Promise { if (!userId) return respond(env, id, token, "无法识别你的 Discord 账号。"); - await removeDiscordLink(env.KV, userId); + await removeDiscordLink(env.DB, userId); await respond(env, id, token, "已解绑你的 GitHub 账号。"); } @@ -374,7 +374,7 @@ async function commentOp( source: string, ): Promise { if (!userId) return respond(env, id, token, "无法识别你的 Discord 账号。"); - const githubUserId = await getDiscordLink(env.KV, userId); + const githubUserId = await getDiscordLink(env.DB, userId); if (!githubUserId) { return respond(env, id, token, "你还没有绑定 GitHub 账号，请先使用 `/gh login`。"); } @@ -487,7 +487,7 @@ async function modalSubmit( const body = d.components?.[0]?.components?.find((c) => c.custom_id === "body")?.value?.trim(); if (!body) return respond(env, id, token, "评论内容不能为空。"); - const githubUserId = await getDiscordLink(env.KV, userId); + const githubUserId = await getDiscordLink(env.DB, userId); if (!githubUserId) { return respond(env, id, token, "你还没有绑定 GitHub 账号，请先使用 `/gh login`。"); } diff --git a/src/github/store.ts b/src/github/store.ts index c564836..afd94ff 100644 --- a/src/github/store.ts +++ b/src/github/store.ts @@ -73,20 +73,27 @@ export async function findUserIdByToken( * as that GitHub account. The actual OAuth token lives under `token:{githubUserId}`. */ export async function saveDiscordLink( - kv: KVNamespace, + db: D1Database, discordUserId: string, githubUserId: string, ): Promise { - await kv.put(`discord-link:${discordUserId}`, githubUserId); + await db + .prepare("INSERT OR REPLACE INTO discord_links (discord_user_id, github_user_id) VALUES (?, ?)") + .bind(discordUserId, githubUserId) + .run(); } export async function getDiscordLink( - kv: KVNamespace, + db: D1Database, discordUserId: string, ): Promise { - return await kv.get(`discord-link:${discordUserId}`, "text"); + const { results } = await db + .prepare("SELECT github_user_id FROM discord_links WHERE discord_user_id = ?") + .bind(discordUserId) + .all<{ github_user_id: string }>(); + return results[0]?.github_user_id ?? null; } -export async function removeDiscordLink(kv: KVNamespace, discordUserId: string): Promise { - await kv.delete(`discord-link:${discordUserId}`); +export async function removeDiscordLink(db: D1Database, discordUserId: string): Promise { + await db.prepare("DELETE FROM discord_links WHERE discord_user_id = ?").bind(discordUserId).run(); } diff --git a/src/lib/send-log.ts b/src/lib/send-log.ts index 2241172..c0a133e 100644 --- a/src/lib/send-log.ts +++ b/src/lib/send-log.ts @@ -10,36 +10,42 @@ export interface SendRecord { error?: string; } -const KEY_PREFIX = "logs:send:"; -const RETENTION_TTL = 3600; -const MAX_READ = 200; - -function randomHex(): string { - const bytes = new Uint8Array(8); - crypto.getRandomValues(bytes); - return Array.from(bytes) - .map((b) => b.toString(16).padStart(2, "0")) - .join(""); -} - -export async function recordSend(kv: KVNamespace, record: SendRecord): Promise { +export async function recordSend(db: D1Database, record: SendRecord): Promise { try { - await kv.put(`${KEY_PREFIX}${record.ts}-${randomHex()}`, JSON.stringify(record), { - expirationTtl: RETENTION_TTL, - }); + await db + .prepare( + "INSERT INTO send_logs (ts, route_id, event, repo, target, ok, error) VALUES (?, ?, ?, ?, ?, ?, ?)", + ) + .bind( + record.ts, + record.routeId, + record.event, + record.repo ?? null, + record.target, + record.ok ? 1 : 0, + record.error ?? null, + ) + .run(); } catch (err) { log.warn({ err }, "Failed to record send log"); } } -export async function getSendLog(kv: KVNamespace, limit = 50): Promise { +export async function getSendLog(db: D1Database, limit = 50): Promise { try { - const list = await kv.list({ prefix: KEY_PREFIX, limit: MAX_READ }); - const records = await Promise.all(list.keys.map((k) => kv.get(k.name, "json"))); - return records - .filter((r): r is SendRecord => r != null) - .sort((a, b) => b.ts - a.ts) - .slice(0, limit); + const { results } = await db + .prepare("SELECT * FROM send_logs ORDER BY ts DESC LIMIT ?") + .bind(limit) + .all<{ ts: number; route_id: string; event: string; repo: string | null; target: string; ok: number; error: string | null }>(); + return results.map((r) => ({ + ts: r.ts, + routeId: r.route_id, + event: r.event, + repo: r.repo ?? undefined, + target: r.target, + ok: r.ok === 1, + error: r.error ?? undefined, + })); } catch (err) { log.warn({ err }, "Failed to load send log"); return []; diff --git a/src/types.ts b/src/types.ts index d133b6f..747f59e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -15,6 +15,7 @@ export interface Env { DISCORD_APPLICATION_ID?: string; ASSETS?: Fetcher; KV: KVNamespace; + DB: D1Database; } export interface Config { diff --git a/src/web/admin-routes.ts b/src/web/admin-routes.ts index c02c815..baec23c 100644 --- a/src/web/admin-routes.ts +++ b/src/web/admin-routes.ts @@ -227,13 +227,13 @@ export function createAdminRoutes(): Hono<{ Bindings: Env }> { if (!s) return c.json({ error: "Unauthorized" }, 401); const limit = Math.min(Math.max(Number(c.req.query("limit") ?? 50), 1), 100); if (s.scope.isSuper) { - return c.json({ logs: await getSendLog(c.env.KV, limit) }); + return c.json({ logs: await getSendLog(c.env.DB, limit) }); } const all = await loadRoutes(c.env.KV); const allowed = new Set( all.filter((r) => r.groupId != null && s.scope.groupIds.has(r.groupId)).map((r) => r.id), ); - const logs = (await getSendLog(c.env.KV, 200)) + const logs = (await getSendLog(c.env.DB, 200)) .filter((l) => allowed.has(l.routeId)) .slice(0, limit); return c.json({ logs }); diff --git a/src/web/oauth-routes.ts b/src/web/oauth-routes.ts index ce941ca..c56966f 100644 --- a/src/web/oauth-routes.ts +++ b/src/web/oauth-routes.ts @@ -83,7 +83,7 @@ export function createOAuthRoutes(): Hono<{ Bindings: Env }> { // Discord account-linking flow: bind the Discord user to this GitHub account. if (pending.discordUserId) { - await saveDiscordLink(c.env.KV, pending.discordUserId, result.userId); + await saveDiscordLink(c.env.DB, pending.discordUserId, result.userId); const isBrowserLink = (c.req.header("accept") ?? "").includes("text/html"); if (isBrowserLink) { return c.html(linkedPage(result.login)); diff --git a/wrangler.jsonc b/wrangler.jsonc index 7fca57d..21cf8ed 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -31,4 +31,11 @@ "id": "53abb6d985a44b80b2b08d510f73928a" }, ], + "d1_databases": [ + { + "binding": "DB", + "database_name": "webhooker", + "database_id": "214a0104-3235-47c0-b7bf-ddda95f3c8ac" + }, + ], }