diff --git a/AGENTS.md b/AGENTS.md index f365056..a260bdf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,7 @@ Core pipeline: GitHub Webhook → Worker (verify + filter + format) → Discord - UI: Vue 3 + Tailwind CSS v4 (`@tailwindcss/vite`) + `@elysola/ui` (Nuxt module, `Rc` prefix); admin console is a client-side SPA (`routeRules: "/admin/**": { ssr: false }`), home/legal pages render server-side - Styling: Elysola UI supplies semantic components and blue theme tokens; `app/assets/css/main.css` retains `--wh-*` RGB-triplet aliases for app-specific Tailwind utilities and bespoke layouts (avoiding collision with Elysola's `--surface`/`--text`/`--border` hex tokens); theme switching supports auto/light/dark via `useTheme` / `ThemeToggle`, toggling `.dark` with inline anti-flash head script - Discord interactions: HTTPS Interactions Endpoint (`POST /discord/interactions`, Ed25519-signed) — no Discord Gateway / Durable Object; bot stays offline, messages always sent via REST -- Storage: D1 is the source of truth for config (routes/groups via `d1_routes`/`d1_groups`, see `server/lib/storage/config-store.ts`), send/audit logs, dedup (`dedup_keys`), delivery state (`delivery_state`) and message tracking (`message_tracking` via `server/lib/storage/d1.ts` `canUseD1` gate). KV keeps only cache + short-lived/ephemeral state (tokens, OAuth state, admin sessions, `msg:*`-adjacent locks, per-group secrets `tenant:*`, invites, i18n overrides) with explicit TTLs. R2 parks oversized queue payloads (`webhooks/YYYY/MM/DD/*.json`, `server/lib/storage/payload.ts`) instead of KV. A `storage-prune` scheduled task cleans up expired dedup/delivery/message-tracking rows +- Storage: D1 is the source of truth for config (routes/groups via `d1_routes`/`d1_groups`, see `server/lib/storage/config-store.ts`; each route's per-group order lives in `d1_routes.position` — migration 0012 — and routes load ordered by `group_id, position, id`, falling back to id order on an unmigrated database), send/audit logs, dedup (`dedup_keys`), delivery state (`delivery_state`) and message tracking (`message_tracking` via `server/lib/storage/d1.ts` `canUseD1` gate). KV keeps only cache + short-lived/ephemeral state (tokens, OAuth state, admin sessions, `msg:*`-adjacent locks, per-group secrets `tenant:*`, invites, i18n overrides) with explicit TTLs. R2 parks oversized queue payloads (`webhooks/YYYY/MM/DD/*.json`, `server/lib/storage/payload.ts`) instead of KV. A `storage-prune` scheduled task cleans up expired dedup/delivery/message-tracking rows - Signature verification: Web Crypto API (HMAC-SHA256 for GitHub/Gitea, Ed25519 for Discord, timing-safe secret-token compare for Telegram) - Webhook providers: pluggable forge adapters under `server/lib/providers/` (github, gitea) — each verifies its own signature format and normalizes its payload to a GitHub-shaped `WebhookEvent`; a `custom` provider accepts arbitrary signed JSON posts (`X-WebHooker-Signature`) as `custom` events; GitLab etc. can be added later - Per-group webhook ingress: optional `POST /webhook/{groupId}` with a per-group secret in KV (`tenant:{groupId}`) — Gitea/classic-GitHub/custom webhooks are verified against the group's secret instead of the operator's global ones; only that group's routes fire. The legacy `POST /webhook` (global secrets, all routes) stays untouched @@ -126,7 +126,7 @@ server/ # Nitro server │ └── metrics.ts # DeliveryMetrics + getDeliveryMetrics (SQL GROUP BY over send_logs: totals, per platform/event/status, duration, attempts, recent failures) ├── storage/ # storage primitives: canUseD1 probe, D1 config store (L1 memory → L2 KV cache → L3 D1), R2 payload store │ ├── d1.ts # canUseD1(db) — prepare+batch probe for D1 availability (falls back to KV) - │ ├── config-store.ts # ConfigStore (loadRoutes/saveRoutes/loadGroups/saveGroups/invalidateCache) backed by d1_groups/d1_routes, seeds from KV + syncs cache + │ ├── config-store.ts # ConfigStore (loadRoutes/saveRoutes/loadGroups/saveGroups/invalidateCache) backed by d1_groups/d1_routes (per-group `position` order when migrated), seeds from KV + syncs cache │ └── payload.ts # PayloadStore backed by R2 (webhooks/YYYY/MM/DD/*.json), no-binding stub throws └── lib/ # shared infra ├── i18n.ts # loadTranslations (KV i18n:{lang} overrides), t() with param interpolation @@ -276,7 +276,7 @@ bunx wrangler d1 create webhooker bunx wrangler queues create webhooker-delivery bunx wrangler queues create webhooker-delivery-dlq # Queues are declared in wrangler.jsonc (QUEUE binding); no env var needed -bun run db:migrate:prod # wrangler d1 migrations apply webhooker --remote (migrations/0001..0011) +bun run db:migrate:prod # wrangler d1 migrations apply webhooker --remote (migrations/0001..0012) # R2 bucket for oversized payloads (P0): bunx wrangler r2 bucket create webhooker-payloads bunx wrangler deploy ``` diff --git a/docs/guide/deployment.md b/docs/guide/deployment.md index 13d2f4f..6fc4436 100644 --- a/docs/guide/deployment.md +++ b/docs/guide/deployment.md @@ -88,6 +88,7 @@ bunx wrangler d1 execute webhooker --remote --file ./migrations/0007_send_logs_i bunx wrangler d1 execute webhooker --remote --file ./migrations/0008_storage_d1.sql bunx wrangler d1 execute webhooker --remote --file ./migrations/0010_fragments.sql bunx wrangler d1 execute webhooker --remote --file ./migrations/0011_feishu_links.sql +bunx wrangler d1 execute webhooker --remote --file ./migrations/0012_route_position.sql ``` ::: diff --git a/docs/zh/guide/deployment.md b/docs/zh/guide/deployment.md index 7402aee..1fc90e3 100644 --- a/docs/zh/guide/deployment.md +++ b/docs/zh/guide/deployment.md @@ -88,6 +88,7 @@ bunx wrangler d1 execute webhooker --remote --file ./migrations/0007_send_logs_i bunx wrangler d1 execute webhooker --remote --file ./migrations/0008_storage_d1.sql bunx wrangler d1 execute webhooker --remote --file ./migrations/0010_fragments.sql bunx wrangler d1 execute webhooker --remote --file ./migrations/0011_feishu_links.sql +bunx wrangler d1 execute webhooker --remote --file ./migrations/0012_route_position.sql ``` ::: diff --git a/migrations/0012_route_position.sql b/migrations/0012_route_position.sql new file mode 100644 index 0000000..c64dc2a --- /dev/null +++ b/migrations/0012_route_position.sql @@ -0,0 +1,5 @@ +-- Persist the user-defined route order inside each group. Routes load ordered by +-- (group_id, position, id); rows written before this migration keep position 0 +-- and therefore stay in id order until their group's routes are saved again. +ALTER TABLE d1_routes ADD COLUMN position INTEGER NOT NULL DEFAULT 0; +CREATE INDEX IF NOT EXISTS idx_d1_routes_group_position ON d1_routes (group_id, position); diff --git a/server/lib/storage/config-store.ts b/server/lib/storage/config-store.ts index 2ca9db1..49e94c9 100644 --- a/server/lib/storage/config-store.ts +++ b/server/lib/storage/config-store.ts @@ -48,12 +48,32 @@ export function d1ConfigStore(db: D1Database, kv: KVNamespace): ConfigStore { let routesCache: { routes: Route[]; expiresAt: number } | null = null; let groupsCache: { groups: Group[]; expiresAt: number } | null = null; + /** + * Run a statement that uses the `position` column (migration 0012) and fall + * back to its legacy form only when the database confirms the column is + * missing. Any other error propagates exactly as before, and nothing is + * remembered, so a database migrated later is picked up on the next call. + */ + async function withPosition(run: (positioned: boolean) => Promise): Promise { + try { + return await run(true); + } catch (err) { + // SQLite says "no such column: position" for reads and "has no column + // named position" for inserts. + if (!/no such column: position|no column named position/i.test(String(err))) throw err; + return run(false); + } + } + async function loadRoutesFromD1(): Promise { - const stmt = db.prepare( - "SELECT id, group_id, name, enabled, filters, targets, stop, fallback, discord_role_ids, ast FROM d1_routes ORDER BY id", - ); - if (typeof stmt.all !== "function") return []; - const { results } = await stmt.all(); + const results = await withPosition(async (positioned) => { + const order = positioned ? "group_id, position, id" : "id"; + const stmt = db.prepare( + `SELECT id, group_id, name, enabled, filters, targets, stop, fallback, discord_role_ids, ast FROM d1_routes ORDER BY ${order}`, + ); + if (typeof stmt.all !== "function") return []; + return (await stmt.all()).results; + }); if (!results || results.length === 0) return []; return results.flatMap((r) => { const filters = parseJson(r.filters, `route:${r.id}:filters`); @@ -115,10 +135,14 @@ export function d1ConfigStore(db: D1Database, kv: KVNamespace): ConfigStore { function routeStatements( routes: Route[], + positioned: boolean, existingRouteKeys?: Set, ): D1PreparedStatement[] { const now = Date.now(); const statements: D1PreparedStatement[] = []; + // A route's position is its index among the routes of its own group, so + // saving one group never shifts the order of another. + const nextPosition = new Map(); // If we have existing routes, delete those not in the new set if (existingRouteKeys) { @@ -138,6 +162,43 @@ export function d1ConfigStore(db: D1Database, kv: KVNamespace): ConfigStore { // Upsert all routes for (const r of routes) { + const groupId = r.groupId ?? ""; + const values = [ + r.id, + groupId, + r.name, + r.enabled ? 1 : 0, + JSON.stringify(r.filters), + JSON.stringify(r.targets), + r.stop ? 1 : 0, + r.fallback ? 1 : 0, + r.discordRoleIds ? JSON.stringify(r.discordRoleIds) : null, + r.ast ? JSON.stringify(r.ast) : null, + ]; + if (positioned) { + const position = nextPosition.get(groupId) ?? 0; + nextPosition.set(groupId, position + 1); + statements.push( + db + .prepare( + `INSERT INTO d1_routes (id, group_id, name, enabled, filters, targets, stop, fallback, discord_role_ids, ast, position, version, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?) + ON CONFLICT(id, group_id) DO UPDATE SET + name = excluded.name, + enabled = excluded.enabled, + filters = excluded.filters, + targets = excluded.targets, + stop = excluded.stop, + fallback = excluded.fallback, + discord_role_ids = excluded.discord_role_ids, + ast = excluded.ast, + position = excluded.position, + updated_at = excluded.updated_at`, + ) + .bind(...values, position, now, now), + ); + continue; + } statements.push( db .prepare( @@ -154,20 +215,7 @@ export function d1ConfigStore(db: D1Database, kv: KVNamespace): ConfigStore { ast = excluded.ast, updated_at = excluded.updated_at`, ) - .bind( - r.id, - r.groupId ?? "", - r.name, - r.enabled ? 1 : 0, - JSON.stringify(r.filters), - JSON.stringify(r.targets), - r.stop ? 1 : 0, - r.fallback ? 1 : 0, - r.discordRoleIds ? JSON.stringify(r.discordRoleIds) : null, - r.ast ? JSON.stringify(r.ast) : null, - now, - now, - ), + .bind(...values, now, now), ); } @@ -214,7 +262,7 @@ export function d1ConfigStore(db: D1Database, kv: KVNamespace): ConfigStore { async function seedRoutesToD1(routes: Route[]): Promise { if (routes.length === 0) return; - await db.batch(routeStatements(routes)); + await withPosition((positioned) => db.batch(routeStatements(routes, positioned))); } async function seedGroupsToD1(groups: Group[]): Promise { @@ -269,7 +317,7 @@ export function d1ConfigStore(db: D1Database, kv: KVNamespace): ConfigStore { async saveRoutes(routes: Route[]): Promise { try { - await db.batch(routeStatements(routes)); + await withPosition((positioned) => db.batch(routeStatements(routes, positioned))); await syncRoutesToKV(routes, KV_CACHE_TTL); } catch (err) { log.warn({ err }, "D1 routes unavailable, falling back to KV"); diff --git a/tests/config-store.test.ts b/tests/config-store.test.ts index b209fde..0df1575 100644 --- a/tests/config-store.test.ts +++ b/tests/config-store.test.ts @@ -3,6 +3,8 @@ import { d1ConfigStore, type ConfigStore } from "../server/lib/storage/config-st import type { Route, Group } from "../server/lib/types"; interface Stmt { + sql: string; + bound: unknown[]; bind: (...args: unknown[]) => Stmt; all: () => Promise<{ results: Array> }>; } @@ -12,6 +14,13 @@ interface FakeDB { batch: (stmts: Stmt[]) => Promise; } +interface FakeDBOptions { + /** Whether `d1_routes` has the `position` column (migration 0012). */ + positionColumn?: boolean; + /** How many batches fail with a transient (non-schema) error first. */ + batchFailures?: number; +} + function route(id: string, groupId: string, overrides: Partial = {}): Route { return { id, @@ -30,18 +39,32 @@ function group(id: string): Group { return { id, name: `group-${id}`, adminIds: [] }; } -function createDB(): { db: FakeDB & D1Database; routesTable: Route[]; groupsTable: Group[] } { +function createDB(options: FakeDBOptions = {}): { + db: FakeDB & D1Database; + routesTable: Route[]; + groupsTable: Group[]; + queries: string[]; + batches: Stmt[][]; +} { const routesTable: Route[] = []; const groupsTable: Group[] = []; + const queries: string[] = []; + const batches: Stmt[][] = []; + let batchFailures = options.batchFailures ?? 0; const db: FakeDB = { prepare(sql: string): Stmt { - let _bound: unknown[] = []; const stmt: Stmt = { + sql, + bound: [], bind(...args: unknown[]): Stmt { - _bound = args; + stmt.bound = args; return stmt; }, async all(): Promise<{ results: Array> }> { + queries.push(sql); + if (sql.includes("position") && !options.positionColumn) { + throw new Error("D1_ERROR: no such column: position at offset 33: SQLITE_ERROR"); + } if (sql.includes("FROM d1_routes")) { return { results: routesTable.map((r) => ({ @@ -74,11 +97,25 @@ function createDB(): { db: FakeDB & D1Database; routesTable: Route[]; groupsTabl return stmt; }, async batch(stmts: Stmt[]): Promise { - void stmts; + batches.push(stmts); + if (batchFailures > 0) { + batchFailures--; + throw new Error("D1_ERROR: network connection lost"); + } + if (!options.positionColumn && stmts.some((st) => st.sql.includes("position,"))) { + throw new Error("D1_ERROR: table d1_routes has no column named position: SQLITE_ERROR"); + } return []; }, }; - return { db: db as FakeDB & D1Database, routesTable, groupsTable }; + return { db: db as FakeDB & D1Database, routesTable, groupsTable, queries, batches }; +} + +/** Bound `(id, position)` pairs of every route upsert in a batch. */ +function upsertPositions(stmts: Stmt[]): Array<[unknown, unknown]> { + return stmts + .filter((s) => s.sql.includes("INSERT INTO d1_routes")) + .map((s) => [s.bound[0], s.sql.includes("position") ? s.bound[10] : undefined]); } function createKV(): { kv: KVNamespace; store: Map } { @@ -172,6 +209,66 @@ describe("d1ConfigStore", () => { expect(second).toHaveLength(2); }); + it("persists the per-group route order when the position column exists", async () => { + const { db, batches } = createDB({ positionColumn: true }); + const { kv } = createKV(); + const cfg: ConfigStore = d1ConfigStore(db, kv); + await cfg.saveRoutes([route("b", "g1"), route("x", "g2"), route("a", "g1")]); + expect(upsertPositions(batches[0]!)).toEqual([ + ["b", 0], + ["x", 0], + ["a", 1], + ]); + }); + + it("loads routes ordered by group and position when the column exists", async () => { + const { db, queries } = createDB({ positionColumn: true }); + const { kv } = createKV(); + await d1ConfigStore(db, kv).loadRoutes(); + expect(queries.some((q) => q.includes("FROM d1_routes ORDER BY group_id, position, id"))).toBe( + true, + ); + }); + + it("falls back to the legacy id order and inserts on an unmigrated database", async () => { + const { db, queries, batches } = createDB(); + const { kv } = createKV(); + const cfg: ConfigStore = d1ConfigStore(db, kv); + await cfg.loadRoutes(); + expect(queries.at(-1)).toContain("FROM d1_routes ORDER BY id"); + await cfg.saveRoutes([route("r1", "g1")]); + expect(upsertPositions(batches.at(-1)!)).toEqual([["r1", undefined]]); + }); + + it("picks up the position column once the database is migrated", async () => { + const options: FakeDBOptions = {}; + const { db, queries } = createDB(options); + const { kv } = createKV(); + const cfg: ConfigStore = d1ConfigStore(db, kv); + await cfg.loadRoutes(); + expect(queries.at(-1)).toContain("ORDER BY id"); + options.positionColumn = true; + cfg.invalidateCache(); + await cfg.loadRoutes(); + expect(queries.at(-1)).toContain("ORDER BY group_id, position, id"); + }); + + it("never retries without positions when D1 fails for another reason", async () => { + const { db, batches } = createDB({ positionColumn: true, batchFailures: 1 }); + const { kv, store } = createKV(); + const cfg: ConfigStore = d1ConfigStore(db, kv); + const routes = [route("b", "g1"), route("a", "g1")]; + await cfg.saveRoutes(routes); + // Only the positioned batch was attempted; the save took the existing + // "D1 unavailable" path instead of writing rows that lose the order. + expect(batches).toHaveLength(1); + expect(upsertPositions(batches[0]!)).toEqual([ + ["b", 0], + ["a", 1], + ]); + expect(store.get("config:routes")).toBe(JSON.stringify(routes)); + }); + it("updating groups does not cascade-delete routes", async () => { const { db, routesTable, groupsTable } = createDB(); const { kv } = createKV();