From 6cdcedde92e229035fb8ee4cc3e0b74547f867c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BF=85=E8=86=80?= Date: Sat, 17 Oct 2026 17:08:31 +0800 Subject: [PATCH] perf(metrics): send all seven queries in one D1 batch Avoid seven sequential network round trips and read totals, breakdowns and recent failures in one consistent transaction. Reuse the failed-send statement and row mapping while retaining group scoping. --- AGENTS.md | 4 +- server/lib/lib/send-log.ts | 32 ++++++---- server/lib/observability/metrics.ts | 96 ++++++++++++++++------------- tests/delivery-metrics.test.ts | 10 +++ 4 files changed, 86 insertions(+), 56 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f02d1d8..832eea8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -127,7 +127,7 @@ server/ # Nitro server │ ├── tenants.ts # Per-group webhook secret CRUD (KV tenant:{groupId}, 32-byte random hex) │ └── richheader.ts # GET /api/richheader: Open Graph page for Telegram avatar link-preview card ├── observability/ # delivery metrics aggregation - │ └── metrics.ts # DeliveryMetrics + getDeliveryMetrics (SQL GROUP BY over send_logs: totals, per platform/event/status, duration, attempts, recent failures) + │ └── metrics.ts # DeliveryMetrics + getDeliveryMetrics (SQL GROUP BY over send_logs in one D1 batch: totals, per platform/event/status, duration, attempts, recent failures) ├── storage/ # storage primitives: canUseD1 probe, D1 config store (L1 memory → L2 KV cache → L3 D1), R2 payload store │ ├── d1.ts # canUseD1(db) — prepare+batch probe for D1 availability (falls back to KV) │ ├── config-store.ts # ConfigStore (loadRoutes/saveRoutes/loadGroups/saveGroups/invalidateCache) backed by d1_groups/d1_routes (per-group `position` order when migrated), seeds from KV + syncs cache @@ -172,7 +172,7 @@ tests/__snapshots__/ # formatter snapshot golden files (toMatchSnapshot) - Route messages to Discord channels/threads and Telegram chats/topics via REST - Edit already-sent messages in place for `workflow_run` / `check_run` progress (stable `updateKey`, `message_tracking` via D1 with KV `msg:*` fallback); `workflow_run` keys on the workflow id while `check_run` keys on check name + commit SHA (falling back to the run id) so providers that emit a fresh run id per phase — e.g. Cloudflare Pages — still edit the same message - Record every dispatch attempt to D1 `send_logs` (route id, event, target, ok/error, duration, error code) -- Aggregate delivery metrics (`server/lib/observability/metrics.ts`) from `send_logs` — totals, ok/failed counts + failure rate, per-platform/per-event/per-status breakdowns, average duration and attempts, recent failures; `getDeliveryMetrics(db, groupId?)` scopes every query by `group_id` when a group is passed +- Aggregate delivery metrics (`server/lib/observability/metrics.ts`) from `send_logs` — totals, ok/failed counts + failure rate, per-platform/per-event/per-status breakdowns, average duration and attempts, recent failures; `getDeliveryMetrics(db, groupId?)` sends all seven queries as one D1 `batch` (a single round trip and a consistent snapshot) and scopes every query by `group_id` when a group is passed - Expose admin observability endpoints — `GET /admin/api/metrics?groupId=` (delivery metrics, optional group scope; recent failures group-scoped for non-super) and `GET /admin/api/delivery/:deliveryId` (all send-log attempts for one delivery, group-scoped) — through the `/admin/api/[...slug]` catch-all route (`server/routes/admin/api/[...slug].ts`) that wires every admin API handler to its method+path - Serve a per-group webhook ingress (`POST /webhook/{groupId}`, per-group secret in KV `tenant:{groupId}`) for Gitea/classic-GitHub/custom senders; only that group's routes fire; dedup keys are provider- and tenant-scoped (`delivery:{provider}:{groupId}:{id}` via `idempotencyStore`, D1 `dedup_keys` with KV fallback) - Issue a per-request correlation id (`requestId`) in webhook responses and dispatch logs diff --git a/server/lib/lib/send-log.ts b/server/lib/lib/send-log.ts index e7dc4b1..001bbaf 100644 --- a/server/lib/lib/send-log.ts +++ b/server/lib/lib/send-log.ts @@ -25,7 +25,7 @@ export interface SendRecord { const COLUMNS = "id, ts, route_id, group_id, event, repo, target, ok, error, status, message_id, delivery_id, platform, actor, action, duration_ms, error_code, attempts, detail"; -interface LogRow { +export interface LogRow { id: number; ts: number; route_id: string; @@ -47,7 +47,7 @@ interface LogRow { detail: string | null; } -function toRecord(r: LogRow): SendRecord { +export function toRecord(r: LogRow): SendRecord { return { id: r.id, ts: r.ts, @@ -147,22 +147,30 @@ export async function getSendLogByDelivery( } } +/** The latest failed sends, optionally of one group (rows map with `toRecord`). */ +export function failedSendLogStatement( + db: D1Database, + limit = 20, + groupId?: string, +): D1PreparedStatement { + return groupId + ? db + .prepare( + `SELECT ${COLUMNS} FROM send_logs WHERE ok = 0 AND group_id = ? ORDER BY ts DESC LIMIT ?`, + ) + .bind(groupId, limit) + : db + .prepare(`SELECT ${COLUMNS} FROM send_logs WHERE ok = 0 ORDER BY ts DESC LIMIT ?`) + .bind(limit); +} + export async function getFailedSendLog( db: D1Database, limit = 20, groupId?: string, ): Promise { try { - const stmt = groupId - ? db - .prepare( - `SELECT ${COLUMNS} FROM send_logs WHERE ok = 0 AND group_id = ? ORDER BY ts DESC LIMIT ?`, - ) - .bind(groupId, limit) - : db - .prepare(`SELECT ${COLUMNS} FROM send_logs WHERE ok = 0 ORDER BY ts DESC LIMIT ?`) - .bind(limit); - const { results } = await stmt.all(); + const { results } = await failedSendLogStatement(db, limit, groupId).all(); return results.map(toRecord); } catch (err) { log.warn({ err }, "Failed to load failed send log"); diff --git a/server/lib/observability/metrics.ts b/server/lib/observability/metrics.ts index 848830a..fe75d16 100644 --- a/server/lib/observability/metrics.ts +++ b/server/lib/observability/metrics.ts @@ -1,5 +1,5 @@ import { log } from "../lib/log"; -import { getFailedSendLog, type SendRecord } from "../lib/send-log"; +import { failedSendLogStatement, toRecord, type LogRow, type SendRecord } from "../lib/send-log"; export interface DeliveryMetrics { total: number; @@ -64,65 +64,77 @@ export async function getDeliveryMetrics( groupId ? stmt.bind(groupId) : stmt; try { - const totals = await bindGroup( - db.prepare(`SELECT COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok FROM send_logs${where}`), - ).all(); - const total = Number(totals.results[0]?.total) || 0; - const ok = Number(totals.results[0]?.ok) || 0; + // One round trip for all seven queries (each D1 call is a network hop), and + // a batch runs as one transaction, so the totals and breakdowns agree. + const results = await db.batch([ + bindGroup( + db.prepare(`SELECT COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok FROM send_logs${where}`), + ), + bindGroup( + db.prepare( + `SELECT platform AS key, COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok + FROM send_logs${where} GROUP BY platform`, + ), + ), + bindGroup( + db.prepare( + `SELECT event AS key, COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok + FROM send_logs${where} GROUP BY event`, + ), + ), + bindGroup( + db.prepare( + `SELECT status, COUNT(*) AS count FROM send_logs + WHERE status IS NOT NULL${and} GROUP BY status`, + ), + ), + bindGroup( + db.prepare( + `SELECT AVG(duration_ms) AS avg FROM send_logs WHERE duration_ms IS NOT NULL${and}`, + ), + ), + bindGroup( + db.prepare( + `SELECT COALESCE(SUM(attempts), 0) AS total, AVG(attempts) AS avg + FROM send_logs WHERE attempts IS NOT NULL${and}`, + ), + ), + failedSendLogStatement(db, 20, groupId), + ]); + const rows = (i: number): T[] => (results[i]?.results ?? []) as T[]; + const totals = rows(0); + const byPlatform = rows(1); + const byEvent = rows(2); + const byStatus = rows(3); + const duration = rows(4); + const attempts = rows(5); + const recentFailures = rows(6).map(toRecord); - const byPlatform = await bindGroup( - db.prepare( - `SELECT platform AS key, COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok - FROM send_logs${where} GROUP BY platform`, - ), - ).all(); - const byEvent = await bindGroup( - db.prepare( - `SELECT event AS key, COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok - FROM send_logs${where} GROUP BY event`, - ), - ).all(); - const byStatus = await bindGroup( - db.prepare( - `SELECT status, COUNT(*) AS count FROM send_logs - WHERE status IS NOT NULL${and} GROUP BY status`, - ), - ).all(); - const duration = await bindGroup( - db.prepare( - `SELECT AVG(duration_ms) AS avg FROM send_logs WHERE duration_ms IS NOT NULL${and}`, - ), - ).all(); - const attempts = await bindGroup( - db.prepare( - `SELECT COALESCE(SUM(attempts), 0) AS total, AVG(attempts) AS avg - FROM send_logs WHERE attempts IS NOT NULL${and}`, - ), - ).all(); - const recentFailures = await getFailedSendLog(db, 20, groupId); + const total = Number(totals[0]?.total) || 0; + const ok = Number(totals[0]?.ok) || 0; return { total, ok, failed: total - ok, failureRate: total > 0 ? (total - ok) / total : 0, - byPlatform: byPlatform.results.map((r) => { + byPlatform: byPlatform.map((r) => { const t = Number(r.total) || 0; const o = Number(r.ok) || 0; return { platform: r.key ?? "unknown", total: t, ok: o, failed: t - o }; }), - byEvent: byEvent.results.map((r) => { + byEvent: byEvent.map((r) => { const t = Number(r.total) || 0; const o = Number(r.ok) || 0; return { event: r.key ?? "unknown", total: t, ok: o, failed: t - o }; }), - byStatus: byStatus.results.map((r) => ({ + byStatus: byStatus.map((r) => ({ status: String(r.status), count: Number(r.count) || 0, })), - avgDurationMs: Number(duration.results[0]?.avg) || 0, - totalAttempts: Number(attempts.results[0]?.total) || 0, - avgAttempts: Number(attempts.results[0]?.avg) || 0, + avgDurationMs: Number(duration[0]?.avg) || 0, + totalAttempts: Number(attempts[0]?.total) || 0, + avgAttempts: Number(attempts[0]?.avg) || 0, recentFailures, }; } catch (err) { diff --git a/tests/delivery-metrics.test.ts b/tests/delivery-metrics.test.ts index b2b4aa4..86ff0b1 100644 --- a/tests/delivery-metrics.test.ts +++ b/tests/delivery-metrics.test.ts @@ -50,6 +50,7 @@ function createMetricsDB(rowsBySql: Record): D1Database { } return { results: [] }; }; + let batches = 0; return { prepare: (sql: string) => ({ all: all(sql), @@ -58,6 +59,13 @@ function createMetricsDB(rowsBySql: Record): D1Database { run: async () => ({ success: true }), }), }), + batch: async (statements: Array<{ all: () => Promise }>) => { + batches++; + return Promise.all(statements.map((s) => s.all())); + }, + get batches(): number { + return batches; + }, } as unknown as D1Database; } @@ -102,6 +110,8 @@ describe("delivery metrics", () => { "COUNT(*) AS total": [{ total: 10, ok: 8 }], }); const m = await getDeliveryMetrics(db); + // Every query goes out in a single batch (one D1 round trip). + expect((db as unknown as { batches: number }).batches).toBe(1); expect(m.total).toBe(10); expect(m.ok).toBe(8); expect(m.failed).toBe(2);