perf(metrics): send all seven queries in one D1 batch

Avoid seven sequential network round trips and read totals, breakdowns and recent failures in one consistent transaction. Reuse the failed-send statement and row mapping while retaining group scoping.
This commit is contained in:
翅膀 committed 2026-10-17 17:08:31 +08:00
1 parent 244b5c6ffe
commit 6cdcedde92
4 files changed
+86 -56

No files matched your search

+2 -2
View File
@@ -127,7 +127,7 @@ server/ # Nitro server
│ ├── tenants.ts # Per-group webhook secret CRUD (KV tenant:{groupId}, 32-byte random hex)
│ └── richheader.ts # GET /api/richheader: Open Graph page for Telegram avatar link-preview card
├── observability/ # delivery metrics aggregation
│ └── metrics.ts # DeliveryMetrics + getDeliveryMetrics (SQL GROUP BY over send_logs: totals, per platform/event/status, duration, attempts, recent failures)
│ └── metrics.ts # DeliveryMetrics + getDeliveryMetrics (SQL GROUP BY over send_logs in one D1 batch: totals, per platform/event/status, duration, attempts, recent failures)
├── storage/ # storage primitives: canUseD1 probe, D1 config store (L1 memory → L2 KV cache → L3 D1), R2 payload store
│ ├── d1.ts # canUseD1(db) — prepare+batch probe for D1 availability (falls back to KV)
│ ├── config-store.ts # ConfigStore (loadRoutes/saveRoutes/loadGroups/saveGroups/invalidateCache) backed by d1_groups/d1_routes (per-group `position` order when migrated), seeds from KV + syncs cache
@@ -172,7 +172,7 @@ tests/__snapshots__/ # formatter snapshot golden files (toMatchSnapshot)
- Route messages to Discord channels/threads and Telegram chats/topics via REST
- Edit already-sent messages in place for `workflow_run` / `check_run` progress (stable `updateKey`, `message_tracking` via D1 with KV `msg:*` fallback); `workflow_run` keys on the workflow id while `check_run` keys on check name + commit SHA (falling back to the run id) so providers that emit a fresh run id per phase — e.g. Cloudflare Pages — still edit the same message
- Record every dispatch attempt to D1 `send_logs` (route id, event, target, ok/error, duration, error code)
- Aggregate delivery metrics (`server/lib/observability/metrics.ts`) from `send_logs` — totals, ok/failed counts + failure rate, per-platform/per-event/per-status breakdowns, average duration and attempts, recent failures; `getDeliveryMetrics(db, groupId?)` scopes every query by `group_id` when a group is passed
- Aggregate delivery metrics (`server/lib/observability/metrics.ts`) from `send_logs` — totals, ok/failed counts + failure rate, per-platform/per-event/per-status breakdowns, average duration and attempts, recent failures; `getDeliveryMetrics(db, groupId?)` sends all seven queries as one D1 `batch` (a single round trip and a consistent snapshot) and scopes every query by `group_id` when a group is passed
- Expose admin observability endpoints — `GET /admin/api/metrics?groupId=` (delivery metrics, optional group scope; recent failures group-scoped for non-super) and `GET /admin/api/delivery/:deliveryId` (all send-log attempts for one delivery, group-scoped) — through the `/admin/api/[...slug]` catch-all route (`server/routes/admin/api/[...slug].ts`) that wires every admin API handler to its method+path
- Serve a per-group webhook ingress (`POST /webhook/{groupId}`, per-group secret in KV `tenant:{groupId}`) for Gitea/classic-GitHub/custom senders; only that group's routes fire; dedup keys are provider- and tenant-scoped (`delivery:{provider}:{groupId}:{id}` via `idempotencyStore`, D1 `dedup_keys` with KV fallback)
- Issue a per-request correlation id (`requestId`) in webhook responses and dispatch logs
+20 -12
View File
@@ -25,7 +25,7 @@ export interface SendRecord {
const COLUMNS =
"id, ts, route_id, group_id, event, repo, target, ok, error, status, message_id, delivery_id, platform, actor, action, duration_ms, error_code, attempts, detail";
interface LogRow {
export interface LogRow {
id: number;
ts: number;
route_id: string;
@@ -47,7 +47,7 @@ interface LogRow {
detail: string | null;
}
function toRecord(r: LogRow): SendRecord {
export function toRecord(r: LogRow): SendRecord {
return {
id: r.id,
ts: r.ts,
@@ -147,22 +147,30 @@ export async function getSendLogByDelivery(
}
}
/** The latest failed sends, optionally of one group (rows map with `toRecord`). */
export function failedSendLogStatement(
db: D1Database,
limit = 20,
groupId?: string,
): D1PreparedStatement {
return groupId
? db
.prepare(
`SELECT ${COLUMNS} FROM send_logs WHERE ok = 0 AND group_id = ? ORDER BY ts DESC LIMIT ?`,
)
.bind(groupId, limit)
: db
.prepare(`SELECT ${COLUMNS} FROM send_logs WHERE ok = 0 ORDER BY ts DESC LIMIT ?`)
.bind(limit);
}
export async function getFailedSendLog(
db: D1Database,
limit = 20,
groupId?: string,
): Promise<SendRecord[]> {
try {
const stmt = groupId
? db
.prepare(
`SELECT ${COLUMNS} FROM send_logs WHERE ok = 0 AND group_id = ? ORDER BY ts DESC LIMIT ?`,
)
.bind(groupId, limit)
: db
.prepare(`SELECT ${COLUMNS} FROM send_logs WHERE ok = 0 ORDER BY ts DESC LIMIT ?`)
.bind(limit);
const { results } = await stmt.all<LogRow>();
const { results } = await failedSendLogStatement(db, limit, groupId).all<LogRow>();
return results.map(toRecord);
} catch (err) {
log.warn({ err }, "Failed to load failed send log");
+54 -42
View File
@@ -1,5 +1,5 @@
import { log } from "../lib/log";
import { getFailedSendLog, type SendRecord } from "../lib/send-log";
import { failedSendLogStatement, toRecord, type LogRow, type SendRecord } from "../lib/send-log";
export interface DeliveryMetrics {
total: number;
@@ -64,65 +64,77 @@ export async function getDeliveryMetrics(
groupId ? stmt.bind(groupId) : stmt;
try {
const totals = await bindGroup(
db.prepare(`SELECT COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok FROM send_logs${where}`),
).all<TotalsRow>();
const total = Number(totals.results[0]?.total) || 0;
const ok = Number(totals.results[0]?.ok) || 0;
// One round trip for all seven queries (each D1 call is a network hop), and
// a batch runs as one transaction, so the totals and breakdowns agree.
const results = await db.batch([
bindGroup(
db.prepare(`SELECT COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok FROM send_logs${where}`),
),
bindGroup(
db.prepare(
`SELECT platform AS key, COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok
FROM send_logs${where} GROUP BY platform`,
),
),
bindGroup(
db.prepare(
`SELECT event AS key, COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok
FROM send_logs${where} GROUP BY event`,
),
),
bindGroup(
db.prepare(
`SELECT status, COUNT(*) AS count FROM send_logs
WHERE status IS NOT NULL${and} GROUP BY status`,
),
),
bindGroup(
db.prepare(
`SELECT AVG(duration_ms) AS avg FROM send_logs WHERE duration_ms IS NOT NULL${and}`,
),
),
bindGroup(
db.prepare(
`SELECT COALESCE(SUM(attempts), 0) AS total, AVG(attempts) AS avg
FROM send_logs WHERE attempts IS NOT NULL${and}`,
),
),
failedSendLogStatement(db, 20, groupId),
]);
const rows = <T>(i: number): T[] => (results[i]?.results ?? []) as T[];
const totals = rows<TotalsRow>(0);
const byPlatform = rows<BreakdownRow>(1);
const byEvent = rows<BreakdownRow>(2);
const byStatus = rows<StatusRow>(3);
const duration = rows<AvgRow>(4);
const attempts = rows<AttemptsRow>(5);
const recentFailures = rows<LogRow>(6).map(toRecord);
const byPlatform = await bindGroup(
db.prepare(
`SELECT platform AS key, COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok
FROM send_logs${where} GROUP BY platform`,
),
).all<BreakdownRow>();
const byEvent = await bindGroup(
db.prepare(
`SELECT event AS key, COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok
FROM send_logs${where} GROUP BY event`,
),
).all<BreakdownRow>();
const byStatus = await bindGroup(
db.prepare(
`SELECT status, COUNT(*) AS count FROM send_logs
WHERE status IS NOT NULL${and} GROUP BY status`,
),
).all<StatusRow>();
const duration = await bindGroup(
db.prepare(
`SELECT AVG(duration_ms) AS avg FROM send_logs WHERE duration_ms IS NOT NULL${and}`,
),
).all<AvgRow>();
const attempts = await bindGroup(
db.prepare(
`SELECT COALESCE(SUM(attempts), 0) AS total, AVG(attempts) AS avg
FROM send_logs WHERE attempts IS NOT NULL${and}`,
),
).all<AttemptsRow>();
const recentFailures = await getFailedSendLog(db, 20, groupId);
const total = Number(totals[0]?.total) || 0;
const ok = Number(totals[0]?.ok) || 0;
return {
total,
ok,
failed: total - ok,
failureRate: total > 0 ? (total - ok) / total : 0,
byPlatform: byPlatform.results.map((r) => {
byPlatform: byPlatform.map((r) => {
const t = Number(r.total) || 0;
const o = Number(r.ok) || 0;
return { platform: r.key ?? "unknown", total: t, ok: o, failed: t - o };
}),
byEvent: byEvent.results.map((r) => {
byEvent: byEvent.map((r) => {
const t = Number(r.total) || 0;
const o = Number(r.ok) || 0;
return { event: r.key ?? "unknown", total: t, ok: o, failed: t - o };
}),
byStatus: byStatus.results.map((r) => ({
byStatus: byStatus.map((r) => ({
status: String(r.status),
count: Number(r.count) || 0,
})),
avgDurationMs: Number(duration.results[0]?.avg) || 0,
totalAttempts: Number(attempts.results[0]?.total) || 0,
avgAttempts: Number(attempts.results[0]?.avg) || 0,
avgDurationMs: Number(duration[0]?.avg) || 0,
totalAttempts: Number(attempts[0]?.total) || 0,
avgAttempts: Number(attempts[0]?.avg) || 0,
recentFailures,
};
} catch (err) {
+10
View File
@@ -50,6 +50,7 @@ function createMetricsDB(rowsBySql: Record<string, unknown[]>): D1Database {
}
return { results: [] };
};
let batches = 0;
return {
prepare: (sql: string) => ({
all: all(sql),
@@ -58,6 +59,13 @@ function createMetricsDB(rowsBySql: Record<string, unknown[]>): D1Database {
run: async () => ({ success: true }),
}),
}),
batch: async (statements: Array<{ all: () => Promise<unknown> }>) => {
batches++;
return Promise.all(statements.map((s) => s.all()));
},
get batches(): number {
return batches;
},
} as unknown as D1Database;
}
@@ -102,6 +110,8 @@ describe("delivery metrics", () => {
"COUNT(*) AS total": [{ total: 10, ok: 8 }],
});
const m = await getDeliveryMetrics(db);
// Every query goes out in a single batch (one D1 round trip).
expect((db as unknown as { batches: number }).batches).toBe(1);
expect(m.total).toBe(10);
expect(m.ok).toBe(8);
expect(m.failed).toBe(2);