mirror of
https://github.com/Elysola/WebHooker.git
synced 2026-11-20 17:13:49 +00:00
perf(metrics): send all seven queries in one D1 batch
Avoid seven sequential network round trips and read totals, breakdowns and recent failures in one consistent transaction. Reuse the failed-send statement and row mapping while retaining group scoping.
This commit is contained in:
1 parent
244b5c6ffe
commit
6cdcedde92
4 files changed
+86
-56
No files matched your search
@@ -127,7 +127,7 @@ server/ # Nitro server
|
||||
│ ├── tenants.ts # Per-group webhook secret CRUD (KV tenant:{groupId}, 32-byte random hex)
|
||||
│ └── richheader.ts # GET /api/richheader: Open Graph page for Telegram avatar link-preview card
|
||||
├── observability/ # delivery metrics aggregation
|
||||
│ └── metrics.ts # DeliveryMetrics + getDeliveryMetrics (SQL GROUP BY over send_logs: totals, per platform/event/status, duration, attempts, recent failures)
|
||||
│ └── metrics.ts # DeliveryMetrics + getDeliveryMetrics (SQL GROUP BY over send_logs in one D1 batch: totals, per platform/event/status, duration, attempts, recent failures)
|
||||
├── storage/ # storage primitives: canUseD1 probe, D1 config store (L1 memory → L2 KV cache → L3 D1), R2 payload store
|
||||
│ ├── d1.ts # canUseD1(db) — prepare+batch probe for D1 availability (falls back to KV)
|
||||
│ ├── config-store.ts # ConfigStore (loadRoutes/saveRoutes/loadGroups/saveGroups/invalidateCache) backed by d1_groups/d1_routes (per-group `position` order when migrated), seeds from KV + syncs cache
|
||||
@@ -172,7 +172,7 @@ tests/__snapshots__/ # formatter snapshot golden files (toMatchSnapshot)
|
||||
- Route messages to Discord channels/threads and Telegram chats/topics via REST
|
||||
- Edit already-sent messages in place for `workflow_run` / `check_run` progress (stable `updateKey`, `message_tracking` via D1 with KV `msg:*` fallback); `workflow_run` keys on the workflow id while `check_run` keys on check name + commit SHA (falling back to the run id) so providers that emit a fresh run id per phase — e.g. Cloudflare Pages — still edit the same message
|
||||
- Record every dispatch attempt to D1 `send_logs` (route id, event, target, ok/error, duration, error code)
|
||||
- Aggregate delivery metrics (`server/lib/observability/metrics.ts`) from `send_logs` — totals, ok/failed counts + failure rate, per-platform/per-event/per-status breakdowns, average duration and attempts, recent failures; `getDeliveryMetrics(db, groupId?)` scopes every query by `group_id` when a group is passed
|
||||
- Aggregate delivery metrics (`server/lib/observability/metrics.ts`) from `send_logs` — totals, ok/failed counts + failure rate, per-platform/per-event/per-status breakdowns, average duration and attempts, recent failures; `getDeliveryMetrics(db, groupId?)` sends all seven queries as one D1 `batch` (a single round trip and a consistent snapshot) and scopes every query by `group_id` when a group is passed
|
||||
- Expose admin observability endpoints — `GET /admin/api/metrics?groupId=` (delivery metrics, optional group scope; recent failures group-scoped for non-super) and `GET /admin/api/delivery/:deliveryId` (all send-log attempts for one delivery, group-scoped) — through the `/admin/api/[...slug]` catch-all route (`server/routes/admin/api/[...slug].ts`) that wires every admin API handler to its method+path
|
||||
- Serve a per-group webhook ingress (`POST /webhook/{groupId}`, per-group secret in KV `tenant:{groupId}`) for Gitea/classic-GitHub/custom senders; only that group's routes fire; dedup keys are provider- and tenant-scoped (`delivery:{provider}:{groupId}:{id}` via `idempotencyStore`, D1 `dedup_keys` with KV fallback)
|
||||
- Issue a per-request correlation id (`requestId`) in webhook responses and dispatch logs
|
||||
|
||||
+20
-12
@@ -25,7 +25,7 @@ export interface SendRecord {
|
||||
const COLUMNS =
|
||||
"id, ts, route_id, group_id, event, repo, target, ok, error, status, message_id, delivery_id, platform, actor, action, duration_ms, error_code, attempts, detail";
|
||||
|
||||
interface LogRow {
|
||||
export interface LogRow {
|
||||
id: number;
|
||||
ts: number;
|
||||
route_id: string;
|
||||
@@ -47,7 +47,7 @@ interface LogRow {
|
||||
detail: string | null;
|
||||
}
|
||||
|
||||
function toRecord(r: LogRow): SendRecord {
|
||||
export function toRecord(r: LogRow): SendRecord {
|
||||
return {
|
||||
id: r.id,
|
||||
ts: r.ts,
|
||||
@@ -147,22 +147,30 @@ export async function getSendLogByDelivery(
|
||||
}
|
||||
}
|
||||
|
||||
/** The latest failed sends, optionally of one group (rows map with `toRecord`). */
|
||||
export function failedSendLogStatement(
|
||||
db: D1Database,
|
||||
limit = 20,
|
||||
groupId?: string,
|
||||
): D1PreparedStatement {
|
||||
return groupId
|
||||
? db
|
||||
.prepare(
|
||||
`SELECT ${COLUMNS} FROM send_logs WHERE ok = 0 AND group_id = ? ORDER BY ts DESC LIMIT ?`,
|
||||
)
|
||||
.bind(groupId, limit)
|
||||
: db
|
||||
.prepare(`SELECT ${COLUMNS} FROM send_logs WHERE ok = 0 ORDER BY ts DESC LIMIT ?`)
|
||||
.bind(limit);
|
||||
}
|
||||
|
||||
export async function getFailedSendLog(
|
||||
db: D1Database,
|
||||
limit = 20,
|
||||
groupId?: string,
|
||||
): Promise<SendRecord[]> {
|
||||
try {
|
||||
const stmt = groupId
|
||||
? db
|
||||
.prepare(
|
||||
`SELECT ${COLUMNS} FROM send_logs WHERE ok = 0 AND group_id = ? ORDER BY ts DESC LIMIT ?`,
|
||||
)
|
||||
.bind(groupId, limit)
|
||||
: db
|
||||
.prepare(`SELECT ${COLUMNS} FROM send_logs WHERE ok = 0 ORDER BY ts DESC LIMIT ?`)
|
||||
.bind(limit);
|
||||
const { results } = await stmt.all<LogRow>();
|
||||
const { results } = await failedSendLogStatement(db, limit, groupId).all<LogRow>();
|
||||
return results.map(toRecord);
|
||||
} catch (err) {
|
||||
log.warn({ err }, "Failed to load failed send log");
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { log } from "../lib/log";
|
||||
import { getFailedSendLog, type SendRecord } from "../lib/send-log";
|
||||
import { failedSendLogStatement, toRecord, type LogRow, type SendRecord } from "../lib/send-log";
|
||||
|
||||
export interface DeliveryMetrics {
|
||||
total: number;
|
||||
@@ -64,65 +64,77 @@ export async function getDeliveryMetrics(
|
||||
groupId ? stmt.bind(groupId) : stmt;
|
||||
|
||||
try {
|
||||
const totals = await bindGroup(
|
||||
db.prepare(`SELECT COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok FROM send_logs${where}`),
|
||||
).all<TotalsRow>();
|
||||
const total = Number(totals.results[0]?.total) || 0;
|
||||
const ok = Number(totals.results[0]?.ok) || 0;
|
||||
// One round trip for all seven queries (each D1 call is a network hop), and
|
||||
// a batch runs as one transaction, so the totals and breakdowns agree.
|
||||
const results = await db.batch([
|
||||
bindGroup(
|
||||
db.prepare(`SELECT COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok FROM send_logs${where}`),
|
||||
),
|
||||
bindGroup(
|
||||
db.prepare(
|
||||
`SELECT platform AS key, COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok
|
||||
FROM send_logs${where} GROUP BY platform`,
|
||||
),
|
||||
),
|
||||
bindGroup(
|
||||
db.prepare(
|
||||
`SELECT event AS key, COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok
|
||||
FROM send_logs${where} GROUP BY event`,
|
||||
),
|
||||
),
|
||||
bindGroup(
|
||||
db.prepare(
|
||||
`SELECT status, COUNT(*) AS count FROM send_logs
|
||||
WHERE status IS NOT NULL${and} GROUP BY status`,
|
||||
),
|
||||
),
|
||||
bindGroup(
|
||||
db.prepare(
|
||||
`SELECT AVG(duration_ms) AS avg FROM send_logs WHERE duration_ms IS NOT NULL${and}`,
|
||||
),
|
||||
),
|
||||
bindGroup(
|
||||
db.prepare(
|
||||
`SELECT COALESCE(SUM(attempts), 0) AS total, AVG(attempts) AS avg
|
||||
FROM send_logs WHERE attempts IS NOT NULL${and}`,
|
||||
),
|
||||
),
|
||||
failedSendLogStatement(db, 20, groupId),
|
||||
]);
|
||||
const rows = <T>(i: number): T[] => (results[i]?.results ?? []) as T[];
|
||||
const totals = rows<TotalsRow>(0);
|
||||
const byPlatform = rows<BreakdownRow>(1);
|
||||
const byEvent = rows<BreakdownRow>(2);
|
||||
const byStatus = rows<StatusRow>(3);
|
||||
const duration = rows<AvgRow>(4);
|
||||
const attempts = rows<AttemptsRow>(5);
|
||||
const recentFailures = rows<LogRow>(6).map(toRecord);
|
||||
|
||||
const byPlatform = await bindGroup(
|
||||
db.prepare(
|
||||
`SELECT platform AS key, COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok
|
||||
FROM send_logs${where} GROUP BY platform`,
|
||||
),
|
||||
).all<BreakdownRow>();
|
||||
const byEvent = await bindGroup(
|
||||
db.prepare(
|
||||
`SELECT event AS key, COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok
|
||||
FROM send_logs${where} GROUP BY event`,
|
||||
),
|
||||
).all<BreakdownRow>();
|
||||
const byStatus = await bindGroup(
|
||||
db.prepare(
|
||||
`SELECT status, COUNT(*) AS count FROM send_logs
|
||||
WHERE status IS NOT NULL${and} GROUP BY status`,
|
||||
),
|
||||
).all<StatusRow>();
|
||||
const duration = await bindGroup(
|
||||
db.prepare(
|
||||
`SELECT AVG(duration_ms) AS avg FROM send_logs WHERE duration_ms IS NOT NULL${and}`,
|
||||
),
|
||||
).all<AvgRow>();
|
||||
const attempts = await bindGroup(
|
||||
db.prepare(
|
||||
`SELECT COALESCE(SUM(attempts), 0) AS total, AVG(attempts) AS avg
|
||||
FROM send_logs WHERE attempts IS NOT NULL${and}`,
|
||||
),
|
||||
).all<AttemptsRow>();
|
||||
const recentFailures = await getFailedSendLog(db, 20, groupId);
|
||||
const total = Number(totals[0]?.total) || 0;
|
||||
const ok = Number(totals[0]?.ok) || 0;
|
||||
|
||||
return {
|
||||
total,
|
||||
ok,
|
||||
failed: total - ok,
|
||||
failureRate: total > 0 ? (total - ok) / total : 0,
|
||||
byPlatform: byPlatform.results.map((r) => {
|
||||
byPlatform: byPlatform.map((r) => {
|
||||
const t = Number(r.total) || 0;
|
||||
const o = Number(r.ok) || 0;
|
||||
return { platform: r.key ?? "unknown", total: t, ok: o, failed: t - o };
|
||||
}),
|
||||
byEvent: byEvent.results.map((r) => {
|
||||
byEvent: byEvent.map((r) => {
|
||||
const t = Number(r.total) || 0;
|
||||
const o = Number(r.ok) || 0;
|
||||
return { event: r.key ?? "unknown", total: t, ok: o, failed: t - o };
|
||||
}),
|
||||
byStatus: byStatus.results.map((r) => ({
|
||||
byStatus: byStatus.map((r) => ({
|
||||
status: String(r.status),
|
||||
count: Number(r.count) || 0,
|
||||
})),
|
||||
avgDurationMs: Number(duration.results[0]?.avg) || 0,
|
||||
totalAttempts: Number(attempts.results[0]?.total) || 0,
|
||||
avgAttempts: Number(attempts.results[0]?.avg) || 0,
|
||||
avgDurationMs: Number(duration[0]?.avg) || 0,
|
||||
totalAttempts: Number(attempts[0]?.total) || 0,
|
||||
avgAttempts: Number(attempts[0]?.avg) || 0,
|
||||
recentFailures,
|
||||
};
|
||||
} catch (err) {
|
||||
|
||||
@@ -50,6 +50,7 @@ function createMetricsDB(rowsBySql: Record<string, unknown[]>): D1Database {
|
||||
}
|
||||
return { results: [] };
|
||||
};
|
||||
let batches = 0;
|
||||
return {
|
||||
prepare: (sql: string) => ({
|
||||
all: all(sql),
|
||||
@@ -58,6 +59,13 @@ function createMetricsDB(rowsBySql: Record<string, unknown[]>): D1Database {
|
||||
run: async () => ({ success: true }),
|
||||
}),
|
||||
}),
|
||||
batch: async (statements: Array<{ all: () => Promise<unknown> }>) => {
|
||||
batches++;
|
||||
return Promise.all(statements.map((s) => s.all()));
|
||||
},
|
||||
get batches(): number {
|
||||
return batches;
|
||||
},
|
||||
} as unknown as D1Database;
|
||||
}
|
||||
|
||||
@@ -102,6 +110,8 @@ describe("delivery metrics", () => {
|
||||
"COUNT(*) AS total": [{ total: 10, ok: 8 }],
|
||||
});
|
||||
const m = await getDeliveryMetrics(db);
|
||||
// Every query goes out in a single batch (one D1 round trip).
|
||||
expect((db as unknown as { batches: number }).batches).toBe(1);
|
||||
expect(m.total).toBe(10);
|
||||
expect(m.ok).toBe(8);
|
||||
expect(m.failed).toBe(2);
|
||||
|
||||
Reference in new issue
Block a user