mirror of
https://github.com/Elysola/WebHooker.git
synced 2026-11-21 01:23:40 +00:00
perf(metrics): send all seven queries in one D1 batch
Avoid seven sequential network round trips and read totals, breakdowns and recent failures in one consistent transaction. Reuse the failed-send statement and row mapping while retaining group scoping.
This commit is contained in:
1 parent
244b5c6ffe
commit
6cdcedde92
4 files changed
+62
-32
No files matched your search
@@ -127,7 +127,7 @@ server/ # Nitro server
|
|||||||
│ ├── tenants.ts # Per-group webhook secret CRUD (KV tenant:{groupId}, 32-byte random hex)
|
│ ├── tenants.ts # Per-group webhook secret CRUD (KV tenant:{groupId}, 32-byte random hex)
|
||||||
│ └── richheader.ts # GET /api/richheader: Open Graph page for Telegram avatar link-preview card
|
│ └── richheader.ts # GET /api/richheader: Open Graph page for Telegram avatar link-preview card
|
||||||
├── observability/ # delivery metrics aggregation
|
├── observability/ # delivery metrics aggregation
|
||||||
│ └── metrics.ts # DeliveryMetrics + getDeliveryMetrics (SQL GROUP BY over send_logs: totals, per platform/event/status, duration, attempts, recent failures)
|
│ └── metrics.ts # DeliveryMetrics + getDeliveryMetrics (SQL GROUP BY over send_logs in one D1 batch: totals, per platform/event/status, duration, attempts, recent failures)
|
||||||
├── storage/ # storage primitives: canUseD1 probe, D1 config store (L1 memory → L2 KV cache → L3 D1), R2 payload store
|
├── storage/ # storage primitives: canUseD1 probe, D1 config store (L1 memory → L2 KV cache → L3 D1), R2 payload store
|
||||||
│ ├── d1.ts # canUseD1(db) — prepare+batch probe for D1 availability (falls back to KV)
|
│ ├── d1.ts # canUseD1(db) — prepare+batch probe for D1 availability (falls back to KV)
|
||||||
│ ├── config-store.ts # ConfigStore (loadRoutes/saveRoutes/loadGroups/saveGroups/invalidateCache) backed by d1_groups/d1_routes (per-group `position` order when migrated), seeds from KV + syncs cache
|
│ ├── config-store.ts # ConfigStore (loadRoutes/saveRoutes/loadGroups/saveGroups/invalidateCache) backed by d1_groups/d1_routes (per-group `position` order when migrated), seeds from KV + syncs cache
|
||||||
@@ -172,7 +172,7 @@ tests/__snapshots__/ # formatter snapshot golden files (toMatchSnapshot)
|
|||||||
- Route messages to Discord channels/threads and Telegram chats/topics via REST
|
- Route messages to Discord channels/threads and Telegram chats/topics via REST
|
||||||
- Edit already-sent messages in place for `workflow_run` / `check_run` progress (stable `updateKey`, `message_tracking` via D1 with KV `msg:*` fallback); `workflow_run` keys on the workflow id while `check_run` keys on check name + commit SHA (falling back to the run id) so providers that emit a fresh run id per phase — e.g. Cloudflare Pages — still edit the same message
|
- Edit already-sent messages in place for `workflow_run` / `check_run` progress (stable `updateKey`, `message_tracking` via D1 with KV `msg:*` fallback); `workflow_run` keys on the workflow id while `check_run` keys on check name + commit SHA (falling back to the run id) so providers that emit a fresh run id per phase — e.g. Cloudflare Pages — still edit the same message
|
||||||
- Record every dispatch attempt to D1 `send_logs` (route id, event, target, ok/error, duration, error code)
|
- Record every dispatch attempt to D1 `send_logs` (route id, event, target, ok/error, duration, error code)
|
||||||
- Aggregate delivery metrics (`server/lib/observability/metrics.ts`) from `send_logs` — totals, ok/failed counts + failure rate, per-platform/per-event/per-status breakdowns, average duration and attempts, recent failures; `getDeliveryMetrics(db, groupId?)` scopes every query by `group_id` when a group is passed
|
- Aggregate delivery metrics (`server/lib/observability/metrics.ts`) from `send_logs` — totals, ok/failed counts + failure rate, per-platform/per-event/per-status breakdowns, average duration and attempts, recent failures; `getDeliveryMetrics(db, groupId?)` sends all seven queries as one D1 `batch` (a single round trip and a consistent snapshot) and scopes every query by `group_id` when a group is passed
|
||||||
- Expose admin observability endpoints — `GET /admin/api/metrics?groupId=` (delivery metrics, optional group scope; recent failures group-scoped for non-super) and `GET /admin/api/delivery/:deliveryId` (all send-log attempts for one delivery, group-scoped) — through the `/admin/api/[...slug]` catch-all route (`server/routes/admin/api/[...slug].ts`) that wires every admin API handler to its method+path
|
- Expose admin observability endpoints — `GET /admin/api/metrics?groupId=` (delivery metrics, optional group scope; recent failures group-scoped for non-super) and `GET /admin/api/delivery/:deliveryId` (all send-log attempts for one delivery, group-scoped) — through the `/admin/api/[...slug]` catch-all route (`server/routes/admin/api/[...slug].ts`) that wires every admin API handler to its method+path
|
||||||
- Serve a per-group webhook ingress (`POST /webhook/{groupId}`, per-group secret in KV `tenant:{groupId}`) for Gitea/classic-GitHub/custom senders; only that group's routes fire; dedup keys are provider- and tenant-scoped (`delivery:{provider}:{groupId}:{id}` via `idempotencyStore`, D1 `dedup_keys` with KV fallback)
|
- Serve a per-group webhook ingress (`POST /webhook/{groupId}`, per-group secret in KV `tenant:{groupId}`) for Gitea/classic-GitHub/custom senders; only that group's routes fire; dedup keys are provider- and tenant-scoped (`delivery:{provider}:{groupId}:{id}` via `idempotencyStore`, D1 `dedup_keys` with KV fallback)
|
||||||
- Issue a per-request correlation id (`requestId`) in webhook responses and dispatch logs
|
- Issue a per-request correlation id (`requestId`) in webhook responses and dispatch logs
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ export interface SendRecord {
|
|||||||
const COLUMNS =
|
const COLUMNS =
|
||||||
"id, ts, route_id, group_id, event, repo, target, ok, error, status, message_id, delivery_id, platform, actor, action, duration_ms, error_code, attempts, detail";
|
"id, ts, route_id, group_id, event, repo, target, ok, error, status, message_id, delivery_id, platform, actor, action, duration_ms, error_code, attempts, detail";
|
||||||
|
|
||||||
interface LogRow {
|
export interface LogRow {
|
||||||
id: number;
|
id: number;
|
||||||
ts: number;
|
ts: number;
|
||||||
route_id: string;
|
route_id: string;
|
||||||
@@ -47,7 +47,7 @@ interface LogRow {
|
|||||||
detail: string | null;
|
detail: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
function toRecord(r: LogRow): SendRecord {
|
export function toRecord(r: LogRow): SendRecord {
|
||||||
return {
|
return {
|
||||||
id: r.id,
|
id: r.id,
|
||||||
ts: r.ts,
|
ts: r.ts,
|
||||||
@@ -147,13 +147,13 @@ export async function getSendLogByDelivery(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function getFailedSendLog(
|
/** The latest failed sends, optionally of one group (rows map with `toRecord`). */
|
||||||
|
export function failedSendLogStatement(
|
||||||
db: D1Database,
|
db: D1Database,
|
||||||
limit = 20,
|
limit = 20,
|
||||||
groupId?: string,
|
groupId?: string,
|
||||||
): Promise<SendRecord[]> {
|
): D1PreparedStatement {
|
||||||
try {
|
return groupId
|
||||||
const stmt = groupId
|
|
||||||
? db
|
? db
|
||||||
.prepare(
|
.prepare(
|
||||||
`SELECT ${COLUMNS} FROM send_logs WHERE ok = 0 AND group_id = ? ORDER BY ts DESC LIMIT ?`,
|
`SELECT ${COLUMNS} FROM send_logs WHERE ok = 0 AND group_id = ? ORDER BY ts DESC LIMIT ?`,
|
||||||
@@ -162,7 +162,15 @@ export async function getFailedSendLog(
|
|||||||
: db
|
: db
|
||||||
.prepare(`SELECT ${COLUMNS} FROM send_logs WHERE ok = 0 ORDER BY ts DESC LIMIT ?`)
|
.prepare(`SELECT ${COLUMNS} FROM send_logs WHERE ok = 0 ORDER BY ts DESC LIMIT ?`)
|
||||||
.bind(limit);
|
.bind(limit);
|
||||||
const { results } = await stmt.all<LogRow>();
|
}
|
||||||
|
|
||||||
|
export async function getFailedSendLog(
|
||||||
|
db: D1Database,
|
||||||
|
limit = 20,
|
||||||
|
groupId?: string,
|
||||||
|
): Promise<SendRecord[]> {
|
||||||
|
try {
|
||||||
|
const { results } = await failedSendLogStatement(db, limit, groupId).all<LogRow>();
|
||||||
return results.map(toRecord);
|
return results.map(toRecord);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
log.warn({ err }, "Failed to load failed send log");
|
log.warn({ err }, "Failed to load failed send log");
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { log } from "../lib/log";
|
import { log } from "../lib/log";
|
||||||
import { getFailedSendLog, type SendRecord } from "../lib/send-log";
|
import { failedSendLogStatement, toRecord, type LogRow, type SendRecord } from "../lib/send-log";
|
||||||
|
|
||||||
export interface DeliveryMetrics {
|
export interface DeliveryMetrics {
|
||||||
total: number;
|
total: number;
|
||||||
@@ -64,65 +64,77 @@ export async function getDeliveryMetrics(
|
|||||||
groupId ? stmt.bind(groupId) : stmt;
|
groupId ? stmt.bind(groupId) : stmt;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const totals = await bindGroup(
|
// One round trip for all seven queries (each D1 call is a network hop), and
|
||||||
|
// a batch runs as one transaction, so the totals and breakdowns agree.
|
||||||
|
const results = await db.batch([
|
||||||
|
bindGroup(
|
||||||
db.prepare(`SELECT COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok FROM send_logs${where}`),
|
db.prepare(`SELECT COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok FROM send_logs${where}`),
|
||||||
).all<TotalsRow>();
|
),
|
||||||
const total = Number(totals.results[0]?.total) || 0;
|
bindGroup(
|
||||||
const ok = Number(totals.results[0]?.ok) || 0;
|
|
||||||
|
|
||||||
const byPlatform = await bindGroup(
|
|
||||||
db.prepare(
|
db.prepare(
|
||||||
`SELECT platform AS key, COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok
|
`SELECT platform AS key, COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok
|
||||||
FROM send_logs${where} GROUP BY platform`,
|
FROM send_logs${where} GROUP BY platform`,
|
||||||
),
|
),
|
||||||
).all<BreakdownRow>();
|
),
|
||||||
const byEvent = await bindGroup(
|
bindGroup(
|
||||||
db.prepare(
|
db.prepare(
|
||||||
`SELECT event AS key, COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok
|
`SELECT event AS key, COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok
|
||||||
FROM send_logs${where} GROUP BY event`,
|
FROM send_logs${where} GROUP BY event`,
|
||||||
),
|
),
|
||||||
).all<BreakdownRow>();
|
),
|
||||||
const byStatus = await bindGroup(
|
bindGroup(
|
||||||
db.prepare(
|
db.prepare(
|
||||||
`SELECT status, COUNT(*) AS count FROM send_logs
|
`SELECT status, COUNT(*) AS count FROM send_logs
|
||||||
WHERE status IS NOT NULL${and} GROUP BY status`,
|
WHERE status IS NOT NULL${and} GROUP BY status`,
|
||||||
),
|
),
|
||||||
).all<StatusRow>();
|
),
|
||||||
const duration = await bindGroup(
|
bindGroup(
|
||||||
db.prepare(
|
db.prepare(
|
||||||
`SELECT AVG(duration_ms) AS avg FROM send_logs WHERE duration_ms IS NOT NULL${and}`,
|
`SELECT AVG(duration_ms) AS avg FROM send_logs WHERE duration_ms IS NOT NULL${and}`,
|
||||||
),
|
),
|
||||||
).all<AvgRow>();
|
),
|
||||||
const attempts = await bindGroup(
|
bindGroup(
|
||||||
db.prepare(
|
db.prepare(
|
||||||
`SELECT COALESCE(SUM(attempts), 0) AS total, AVG(attempts) AS avg
|
`SELECT COALESCE(SUM(attempts), 0) AS total, AVG(attempts) AS avg
|
||||||
FROM send_logs WHERE attempts IS NOT NULL${and}`,
|
FROM send_logs WHERE attempts IS NOT NULL${and}`,
|
||||||
),
|
),
|
||||||
).all<AttemptsRow>();
|
),
|
||||||
const recentFailures = await getFailedSendLog(db, 20, groupId);
|
failedSendLogStatement(db, 20, groupId),
|
||||||
|
]);
|
||||||
|
const rows = <T>(i: number): T[] => (results[i]?.results ?? []) as T[];
|
||||||
|
const totals = rows<TotalsRow>(0);
|
||||||
|
const byPlatform = rows<BreakdownRow>(1);
|
||||||
|
const byEvent = rows<BreakdownRow>(2);
|
||||||
|
const byStatus = rows<StatusRow>(3);
|
||||||
|
const duration = rows<AvgRow>(4);
|
||||||
|
const attempts = rows<AttemptsRow>(5);
|
||||||
|
const recentFailures = rows<LogRow>(6).map(toRecord);
|
||||||
|
|
||||||
|
const total = Number(totals[0]?.total) || 0;
|
||||||
|
const ok = Number(totals[0]?.ok) || 0;
|
||||||
|
|
||||||
return {
|
return {
|
||||||
total,
|
total,
|
||||||
ok,
|
ok,
|
||||||
failed: total - ok,
|
failed: total - ok,
|
||||||
failureRate: total > 0 ? (total - ok) / total : 0,
|
failureRate: total > 0 ? (total - ok) / total : 0,
|
||||||
byPlatform: byPlatform.results.map((r) => {
|
byPlatform: byPlatform.map((r) => {
|
||||||
const t = Number(r.total) || 0;
|
const t = Number(r.total) || 0;
|
||||||
const o = Number(r.ok) || 0;
|
const o = Number(r.ok) || 0;
|
||||||
return { platform: r.key ?? "unknown", total: t, ok: o, failed: t - o };
|
return { platform: r.key ?? "unknown", total: t, ok: o, failed: t - o };
|
||||||
}),
|
}),
|
||||||
byEvent: byEvent.results.map((r) => {
|
byEvent: byEvent.map((r) => {
|
||||||
const t = Number(r.total) || 0;
|
const t = Number(r.total) || 0;
|
||||||
const o = Number(r.ok) || 0;
|
const o = Number(r.ok) || 0;
|
||||||
return { event: r.key ?? "unknown", total: t, ok: o, failed: t - o };
|
return { event: r.key ?? "unknown", total: t, ok: o, failed: t - o };
|
||||||
}),
|
}),
|
||||||
byStatus: byStatus.results.map((r) => ({
|
byStatus: byStatus.map((r) => ({
|
||||||
status: String(r.status),
|
status: String(r.status),
|
||||||
count: Number(r.count) || 0,
|
count: Number(r.count) || 0,
|
||||||
})),
|
})),
|
||||||
avgDurationMs: Number(duration.results[0]?.avg) || 0,
|
avgDurationMs: Number(duration[0]?.avg) || 0,
|
||||||
totalAttempts: Number(attempts.results[0]?.total) || 0,
|
totalAttempts: Number(attempts[0]?.total) || 0,
|
||||||
avgAttempts: Number(attempts.results[0]?.avg) || 0,
|
avgAttempts: Number(attempts[0]?.avg) || 0,
|
||||||
recentFailures,
|
recentFailures,
|
||||||
};
|
};
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|||||||
@@ -50,6 +50,7 @@ function createMetricsDB(rowsBySql: Record<string, unknown[]>): D1Database {
|
|||||||
}
|
}
|
||||||
return { results: [] };
|
return { results: [] };
|
||||||
};
|
};
|
||||||
|
let batches = 0;
|
||||||
return {
|
return {
|
||||||
prepare: (sql: string) => ({
|
prepare: (sql: string) => ({
|
||||||
all: all(sql),
|
all: all(sql),
|
||||||
@@ -58,6 +59,13 @@ function createMetricsDB(rowsBySql: Record<string, unknown[]>): D1Database {
|
|||||||
run: async () => ({ success: true }),
|
run: async () => ({ success: true }),
|
||||||
}),
|
}),
|
||||||
}),
|
}),
|
||||||
|
batch: async (statements: Array<{ all: () => Promise<unknown> }>) => {
|
||||||
|
batches++;
|
||||||
|
return Promise.all(statements.map((s) => s.all()));
|
||||||
|
},
|
||||||
|
get batches(): number {
|
||||||
|
return batches;
|
||||||
|
},
|
||||||
} as unknown as D1Database;
|
} as unknown as D1Database;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -102,6 +110,8 @@ describe("delivery metrics", () => {
|
|||||||
"COUNT(*) AS total": [{ total: 10, ok: 8 }],
|
"COUNT(*) AS total": [{ total: 10, ok: 8 }],
|
||||||
});
|
});
|
||||||
const m = await getDeliveryMetrics(db);
|
const m = await getDeliveryMetrics(db);
|
||||||
|
// Every query goes out in a single batch (one D1 round trip).
|
||||||
|
expect((db as unknown as { batches: number }).batches).toBe(1);
|
||||||
expect(m.total).toBe(10);
|
expect(m.total).toBe(10);
|
||||||
expect(m.ok).toBe(8);
|
expect(m.ok).toBe(8);
|
||||||
expect(m.failed).toBe(2);
|
expect(m.failed).toBe(2);
|
||||||
|
|||||||
Reference in new issue
Block a user