import type { H3Event } from "h3"; import { getHeader, readRawBody, setResponseStatus } from "h3"; import type { Env, Group } from "./types"; import { detectProvider } from "./providers"; import { dispatchEvent } from "./core/dispatch"; import { loadConfigSnapshot, loadGroupFresh, initConfigStore } from "./config"; import { ensureInstallationGroup } from "./web/groups"; import { getTenantSecret } from "./web/tenants"; import { NONCE_TTL_SECONDS, ReplayPendingError, replayNonceKey } from "./providers/custom"; import { recordAudit } from "./lib/audit"; import { cfEnv, cfWaitUntil, headersFrom } from "./cf"; import { log } from "./lib/log"; import { claimOnce, confirmClaim, deliveryKey, idempotencyStore } from "./lib/idempotency"; import { deliveryClaimStore } from "./storage/state-store"; import { newCorrelationId } from "./lib/correlation"; import { deliveryStateKey, enqueueWebhook, hadPermanentFailure, isRetryableFailure, markTargetsDone, targetDoneChecker, type DeliveryMessage, } from "./queue/delivery"; const MAX_BODY_SIZE = 1024 * 1024; /** How long a delivery's dedup claim lasts when it is queued. */ const QUEUED_CLAIM_SECONDS = 120; /** How long it lasts for an inline dispatch, which may run for minutes. */ const INLINE_CLAIM_SECONDS = 600; /** An installation's auto-provisioning is remembered as long as delivery state (7 days). */ const PROVISIONED_TTL_SECONDS = 7 * 86400; /** Sending time of one inline dispatch: within its claim, with a margin. */ const INLINE_SEND_BUDGET_MS = 8 * 60_000; /** Result of an inline dispatch: whether the sender should retry. */ interface InlineOutcome { retry: boolean; /** A target failed for good (in this or an earlier attempt). */ failed: boolean; } export interface WebhookResult { status: 200 | 400 | 401 | 404 | 413 | 503; body: unknown; } /** * Core webhook processing. Without `tenantId` this is the legacy global * endpoint (`POST /webhook`): events verify against the operator's global * secrets and may dispatch into every route. With a `tenantId` (a group id, * `POST /webhook/{groupId}`) the group's own secret is used for verification * (GITHUB_WEBHOOK_SECRET/GITEA_WEBHOOK_SECRET are overridden per request) and * only that group's routes are eligible. * * A tenant-signed payload is only as trustworthy as the tenant: its claimed * GitHub App installation is dropped, so it can neither borrow the operator's * App credentials nor provision or rebind installation groups. */ export async function processWebhook( env: Env, body: string, headers: Record, waitUntil: (promise: Promise) => void, tenantId?: string, ): Promise { const requestId = newCorrelationId(); let effectiveEnv = env; initConfigStore(env); // Until the request is verified, nothing but the receiving group's own row // is read (fresh): an unsigned flood must not read the whole configuration // each time. The full snapshot is read once the event is accepted. const provider = detectProvider(headers); if (!provider) { return { status: 400, body: { error: "Unknown webhook provider" } }; } let tenant: Group | undefined; if (tenantId) { const found = await loadGroupFresh(env, tenantId); if (!found.group) { // Unreadable is not missing: the sender is asked to retry. if (found.unreadable) { return { status: 503, body: { error: "Group configuration unreadable; retry later" } }; } return { status: 404, body: { error: "Group not found" } }; } tenant = found.group; const secret = await getTenantSecret(env.KV, tenant); if (!secret) { return { status: 404, body: { error: "Webhook disabled for this group" } }; } effectiveEnv = { ...env, GITHUB_WEBHOOK_SECRET: secret, GITEA_WEBHOOK_SECRET: secret }; } let verified = false; try { verified = await provider.verify(body, headers, effectiveEnv); } catch (err) { if (err instanceof ReplayPendingError) { // The nonce is held by an attempt not (yet) accepted: not a replay. return { status: 503, body: { error: "Request is already being processed; retry later" } }; } // A malformed secret or an unavailable crypto implementation must fail as // a clean 401, never an uncaught 500. log.warn( { provider: provider.id, requestId, err: String(err) }, "Webhook signature verification failed", ); return { status: 401, body: { error: "Invalid signature" } }; } if (!verified) { // Log the actual cause: a missing provider secret is a deployment problem, // while a mismatched signature usually means the sender used the wrong secret. const secret = provider.id === "gitea" ? effectiveEnv.GITEA_WEBHOOK_SECRET : effectiveEnv.GITHUB_WEBHOOK_SECRET; if (!secret) { log.warn( { provider: provider.id, requestId }, "Webhook rejected: provider secret is not configured", ); } else { log.warn({ provider: provider.id, requestId }, "Webhook rejected: invalid signature"); } return { status: 401, body: { error: "Invalid signature" } }; } // From here on, one-time claims are held (the custom-webhook nonce taken // by `verify`, the delivery dedup key). Only once the event is accepted // (queued, or dispatched without a transient failure) are they confirmed; // otherwise they are given back, so the sender's retry is not refused. A // claim that could not be given back is never mistaken for an accepted // event: a retry finds it unconfirmed and is asked to come back later. const claims = deliveryClaimStore(env); const held: { key: string; ttl: number }[] = []; const nonce = headers["x-webhooker-nonce"]; if (provider.id === "custom" && nonce && headers["x-webhooker-timestamp"]) { held.push({ key: await replayNonceKey(effectiveEnv.GITHUB_WEBHOOK_SECRET, nonce), ttl: NONCE_TTL_SECONDS, }); } const releaseClaims = async (): Promise => { for (const { key } of held.splice(0)) { await claims.release(key).catch((err: unknown) => { log.warn({ requestId, key, err }, "Failed to release a webhook claim"); }); } }; const confirmClaims = async (): Promise => { for (const { key, ttl } of held.splice(0)) { // Unconfirmed, a claim only delays a duplicate until it expires. await confirmClaim(claims, key, ttl).catch((err: unknown) => { log.warn({ requestId, key, err }, "Failed to confirm a webhook claim"); }); } }; const retryLater = (error: string): WebhookResult => ({ status: 503, body: { error, requestId }, }); try { const event = provider.parse(body, headers); if (!event) { await releaseClaims(); return { status: 400, body: { error: "Invalid event" } }; } if (tenantId) event.installationId = undefined; // Dedup via the idempotency store: a provider/tenant-scoped key means // different accounts may reuse a delivery id without colliding, while // retries of the same delivery never dispatch twice. An inline dispatch // holds its claim for as long as it may run. // A custom sender's ids are namespaced (`d:`) as its delivery identity is // below: no id it picks can take the key of a nonce-identified request. const dedupKey = event.deliveryId ? deliveryKey( provider.id, tenantId, provider.id === "custom" ? `d:${event.deliveryId}` : event.deliveryId, tenant?.instance, ) : null; if (dedupKey) { const ttl = env.QUEUE ? QUEUED_CLAIM_SECONDS : INLINE_CLAIM_SECONDS; const state = await claimOnce(claims, dedupKey, ttl); if (state === "done") { // Already accepted: this request's own nonce is used up as well. await confirmClaims(); return { status: 200, body: { ok: true, duplicate: true, requestId } }; } if (state === "pending") { // Still in flight elsewhere, or left over from an attempt that was // not accepted: the event may not be on its way yet. await releaseClaims(); return retryLater("Delivery is already being processed; retry later"); } held.push({ key: dedupKey, ttl }); } // Provisioning runs only for an event not yet accepted (a duplicate // returned above): a redelivery must not redo a binding an admin undid. // Auto-provision GitHub App installations so tenant isolation is // configured without manual id entry: a group is created (or existing // matching groups are bound) before the event is dispatched. Only events // signed with the operator's own secret (the global endpoint) may do this. if ( !tenantId && provider.id === "github" && event.event === "installation" && event.payload.action === "created" && event.installationId != null ) { const install = event.payload.installation as { account?: { login?: string } } | undefined; const account = install?.account?.login ?? ""; // Done once per installation, on record as long as delivery state is // kept: a redelivery after the dedup claim lapsed (minutes) must not // redo a binding an admin undid since. Recorded only once it succeeded, // so a provisioning that failed midway is completed by the retry. Kept // in D1 with the bindings it guards, whichever store deliveries use. const doneKey = `installation-provisioned:${event.installationId}`; const records = idempotencyStore(env.DB, env.KV); try { if (!(await records.has(doneKey))) { const group = await ensureInstallationGroup( env.KV, event.installationId, account, env.DB, ); await records.claim(doneKey, PROVISIONED_TTL_SECONDS); if (group) { await recordAudit(env.DB, { ts: Date.now(), actorLogin: account || undefined, action: "installation.created", targetType: "group", targetId: group.id, groupId: group.id, groupInstance: group.instance, }); } } } catch (err) { // Not accepted: the consumer never provisions, so a sender retry is // the only way the group gets created (provisioning is idempotent). log.warn( { err, installationId: event.installationId }, "Failed to auto-provision installation group", ); await releaseClaims(); return retryLater("Installation provisioning failed; retry later"); } } // A stable identity across the sender's retries: the delivery id, else the // replay-protection nonce of a signed request. Queued or inline, it keys // the delivery's state and target ledger, so a retry of a request whose // first attempt went through after all is recognized, not sent again. // A custom sender picks its delivery ids freely: they get a namespace of // their own, so none can coincide with a nonce-derived identity. const nonceKey = held.find((h) => h.key.startsWith("nonce:"))?.key; const identity = event.deliveryId != null ? provider.id === "custom" ? `d:${event.deliveryId}` : event.deliveryId : nonceKey ? `n:${nonceKey}` : undefined; // Inline, a request identified by its nonce alone is held for the whole // dispatch, as a delivery id is above: its nonce claim is short (shorter // than a long dispatch), and a copy arriving meanwhile must not send the // targets not yet on the ledger a second time. if (!env.QUEUE && event.deliveryId == null && identity) { const key = deliveryKey(provider.id, tenantId, identity, tenant?.instance); const state = await claimOnce(claims, key, INLINE_CLAIM_SECONDS); if (state === "done") { await confirmClaims(); return { status: 200, body: { ok: true, duplicate: true, requestId } }; } if (state === "pending") { await releaseClaims(); return retryLater("Delivery is already being processed; retry later"); } held.push({ key, ttl: INLINE_CLAIM_SECONDS }); } if (env.QUEUE) { const message: DeliveryMessage = { deliveryId: identity ?? requestId, groupId: tenantId, groupInstance: tenant?.instance, provider: provider.id, event: event.event, payload: event.payload, installationId: event.installationId, receivedAt: Date.now(), requestId, stateVersion: 2, }; // Acknowledge only once the event is durably queued. await enqueueWebhook(env, message); await confirmClaims(); return { status: 200, body: { ok: true, requestId } }; } // Inline dispatch (no queue): there is no durable retry, so the sender // is answered only after the dispatch, and asked to retry while a target // failed transiently. A delivery id gets a target ledger, so targets // already sent are skipped when the sender redelivers. // One consistent snapshot of routes and groups serves the dispatch — read // fresh: a route or target removed moments ago in another isolate must // not still be dispatched to from this isolate's cache. const snapshot = await loadConfigSnapshot(env, { fresh: true }); if (tenantId) { const current = snapshot.groups.find((g) => g.id === tenantId); if (!current || current.instance !== tenant?.instance) { // Gone or replaced since its row was read: not this group's event now. await releaseClaims(); if (snapshot.config.unreadableGroups?.includes(tenantId)) { return retryLater("Group configuration unreadable; retry later"); } return { status: 404, body: { error: "Group not found" } }; } } const scoped = tenantId ? { ...snapshot.config, routes: snapshot.config.routes.filter((r) => r.groupId === tenantId), } : snapshot.config; const ledgerKey = identity ? deliveryStateKey(provider.id, tenantId, identity, tenant?.instance) : null; const run = (async (): Promise => { const recorded = new Set(); // Dispatched under the delivery's identity (as a queued copy is), so the // platform idempotency keys stay the same when the sender retries. const delivered = identity ? { ...event, deliveryId: identity } : event; // Sending stops well before the claim held for this dispatch expires // (a target takes under a minute at worst): a copy arriving after that // could otherwise send the targets still in flight. What is left is a // transient failure, so the sender retries it (the ledger skips the rest). const sendUntil = Date.now() + INLINE_SEND_BUDGET_MS; const summary = await dispatchEvent(scoped, delivered, env, snapshot.groups, { tenantId, beforeTarget: async (): Promise => Date.now() < sendUntil, skipTarget: ledgerKey ? targetDoneChecker(env, ledgerKey) : undefined, afterTarget: ledgerKey ? async (targetKey): Promise => { await markTargetsDone(env, ledgerKey, [targetKey]); recorded.add(targetKey); } : undefined, }); const permanent = summary.failures.filter((f) => !isRetryableFailure(f)); if (ledgerKey) { await markTargetsDone( env, ledgerKey, [...(summary.succeeded ?? []), ...permanent.flatMap((f) => f.key ?? [])].filter( (k) => !recorded.has(k), ), { permanentFailure: permanent.length > 0 }, ); } // A transient failure is always retried: without a ledger the retry // sends the other targets again too — duplicates rather than a loss. return { retry: summary.failures.some(isRetryableFailure), // Targets that failed for good in an earlier attempt still count. failed: permanent.length > 0 || (ledgerKey != null && (await hadPermanentFailure(env, ledgerKey))), }; })(); // The dispatch keeps running should the sender hang up meanwhile. waitUntil(run.catch(() => undefined)); let outcome: InlineOutcome; try { outcome = await run; } catch (err) { log.error({ requestId, err }, "Dispatch failed"); await releaseClaims(); return retryLater("Dispatch failed; retry later"); } if (outcome.retry) { log.warn({ requestId }, "Inline dispatch failed transiently"); await releaseClaims(); return retryLater("Some targets failed temporarily; retry later"); } await confirmClaims(); // Accepted for good; a target that can never take the event is reported. return outcome.failed ? { status: 200, body: { ok: true, requestId, failed: true } } : { status: 200, body: { ok: true, requestId } }; } catch (err) { log.error({ requestId, err }, "Failed to accept webhook"); await releaseClaims(); return retryLater("Temporarily unable to accept webhook"); } } /** h3 wrapper for `POST /webhook` / `POST /webhook/:groupId`. */ export async function handleWebhookRequest(event: H3Event, tenantId?: string): Promise { const contentLength = Number(getHeader(event, "content-length") ?? 0); if (contentLength > MAX_BODY_SIZE) { setResponseStatus(event, 413); return { error: "Request too large" }; } const body = (await readRawBody(event, "utf8")) ?? ""; if (body.length > MAX_BODY_SIZE) { setResponseStatus(event, 413); return { error: "Request too large" }; } const result = await processWebhook( cfEnv(event), body, headersFrom(event), cfWaitUntil(event), tenantId, ); setResponseStatus(event, result.status); return result.body; }