diff --git a/server/lib/core/dispatch.ts b/server/lib/core/dispatch.ts index 4d38bdc..a77f94d 100644 --- a/server/lib/core/dispatch.ts +++ b/server/lib/core/dispatch.ts @@ -1,27 +1,64 @@ -import type { Config, WebhookEvent, Env, Route, Group, NeutralMessage } from "../types"; +import type { + Config, + WebhookEvent, + Env, + Route, + RouteTarget, + Group, + NeutralMessage, +} from "../types"; import { formatEvent } from "../formatters"; import { emojiPrefix, forgeInfo } from "../formatters/helpers"; -import { matchRoute, eventOwners } from "../events/match"; +import { matchRoute as matchRouteUnsafe, eventOwners } from "../events/match"; import { log } from "../lib/log"; import { loadTranslations, t as translate, type Translations } from "../lib/i18n"; import type { SendRecord } from "../lib/send-log"; import { recordSendBatch } from "../lib/send-log-batch"; import { explainRoute } from "../config/schema"; -import { messageTracker } from "../lib/message-tracker"; +import { chainLock, messageTracker, type MessageTracker } from "../lib/message-tracker"; +import { idempotencyStore, instanceScope, tenantScope } from "../lib/idempotency"; import { loadGroups, groupAcceptsOwners, groupAcceptsProvider, groupAcceptsInstallation, + groupAcceptsGlobalIngress, } from "../web/groups"; -import { getDriver } from "../drivers"; -import type { SendResult } from "../drivers/types"; -import type { DispatchFailure, DispatchSummary } from "../queue/delivery"; +import { destinationOf, getDriver } from "../drivers"; +import type { PlatformDriver, SendResult } from "../drivers/types"; +import { isRetryableFailure, type DispatchFailure, type DispatchSummary } from "../queue/delivery"; import { getCheckSuiteBuildLogUrl } from "../github/check-run"; import { getWorkflowRunJobs } from "../github/workflow-jobs"; +export interface DispatchOptions { + /** + * The group whose own webhook ingress (`/webhook/{groupId}`) received the + * event; undefined for the operator's global endpoint. Tenant events only + * reach that group, and never use the operator's GitHub App credentials. + */ + tenantId?: string; + /** + * True for a route × target an earlier attempt of the same delivery already + * finished (sent, or failed for good); it is not attempted again. + */ + skipTarget?: (targetKey: string) => Promise; + /** + * Asked before each send; false means this dispatch lost the right to send + * (its delivery lease was taken over): the target is reported as a + * transient failure and left to the new holder. + */ + beforeTarget?: () => Promise; + /** + * Told right after a target was sent, so the caller can record it at once + * (should this invocation die mid-dispatch, it is not sent again). Errors + * are logged and otherwise ignored: the summary still lists the target. + */ + afterTarget?: (targetKey: string) => Promise; +} + /** One dispatch attempt (route × target), collected for the group webhook log. */ interface DispatchAttempt { + key: string; groupId?: string; routeId: string; routeName: string; @@ -30,6 +67,321 @@ interface DispatchAttempt { error?: string; errorCode?: string; status?: number; + /** Finished by an earlier attempt of the delivery (skipped now). */ + earlier?: boolean; +} + +/** + * Match one route, isolating failures: a filter that throws on an unusual + * payload counts as "no match" for that route only, never stopping the + * dispatch of every other route and group. + */ +function matchRoute(route: Route, event: WebhookEvent): boolean { + try { + return matchRouteUnsafe(route, event); + } catch (err) { + log.warn({ routeId: route.id, groupId: route.groupId, err }, "Route filter failed to evaluate"); + return false; + } +} + +/** Human-readable target, also used in send logs. */ +function describeTarget(target: RouteTarget): string { + if (target.platform === "telegram") { + return target.topicId ? `${target.chatId}/${target.topicId}` : (target.chatId ?? ""); + } + if (target.platform === "feishu") return target.chatId ?? ""; + return target.threadId ? `${target.channelId}/${target.threadId}` : (target.channelId ?? ""); +} + +/** + * Identity of a route × target within a delivery. It names the group by its + * immutable `instance` (falling back to the id for groups without one), so a + * retry after any number of renames still recognizes targets already done. + */ +async function sha256Hex(text: string): Promise { + const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text)); + return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, "0")).join(""); +} + +function targetKeyOf(group: Group | undefined, route: Route, target: RouteTarget): string { + const owner = group?.instance ? instanceScope(group.instance) : (route.groupId ?? ""); + return `${owner}:${route.id}:${target.platform ?? "discord"}:${describeTarget(target)}`; +} + +/** + * The forge instance an event came from (provider + repository host), so two + * sources with the same `owner/repo` never share an update chain. + */ +function eventOrigin(event: WebhookEvent): string { + const html = (event.payload.repository as { html_url?: string } | undefined)?.html_url; + let host = ""; + if (html) { + try { + host = new URL(html).host.toLowerCase(); + } catch { + host = ""; + } + } + return `${event.provider ?? "github"}@${host || "-"}`; +} + +const VERSION_SOURCES = [ + "workflow_run", + "check_run", + "check_suite", + "deployment_status", + "review", + "comment", + "deployment", +] as const; + +/** Lifecycle stage of a tracked object: later stages win a tie in event time. */ +function stageOf(status: unknown): number { + // `status` of runs and checks, `state` of deployment statuses. + if ( + status === "completed" || + status === "success" || + status === "failure" || + status === "error" || + status === "inactive" + ) { + return 2; + } + if (status === "in_progress") return 1; + return 0; +} + +/** + * Version of the object an update chain tracks — its event time (ms), with + * its stage (queued < in progress < completed < deleted/dismissed) breaking a + * tie — so a stale event delivered late never overwrites a newer state, not + * even one stamped with the same time. Without a time it is the stage alone + * (below 4, see `outdated`); 0 when nothing is known. + */ +function updateVersion(event: WebhookEvent): number { + for (const name of VERSION_SOURCES) { + const obj = event.payload[name] as Record | undefined; + if (!obj || typeof obj !== "object") continue; + // A deletion or dismissal ends the object (its time is that of the last + // change): nothing stamped the same time may bring it back. + const action = event.payload.action; + const stage = + action === "deleted" || action === "dismissed" ? 3 : stageOf(obj.status ?? obj.state); + for (const field of [ + "updated_at", + "completed_at", + "submitted_at", + "started_at", + "created_at", + ]) { + const value = obj[field]; + if (typeof value === "string") { + const ms = Date.parse(value); + if (Number.isFinite(ms) && ms > 0) return ms * 4 + stage; + } + } + return stage; + } + return 0; +} + +/** + * Whether an event at `version` is older than what the chain shows. Without + * an event time (a version below 4), only the stage decides: an untimed + * queued event never undoes a completion. + */ +function outdated(stored: number, version: number): boolean { + if (version >= 4) return stored > version; + return stored % 4 > version; +} + +/** + * The version a chain keeps once an event at `version` was shown over + * `stored`: never lower. An untimed event keeps the stored time but moves + * the stage on (a later untimed queued event must find the completion). + */ +function shownVersion(stored: number, version: number): number { + if (version >= 4) return Math.max(stored, version); + return stored - (stored % 4) + Math.max(stored % 4, version); +} + +function notModified(result: SendResult): boolean { + return /not modified/i.test(result.error ?? ""); +} + +/** The tracked message no longer exists (deleted / not editable): send a new one. */ +function messageGone(result: SendResult): boolean { + if (result.status === 404) return true; + // Sent by another bot (the deployment switched Discord applications): it + // can never be edited again — a new message continues the chain. + if ( + /cannot edit a message authored by another user|"code"\s*:\s*50005\b/i.test(result.error ?? "") + ) { + return true; + } + return /unknown message|message to edit not found|message can't be edited|no text in the message to edit|message not found|has been (deleted|recalled)/i.test( + result.error ?? "", + ); +} + +const LOCK_ATTEMPTS = 5; +/** + * Lifetime of an update-chain lock. It must outlast the longest edit + send + * a holder can make (every platform call is bounded: 3 attempts × 15 s, plus + * rate-limit waits of ≤ 10 s, plus a Feishu token fetch — under 90 s each), + * so a slow holder can never land its edit after a newer holder's. + */ +const LOCK_TTL_SECONDS = 240; + +/** + * Send or edit the message of an update chain (workflow progress, edited + * comments, ...). A best-effort KV lock serializes concurrent events of the + * same chain; an existing message is edited unless the incoming event is + * older than what it shows; a new message is sent only when there is none or + * the old one is gone — never after a transient edit failure, which would + * leave two messages for one chain. + */ +async function deliverUpdate( + env: Env, + tracker: MessageTracker, + driver: PlatformDriver, + message: NeutralMessage, + target: RouteTarget, + chain: { id: string; target: string; legacyId: string; legacyTarget: string }, + version: number, +): Promise { + const chainId = chain.id; + const trackTarget = chain.target; + const lockKey = `msg:lock:${chainId}:${trackTarget}`; + const lock = chainLock(env.DB, env.KV); + let token: string | null = null; + for (let attempt = 0; attempt < LOCK_ATTEMPTS && !token; attempt++) { + token = await lock.acquire(lockKey, LOCK_TTL_SECONDS); + if (!token) await new Promise((r) => setTimeout(r, 100 * (attempt + 1))); + } + /** + * Bookkeeping after the platform call succeeded. A failure here must not + * turn the delivered message into a failed (and so resent) target: it is + * logged, and at worst the chain's next update posts a new message. + */ + const track = async (write: () => Promise): Promise => { + for (let attempt = 1; ; attempt++) { + try { + await write(); + return true; + } catch (err) { + if (attempt < 3) continue; + log.warn({ chain: chainId, target: trackTarget, err }, "Failed to track an update message"); + return false; + } + } + }; + if (!token) { + // Another event of this chain is still being delivered: going ahead could + // let an older state finish last. Report a transient failure instead (the + // queue retries it; the version check then skips it if it is outdated). + return { + ok: false, + error: "Another update of this message is in progress", + errorCode: "RATE_LIMITED", + attempts: 0, + }; + } + try { + let existing = await tracker.get(chainId, trackTarget); + // Read first: only an old entry that is there is taken under its lock (a + // new chain with nothing to adopt costs one read, not a lock taken and + // released). + if (!existing && chain.legacyId && (await tracker.get(chain.legacyId, chain.legacyTarget))) { + // Chains tracked before keys carried the forge origin and platform. + // Only adopted for github.com, the one forge where `owner/repo` alone is + // unambiguous; elsewhere the old message could belong to another host. + // Several new chains may map to one old key (e.g. check runs of two + // suites): taken under the old key's lock, so exactly one adopts it. + const legacyLockKey = `msg:lock:legacy:${chain.legacyId}:${chain.legacyTarget}`; + let legacyToken: string | null = null; + for (let attempt = 0; attempt < LOCK_ATTEMPTS && !legacyToken; attempt++) { + legacyToken = await lock.acquire(legacyLockKey, LOCK_TTL_SECONDS); + if (!legacyToken) await new Promise((r) => setTimeout(r, 100 * (attempt + 1))); + } + if (!legacyToken) { + return { + ok: false, + error: "Another update of this message is in progress", + errorCode: "RATE_LIMITED", + attempts: 0, + }; + } + try { + const legacy = await tracker.get(chain.legacyId, chain.legacyTarget); + if (legacy) { + // One transaction: the old entry never survives its adoption (a + // second chain would take the same message too). + await tracker.move( + { eventId: chain.legacyId, targetId: chain.legacyTarget }, + { eventId: chainId, targetId: trackTarget }, + legacy, + ); + existing = legacy; + } + } finally { + await lock.release(legacyLockKey, legacyToken).catch(() => undefined); + } + } + if (existing) { + if (outdated(existing.version, version)) { + // A newer event already updated this message. + return { ok: true, messageId: existing.messageId, attempts: 0 }; + } + const result = await driver.edit(message, target, env, existing.messageId); + if (result.ok || notModified(result)) { + // Never lower the stored version (another holder may have moved on). + const tracked = await track(async () => { + const latest = await tracker.get(chainId, trackTarget); + await tracker.set(chainId, trackTarget, { + messageId: existing.messageId, + version: shownVersion(Math.max(existing.version, latest?.version ?? 0), version), + }); + }); + // Edited, but the version shown is not on record: a late older event + // would edit it back. Reported as transient — the retry repeats the + // (harmless) edit and records the version. + if (!tracked) { + return { + ok: false, + error: "Edited, but its version could not be recorded", + messageId: existing.messageId, + attempts: result.attempts, + }; + } + return { ...result, ok: true, messageId: existing.messageId }; + } + if (!messageGone(result)) return result; + await tracker.delete(chainId, trackTarget); + } + const result = await driver.send(message, target, env); + if (result.ok && result.messageId) { + const messageId = result.messageId; + const tracked = await track(() => tracker.set(chainId, trackTarget, { messageId, version })); + // Untracked, the chain's next update would post a second message. Where + // the platform dedupes sends by the message's key (Discord nonce, Feishu + // uuid), the retry gets this very message back and records it; Telegram + // cannot, so there the delivery stands (one extra message at worst). + if (!tracked && message.sendKey && driver.id !== "telegram") { + return { + ok: false, + error: "Sent, but its message id could not be recorded", + messageId, + attempts: result.attempts, + }; + } + } + return result; + } finally { + // Release only our own lock (it may have expired and been taken over). + await lock.release(lockKey, token).catch(() => undefined); + } } export async function dispatchEvent( @@ -37,64 +389,112 @@ export async function dispatchEvent( event: WebhookEvent, env: Env, groups?: Group[], + options: DispatchOptions = {}, ): Promise { + const { tenantId, skipTarget, beforeTarget, afterTarget } = options; const loadedGroups = groups ?? (await loadGroups(env.KV)); const groupById = new Map(loadedGroups.map((g) => [g.id, g])); const tracker = messageTracker(env.DB, env.KV); - // Message language is configured per group (Group.lang), not per route. - const langs = [...new Set(loadedGroups.map((g) => g.lang ?? "en"))]; - const trMap = new Map(); - await Promise.all( - langs.map(async (lang) => { - trMap.set(lang, await loadTranslations(lang, env.KV)); - }), - ); + // Message language is configured per group (Group.lang), not per route; + // loaded when a message is built in it (a group nothing is sent to costs + // no KV read), once per language. + const trMap = new Map>(); + const trFor = (lang: string): Promise => { + let tr = trMap.get(lang); + if (!tr) { + tr = loadTranslations(lang, env.KV); + trMap.set(lang, tr); + } + return tr; + }; const owners = eventOwners(event); const accepted = (route: Route): boolean => { + if (tenantId !== undefined) { + // Per-group ingress: authenticated by that group's own secret, so only + // its routes are eligible; the installation claim was already dropped. + if (route.groupId !== tenantId) return false; + const group = groupById.get(tenantId); + if (!group) return false; + return groupAcceptsOwners(group, owners) && groupAcceptsProvider(group, event.provider); + } if (!route.groupId) return true; + // A route whose group is gone (deleted, or not loaded) has no source + // restrictions to check, so it must not fire at all. const group = groupById.get(route.groupId); - if (!group) return true; + if (!group) return false; + if (!groupAcceptsGlobalIngress(group)) return false; if (!groupAcceptsInstallation(group, event.installationId)) return false; if (!groupAcceptsOwners(group, owners)) return false; return groupAcceptsProvider(group, event.provider); }; // `stop` and `fallback` act within a route's own group (its routing table), so - // one group's routes can never silence another group's. + // one group's routes can never silence another group's. Source checks run + // before matching: another tenant's filters never see this event. const scopeOf = (route: Route): string => route.groupId ?? ""; - const regularMatched = new Set( - config.routes - .filter((route) => !route.fallback && matchRoute(route, event) && accepted(route)) - .map(scopeOf), + // A group with an unreadable route is left out whole: without its full + // routing table, neither its fallback nor its `stop` can be judged (it is + // reported as a transient failure below). + const unreadable = new Set(config.unreadableGroups ?? []); + const eligible = config.routes.filter((r) => !unreadable.has(r.groupId ?? "") && accepted(r)); + // Each regular route is matched once (filters may run regexes and JSONPath + // over the payload): the result decides both the group's fallbacks and the + // dispatch below. + const matchedRegular = new Set( + eligible.filter((route) => !route.fallback && matchRoute(route, event)), ); + const regularMatched = new Set([...matchedRegular].map(scopeOf)); const stopped = new Set(); const attempts: DispatchAttempt[] = []; + // Targets an earlier attempt finished (skipped now): only for the group + // summaries, which describe the whole delivery. + const earlier: DispatchAttempt[] = []; const sendLogs: SendRecord[] = []; + // The operator's GitHub App may only enrich events the operator's own + // webhook secret vouched for. + const appId = tenantId === undefined ? env.GITHUB_APP_ID : undefined; + const privateKey = tenantId === undefined ? env.GITHUB_PRIVATE_KEY : undefined; + let workflowJobsPromise: Promise | undefined; function ensureWorkflowJobs(): Promise { if (event.event !== "workflow_run") return Promise.resolve(); workflowJobsPromise ??= (async (): Promise => { const run = event.payload.workflow_run as { jobs_url?: string; jobs?: unknown[] } | undefined; if (!run?.jobs_url || run.jobs?.length) return; - const jobs = await getWorkflowRunJobs( - run.jobs_url, - env.GITHUB_APP_ID, - env.GITHUB_PRIVATE_KEY, - event.installationId, - ); + const jobs = await getWorkflowRunJobs(run.jobs_url, appId, privateKey, event.installationId); if (jobs) run.jobs = jobs; })(); return workflowJobsPromise; } + // One lookup per dispatch, shared by every route (each would otherwise ask + // for its own App token and check list — calls that count against the + // invocation's limit before any message is sent). + let buildLogPromise: Promise | undefined; + function checkSuiteBuildLog(): Promise { + buildLogPromise ??= (async (): Promise => { + const suite = event.payload.check_suite as + { conclusion?: string; check_runs_url?: string } | undefined; + if (event.event !== "check_suite" || suite?.conclusion !== "failure") return undefined; + if (!suite.check_runs_url) return undefined; + return getCheckSuiteBuildLogUrl( + suite.check_runs_url, + appId, + privateKey, + event.installationId, + ); + })().catch(() => undefined); + return buildLogPromise; + } + const tasks: Promise[] = []; - for (const route of config.routes) { + for (const route of eligible) { const scope = scopeOf(route); - if (stopped.has(scope) || !accepted(route)) continue; + if (stopped.has(scope)) continue; if (route.fallback) { if (!regularMatched.has(scope) && matchRoute(route, event)) { tasks.push(processRoute(route)); @@ -102,25 +502,81 @@ export async function dispatchEvent( } continue; } - if (matchRoute(route, event)) { + if (matchedRegular.has(route)) { tasks.push(processRoute(route)); if (route.stop) stopped.add(scope); } } - await Promise.allSettled(tasks); + // A group with an unreadable route may have had a match we cannot see: + // report it as a transient failure (retried, and visible in the send log) + // rather than let the event pass for delivered. + for (const gid of config.unreadableGroups ?? []) { + // A group whose own row is unreadable has no source restrictions to + // check: it fails for every event it could have received. + if ( + groupById.has(gid) + ? !accepted({ groupId: gid } as Route) + : tenantId !== undefined && tenantId !== gid + ) { + continue; + } + const error = + "The configuration of this group could not be read; repair it to deliver the event"; + log.error({ groupId: gid, event: event.event }, error); + attempts.push({ + key: `unreadable:${gid}`, + groupId: gid, + routeId: "", + routeName: "", + target: "", + ok: false, + error, + }); + sendLogs.push({ + ts: Date.now(), + routeId: "", + groupId: gid, + groupInstance: groupById.get(gid)?.instance, + event: event.event, + target: "", + ok: false, + error, + deliveryId: event.deliveryId, + }); + } + const settled = await Promise.allSettled(tasks); + for (const r of settled) { + if (r.status !== "rejected") continue; + // Never let a crash pass for success: record it as a transient failure + // (no error code), so a queued delivery is retried instead of acked. + log.error({ err: r.reason }, "Route dispatch crashed"); + attempts.push({ + key: "crash", + routeId: "", + routeName: "", + target: "", + ok: false, + error: r.reason instanceof Error ? r.reason.message : String(r.reason), + }); + } await recordSendBatch(env.DB, sendLogs); - await sendGroupLogs(attempts); + await sendGroupLogs([...attempts, ...earlier]); const failures: DispatchFailure[] = attempts .filter((a) => !a.ok) .map((a) => ({ + key: a.key, target: a.target, error: a.error, errorCode: a.errorCode, status: a.status, })); - return { attempts: attempts.length, failures }; + return { + attempts: attempts.length, + failures, + succeeded: attempts.filter((a) => a.ok).map((a) => a.key), + }; async function sendGroupLogs(list: DispatchAttempt[]): Promise { const byGroup = new Map(); @@ -130,26 +586,48 @@ export async function dispatchEvent( if (bucket) bucket.push(a); else byGroup.set(a.groupId, [a]); } - for (const [groupId, entries] of byGroup) { + for (const [groupId, unordered] of byGroup) { const group = groupById.get(groupId); const target = group?.logTarget; if (!group || !target) continue; - const tr = trMap.get(group.lang ?? "en")!; + // Nothing was attempted for this group now (a redelivery of a finished + // delivery): its summary was posted when it finished. + if (unordered.every((a) => a.earlier)) continue; + let claimed: string | null = null; try { - const allOk = entries.every((a) => a.ok); + const tr = await trFor(group.lang ?? "en"); + // In a stable order (targets finish in any order), so the same outcome + // always reads — and deduplicates — the same. + const entries = [...unordered].sort((a, b) => { + const x = `${a.routeId}|${a.key}`; + const y = `${b.routeId}|${b.key}`; + return x < y ? -1 : x > y ? 1 : 0; + }); + // Green only when every target is known to have succeeded: one an + // earlier attempt finished may have failed for good. + const allOk = entries.every((a) => a.ok && !a.earlier); + const anyFailed = entries.some((a) => !a.ok); const routeLines = entries .slice(0, 10) .map((a) => - a.ok - ? emojiPrefix("✅", true) + - translate("log.route_ok", { route: a.routeName, target: a.target }, undefined, tr) - : emojiPrefix("❌", true) + + a.earlier + ? emojiPrefix("⏭️", true) + translate( - "log.route_fail", - { route: a.routeName, target: a.target, error: a.error ?? "?" }, + "log.route_earlier", + { route: a.routeName, target: a.target }, undefined, tr, - ), + ) + : a.ok + ? emojiPrefix("✅", true) + + translate("log.route_ok", { route: a.routeName, target: a.target }, undefined, tr) + : emojiPrefix("❌", true) + + translate( + "log.route_fail", + { route: a.routeName, target: a.target, error: a.error ?? "?" }, + undefined, + tr, + ), ); if (entries.length > 10) routeLines.push(`… +${entries.length - 10}`); const message: NeutralMessage = { @@ -164,7 +642,7 @@ export async function dispatchEvent( undefined, tr, ), - color: allOk ? 0x3fb950 : 0xf85149, + color: allOk ? 0x3fb950 : anyFailed ? 0xf85149 : 0x8b949e, fields: [ { name: translate("log.routes", {}, undefined, tr), @@ -179,48 +657,117 @@ export async function dispatchEvent( ], timestamp: new Date().toISOString(), }; + // A retry of the same delivery that reaches the same outcome posts no + // second summary (a changed outcome — say, a recovered target — does). + // Keyed by where each route×target stands — finished (sent, failed + // for good, or by an earlier attempt) or still to retry — so a retry + // that skips the finished ones and leaves the rest as they were posts + // nothing new; not by the truncated, worded lines. + const outcome = entries + .map( + (a) => + `${a.routeId}|${a.key}|${a.ok || a.earlier || !isRetryableFailure(a) ? "done" : "retry"}`, + ) + .join("\n"); + const summaryKey = event.deliveryId + ? `group-log:${await sha256Hex( + `${event.provider ?? "github"}|${tenantScope( + tenantId, + tenantId ? groupById.get(tenantId)?.instance : undefined, + )}|${event.deliveryId}|${group.instance ?? groupId}|${outcome}`, + )}` + : null; + if (summaryKey && !(await idempotencyStore(env.DB, env.KV).claim(summaryKey, 86400))) { + continue; + } + claimed = summaryKey; const result = await getDriver(target).send(message, target, env); if (!result.ok) { log.warn({ groupId, error: result.error }, "Failed to send group webhook log"); + } else { + claimed = null; } } catch (err) { log.error({ groupId, err }, "Group webhook log send failed"); } + // Not posted (a failed or thrown send): a retry may try again. + if (claimed) { + await idempotencyStore(env.DB, env.KV) + .release(claimed) + .catch(() => undefined); + } } } - async function processRoute(route: Route): Promise { - const targets = route.targets && route.targets.length > 0 ? route.targets : []; - if (targets.length === 0) return; + function baseRecord( + route: Route, + targetStr: string, + detail: Record, + ): Omit { + return { + ts: Date.now(), + routeId: route.id, + groupId: route.groupId, + groupInstance: route.groupId ? groupById.get(route.groupId)?.instance : undefined, + event: event.event, + repo: (event.payload.repository as { full_name?: string } | undefined)?.full_name, + target: targetStr, + deliveryId: event.deliveryId, + actor: (event.payload.sender as { login?: string } | undefined)?.login, + action: event.payload.action as string | undefined, + detail, + }; + } + function recordFailure( + route: Route, + target: RouteTarget, + detail: Record, + failure: { error: string; errorCode?: string; status?: number; platform?: string }, + extra: { durationMs?: number; attempts?: number } = {}, + logged = true, + ): void { + const targetStr = describeTarget(target); + attempts.push({ + key: targetKeyOf(groupById.get(route.groupId ?? ""), route, target), + groupId: route.groupId, + routeId: route.id, + routeName: route.name, + target: targetStr, + ok: false, + error: failure.error, + errorCode: failure.errorCode, + status: failure.status, + }); + if (!logged) return; + sendLogs.push({ + ...baseRecord(route, targetStr, detail), + ok: false, + error: failure.error, + status: failure.status, + platform: failure.platform, + errorCode: failure.errorCode, + ...extra, + }); + } + + async function buildRouteMessage(route: Route): Promise { await ensureWorkflowJobs(); const group = route.groupId ? groupById.get(route.groupId) : undefined; - const tr = trMap.get(group?.lang ?? "en")!; + const tr = await trFor(group?.lang ?? "en"); const showEmoji = group?.emoji !== false; const message = formatEvent(route, event, tr, showEmoji); + message.emoji = showEmoji; if (event.event === "check_suite") { - const suite = event.payload.check_suite as - | { - conclusion?: string; - check_runs_url?: string; - } - | undefined; - if (suite?.conclusion === "failure" && suite.check_runs_url) { - const buildLogUrl = await getCheckSuiteBuildLogUrl( - suite.check_runs_url, - env.GITHUB_APP_ID, - env.GITHUB_PRIVATE_KEY, - event.installationId, - ); - if (buildLogUrl) { - message.fields = message.fields ?? []; - message.fields.push({ - name: translate("fields.build_log", {}, undefined, tr), - value: `[${translate("fields.build_log", {}, undefined, tr)}](${buildLogUrl})`, - inline: false, - }); - } + const buildLogUrl = await checkSuiteBuildLog(); + if (buildLogUrl) { + message.fields = message.fields ?? []; + message.fields.push({ + name: translate("fields.build_log", {}, undefined, tr), + value: `[${translate("fields.build_log", {}, undefined, tr)}](${buildLogUrl})`, + inline: false, + }); } } if (group?.forgeSources?.length) { @@ -229,6 +776,69 @@ export async function dispatchEvent( if (route.discordRoleIds?.length) { message.mentionRoleIds = route.discordRoleIds; } + return message; + } + + async function processRoute(route: Route): Promise { + const targets = route.targets && route.targets.length > 0 ? route.targets : []; + if (targets.length === 0) return; + + const pending: RouteTarget[] = []; + // A destination listed twice (stored before that was refused) is sent once. + const seen = new Set(); + for (const target of targets) { + const key = targetKeyOf(groupById.get(route.groupId ?? ""), route, target); + const destination = destinationOf(target); + if (seen.has(destination)) continue; + seen.add(destination); + if (skipTarget) { + let done: boolean; + try { + done = await skipTarget(key); + } catch (err) { + // Unknown whether it was already sent: retry later rather than + // resend now or drop it. + recordFailure( + route, + target, + { match: explainRoute(route) }, + { + error: `Delivery ledger unavailable: ${err instanceof Error ? err.message : String(err)}`, + }, + ); + continue; + } + if (done) { + earlier.push({ + key, + groupId: route.groupId, + routeId: route.id, + routeName: route.name, + target: describeTarget(target), + ok: true, + earlier: true, + }); + continue; + } + } + pending.push(target); + } + if (pending.length === 0) return; + + let message: NeutralMessage; + try { + message = await buildRouteMessage(route); + } catch (err) { + // A payload the formatter cannot handle fails every target of the + // route explicitly (and permanently) instead of vanishing. + const error = `Format failed: ${err instanceof Error ? err.message : String(err)}`; + log.error({ routeId: route.id, event: event.event, err }, "Failed to format event"); + const detail = { match: explainRoute(route), provider: event.provider }; + for (const target of pending) { + recordFailure(route, target, detail, { error, errorCode: "FORMAT" }); + } + return; + } const detail: Record = { title: message.title, @@ -238,179 +848,106 @@ export async function dispatchEvent( provider: event.provider, installationId: event.installationId, }; + const version = message.updateKey ? updateVersion(event) : 0; - for (const target of targets) { - const targetStr = - target.platform === "telegram" - ? target.topicId - ? `${target.chatId}/${target.topicId}` - : (target.chatId ?? "") - : target.platform === "feishu" - ? (target.chatId ?? "") - : target.threadId - ? `${target.channelId}/${target.threadId}` - : (target.channelId ?? ""); - - const base: { - ts: number; - routeId: string; - groupId: string | undefined; - event: string; - repo: string | undefined; - target: string; - deliveryId: string | undefined; - actor: string | undefined; - action: string | undefined; - detail: Record; - } = { - ts: Date.now(), - routeId: route.id, - groupId: route.groupId, - event: event.event, - repo: (event.payload.repository as { full_name?: string } | undefined)?.full_name, - target: targetStr, - deliveryId: event.deliveryId, - actor: (event.payload.sender as { login?: string } | undefined)?.login, - action: event.payload.action as string | undefined, - detail, - }; - + for (const target of pending) { + const targetStr = describeTarget(target); + if (beforeTarget && !(await beforeTarget())) { + // Retried later (a transient failure), but not a send log row: the + // target was not attempted, and a large fan-out continued over several + // attempts would log it again each time. + recordFailure( + route, + target, + detail, + { error: "Delivery paused before this target (lease lost or out of time); sent later" }, + {}, + false, + ); + continue; + } const started = Date.now(); + let driver: PlatformDriver | undefined; try { - const driver = getDriver(target); + driver = getDriver(target); + // The same key in every attempt of this delivery to this target. + const outgoing: NeutralMessage = event.deliveryId + ? { + ...message, + // Scoped like the delivery itself (global endpoint vs the + // receiving group's instance): two endpoints' events sharing a + // delivery id are two messages. + sendKey: await sha256Hex( + `${event.provider ?? "github"}|${tenantScope( + tenantId, + tenantId ? groupById.get(tenantId)?.instance : undefined, + )}|${event.deliveryId}|${targetKeyOf( + groupById.get(route.groupId ?? ""), + route, + target, + )}`, + ), + } + : message; let result: SendResult; if (message.updateKey) { const groupPrefix = route.groupId ? `${route.groupId}:` : ""; - const eventId = `${groupPrefix}${route.id}:${message.updateKey}`; - const kvKey = `msg:${eventId}:${targetStr}`; - const lockKey = `msg:lock:${kvKey}`; - - // Acquire a short-lived lock so concurrent events for the same - // updateKey don't race (both read null, both send, the later put - // overwrites). KV is eventually consistent so the lock is - // best-effort; a retry loop further shrinks the window. - let locked = false; - for (let attempt = 0; attempt < 3; attempt++) { - const holder = await env.KV.get(lockKey); - if (holder) { - const existing = await tracker.get(eventId, targetStr); - if (existing) { - result = await driver.edit(message, target, env, existing); - if (result.ok || /not modified/i.test(result.error ?? "")) { - const ok = result.ok || /not modified/i.test(result.error ?? ""); - attempts.push({ - groupId: route.groupId, - routeId: route.id, - routeName: route.name, - target: targetStr, - ok: true, - }); - sendLogs.push({ - ...base, - ok: true, - status: result.status, - messageId: existing, - platform: driver.id, - attempts: result.attempts, - durationMs: Date.now() - started, - errorCode: result.errorCode, - }); - if (!ok) await tracker.delete(eventId, targetStr); - continue; - } - await tracker.delete(eventId, targetStr); - break; - } - await new Promise((r) => setTimeout(r, 50 * (attempt + 1))); - continue; - } - await env.KV.put(lockKey, "1", { expirationTtl: 60 }); - locked = true; - break; - } - - try { - const existingId = await tracker.get(eventId, targetStr); - if (existingId) { - result = await driver.edit(message, target, env, existingId); - if (result.ok) { - attempts.push({ - groupId: route.groupId, - routeId: route.id, - routeName: route.name, - target: targetStr, - ok: true, - }); - sendLogs.push({ - ...base, - ok: true, - status: result.status, - messageId: existingId, - platform: driver.id, - attempts: result.attempts, - durationMs: Date.now() - started, - errorCode: result.errorCode, - }); - continue; - } - if (/not modified/i.test(result.error ?? "")) { - attempts.push({ - groupId: route.groupId, - routeId: route.id, - routeName: route.name, - target: targetStr, - ok: true, - }); - sendLogs.push({ - ...base, - ok: true, - status: result.status, - messageId: existingId, - platform: driver.id, - attempts: result.attempts, - durationMs: Date.now() - started, - errorCode: result.errorCode, - }); - continue; - } - await tracker.delete(eventId, targetStr); - } - result = await driver.send(message, target, env); - if (result.ok && result.messageId) { - await tracker.set(eventId, targetStr, result.messageId); - } - } finally { - if (locked) await env.KV.delete(lockKey); - } + // Chains are owned by the group instance: they survive renames, and a + // group re-created under the id never edits the old group's messages. + const owner = groupById.get(route.groupId ?? "")?.instance; + const legacyOwner = groupById.get(route.groupId ?? "")?.legacyId; + const chainPrefix = owner ? `${instanceScope(owner)}:` : groupPrefix; + result = await deliverUpdate( + env, + tracker, + driver, + outgoing, + target, + { + id: `${chainPrefix}${route.id}:${eventOrigin(event)}:${message.updateKey}`, + target: `${driver.id}:${targetStr}`, + // Chains from before instances were keyed by the id the group + // had back then: only that group (its `legacyId`) adopts them, + // never a group created since under a reused id. + // (An empty `legacyId` marks the group seeded for routes that + // had no group: their chains carried no group prefix.) + legacyId: + eventOrigin(event) === "github@github.com" && legacyOwner != null + ? `${legacyOwner ? `${legacyOwner}:` : ""}${route.id}:${ + message.legacyUpdateKey ?? message.updateKey + }` + : "", + legacyTarget: targetStr, + }, + version, + ); } else { - result = await driver.send(message, target, env); + result = await driver.send(outgoing, target, env); } const durationMs = Date.now() - started; if (!result.ok) { - const error = result.error ?? "Send failed"; - attempts.push({ - groupId: route.groupId, - routeId: route.id, - routeName: route.name, - target: targetStr, - ok: false, - error, - errorCode: result.errorCode, - status: result.status, - }); - sendLogs.push({ - ...base, - ok: false, - error, - durationMs, - status: result.status, - platform: driver.id, - attempts: result.attempts, - errorCode: result.errorCode, - }); + recordFailure( + route, + target, + detail, + { + error: result.error ?? "Send failed", + errorCode: result.errorCode, + status: result.status, + platform: driver.id, + }, + { durationMs, attempts: result.attempts }, + ); continue; } + const key = targetKeyOf(groupById.get(route.groupId ?? ""), route, target); + if (afterTarget) { + await afterTarget(key).catch((err: unknown) => { + log.warn({ routeId: route.id, target: targetStr, err }, "Failed to record sent target"); + }); + } attempts.push({ + key, groupId: route.groupId, routeId: route.id, routeName: route.name, @@ -418,7 +955,7 @@ export async function dispatchEvent( ok: true, }); sendLogs.push({ - ...base, + ...baseRecord(route, targetStr, detail), ok: true, status: result.status, messageId: result.messageId, @@ -428,22 +965,14 @@ export async function dispatchEvent( errorCode: result.errorCode, }); } catch (err) { - const durationMs = Date.now() - started; const error = err instanceof Error ? err.message : String(err); - attempts.push({ - groupId: route.groupId, - routeId: route.id, - routeName: route.name, - target: targetStr, - ok: false, - error, - }); - sendLogs.push({ - ...base, - ok: false, - error, - durationMs, - }); + recordFailure( + route, + target, + detail, + { error, platform: driver?.id }, + { durationMs: Date.now() - started }, + ); log.error({ routeId: route.id, target: targetStr, err }, "Route failed"); } } diff --git a/server/lib/lib/message-tracker.ts b/server/lib/lib/message-tracker.ts index ac282f1..2cbfe29 100644 --- a/server/lib/lib/message-tracker.ts +++ b/server/lib/lib/message-tracker.ts @@ -1,40 +1,107 @@ -import { canUseD1 } from "../storage/d1"; +import { canUseD1, isMissingTable, kvFallbackActive, noteKvFallback } from "../storage/d1"; +import { log } from "./log"; -export interface MessageTracker { - get(eventId: string, targetId: string): Promise; - set(eventId: string, targetId: string, messageId: string): Promise; - delete(eventId: string, targetId: string): Promise; +/** + * A tracked platform message: its id plus the version (event time, ms) of the + * update it currently shows, so an older event arriving late never overwrites + * a newer state. Entries written before versions existed carry `version: 0`. + */ +export interface TrackedMessage { + messageId: string; + version: number; } -const MESSAGE_KEY_TTL_SECONDS = 86400; +export interface MessageTracker { + get(eventId: string, targetId: string): Promise; + /** Store (or refresh) an entry; refreshing also renews its retention. */ + set(eventId: string, targetId: string, entry: TrackedMessage): Promise; + delete(eventId: string, targetId: string): Promise; + /** + * Re-file an entry under a new key, dropping the old one — in one D1 + * transaction, so the entry is never left under both (two owners). + */ + move( + from: { eventId: string; targetId: string }, + to: { eventId: string; targetId: string }, + entry: TrackedMessage, + ): Promise; +} + +/** KV entries outlive the D1 retention of message_tracking rows (30 days). */ +const MESSAGE_KEY_TTL_SECONDS = 30 * 86400; + +/** Stored value: JSON `{ id, v }`, or a bare message id from older versions. */ +export function decodeTracked(value: string | null | undefined): TrackedMessage | null { + if (!value) return null; + if (value.startsWith("{")) { + try { + const parsed = JSON.parse(value) as { id?: unknown; v?: unknown }; + if (typeof parsed.id === "string" && parsed.id) { + return { messageId: parsed.id, version: Number(parsed.v) || 0 }; + } + } catch { + // fall through: treat as a bare id + } + } + return { messageId: value, version: 0 }; +} + +export function encodeTracked(entry: TrackedMessage): string { + return JSON.stringify({ id: entry.messageId, v: entry.version }); +} + +/** + * What a moved entry leaves under its old key in D1: a tombstone. Without it, + * a KV copy of the old entry (written while the table was missing, its delete + * not yet seen everywhere) could be read back and adopted a second time. + */ +const MOVED = '{"moved":true}'; export function kvMessageTracker(kv: KVNamespace): MessageTracker { const key = (eventId: string, targetId: string): string => `msg:${eventId}:${targetId}`; return { - async get(eventId: string, targetId: string): Promise { - return kv.get(key(eventId, targetId)); + async get(eventId, targetId): Promise { + return decodeTracked(await kv.get(key(eventId, targetId))); }, - async set(eventId: string, targetId: string, messageId: string): Promise { - await kv.put(key(eventId, targetId), messageId, { + async set(eventId, targetId, entry): Promise { + await kv.put(key(eventId, targetId), encodeTracked(entry), { expirationTtl: MESSAGE_KEY_TTL_SECONDS, }); }, - async delete(eventId: string, targetId: string): Promise { + async delete(eventId, targetId): Promise { await kv.delete(key(eventId, targetId)); }, + async move(from, to, entry): Promise { + // No transactions in KV: the old key goes first, so a failure leaves + // the entry lost (a new message later) rather than owned twice. + await kv.delete(key(from.eventId, from.targetId)); + await kv.put(key(to.eventId, to.targetId), encodeTracked(entry), { + expirationTtl: MESSAGE_KEY_TTL_SECONDS, + }); + }, }; } +/** A D1 lookup: the entry, and whether its key was moved away (a tombstone). */ +async function d1Lookup( + db: D1Database, + eventId: string, + targetId: string, +): Promise<{ entry: TrackedMessage | null; moved: boolean }> { + const row = await db + .prepare("SELECT message_id FROM message_tracking WHERE event_id = ? AND target_id = ?") + .bind(eventId, targetId) + .first<{ message_id: string }>(); + if (row?.message_id === MOVED) return { entry: null, moved: true }; + return { entry: decodeTracked(row?.message_id), moved: false }; +} + export function d1MessageTracker(db: D1Database): MessageTracker { return { - async get(eventId: string, targetId: string): Promise { - const row = await db - .prepare("SELECT message_id FROM message_tracking WHERE event_id = ? AND target_id = ?") - .bind(eventId, targetId) - .first<{ message_id: string }>(); - return row?.message_id ?? null; + async get(eventId, targetId): Promise { + return (await d1Lookup(db, eventId, targetId)).entry; }, - async set(eventId: string, targetId: string, messageId: string): Promise { + async set(eventId, targetId, entry): Promise { await db .prepare( `INSERT INTO message_tracking (event_id, target_id, message_id, updated_at) @@ -42,18 +109,178 @@ export function d1MessageTracker(db: D1Database): MessageTracker { ON CONFLICT(event_id, target_id) DO UPDATE SET message_id = excluded.message_id, updated_at = excluded.updated_at`, ) - .bind(eventId, targetId, messageId, Date.now()) + .bind(eventId, targetId, encodeTracked(entry), Date.now()) .run(); }, - async delete(eventId: string, targetId: string): Promise { + async delete(eventId, targetId): Promise { await db .prepare("DELETE FROM message_tracking WHERE event_id = ? AND target_id = ?") .bind(eventId, targetId) .run(); }, + async move(from, to, entry): Promise { + await db.batch([ + db + .prepare( + `INSERT INTO message_tracking (event_id, target_id, message_id, updated_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(event_id, target_id) DO UPDATE SET + message_id = excluded.message_id, updated_at = excluded.updated_at`, + ) + .bind(to.eventId, to.targetId, encodeTracked(entry), Date.now()), + // The old key keeps a tombstone (pruned with the other rows). + db + .prepare( + `INSERT INTO message_tracking (event_id, target_id, message_id, updated_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(event_id, target_id) DO UPDATE SET + message_id = excluded.message_id, updated_at = excluded.updated_at`, + ) + .bind(from.eventId, from.targetId, MOVED, Date.now()), + ]); + }, + }; +} + +/** D1 first; degrades to KV only while the D1 table is missing (unmigrated). */ +function withKvFallback( + primary: MessageTracker, + fallback: MessageTracker, + kvNs: KVNamespace, + db: D1Database, +): MessageTracker { + async function run(op: string, d1: () => Promise, kv: () => Promise): Promise { + try { + return await d1(); + } catch (err) { + if (!isMissingTable(err)) throw err; + log.warn({ err, op }, "D1 message_tracking table missing, falling back to KV"); + await noteKvFallback(kvNs); + return kv(); + } + } + return { + get: (e, t) => + run( + "get", + async () => { + const { entry, moved } = await d1Lookup(db, e, t); + // A moved key never falls back to its KV copy. + if (entry || moved) return entry; + return (await kvFallbackActive(kvNs)) ? fallback.get(e, t) : null; + }, + () => fallback.get(e, t), + ), + set: (e, t, entry) => + run( + "set", + () => primary.set(e, t, entry), + () => fallback.set(e, t, entry), + ), + delete: (e, t) => + run( + "delete", + async () => { + await primary.delete(e, t); + // Otherwise the KV copy from the missing-table period would resurface. + if (await kvFallbackActive(kvNs)) await fallback.delete(e, t); + }, + () => fallback.delete(e, t), + ), + move: (from, to, entry) => + run( + "move", + async () => { + // D1 first (the new entry and the old key's tombstone, together): + // from then on the tombstone keeps any KV copy from being adopted + // again. The KV copy goes after — removing it first would lose the + // entry altogether should the D1 write fail. + await primary.move(from, to, entry); + // Cleanup only (the tombstone already guards): a failure here never + // fails the move that went through. + try { + if (await kvFallbackActive(kvNs)) await fallback.delete(from.eventId, from.targetId); + } catch { + // left to expire + } + }, + () => fallback.move(from, to, entry), + ), }; } export function messageTracker(db: D1Database, kv: KVNamespace): MessageTracker { - return canUseD1(db) ? d1MessageTracker(db) : kvMessageTracker(kv); + if (canUseD1(db)) return withKvFallback(d1MessageTracker(db), kvMessageTracker(kv), kv, db); + // Without D1, tracking lives in KV: marked, so a D1 binding added later + // still finds the messages to edit. + const tracker = kvMessageTracker(kv); + return { + ...tracker, + async set(eventId, targetId, entry): Promise { + await noteKvFallback(kv); + return tracker.set(eventId, targetId, entry); + }, + async move(from, to, entry): Promise { + await noteKvFallback(kv); + return tracker.move(from, to, entry); + }, + }; +} + +/** + * Serializes the updates of one message chain. With D1 the lock is a + * `dedup_keys` row taken by an atomic conditional UPSERT (taken over only once + * expired) and released only by its holder, identified by the claim time it + * wrote. Without D1 it degrades to a best-effort KV marker. + */ +export interface ChainLock { + /** A holder token, or null while someone else holds the lock. */ + acquire(key: string, ttlSeconds: number): Promise; + release(key: string, token: string): Promise; +} + +export function chainLock(db: D1Database, kv: KVNamespace): ChainLock { + const kvLock: ChainLock = { + async acquire(key, ttlSeconds) { + if (await kv.get(key)) return null; + const token = crypto.randomUUID(); + await kv.put(key, token, { expirationTtl: Math.max(60, ttlSeconds) }); + return token; + }, + async release(key, token) { + if ((await kv.get(key)) === token) await kv.delete(key); + }, + }; + if (!canUseD1(db)) return kvLock; + return { + async acquire(key, ttlSeconds): Promise { + const now = Date.now(); + try { + const result = await db + .prepare( + `INSERT INTO dedup_keys (key, claimed_at, expires_at) VALUES (?, ?, ?) + ON CONFLICT(key) DO UPDATE SET + claimed_at = excluded.claimed_at, expires_at = excluded.expires_at + WHERE dedup_keys.expires_at < excluded.claimed_at`, + ) + .bind(key, now, now + ttlSeconds * 1000) + .run(); + return (result.meta?.changes ?? 0) > 0 ? String(now) : null; + } catch (err) { + if (!isMissingTable(err)) throw err; + return kvLock.acquire(key, ttlSeconds); + } + }, + async release(key, token): Promise { + try { + await db + .prepare("DELETE FROM dedup_keys WHERE key = ? AND claimed_at = ?") + .bind(key, Number(token)) + .run(); + } catch (err) { + if (!isMissingTable(err)) throw err; + await kvLock.release(key, token); + } + }, + }; } diff --git a/tests/d1-stores.test.ts b/tests/d1-stores.test.ts index de33af8..0ca5a13 100644 --- a/tests/d1-stores.test.ts +++ b/tests/d1-stores.test.ts @@ -9,7 +9,16 @@ import { kvMessageTracker, messageTracker, } from "../server/lib/lib/message-tracker"; -import { getDeliveryState, setDeliveryState } from "../server/lib/queue/delivery"; +import { + acquireDelivery, + finishLease, + getDeliveryState, + renewLease, + setDeliveryState, +} from "../server/lib/queue/delivery"; +import { createMemoryKV, createSqliteD1 } from "./d1-sqlite"; +import { chainLock } from "../server/lib/lib/message-tracker"; +import { recordSendBatch } from "../server/lib/lib/send-log-batch"; import { canUseD1 } from "../server/lib/storage/d1"; import type { Env } from "../server/lib/types"; @@ -159,17 +168,20 @@ describe("d1MessageTracker", () => { it("round-trips a message id", async () => { const { db } = createMockD1(); const tracker = d1MessageTracker(db); - await tracker.set("evt-1", "discord:123", "message-42"); - await expect(tracker.get("evt-1", "discord:123")).resolves.toBe("message-42"); + await tracker.set("evt-1", "discord:123", { messageId: "message-42", version: 7 }); + await expect(tracker.get("evt-1", "discord:123")).resolves.toEqual({ + messageId: "message-42", + version: 7, + }); await expect(tracker.get("evt-1", "discord:999")).resolves.toBeNull(); }); it("updates on re-set and deletes on delete", async () => { const { db } = createMockD1(); const tracker = d1MessageTracker(db); - await tracker.set("evt-1", "tg:9", "old"); - await tracker.set("evt-1", "tg:9", "new"); - await expect(tracker.get("evt-1", "tg:9")).resolves.toBe("new"); + await tracker.set("evt-1", "tg:9", { messageId: "old", version: 1 }); + await tracker.set("evt-1", "tg:9", { messageId: "new", version: 2 }); + await expect(tracker.get("evt-1", "tg:9")).resolves.toEqual({ messageId: "new", version: 2 }); await tracker.delete("evt-1", "tg:9"); await expect(tracker.get("evt-1", "tg:9")).resolves.toBeNull(); }); @@ -227,8 +239,11 @@ describe("factory fallback decision", () => { const db = {} as D1Database; const kv = createMockKV(); const tracker = messageTracker(db, kv); - await tracker.set("evt-1", "discord:1", "m1"); - await expect(tracker.get("evt-1", "discord:1")).resolves.toBe("m1"); + await tracker.set("evt-1", "discord:1", { messageId: "m1", version: 0 }); + await expect(tracker.get("evt-1", "discord:1")).resolves.toEqual({ + messageId: "m1", + version: 0, + }); await tracker.delete("evt-1", "discord:1"); await expect(tracker.get("evt-1", "discord:1")).resolves.toBeNull(); }); @@ -240,8 +255,92 @@ describe("factory fallback decision", () => { await expect(idem.has("delivery:github:g1:e1")).resolves.toBe(true); const msg = kvMessageTracker(kv); - await msg.set("evt-1", "discord:1", "m1"); - await expect(msg.get("evt-1", "discord:1")).resolves.toBe("m1"); + await msg.set("evt-1", "discord:1", { messageId: "m1", version: 3 }); + await expect(msg.get("evt-1", "discord:1")).resolves.toEqual({ messageId: "m1", version: 3 }); + }); + + it("reads message ids stored before versions existed", async () => { + const kv = createMockKV(); + await kv.put("msg:evt-1:discord:1", "legacy-id"); + await expect(kvMessageTracker(kv).get("evt-1", "discord:1")).resolves.toEqual({ + messageId: "legacy-id", + version: 0, + }); + }); +}); + +describe("D1 failures degrade to KV", () => { + it("idempotency claims fall back to KV when the D1 table is missing", async () => { + const { db } = createSqliteD1({ + migrations: (f) => !f.startsWith("0008") && !f.startsWith("0013"), + }); + const { kv } = createMemoryKV(); + const store = idempotencyStore(db, kv); + await expect(store.claim("k", 120)).resolves.toBe(true); + await expect(store.claim("k", 120)).resolves.toBe(false); + await store.release("k"); + await expect(store.claim("k", 120)).resolves.toBe(true); + }); + + it("message tracking falls back to KV when the D1 table is missing", async () => { + const { db } = createSqliteD1({ + migrations: (f) => !f.startsWith("0008") && !f.startsWith("0013"), + }); + const { kv } = createMemoryKV(); + const tracker = messageTracker(db, kv); + await tracker.set("e", "t", { messageId: "m", version: 1 }); + await expect(tracker.get("e", "t")).resolves.toEqual({ messageId: "m", version: 1 }); + }); +}); + +describe("acquireDelivery", () => { + function env(): Env { + return { KV: createMemoryKV().kv, DB: createSqliteD1().db } as Env; + } + + it("lets exactly one consumer take a delivery", async () => { + const e = env(); + expect((await acquireDelivery(e, "k")).status).toBe("acquired"); + expect((await acquireDelivery(e, "k")).status).toBe("busy"); + }); + + it("hands a waiting retry to the next consumer", async () => { + const e = env(); + await acquireDelivery(e, "k"); + await setDeliveryState(e, "k", "retrying"); + expect((await acquireDelivery(e, "k")).status).toBe("acquired"); + }); + + it("reports finished deliveries", async () => { + const e = env(); + for (const status of ["delivered", "failed", "dead"] as const) { + await setDeliveryState(e, `k-${status}`, status); + expect((await acquireDelivery(e, `k-${status}`)).status).toBe("finished"); + } + }); + + it("fences a holder whose lease was taken over", async () => { + const { db, sqlite } = createSqliteD1(); + const e = { KV: createMemoryKV().kv, DB: db } as Env; + const first = await acquireDelivery(e, "k"); + sqlite.exec("UPDATE delivery_state SET updated_at = 0"); + // A takeover happens minutes later in practice; never in the same ms. + await Bun.sleep(2); + const second = await acquireDelivery(e, "k"); + expect(second.status).toBe("acquired"); + // The old holder can neither renew nor overwrite the new holder's outcome. + expect(await renewLease(e, "k", first.token)).toBeNull(); + expect(await finishLease(e, "k", first.token, "delivered")).toBe(false); + expect(await finishLease(e, "k", second.token, "delivered")).toBe(true); + expect(await getDeliveryState(e, "k")).toBe("delivered"); + }); + + it("takes over a lease whose holder went silent", async () => { + const { db, sqlite } = createSqliteD1(); + const e = { KV: createMemoryKV().kv, DB: db } as Env; + await acquireDelivery(e, "k"); + sqlite.exec("UPDATE delivery_state SET updated_at = 0"); + expect((await acquireDelivery(e, "k")).status).toBe("acquired"); }); }); @@ -259,3 +358,52 @@ describe("delivery state falls back to KV without batch", () => { await expect(getDeliveryState(env, "delivery-state:github:global:nope")).resolves.toBeNull(); }); }); + +describe("KV fallback transition", () => { + it("keeps honoring and releasing claims made in KV before the table existed", async () => { + const { db: noTable } = createSqliteD1({ + migrations: (f) => !f.startsWith("0008") && !f.startsWith("0013"), + }); + const { kv } = createMemoryKV(); + await idempotencyStore(noTable, kv).claim("k", 600); + // The migration is applied afterwards (same KV, a database with the table). + const { db } = createSqliteD1(); + const store = idempotencyStore(db, kv); + expect(await store.has("k")).toBe(true); + expect(await store.claim("k", 600)).toBe(false); + await store.release("k"); + expect(await store.claim("k", 600)).toBe(true); + }); +}); + +describe("chainLock", () => { + it("admits one holder at a time and only the holder can release", async () => { + const { db } = createSqliteD1(); + const lock = chainLock(db, createMemoryKV().kv); + const token = await lock.acquire("chain", 60); + expect(token).not.toBeNull(); + expect(await lock.acquire("chain", 60)).toBeNull(); + await lock.release("chain", "999"); + expect(await lock.acquire("chain", 60)).toBeNull(); + await lock.release("chain", token!); + expect(await lock.acquire("chain", 60)).not.toBeNull(); + }); +}); + +describe("send log group attribution", () => { + it("files rows under the id the group instance has at insert time", async () => { + const { db, sqlite } = createSqliteD1(); + sqlite.exec( + `INSERT INTO d1_groups (id, name, data, version, created_at, updated_at) VALUES ('renamed', 'R', '{"id":"renamed","name":"R","adminIds":[],"instance":"abc"}', 1, 0, 0)`, + ); + const base = { ts: 1, routeId: "r", event: "push", target: "t", ok: true }; + await recordSendBatch(db, [ + { ...base, groupId: "old-id", groupInstance: "abc" }, + { ...base, groupId: "deleted", groupInstance: "gone" }, + ]); + expect(sqlite.query("SELECT group_id FROM send_logs ORDER BY id").all()).toEqual([ + { group_id: "renamed" }, + { group_id: null }, + ]); + }); +}); diff --git a/tests/discord.test.ts b/tests/discord.test.ts index ab3b42e..3a7367c 100644 --- a/tests/discord.test.ts +++ b/tests/discord.test.ts @@ -245,7 +245,8 @@ describe("dispatchEvent fallback routing", () => { route("omega-catch-all-2", "omega", { fallback: true, filters: [] }, "306"), ]; - await dispatchEvent({ ...baseConfig, routes }, { event: "push", payload: {} }, env); + const groups = ["alpha", "zeta", "omega"].map((id) => ({ id, name: id, adminIds: [] })); + await dispatchEvent({ ...baseConfig, routes }, { event: "push", payload: {} }, env, groups); const hits = (channel: string): number => sent.filter((u) => u.includes(`/${channel}/`)).length; // alpha stops after its first match without silencing zeta. expect(hits("301")).toBe(1); @@ -258,6 +259,52 @@ describe("dispatchEvent fallback routing", () => { expect(hits("306")).toBe(0); }); + it("never fires routes whose group no longer exists", async () => { + const sent: string[] = []; + mockFetch((url) => { + sent.push(url); + return new Response("{}", { status: 200 }); + }); + const env = createEnv({ KV: createMockKV(), DB: createMockDB() }); + const routes: Route[] = [ + { + id: "orphan", + name: "orphan", + enabled: true, + filters: [{ type: "event", match: "push" }], + targets: [{ channelId: "401" }], + groupId: "deleted", + }, + ]; + await dispatchEvent({ ...baseConfig, routes }, { event: "push", payload: {} }, env, []); + expect(sent).toHaveLength(0); + }); + + it("keeps isolated groups off the global endpoint but serves their own ingress", async () => { + const sent: string[] = []; + mockFetch((url) => { + sent.push(url); + return new Response("{}", { status: 200 }); + }); + const env = createEnv({ KV: createMockKV(), DB: createMockDB() }); + const routes: Route[] = [ + { + id: "mine", + name: "mine", + enabled: true, + filters: [{ type: "event", match: "push" }], + targets: [{ channelId: "501" }], + groupId: "u-1", + }, + ]; + const groups = [{ id: "u-1", name: "me", adminIds: [], isolated: true }]; + const event = { event: "push", payload: {} }; + await dispatchEvent({ ...baseConfig, routes }, event, env, groups); + expect(sent).toHaveLength(0); + await dispatchEvent({ ...baseConfig, routes }, event, env, groups, { tenantId: "u-1" }); + expect(sent.filter((u) => u.includes("/501/"))).toHaveLength(1); + }); + it("prepends role mentions to the Discord message content", async () => { const bodies: string[] = []; mockFetch((url, init) => { diff --git a/tests/message-tracker.test.ts b/tests/message-tracker.test.ts index 5eebf48..46dd3b6 100644 --- a/tests/message-tracker.test.ts +++ b/tests/message-tracker.test.ts @@ -21,8 +21,11 @@ describe("kvMessageTracker", () => { it("stores and retrieves a message id by event and target", async () => { const kv = createMockKV(); const tracker = kvMessageTracker(kv); - await tracker.set("route-1:workflow-9", "channel-1", "msg-42"); - expect(await tracker.get("route-1:workflow-9", "channel-1")).toBe("msg-42"); + await tracker.set("route-1:workflow-9", "channel-1", { messageId: "msg-42", version: 5 }); + expect(await tracker.get("route-1:workflow-9", "channel-1")).toEqual({ + messageId: "msg-42", + version: 5, + }); }); it("returns null for an unknown event/target", async () => { @@ -31,7 +34,7 @@ describe("kvMessageTracker", () => { expect(await tracker.get("missing", "channel-1")).toBeNull(); }); - it("uses the msg:{eventId}:{targetId} key with a 24h TTL", async () => { + it("uses the msg:{eventId}:{targetId} key with a 30-day TTL", async () => { const kv = createMockKV(); const keys = new Map(); const ttls = new Map(); @@ -44,15 +47,15 @@ describe("kvMessageTracker", () => { ttls.set(key, opts?.expirationTtl ?? 0); }; const tracker = kvMessageTracker(kv); - await tracker.set("event-1", "target-1", "msg-7"); + await tracker.set("event-1", "target-1", { messageId: "msg-7", version: 0 }); expect(keys.has("msg:event-1:target-1")).toBe(true); - expect(ttls.get("msg:event-1:target-1")).toBe(86400); + expect(ttls.get("msg:event-1:target-1")).toBe(30 * 86400); }); it("deletes tracked entries", async () => { const kv = createMockKV(); const tracker = kvMessageTracker(kv); - await tracker.set("event-1", "target-1", "msg-7"); + await tracker.set("event-1", "target-1", { messageId: "msg-7", version: 0 }); await tracker.delete("event-1", "target-1"); expect(await tracker.get("event-1", "target-1")).toBeNull(); });