fix(security): harden webhook delivery pipeline and group storage (#11)

* fix(log): preserve structured error details

Error objects lose their useful details when serialized as plain JSON.
Keep the message, stack and cause in structured log output.

* fix(runtime): bound shared in-flight waits

A canceled request can leave its shared promise pending forever.
Bound joins and coordinate a single replacement so later requests
make progress without multiplying upstream work.

* fix(bot): reject oversized callback bodies

Reject declared oversized bot callbacks before reading their bodies.
This keeps unauthenticated requests from wasting worker resources.

* test(d1): exercise real sqlite transactions

Exercise real foreign keys, transactions and conditional statements
instead of mocks that cannot reproduce lost updates or cascades.

* fix(storage): preserve claims across kv fallback

Only missing tables justify KV fallback; other failures must surface.
Remember fallback writes so migration to D1 cannot forget claims
and replay completed work. Share marker checks without extra writes.

* fix(filter): use isolated linear-time patterns

Backtracking patterns can exhaust worker CPU on untrusted payloads.
Use linear-time matching and separate bounded cache namespaces so
glob and regex entries cannot corrupt each other.

* fix(filter): read each event branch correctly

Deployment and check branches live in event-specific nested fields.
Read those fields and remove only the actual leading ref prefix.

* fix(filter): read only own payload properties

Inherited object properties are not webhook payload data.
Read own properties only so JSONPath cannot traverse prototypes.

* fix(filter): validate executable condition trees

Malformed trees and unsupported patterns must not silently turn an
exclude rule into a pass. Validate the shape and only the patterns
the evaluator actually executes.

* fix(i18n): bound and cache translation loads

Repeated misses and simultaneous override loads spend KV reads for
the same language. Cache confirmed misses and share bounded loads
without caching read failures as missing translations.

* i18n(bot): explain command and account failures

Explain account linking, command failures and skipped delivery
outcomes consistently in both supported languages.

* fix(github): bound and reuse app token fetches

Repeated token exchanges waste requests and JWT work for every event.
Reuse valid installation tokens, share refreshes and back off briefly
after failures while keeping every shared wait bounded.

* fix(github): restrict enrichment request urls

Webhook-provided URLs must never receive an installation token on
another host. Restrict enrichment to GitHub repository API paths,
reject redirects and bound optional requests.

* fix(formatter): preserve invalid code points

Out-of-range numeric entities can throw while formatting a delivery.
Keep invalid code points verbatim instead of aborting the message.

* fix(formatter): preserve bounded markdown text

Code spans, fences, indentation and autolinks are Markdown content,
not HTML tags to strip. Preserve them while bounding cleanup input
and reusing results across routes for the same event.

* fix(formatter): strip only leading ref prefixes

A short branch or tag may itself contain refs/heads or refs/tags.
Strip one leading full-ref prefix at the caller and keep short names
unchanged when generating labels and links.

* fix(gitea): link push authors to their forge

A Gitea author is not the same-named GitHub account.
Build author profile links from the originating forge.

* fix(formatter): render deployment state values

The deployment status object stringifies as [object Object].
Render its actual state in the status field.

* fix(formatter): read security severity levels

The scanner severity is not the security severity of an alert.
Prefer the security level so critical and high alerts get the right
label and color.

* fix(formatter): hide actions on other forges

Action buttons call the GitHub API as the linked user.
Hide them for other forges to avoid operating on a same-named
GitHub repository.

* fix(check): isolate updates by check suite

Same-named checks from different suites must not edit one message.
Include the suite or app identity and retain the former key for
controlled adoption of existing tracked messages.

* fix(logs): scope queries before limiting results

Filter authorized groups in SQL before LIMIT and aggregate only
visible rows. Preserve current group attribution for late writes,
surface read failures and combine related totals in one scan.

* perf(d1): index pruning and failed send queries

Pruning and failure views should not scan all historical rows.
Index the high-value access paths without adding a write-heavy
index for infrequent delivery-detail lookups.

* fix(discord): bound individual api requests

An unbounded API request can outlive an update-chain lock.
Bound each request so another holder cannot race a late edit.

* fix(discord): defer long rate-limit waits

Long rate-limit sleeps block delivery progress inside a worker.
Return a retryable 429 for the queue when the wait exceeds the bound.

* fix(discord): encode channel path segments

Stored channel and message ids are path segments, not URL paths.
Encode them so aliases cannot address another Discord resource.

* fix(discord): reuse nonces after lost responses

A successful create may lose its response. Reuse a stable nonce
with enforce_nonce so retries recover that message instead of
posting duplicates or accepting an unreadable success.

* fix(discord): constrain embeds and role mentions

Respect combined embed and component limits as well as field caps.
Restrict allowed mentions to validated route roles so payload text
cannot notify everyone.

* fix(discord): resume bounded command syncs

Application changes, command rollbacks and large guild lists can
leave commands stale indefinitely. Track the current hash per app
and resume bounded paginated work without needless KV writes.

* fix(github): verify installation management

An installation id is user-controlled, and repository access is not
installation management. Verify the owning account or an active
organization admin using the user token and read:org scope.

* fix(github): resolve immutable member ids

A login can be renamed and reclaimed before its first sign-in.
Resolve the current immutable user id when membership is granted
so a future holder of the login cannot inherit access.

* fix(auth): revoke stale sessions and tokens

Revoked or rotated credentials must stop working despite stale KV
replicas. Track the live token and revocations in D1 and preserve
immutable identity bindings rather than trusting a recycled login.

* fix(auth): bind sessions and tokens to identity

Automatic retries can repeat non-idempotent user actions.
Disable those retries and expose a write hook so bot handlers can
record completion immediately before the GitHub write.

* fix(discord): defer and deduplicate interactions

Acknowledge slow actions before Discord times out and deduplicate
interaction writes. Preserve comment text and reject edits whose
source changed after the modal opened.

* fix(telegram): always verify webhook secrets

Webhook verification must not silently disable itself when the
optional secret is absent. Derive a stable secret from the bot
token and reject callbacks when verification is unavailable.

* fix(telegram): bound individual api requests

Bound individual platform calls so they cannot outlive the lock
protecting a message update or stall a whole delivery invocation.

* fix(telegram): honor nested rate-limit delays

Telegram places retry_after inside parameters, not at the top level.
Read that value and defer long waits to the delivery queue.

* fix(telegram): report unreadable send results

An unreadable successful response may already have sent the message.
Report an explicit uncertain result instead of retrying a create
without an idempotency key or pretending it succeeded.

* fix(telegram): keep editable messages as text

Guessing photo versus text from message content makes later edits
use the wrong API and a smaller caption budget. Always send text
and use the avatar as a link preview.

* fix(telegram): render hard breaks as newlines

Telegram HTML does not support br tags. Render Markdown hard breaks
as plain newlines so otherwise valid messages are accepted.

* fix(telegram): clamp visible html characters

Telegram counts visible characters, not HTML source bytes.
Truncate without splitting tags, entities or surrogate pairs.

* fix(telegram): honor the author emoji toggle

The author marker must obey the group emoji preference just like
formatter-added decoration.

* fix(telegram): keep completed commands claimed

Redelivery after a completed write must never post another comment.
Keep completion claims despite reply failures, scope updates per bot
and only issue account-login links in private chats.

* fix(feishu): bound individual api requests

Bound individual platform calls so they cannot outlive the lock
protecting a message update or stall a whole delivery invocation.

* fix(feishu): classify retryable api failures

Business error codes can be transient even in HTTP 200 responses.
Classify token and rate-limit errors for retry and defer long waits.

* fix(feishu): retry sends with stable uuids

An unreadable response does not prove a send failed.
Reuse the same uuid for retries so recovering the result cannot
create a second Feishu message.

* fix(feishu): refresh and evict tokens safely

Separate app token caches, stop reusing refused tokens and share
refreshes. Cache failures briefly and keep valid tokens in memory
when a best-effort KV cache write fails.

* fix(feishu): use the card update endpoint

Delayed card updates use the card update endpoint; the config
endpoint cannot update an interaction response.

* fix(feishu): neutralize untrusted mentions

Forge text is untrusted and can contain at-all tags.
Neutralize mentions in card bodies and footers before rendering.

* fix(feishu): authenticate and dedupe callbacks

Authenticate signed or token-verified callbacks before dispatching
commands. Decode encrypted envelopes and handle card actions using
the correct fields without claiming irrelevant chat events.

* fix(target): normalize actual destinations

Different target spellings can name the same actual destination.
Normalize channel, thread and topic identities for ownership checks
and per-route duplicate suppression.

* fix(config): transact per-group configuration

Whole-table writes and cached snapshots can overwrite other groups
or resurrect removed configuration. Use per-group guarded writes,
one-time identity migration and explicit unreadable-row failures.

* fix(group): preserve identity through lifecycle

Group names are mutable and reusable, so they cannot own durable
state. Preserve immutable identities across lifecycle operations and
commit invite use with membership under transaction-time guards.

* fix(oauth): confirm account links and installs

A login link or installation id is not proof of account ownership.
Verify installation management, bind browser state and require an
explicit account-link confirmation before granting chat access.

* fix(queue): fence leases and record target outcomes

Retries must remember completed targets and distinguish active
lease holders. Fence state updates, preserve permanent outcomes and
reject missing parked payloads instead of dispatching empty data.

* fix(dispatch): isolate and resume tenant sends

Check tenant sources before filters and never dispatch orphan routes.
Record each target outcome, isolate update chains by forge and group
identity, and keep stale events from overwriting newer messages.

* fix(queue): bound batches and resume deliveries

A queue invocation has shared query and subrequest limits.
Use a small batch and shared send budget, renew fenced leases and
resume only unfinished work without exhausting unstarted messages.

* fix(admin): guard revisions and tenant targets

Read-time authorization alone cannot protect concurrent edits.
Guard revisions, current roles and destination ownership in the
write transaction, and use per-group endpoints for isolated changes.

* fix(webhook): confirm only accepted deliveries

Acknowledging before queue acceptance permanently loses deliveries.
Confirm only accepted work, release failed claims, scope replay
identities and strip tenant-supplied installation authority.

* fix(storage): prune abandoned parked payloads

Oversized payloads can remain after interrupted enqueue attempts.
Prune objects beyond their retry lifetime without deleting payloads
that a current delivery may still need.

* fix(console): expose api conflict statuses

Callers need the HTTP status to distinguish stale edits from other
failures and reload current data after a conflict.

* refactor(console): model group revisions

Carry server revisions explicitly so console writes can reject
changes based on an outdated group snapshot.

* fix(console): send loaded route revisions

Send the revision actually loaded with the route list and adopt the
server response after a save. Reload on conflict instead of silently
overwriting another editor.

* fix(console): guard fragment saves by revision

Fragments share a group version with other configuration writes.
Submit the loaded revision and reload on conflict to preserve
concurrent edits.

* fix(console): save groups with guarded writes

Whole-list saves can overwrite unrelated groups. Use per-group
revision-aware writes, retain rename results and confirm deletion
on the exact group, routes and version shown to the user.

* fix(console): reject edits of stale routes

A conflict reload can leave an older route editor open.
Compare its original value before saving so it cannot overwrite
the newly loaded route.

* fix(console): show overview loading failures

Failed overview requests must not look like an empty dashboard or
a successful refresh. Show the reason while retaining prior data.

* fix(console): explain rejected install bindings

Explain why an installation binding was refused instead of silently
returning to the console with no visible outcome.

* fix(console): allow selecting a group named all

The all sentinel collides with a valid group id.
Use a value that cannot be a group id so that group stays selectable.

* fix(console): discard stale log detail responses

An older detail request can finish after a newer selection or close.
Discard it so the dialog never shows the previously clicked record.

* fix(console): retain legacy group owners

Groups predating member roles store owners in adminIds.
Load those as owners so adding a member never drops all old owners.

* fix(console): lock member edits during saves

Edits made during a save are overwritten by its result.
Disable member controls and guard handlers until the write settles.

* fix(console): show invite loading failures

Failed invite loads must not look like no pending invites.
Show the error and discard responses from a previously shown group.

* fix(console): retain legacy discord log targets

A stored log target without platform means Discord.
Keep that meaning in the editor instead of clearing the target.

* fix(console): retain undisplayed forge sources

An editor that only displays some providers must preserve the others.
Keep stored custom sources when saving unrelated group settings.

* fix(console): lock the group editor while saving

Prevent edits while a group save is in flight and make super-only
installation settings readonly instead of silently ignoring edits.

* fix(console): lock route edits and show failures

Lock the route editor while saving and show fragment load failures
with a retry action rather than presenting a false empty list.

* fix(config): expose the legal contact override

Declare the public runtime key so the documented environment
override reaches legal pages in deployed Nuxt builds.

* build(deploy): migrate d1 before deployment

Apply pending D1 migrations before deploying code that requires
them. Stop deployment when migration fails instead of running
against an incomplete schema.

* docs(worker): align guides with runtime behavior

Synchronize both language guides, API references and examples with
the implemented authorization, storage and delivery behavior.
Document migration requirements and accepted operational limits.

* fix(storage): repair orphaned KV routes

Repair orphaned KV routes before serving KV-only configuration.

* fix(security): fully escape forge HTML

---------

Co-authored-by: RhenCloud <i@rhen.cloud>
This commit is contained in:
翅膀喵andRhenCloud authored and GitHub committed 2026-10-24 16:19:06 +08:00
1 parent dbf162bf74
commit 978c53eeb3
163 files changed
+18459 -3607

No files matched your search

+13
View File
@@ -56,3 +56,16 @@ export function headersFrom(event: H3Event): Record<string, string> {
});
return out;
}
/** Largest body the bot callback routes read (each handler checks it again). */
export const MAX_CALLBACK_BODY = 1024 * 1024;
/**
* A body declared larger than `max` bytes: answered 413 before the route
* reads (and copies) it. Bodies without a declared length are still checked
* by the handler once read.
*/
export function declaresBodyOver(event: H3Event, max = MAX_CALLBACK_BODY): boolean {
const length = Number(event.headers.get("content-length"));
return Number.isFinite(length) && length > max;
}
+72 -51
View File
@@ -1,19 +1,31 @@
import type { Env, Config, Route } from "./types";
import { log } from "./lib/log";
import { migrateRoutes, validateRoutes } from "./config/schema";
import { d1ConfigStore, type ConfigStore } from "./storage/config-store";
import type { Env, Config, Group, Route } from "./types";
import {
createConfigStore,
kvConfigStore,
type ConfigStore,
type GuardedWrite,
type LoadOptions,
} from "./storage/config-store";
import { canUseD1 } from "./storage/d1";
const CONFIG_CACHE_TTL = 300_000;
const ROUTES_KEY = "config:routes";
let configCache: { config: Config; expiresAt: number } | null = null;
const configStores = new WeakMap<KVNamespace, ConfigStore>();
export function getConfigStore(kv: KVNamespace): ConfigStore | null {
return configStores.get(kv) ?? null;
/**
* The configuration store bound to this KV namespace. Every request, queue
* batch and scheduled task registers the D1-backed store first (see
* `initConfigStore`); a namespace that was never registered — a test, or a
* deployment without D1 — reads and writes the KV-only store.
*/
export function getConfigStore(kv: KVNamespace): ConfigStore {
return configStores.get(kv) ?? kvConfigStore(kv);
}
export function initConfigStore(env: Env): ConfigStore {
return ensureStore(env);
const existing = configStores.get(env.KV);
if (existing && (existing.kind === "d1" || !canUseD1(env.DB))) return existing;
const store = createConfigStore(env.DB, env.KV);
configStores.set(env.KV, store);
return store;
}
export function resetConfigStore(kv?: KVNamespace): void {
@@ -23,52 +35,64 @@ export function resetConfigStore(kv?: KVNamespace): void {
}
}
function ensureStore(env: Env): ConfigStore {
let store = configStores.get(env.KV);
if (!store) {
store = d1ConfigStore(env.DB, env.KV);
configStores.set(env.KV, store);
}
return store;
}
export async function loadRoutes(kv: KVNamespace): Promise<Route[]> {
const store = configStores.get(kv);
if (store) return store.loadRoutes();
try {
const stored = await kv.get<Route[]>(ROUTES_KEY, "json");
if (stored) return validateRoutes(migrateRoutes(stored));
} catch (err) {
log.warn({ err }, "Failed to load routes from KV");
}
return [];
export async function loadRoutes(kv: KVNamespace, opts?: LoadOptions): Promise<Route[]> {
return getConfigStore(kv).loadRoutes(opts);
}
export async function saveRoutes(kv: KVNamespace, routes: Route[]): Promise<void> {
const store = configStores.get(kv);
if (store) {
await store.saveRoutes(routes);
configCache = null;
return;
}
await kv.put(ROUTES_KEY, JSON.stringify(routes));
configCache = null;
await getConfigStore(kv).saveRoutes(routes);
}
export function invalidateConfigCache(): void {
configCache = null;
export async function saveGroupRoutes(
kv: KVNamespace,
groupId: string,
routes: Route[],
opts?: GuardedWrite<Route[]>,
): Promise<Route[]> {
return getConfigStore(kv).saveGroupRoutes(groupId, routes, opts);
}
/** Drop every cached configuration snapshot held by this isolate. */
export function invalidateConfigCache(kv?: KVNamespace): void {
if (kv) configStores.get(kv)?.invalidateCache();
}
/** One group, read fresh (its row alone): see `ConfigStore.loadGroup`. */
export function loadGroupFresh(
env: Env,
id: string,
): Promise<{ group: Group | null; unreadable: boolean }> {
return initConfigStore(env).loadGroup(id);
}
/**
* Configuration and groups of one consistent moment, for a dispatch that
* needs both (routes of a renamed group must never meet its old id, nor new
* targets meet stale source restrictions).
*/
export async function loadConfigSnapshot(
env: Env,
opts?: LoadOptions,
): Promise<{ config: Config; groups: Group[] }> {
const store = initConfigStore(env);
const { routes, groups, unreadable } = await store.loadSnapshot(opts);
const config = buildConfig(env, routes);
if (unreadable?.length) config.unreadableGroups = unreadable;
return { config, groups };
}
/**
* The configuration for one dispatch. Each call returns a fresh object with
* its own `routes` array, so a caller narrowing the routes (tenant scoping)
* can never leak that change into another request served by this isolate.
*/
export async function loadConfig(env: Env): Promise<Config> {
if (configCache && Date.now() < configCache.expiresAt) {
return configCache.config;
}
initConfigStore(env);
return buildConfig(env, await loadRoutes(env.KV));
}
ensureStore(env);
const routes = await loadRoutes(env.KV);
const config: Config = {
function buildConfig(env: Env, routes: Route[]): Config {
return {
baseUrl: env.BASE_URL ?? "https://webhooker.example.workers.dev",
github: {
webhookSecret: env.GITHUB_WEBHOOK_SECRET,
@@ -80,9 +104,6 @@ export async function loadConfig(env: Env): Promise<Config> {
discord: {
token: env.DISCORD_TOKEN ?? "",
},
routes,
routes: [...routes],
};
configCache = { config, expiresAt: Date.now() + CONFIG_CACHE_TTL };
return config;
}
+6
View File
@@ -61,6 +61,7 @@ export const routeSchema = v.object({
export const groupMemberSchema = v.object({
login: v.string(),
role: v.picklist(["owner", "admin", "viewer"]),
userId: v.optional(v.string()),
});
export const forgeSourceSchema = v.object({
@@ -77,6 +78,11 @@ export const groupSchema = v.object({
owners: v.optional(v.array(v.string())),
providers: v.optional(v.array(v.picklist(["github", "gitea", "gitlab", "custom"]))),
installationId: v.optional(v.number()),
isolated: v.optional(v.boolean()),
instance: v.optional(v.string()),
instanceBackfilled: v.optional(v.boolean()),
legacyId: v.optional(v.string()),
webhookSecret: v.optional(v.nullable(v.string())),
emoji: v.optional(v.boolean()),
forgeSources: v.optional(v.array(forgeSourceSchema)),
lang: v.optional(v.string()),
File diff suppressed because it is too large. Load diff
+138 -35
View File
@@ -75,16 +75,44 @@ export const APP_COMMANDS = [
{ name: MSG_CMD_DEL, type: COMMAND_TYPE.MESSAGE },
];
async function shortHash(text: string): Promise<string> {
const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text));
return [...new Uint8Array(digest).slice(0, 8)]
.map((b) => b.toString(16).padStart(2, "0"))
.join("");
}
/**
* Registration markers are kept per application (and guild) and hold the hash
* of the command definitions last registered there: another application, or
* definitions that differ from the registered ones — newer or rolled back —
* register again instead of being skipped.
*/
function commandsHash(): Promise<string> {
return shortHash(JSON.stringify(APP_COMMANDS));
}
const applicationIds = new Map<string, string>();
export async function getApplicationId(env: Env): Promise<string | null> {
if (env.DISCORD_APPLICATION_ID) return env.DISCORD_APPLICATION_ID;
const token = env.DISCORD_TOKEN ?? "";
if (!token) return null;
// Cached per bot token: a token of another application never gets the
// id resolved for the previous one. Held in memory too (one KV read per
// isolate, not one per sync run).
const cacheKey = `config:discord-app-id:${await shortHash(token)}`;
const known = applicationIds.get(cacheKey);
if (known) return known;
try {
const cached = await env.KV.get("config:discord-app-id");
if (cached) return cached;
const cached = await env.KV.get(cacheKey);
if (cached) {
applicationIds.set(cacheKey, cached);
return cached;
}
} catch {
// fall through to the API
}
const token = env.DISCORD_TOKEN ?? "";
if (!token) return null;
const res = await fetch(`${DISCORD_API}/oauth2/applications/@me`, {
headers: { Authorization: `Bot ${token}` },
});
@@ -95,7 +123,8 @@ export async function getApplicationId(env: Env): Promise<string | null> {
const app = (await res.json()) as { id?: string };
if (app.id) {
try {
await env.KV.put("config:discord-app-id", app.id);
applicationIds.set(cacheKey, app.id);
await env.KV.put(cacheKey, app.id);
} catch {
// cache is best-effort
}
@@ -107,13 +136,15 @@ export async function getApplicationId(env: Env): Promise<string | null> {
export async function registerGlobalCommands(env: Env): Promise<void> {
const token = env.DISCORD_TOKEN ?? "";
if (!token) return;
const appId = await getApplicationId(env);
if (!appId) return;
const marker = `cmd:registered:global:${appId}`;
const hash = await commandsHash();
try {
if (await env.KV.get("cmd:registered:global")) return;
if ((await env.KV.get(marker)) === hash) return;
} catch {
// fall through and register
}
const appId = await getApplicationId(env);
if (!appId) return;
const res = await fetch(`${DISCORD_API}/applications/${appId}/commands`, {
method: "PUT",
headers: { Authorization: `Bot ${token}`, "Content-Type": "application/json" },
@@ -121,7 +152,7 @@ export async function registerGlobalCommands(env: Env): Promise<void> {
});
if (res.ok) {
try {
await env.KV.put("cmd:registered:global", "1", { expirationTtl: 86400 });
await env.KV.put(marker, hash, { expirationTtl: 86400 });
} catch {
// best-effort
}
@@ -132,38 +163,110 @@ export async function registerGlobalCommands(env: Env): Promise<void> {
}
}
const GUILD_PAGE_SIZE = 200;
/**
* Discord calls one run makes for guild commands — pages listed and
* registrations together — well within a Worker invocation's subrequest
* limit (50 on the Free plan, shared with the run's other work). A run picks
* up where the previous one stopped, so a bot in any number of guilds keeps
* making progress.
*/
const GUILD_CALLS_PER_RUN = 30;
/** Per application: which guild holds which definitions, and where the walk is. */
interface GuildRecord {
hashes: Record<string, string>;
/** The guild after which the next run continues ("" from the start). */
after: string;
}
export async function syncGuildCommands(env: Env): Promise<void> {
const token = env.DISCORD_TOKEN ?? "";
if (!token) return;
const appId = await getApplicationId(env);
if (!appId) return;
const res = await fetch(`${DISCORD_API}/users/@me/guilds`, {
headers: { Authorization: `Bot ${token}` },
});
if (!res.ok) {
const err = await res.text();
log.warn({ status: res.status, err }, "Failed to list guilds");
return;
}
const guilds = (await res.json()) as Array<{ id: string }>;
for (const guild of guilds) {
try {
if (await env.KV.get(`cmd:guild:${guild.id}`)) continue;
const r = await fetch(`${DISCORD_API}/applications/${appId}/guilds/${guild.id}/commands`, {
method: "PUT",
headers: { Authorization: `Bot ${token}`, "Content-Type": "application/json" },
body: JSON.stringify(APP_COMMANDS),
});
if (r.ok) {
await env.KV.put(`cmd:guild:${guild.id}`, "1");
log.info({ guildId: guild.id }, "Registered guild application commands");
} else {
const err = await r.text();
log.warn({ guildId: guild.id, status: r.status, err }, "Command registration failed");
}
} catch (err) {
log.warn({ guildId: guild.id, err: String(err) }, "Command registration failed");
const hash = await commandsHash();
// One KV record per application, read and written once per run (not one
// read per guild).
const recordKey = `cmd:guilds:${appId}`;
const stored = await env.KV.get<GuildRecord | Record<string, string>>(recordKey, "json").catch(
() => null,
);
const record: GuildRecord =
stored && typeof stored === "object" && "hashes" in stored
? {
hashes: { ...(stored as GuildRecord).hashes },
after: (stored as GuildRecord).after ?? "",
}
: { hashes: { ...((stored as Record<string, string> | null) ?? {}) }, after: "" };
const startedAfter = record.after;
let calls = GUILD_CALLS_PER_RUN;
let changed = false;
/** Stopped at a guild still to register: where to resume must be kept. */
let pending = false;
/** A guild needed registering this run (successful or not). */
let attempted = false;
walk: while (calls > 0) {
calls--;
const res = await fetch(
`${DISCORD_API}/users/@me/guilds?limit=${GUILD_PAGE_SIZE}${record.after ? `&after=${record.after}` : ""}`,
{ headers: { Authorization: `Bot ${token}` } },
);
if (!res.ok) {
const err = await res.text();
log.warn({ status: res.status, err }, "Failed to list guilds");
break;
}
const page = (await res.json()) as Array<{ id: string }>;
for (const guild of page) {
if (record.hashes[guild.id] !== hash) {
attempted = true;
// Out of calls: the next run resumes at this guild.
if (calls <= 0) {
pending = true;
break walk;
}
calls--;
try {
const r = await fetch(
`${DISCORD_API}/applications/${appId}/guilds/${guild.id}/commands`,
{
method: "PUT",
headers: { Authorization: `Bot ${token}`, "Content-Type": "application/json" },
body: JSON.stringify(APP_COMMANDS),
},
);
if (r.ok) {
record.hashes[guild.id] = hash;
changed = true;
log.info({ guildId: guild.id }, "Registered guild application commands");
} else {
const err = await r.text();
log.warn({ guildId: guild.id, status: r.status, err }, "Command registration failed");
}
} catch (err) {
log.warn({ guildId: guild.id, err: String(err) }, "Command registration failed");
}
}
record.after = guild.id;
}
if (page.length < GUILD_PAGE_SIZE) {
// The end of the list: the next run starts over (picking up guilds
// joined meanwhile, and redoing ones whose registration failed).
record.after = "";
break;
}
}
// Out of calls right after a full page, with work found on the way: the
// walk must resume past it (or the guilds after it would never be reached).
if (calls <= 0 && record.after !== "" && attempted) pending = true;
// Written only when something changed — a registration, a walk stopped
// with work left, or the cursor reset after one: a run that only checked
// writes nothing (KV writes are scarce; this runs every few minutes).
if (changed || pending || (record.after === "" && startedAfter !== "")) {
await env.KV.put(recordKey, JSON.stringify(record)).catch((err: unknown) =>
log.warn({ err: String(err) }, "Failed to record guild command registrations"),
);
}
}
+7 -1
View File
@@ -12,7 +12,13 @@ export class DiscordDriver implements PlatformDriver {
return { ok: false, error: "target.channelId is required", errorCode: "NO_TARGET" };
}
const token = env.DISCORD_TOKEN ?? "";
return sendMessage(token, channelId, renderNeutralMessage(message), target.threadId);
return sendMessage(
token,
channelId,
renderNeutralMessage(message),
target.threadId,
message.sendKey,
);
}
async edit(
+226 -67
View File
@@ -9,6 +9,7 @@ import {
closePullRequestAsUser,
} from "../../github/oauth";
import { getDiscordLink, removeDiscordLink } from "../../github/store";
import { idempotencyStore } from "../../lib/idempotency";
import type { Env } from "../../types";
import { MSG_CMD_ADD, MSG_CMD_EDIT, MSG_CMD_DEL } from "./commands";
@@ -22,16 +23,84 @@ const EPHEMERAL = 64;
// Modal custom_id encodings (delimiter '|' never appears in owner/repo).
const MODAL_ADD = "ghc|add|"; // ghc|add|owner|repo|issueNumber
const MODAL_EDIT = "ghc|edit|"; // ghc|edit|owner|repo|commentId
const MODAL_EDIT = "ghc|edit|"; // ghc|edit|owner|repo|commentId (read for older modals)
/** ghc|ed|owner|repo|commentId|base: an edit that remembers the text it was opened on. */
const MODAL_EDIT_BASE = "ghc|ed|";
/**
* ghc|m|{context id}: the context kept in KV instead — only when the custom
* id would pass Discord's 100 characters (a long owner/repo). KV writes are
* scarce (Workers Free), so the context travels in the custom id whenever
* it fits.
*/
const MODAL_CTX = "ghc|m|";
const MAX_CUSTOM_ID = 100;
/** Hex characters of the comment hash an edit modal remembers. */
const BASE_HASH_LENGTH = 16;
const MODAL_CTX_TTL_SECONDS = 86400;
interface ModalContext {
op: "add" | "edit";
owner: string;
repo: string;
/** Issue/PR number (add) or comment id (edit). */
ref: string;
/** SHA-256 (hex, possibly shortened) of the comment as the edit modal showed it. */
base?: string;
}
async function sha256Hex(text: string): Promise<string> {
const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text));
return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, "0")).join("");
}
async function modalCustomId(env: Env, context: ModalContext): Promise<string> {
const inline =
context.op === "add"
? `${MODAL_ADD}${context.owner}|${context.repo}|${context.ref}`
: `${MODAL_EDIT_BASE}${context.owner}|${context.repo}|${context.ref}|${context.base ?? ""}`;
if (inline.length <= MAX_CUSTOM_ID) return inline;
const contextId = crypto.randomUUID().replace(/-/g, "");
await env.KV.put(`discord-modal:${contextId}`, JSON.stringify(context), {
expirationTtl: MODAL_CTX_TTL_SECONDS,
});
return `${MODAL_CTX}${contextId}`;
}
async function modalContext(env: Env, customId: string): Promise<ModalContext | null> {
if (customId.startsWith(MODAL_CTX)) {
return env.KV.get<ModalContext>(`discord-modal:${customId.slice(MODAL_CTX.length)}`, "json");
}
if (customId.startsWith(MODAL_EDIT_BASE)) {
const rest = customId.slice(MODAL_EDIT_BASE.length);
const cut = rest.lastIndexOf("|");
const [owner, repo, ref] = splitThree(rest.slice(0, cut));
const base = rest.slice(cut + 1);
return cut > 0 && owner && repo && ref ? { op: "edit", owner, repo, ref, base } : null;
}
for (const [prefix, op] of [
[MODAL_ADD, "add"],
[MODAL_EDIT, "edit"],
] as const) {
if (!customId.startsWith(prefix)) continue;
const [owner, repo, ref] = splitThree(customId.slice(prefix.length));
return owner && repo && ref ? { op, owner, repo, ref } : null;
}
return null;
}
// PR notification button custom_id encodings.
const BTN_MERGE = "ghpr|merge|"; // ghpr|merge|owner|repo|pullNumber
const BTN_CLOSE = "ghpr|close|"; // ghpr|close|owner|repo|pullNumber
// Comment link (has the comment id); check this BEFORE the plain issue regex.
// Both only accept github.com itself (not e.g. `evilgithub.com`): the actions
// run against the GitHub API.
const GITHUB_COMMENT_RE =
/github\.com\/([^/\s]+)\/([^/\s]+)\/(?:issues|pull)\/\d+#issuecomment-(\d+)/;
const GITHUB_ISSUE_RE = /github\.com\/([^/\s]+)\/([^/\s]+)\/(?:issues|pull)\/(\d+)/;
/https?:\/\/(?:www\.)?github\.com\/([^/\s]+)\/([^/\s]+)\/(?:issues|pull)\/\d+#issuecomment-(\d+)/;
const GITHUB_ISSUE_RE =
/https?:\/\/(?:www\.)?github\.com\/([^/\s]+)\/([^/\s]+)\/(?:issues|pull)\/(\d+)/;
/** Discord caps a text input's value (and max_length) at 4000 characters. */
const MAX_MODAL_TEXT = 4000;
/**
* Split a pipe-delimited custom-id suffix into [owner, repo, number]. The
@@ -46,13 +115,23 @@ function splitThree(rest: string): [string, string, string] {
return [owner ?? "", repo ?? "", number ?? ""];
}
interface DiscordUser {
id?: string;
username?: string;
global_name?: string | null;
}
interface Interaction {
id: string;
/** The application the interaction belongs to (follow-up webhooks use it). */
application_id?: string;
token: string;
type: number;
channel_id?: string;
member?: { user?: { id?: string } };
user?: { id?: string };
member?: { user?: DiscordUser };
user?: DiscordUser;
/** The message a component (button) is attached to. */
message?: { id?: string };
data?: Record<string, unknown>;
}
@@ -146,6 +225,15 @@ export async function handleInteractionRequest(request: Request, env: Env): Prom
});
}
// Discord does not redeliver interactions, but a replayed request (same
// signed body within the timestamp window) must not run its action twice.
if (
interaction.id &&
!(await idempotencyStore(env.DB, env.KV).claim(`bot:discord:${interaction.id}`, 3600))
) {
return new Response(null, { status: 202 });
}
// Handle the interaction via the callback webhook; respond 202 with no body
// as required for interactions received over the HTTP endpoint.
await handleInteraction(env, interaction).catch((err) =>
@@ -155,19 +243,22 @@ export async function handleInteractionRequest(request: Request, env: Env): Prom
}
async function handleInteraction(env: Env, interaction: Interaction): Promise<void> {
const userId = interaction.member?.user?.id ?? interaction.user?.id ?? null;
const user = interaction.member?.user ?? interaction.user;
const userId = user?.id ?? null;
const id = interaction.id;
const token = interaction.token;
const applicationId = interaction.application_id ?? env.DISCORD_APPLICATION_ID ?? "";
if (interaction.type === INTERACTION_TYPE.BUTTON) {
const data = interaction.data as { custom_id?: string; message?: { id?: string } };
const data = interaction.data as { custom_id?: string };
return handleButton(
env,
id,
token,
applicationId,
userId,
interaction.channel_id,
data.message?.id,
interaction.message?.id,
data.custom_id,
);
}
@@ -203,13 +294,15 @@ async function handleInteraction(env: Env, interaction: Interaction): Promise<vo
if (!op) return;
const target = data.target_id ? data.resolved?.messages?.[data.target_id] : undefined;
const source = target?.embeds?.[0]?.url ?? target?.content ?? "";
return commentOp(env, id, token, userId, op, source);
return commentOp(env, id, token, applicationId, userId, op, source);
}
// Slash command /gh ...
if (data.name === "gh" && data.type === COMMAND_TYPE.CHAT_INPUT) {
const top = data.options?.[0];
if (top?.name === "login") return cmdLogin(env, id, token, userId);
if (top?.name === "login") {
return cmdLogin(env, id, token, userId, user?.global_name || user?.username);
}
if (top?.name === "logout") return cmdLogout(env, id, token, userId);
if (top?.name === "comment") {
const sub = top.options?.[0];
@@ -223,7 +316,7 @@ async function handleInteraction(env: Env, interaction: Interaction): Promise<vo
: null;
if (!op) return;
const link = sub?.options?.find((o) => o.name === "link")?.value ?? "";
return commentOp(env, id, token, userId, op, link);
return commentOp(env, id, token, applicationId, userId, op, link);
}
return;
}
@@ -231,7 +324,7 @@ async function handleInteraction(env: Env, interaction: Interaction): Promise<vo
}
if (interaction.type === INTERACTION_TYPE.MODAL_SUBMIT) {
return modalSubmit(env, id, token, userId, interaction.data);
return modalSubmit(env, id, token, applicationId, userId, interaction.data);
}
}
@@ -243,12 +336,13 @@ async function respond(env: Env, id: string, token: string, content: string): Pr
});
}
/** Answer an interaction; whether Discord accepted the answer. */
async function interactionCallback(
env: Env,
id: string,
token: string,
body: unknown,
): Promise<void> {
): Promise<boolean> {
const res = await fetch(`${DISCORD_API}/interactions/${id}/${token}/callback`, {
method: "POST",
headers: { "Content-Type": "application/json" },
@@ -258,11 +352,23 @@ async function interactionCallback(
const err = await res.text();
log.warn({ status: res.status, err }, "Interaction callback failed");
}
return res.ok;
}
/** Replace the deferred (ephemeral) response body with the final result. */
async function updateOriginal(env: Env, id: string, token: string, content: string): Promise<void> {
const res = await fetch(`${DISCORD_API}/interactions/${id}/${token}/messages/@original`, {
/**
* Replace the deferred (ephemeral) response body with the final result. The
* original response is edited through the application's follow-up webhook.
*/
async function updateOriginal(
applicationId: string,
token: string,
content: string,
): Promise<void> {
if (!applicationId) {
log.warn("Cannot update interaction response: application id unknown");
return;
}
const res = await fetch(`${DISCORD_API}/webhooks/${applicationId}/${token}/messages/@original`, {
method: "PATCH",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ content }),
@@ -273,6 +379,35 @@ async function updateOriginal(env: Env, id: string, token: string, content: stri
}
}
/**
* Acknowledge first (deferred, ephemeral), then run the GitHub action and
* show its outcome: Discord fails an interaction not answered within 3
* seconds, and a slow GitHub call must not leave a done action looking failed
* (the user would submit it again).
*/
async function deferThen(
env: Env,
id: string,
token: string,
applicationId: string,
work: () => Promise<string>,
): Promise<void> {
// Not acknowledged: Discord shows the interaction as failed, so acting on
// GitHub now would leave a done action looking failed (and resubmitted).
const acked = await interactionCallback(env, id, token, {
type: CALLBACK_TYPE.DEFERRED_MESSAGE,
data: { flags: EPHEMERAL },
});
if (!acked) return;
let content: string;
try {
content = await work();
} catch (err) {
content = errText(err);
}
await updateOriginal(applicationId, token, content);
}
/**
* PR notification buttons: merge or close the PR as the clicker's linked
* GitHub account. The clicker must have run `/gh login` first.
@@ -281,6 +416,7 @@ async function handleButton(
env: Env,
id: string,
token: string,
applicationId: string,
userId: string | null,
channelId: string | undefined,
messageId: string | undefined,
@@ -308,17 +444,18 @@ async function handleButton(
if (!owner || !repo || !number) return;
// Acknowledge first (deferred, ephemeral) so the clicker sees a spinner
// while the GitHub API call runs.
await interactionCallback(env, id, token, {
// while the GitHub API call runs; unacknowledged, nothing is done.
const acked = await interactionCallback(env, id, token, {
type: CALLBACK_TYPE.DEFERRED_MESSAGE,
data: { flags: EPHEMERAL },
});
if (!acked) return;
try {
if (op === "merge") {
await mergePullRequestAsUser(env.KV, githubUserId, owner, repo, Number(number));
await mergePullRequestAsUser(env, githubUserId, owner, repo, Number(number));
} else {
await closePullRequestAsUser(env.KV, githubUserId, owner, repo, Number(number));
await closePullRequestAsUser(env, githubUserId, owner, repo, Number(number));
}
// Remove the buttons from the notification so nobody double-clicks.
if (channelId && messageId) {
@@ -332,13 +469,19 @@ async function handleButton(
}).catch((err) => log.warn({ err: String(err) }, "Failed to strip PR buttons"));
}
const label = op === "merge" ? "合并" : "关闭";
await updateOriginal(env, id, token, `✅ 已${label} PR ${owner}/${repo}#${number}`);
await updateOriginal(applicationId, token, `✅ 已${label} PR ${owner}/${repo}#${number}`);
} catch (err) {
await updateOriginal(env, id, token, errText(err));
await updateOriginal(applicationId, token, errText(err));
}
}
async function cmdLogin(env: Env, id: string, token: string, userId: string | null): Promise<void> {
async function cmdLogin(
env: Env,
id: string,
token: string,
userId: string | null,
name: string | undefined,
): Promise<void> {
if (!userId) return respond(env, id, token, "无法识别你的 Discord 账号。");
const clientId = env.GITHUB_CLIENT_ID;
if (!clientId) return respond(env, id, token, "服务器未配置 GitHub OAuth(GITHUB_CLIENT_ID)。");
@@ -346,7 +489,11 @@ async function cmdLogin(env: Env, id: string, token: string, userId: string | nu
const state = crypto.randomUUID().replace(/-/g, "");
await env.KV.put(
`state:${state}`,
JSON.stringify({ redirectTo: "/", discordUserId: userId, expiresAt: Date.now() + 600_000 }),
JSON.stringify({
redirectTo: "/",
link: { platform: "discord", userId, name },
expiresAt: Date.now() + 600_000,
}),
{ expirationTtl: 600 },
);
const url = getOAuthURL(clientId, state);
@@ -386,6 +533,7 @@ async function commentOp(
env: Env,
id: string,
token: string,
applicationId: string,
userId: string | null,
op: "add" | "edit" | "del",
source: string,
@@ -409,7 +557,7 @@ async function commentOp(
env,
id,
token,
`${MODAL_ADD}${m[1]}|${m[2]}|${m[3]}`,
await modalCustomId(env, { op: "add", owner: m[1]!, repo: m[2]!, ref: m[3]! }),
`评论 ${m[1]}/${m[2]}#${m[3]}`,
);
}
@@ -427,27 +575,41 @@ async function commentOp(
const [, owner, repo, commentId] = m;
if (op === "del") {
try {
await deleteCommentAsUser(env.KV, githubUserId, owner!, repo!, Number(commentId));
return respond(env, id, token, `已删除评论 ${owner}/${repo}#issuecomment-${commentId}。`);
} catch (err) {
return respond(env, id, token, errText(err));
}
return deferThen(env, id, token, applicationId, async () => {
await deleteCommentAsUser(env, githubUserId, owner!, repo!, Number(commentId));
return `已删除评论 ${owner}/${repo}#issuecomment-${commentId}。`;
});
}
// edit: fetch current body to prefill the modal.
let prefill = "";
try {
const { body } = await getCommentAsUser(env.KV, githubUserId, owner!, repo!, Number(commentId));
const { body } = await getCommentAsUser(env, githubUserId, owner!, repo!, Number(commentId));
prefill = body;
} catch (err) {
return respond(env, id, token, errText(err));
}
// The modal can only hold part of a longer comment, and saving it would
// replace the whole comment with that part: such comments are edited on GitHub.
if (prefill.length > MAX_MODAL_TEXT) {
return respond(
env,
id,
token,
`这条评论超过 ${MAX_MODAL_TEXT} 字,无法在 Discord 中完整编辑,请到 GitHub 上编辑:${m[0]}`,
);
}
return openCommentModal(
env,
id,
token,
`${MODAL_EDIT}${owner}|${repo}|${commentId}`,
await modalCustomId(env, {
op: "edit",
owner: owner!,
repo: repo!,
ref: commentId!,
base: (await sha256Hex(prefill)).slice(0, BASE_HASH_LENGTH),
}),
`编辑评论 #${commentId}`,
prefill,
);
@@ -477,8 +639,8 @@ async function openCommentModal(
label: "评论内容",
style: 2,
required: true,
max_length: 2000,
value: prefill.slice(0, 2000) || undefined,
max_length: MAX_MODAL_TEXT,
value: prefill || undefined,
},
],
},
@@ -491,6 +653,7 @@ async function modalSubmit(
env: Env,
id: string,
token: string,
applicationId: string,
userId: string | null,
data: unknown,
): Promise<void> {
@@ -501,51 +664,47 @@ async function modalSubmit(
const customId = d.custom_id;
if (!userId || !customId) return;
const body = d.components?.[0]?.components?.find((c) => c.custom_id === "body")?.value?.trim();
if (!body) return respond(env, id, token, "评论内容不能为空。");
// Submitted as written (an indented first line is a Markdown code block):
// blank only decides emptiness.
const body = d.components?.[0]?.components?.find((c) => c.custom_id === "body")?.value ?? "";
if (!body.trim()) return respond(env, id, token, "评论内容不能为空。");
const githubUserId = await getDiscordLink(env.DB, userId);
if (!githubUserId) {
return respond(env, id, token, "你还没有绑定 GitHub 账号,请先使用 `/gh login`。");
}
// ghc|add|owner|repo|issueNumber
if (customId.startsWith(MODAL_ADD)) {
const [owner, repo, number] = splitThree(customId.slice(MODAL_ADD.length));
if (!owner || !repo || !number)
return respond(env, id, token, "内部错误:无法解析目标 issue。");
try {
const context = await modalContext(env, customId);
if (!context) {
return respond(env, id, token, "这个评论框已失效,请重新打开。");
}
const { owner, repo, ref } = context;
if (context.op === "add") {
return deferThen(env, id, token, applicationId, async () => {
const { htmlUrl, login } = await commentAsUser(
env.KV,
env,
githubUserId,
owner,
repo,
Number(number),
Number(ref),
body,
);
return respond(env, id, token, `已以 **@${login}** 身份评论:${htmlUrl}`);
} catch (err) {
return respond(env, id, token, errText(err));
}
return `已以 **@${login}** 身份评论:${htmlUrl}`;
});
}
// ghc|edit|owner|repo|commentId
if (customId.startsWith(MODAL_EDIT)) {
const [owner, repo, commentId] = splitThree(customId.slice(MODAL_EDIT.length));
if (!owner || !repo || !commentId)
return respond(env, id, token, "内部错误:无法解析目标评论。");
try {
const { htmlUrl } = await editCommentAsUser(
env.KV,
githubUserId,
owner,
repo,
Number(commentId),
body,
);
return respond(env, id, token, `已更新评论:${htmlUrl}`);
} catch (err) {
return respond(env, id, token, errText(err));
return deferThen(env, id, token, applicationId, async () => {
// Changed on GitHub since the modal showed it: saving would silently
// replace those changes (GitHub has no conditional update — this check
// narrows the window to the moment between it and the save).
if (context.base) {
const { body: current } = await getCommentAsUser(env, githubUserId, owner, repo, Number(ref));
if ((await sha256Hex(current)).slice(0, context.base.length) !== context.base) {
return "这条评论在你打开编辑框后已被修改。为避免覆盖这些修改,请重新打开编辑框。";
}
}
}
const { htmlUrl } = await editCommentAsUser(env, githubUserId, owner, repo, Number(ref), body);
return `已更新评论:${htmlUrl}`;
});
}
+68 -8
View File
@@ -9,6 +9,54 @@ import {
splitMessageTitle,
} from "../../formatters/helpers";
/** Discord's cap on the combined characters of all embeds in a message. */
const MAX_EMBED_TOTAL = 6000;
/** Discord's caps on a button's custom_id and label. */
const MAX_CUSTOM_ID = 100;
const MAX_BUTTON_LABEL = 80;
type Embed = NonNullable<FormattedMessage["embeds"]>[number];
function embedLength(embed: Embed): number {
return (
(embed.title?.length ?? 0) +
(embed.description?.length ?? 0) +
(embed.author?.name.length ?? 0) +
(embed.footer?.text.length ?? 0) +
(embed.fields ?? []).reduce((n, f) => n + f.name.length + f.value.length, 0)
);
}
/** Description length kept before any field is given up. */
const MIN_DESCRIPTION = 1024;
/** Shorten the description by `over` characters (never below `floor`); returns what is left over. */
function shrinkDescription(embed: Embed, over: number, floor: number): number {
const text = embed.description;
if (!text || over <= 0 || text.length <= floor) return over;
const keep = Math.max(floor, text.length - over - 1);
embed.description = keep > 0 ? `${text.slice(0, keep)}…` : undefined;
return over - (text.length - (embed.description?.length ?? 0));
}
/**
* Fit an embed into the 6000-character total so the request is never
* rejected whole: a long description shrinks first (down to 1024 characters),
* then trailing fields are dropped, then the description shrinks further.
*/
function fitEmbed(embed: Embed): Embed {
let over = embedLength(embed) - MAX_EMBED_TOTAL;
if (over <= 0) return embed;
const out: Embed = { ...embed, fields: embed.fields ? [...embed.fields] : undefined };
over = shrinkDescription(out, over, MIN_DESCRIPTION);
while (over > 0 && out.fields && out.fields.length > 0) {
const f = out.fields.pop()!;
over -= f.name.length + f.value.length;
}
shrinkDescription(out, over, 0);
return out;
}
function toStyle(style: NeutralActionStyle): number {
switch (style) {
case "primary":
@@ -20,10 +68,16 @@ function toStyle(style: NeutralActionStyle): number {
}
}
/**
* Most roles one message mentions: well within Discord's 100 allowed-mention
* roles and 2000 content characters (a mention takes at most 24 with its
* space). Routes are validated against it; this is the safety net.
*/
export const MAX_ROLE_MENTIONS = 50;
export function renderNeutralMessage(message: NeutralMessage): FormattedMessage {
const content = message.mentionRoleIds?.length
? message.mentionRoleIds.map((id) => `<@&${id}>`).join(" ")
: undefined;
const roleIds = [...new Set(message.mentionRoleIds ?? [])].slice(0, MAX_ROLE_MENTIONS);
const content = roleIds.length ? roleIds.map((id) => `<@&${id}>`).join(" ") : undefined;
// Discord embed titles can only be linked as a whole, so only the repo head
// goes into the title (linked to the event's object URL); the `: subject`
@@ -41,10 +95,16 @@ export function renderNeutralMessage(message: NeutralMessage): FormattedMessage
[message.forge?.name, message.footer].filter(Boolean).join(" · "),
MAX_FOOTER,
);
// Buttons whose id would exceed Discord's limit (very long repo names) are
// left out rather than failing the whole message.
const actions = (message.actions ?? []).filter((a) => a.id.length <= MAX_CUSTOM_ID);
return {
content,
// Only the route's own roles may ping — never @everyone, @here or users,
// whatever text ends up in the message.
allowed_mentions: { parse: [], roles: roleIds },
embeds: [
{
fitEmbed({
title: cap(head, MAX_TITLE),
url: message.url,
color: message.color,
@@ -63,16 +123,16 @@ export function renderNeutralMessage(message: NeutralMessage): FormattedMessage
})),
footer: footerText ? { text: footerText, icon_url: message.forge?.iconUrl } : undefined,
timestamp: message.timestamp,
},
}),
],
components: message.actions?.length
components: actions.length
? [
{
type: 1,
components: message.actions.map((action) => ({
components: actions.map((action) => ({
type: 2,
style: toStyle(action.style),
label: action.label,
label: cap(action.label, MAX_BUTTON_LABEL),
custom_id: action.id,
})),
},
+45 -18
View File
@@ -7,6 +7,18 @@ interface DiscordMessage {
id?: string;
}
/**
* Longest rate-limit wait (seconds) slept inside one request; a longer one is
* returned as a 429 failure so the delivery queue retries it later instead.
*/
const MAX_INLINE_RETRY_AFTER = 10;
/**
* Longest a single API call may take. Bounded, so a message edit always ends
* well within the update-chain lock it runs under (see `deliverUpdate`).
*/
export const REQUEST_TIMEOUT_MS = 15_000;
async function request(
url: string,
method: string,
@@ -23,13 +35,24 @@ async function request(
"Content-Type": "application/json",
},
body: JSON.stringify(message),
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
});
if (res.status === 429) {
const rateLimit = (await res.json()) as { retry_after?: number };
const retryAfter = (rateLimit.retry_after ?? 1) * 1000;
const rateLimit = (await res.json().catch(() => ({}))) as { retry_after?: number };
const seconds = Number(rateLimit.retry_after);
const retryAfter = Number.isFinite(seconds) && seconds > 0 ? seconds : 1;
log.warn({ retryAfter, attempt }, "Rate limited");
await new Promise((r) => setTimeout(r, retryAfter));
if (retryAfter > MAX_INLINE_RETRY_AFTER || attempt === 2) {
return {
ok: false,
error: `Rate limited (retry_after=${retryAfter})`,
errorCode: "RATE_LIMITED",
status: 429,
attempts: attempt + 1,
};
}
await new Promise((r) => setTimeout(r, retryAfter * 1000));
continue;
}
@@ -58,15 +81,11 @@ async function request(
};
}
let messageId: string | undefined;
try {
const data = (await res.json()) as DiscordMessage;
messageId = data.id;
} catch {
// ignore malformed success body
}
return { ok: true, status: res.status, messageId, attempts: attempt + 1 };
// A success whose body cannot be read (cut off, timed out) is not known
// to have happened: tried again — a create carries its nonce, so it is
// never doubled.
const data = (await res.json()) as DiscordMessage | null;
return { ok: true, status: res.status, messageId: data?.id, attempts: attempt + 1 };
} catch (err) {
log.error({ err, attempt, channelId }, "Failed to send message");
if (attempt === 2)
@@ -83,11 +102,19 @@ export async function sendMessage(
channelId: string,
message: unknown,
threadId?: string,
sendKey?: string,
): Promise<SendResult> {
const url = threadId
? `${DISCORD_API}/channels/${threadId}/messages`
: `${DISCORD_API}/channels/${channelId}/messages`;
return request(url, "POST", token, message, channelId);
// Ids are path segments: encoded, so no stored value can address another
// resource (`x/../123` is never the channel `123`).
const url = `${DISCORD_API}/channels/${encodeURIComponent(threadId || channelId)}/messages`;
// One nonce for every attempt of this send — and, given a `sendKey`, for
// every attempt of the delivery: when a create went through but its
// response was lost, Discord answers the retry with that same message
// instead of posting a second one (`enforce_nonce`, a few minutes' window).
const nonce = (sendKey ?? crypto.randomUUID().replace(/-/g, "")).slice(0, 25);
const body =
message && typeof message === "object" ? { ...message, nonce, enforce_nonce: true } : message;
return request(url, "POST", token, body, channelId);
}
export async function editMessage(
@@ -97,7 +124,7 @@ export async function editMessage(
message: unknown,
threadId?: string,
): Promise<SendResult> {
const base = threadId ?? channelId;
const url = `${DISCORD_API}/channels/${base}/messages/${messageId}`;
const base = encodeURIComponent(threadId || channelId);
const url = `${DISCORD_API}/channels/${base}/messages/${encodeURIComponent(messageId)}`;
return request(url, "PATCH", token, message, channelId);
}
+13 -3
View File
@@ -1,6 +1,6 @@
import type { RouteTarget, Env, NeutralMessage } from "../../types";
import type { PlatformDriver, SendResult } from "../types";
import { getTenantAccessToken, sendMessage, updateMessage } from "./rest";
import { forgetRejectedToken, getTenantAccessToken, sendMessage, updateMessage } from "./rest";
import { renderNeutralMessage } from "./render";
export class FeishuDriver implements PlatformDriver {
@@ -17,9 +17,14 @@ export class FeishuDriver implements PlatformDriver {
ok: false,
error: tokenRes.error ?? "No token",
errorCode: tokenRes.errorCode ?? "NO_TOKEN",
status: tokenRes.status,
};
}
return sendMessage(tokenRes.token, chatId, renderNeutralMessage(message));
return forgetRejectedToken(
await sendMessage(tokenRes.token, chatId, renderNeutralMessage(message), message.sendKey),
env,
tokenRes.token,
);
}
async edit(
@@ -38,8 +43,13 @@ export class FeishuDriver implements PlatformDriver {
ok: false,
error: tokenRes.error ?? "No token",
errorCode: tokenRes.errorCode ?? "NO_TOKEN",
status: tokenRes.status,
};
}
return updateMessage(tokenRes.token, messageId, renderNeutralMessage(message));
return forgetRejectedToken(
await updateMessage(tokenRes.token, messageId, renderNeutralMessage(message)),
env,
tokenRes.token,
);
}
}
+15 -2
View File
@@ -26,8 +26,21 @@ function colorToTemplate(color?: number): FeishuTemplate {
return "blue";
}
/**
* Text from the forge (titles, bodies, names) is anyone's to write: an
* `<at id=all></at>` in an issue title would make the bot notify the whole
* chat. The card never mentions anyone itself, so every `<at` tag is broken
* up (a zero-width space after `<`) and shows as text.
*/
function noMentions(content: string): string {
// Escape every angle bracket, not only `<at`. Replacing just the mention
// prefix still leaves values such as `<script` in Feishu markdown, where an
// incomplete tag can be interpreted as HTML by the client.
return content.replace(/</g, "&lt;").replace(/>/g, "&gt;");
}
function mdText(content: string): { tag: "lark_md"; content: string } {
return { tag: "lark_md", content: cap(content, MAX_MARKDOWN) };
return { tag: "lark_md", content: cap(noMentions(content), MAX_MARKDOWN) };
}
function divMarkdown(content: string): { tag: "div"; text: { tag: "lark_md"; content: string } } {
@@ -92,7 +105,7 @@ export function renderNeutralMessage(message: NeutralMessage): Record<string, un
if (elements.length) elements.push({ tag: "hr" });
elements.push({
tag: "note",
elements: [{ tag: "lark_md", content: cap(meta.join(" · "), MAX_NOTE) }],
elements: [{ tag: "lark_md", content: cap(noMentions(meta.join(" · ")), MAX_NOTE) }],
});
}
+166 -9
View File
@@ -1,9 +1,16 @@
import { log } from "../../lib/log";
import { sharedFlight, type Flight } from "../../lib/shared-flight";
import type { Env } from "../../types";
import type { SendResult } from "../types";
const FEISHU_API = "https://open.feishu.cn";
const TOKEN_KEY = "feishu:token";
/**
* The cached tenant token of the configured app: switching apps (another
* FEISHU_APP_ID) never sends with the previous app's token.
*/
export function tokenKey(env: Env): string {
return `feishu:token:${env.FEISHU_APP_ID?.trim() ?? ""}`;
}
interface FeishuResponse {
code?: number;
@@ -12,6 +19,29 @@ interface FeishuResponse {
error?: { message?: string };
}
/** Feishu business codes for "slow down" (group message rate limit, API frequency limit). */
const RATE_LIMIT_CODES = new Set([230020, 99991400]);
/** Feishu business codes for an invalid or expired tenant access token. */
const TOKEN_CODES = new Set([99991661, 99991663, 99991664, 99991668]);
/**
* The error code of a failed call: transient business codes map to the codes
* the delivery queue retries (`RATE_LIMITED`, `FEISHU_TOKEN`), whatever HTTP
* status Feishu answered with.
*/
function errorCodeOf(status: number, code: number | undefined): string {
if (code != null && RATE_LIMIT_CODES.has(code)) return "RATE_LIMITED";
if (code != null && TOKEN_CODES.has(code)) return "FEISHU_TOKEN";
if (status >= 500) return "FEISHU_5XX";
return code != null && code !== 0 ? `FEISHU_${code}` : "FEISHU_ERROR";
}
/**
* Longest a single API call may take. Bounded, so a message edit always ends
* well within the update-chain lock it runs under (see `deliverUpdate`).
*/
export const REQUEST_TIMEOUT_MS = 15_000;
async function feishuRequest(
url: string,
method: string,
@@ -33,18 +63,35 @@ async function feishuRequest(
method,
headers,
body: JSON.stringify(body),
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
});
lastStatus = res.status;
if (res.status === 429) {
const retryAfter = Number(res.headers.get("retry-after") || "1");
const header = Number(res.headers.get("retry-after") || "1");
const retryAfter = Number.isFinite(header) && header > 0 ? header : 1;
lastError = `Rate limited (retry_after=${retryAfter})`;
log.warn({ retryAfter, attempt, label }, "Feishu rate limited");
if (retryAfter > 10 || attempt === 2) {
return {
ok: false,
error: lastError,
errorCode: "RATE_LIMITED",
status: 429,
attempts: attempt + 1,
};
}
await new Promise((r) => setTimeout(r, retryAfter * 1000));
continue;
}
const data = (await res.json().catch(() => null)) as FeishuResponse | null;
// A 200 whose body cannot be read (or carries no result code) is not
// known to have succeeded: tried again (the request's uuid keeps a
// send from doubling).
if (res.ok && (!data || typeof data.code !== "number")) {
throw new Error("Feishu answered without a result code");
}
if (!res.ok) {
lastError = data?.msg ?? data?.error?.message ?? `HTTP ${res.status}`;
@@ -55,7 +102,7 @@ async function feishuRequest(
return {
ok: false,
error: lastError,
errorCode: res.status >= 500 ? "FEISHU_5XX" : "FEISHU_ERROR",
errorCode: errorCodeOf(res.status, data?.code),
status: res.status,
attempts: attempt + 1,
};
@@ -70,7 +117,7 @@ async function feishuRequest(
return {
ok: false,
error: lastError,
errorCode: `FEISHU_${data.code}`,
errorCode: errorCodeOf(res.status, data.code),
status: res.status,
attempts: attempt + 1,
};
@@ -107,6 +154,15 @@ async function feishuRequest(
};
}
/** How long this isolate uses a token before reading KV again. */
const HELD_TOKEN_MS = 5 * 60_000;
/**
* A failed token request is answered from memory this long: targets sent one
* after another in the same delivery then fail at once instead of each
* reading KV and asking Feishu again (the delivery is retried later anyway).
*/
const FAILED_TOKEN_MS = 5_000;
interface TokenResult {
ok: boolean;
token?: string;
@@ -115,6 +171,52 @@ interface TokenResult {
status?: number;
}
/**
* Tokens this isolate holds (per cache key), and refreshes in flight: routes
* sending in parallel share one token request and one KV write instead of
* each fetching and storing its own.
*/
const heldTokens = new Map<string, { token: string; until: number }>();
const refreshing = new Map<string, Flight<TokenResult>>();
const failedTokens = new Map<string, { result: TokenResult; until: number }>();
/** Tokens Feishu refused (bounded): never taken from KV again. */
const rejected = new Set<string>();
/** Refused tokens whose KV copy this isolate removed (or is removing). */
const cleared = new Set<string>();
/**
* A rejected token is dropped from the cache, so the retry fetches a new one.
* From memory every time; from KV once per isolate (every target refused
* with it would otherwise delete the same key again) — tried again by the
* next refusal when that delete fails — and never a newer token another
* request stored meanwhile.
*/
export async function forgetRejectedToken(
result: SendResult,
env: Env,
token: string,
): Promise<SendResult> {
if (result.errorCode !== "FEISHU_TOKEN") return result;
const key = tokenKey(env);
if (heldTokens.get(key)?.token === token) heldTokens.delete(key);
if (!rejected.has(token)) {
if (rejected.size >= 100) {
rejected.clear();
cleared.clear();
}
rejected.add(token);
}
if (!cleared.has(token)) {
cleared.add(token);
try {
if ((await env.KV.get(key)) === token) await env.KV.delete(key);
} catch {
cleared.delete(token);
}
}
return result;
}
export async function getTenantAccessToken(env: Env): Promise<TokenResult> {
const appId = env.FEISHU_APP_ID?.trim();
const appSecret = env.FEISHU_APP_SECRET?.trim();
@@ -126,8 +228,39 @@ export async function getTenantAccessToken(env: Env): Promise<TokenResult> {
};
}
const cached = await env.KV.get(TOKEN_KEY);
if (cached) return { ok: true, token: cached };
const key = tokenKey(env);
const held = heldTokens.get(key);
if (held && Date.now() < held.until) return { ok: true, token: held.token };
const failed = failedTokens.get(key);
if (failed && Date.now() < failed.until) return failed.result;
return sharedFlight(
refreshing,
key,
async () => {
const result = await fetchTenantAccessToken(env, appId, appSecret);
if (result.ok) failedTokens.delete(key);
else failedTokens.set(key, { result, until: Date.now() + FAILED_TOKEN_MS });
return result;
},
REQUEST_TIMEOUT_MS + 5_000,
);
}
async function fetchTenantAccessToken(
env: Env,
appId: string,
appSecret: string,
): Promise<TokenResult> {
const key = tokenKey(env);
const cached = await env.KV.get(key);
// A copy Feishu already refused (its KV delete failed or has not spread
// yet) is not used again: a new token is requested instead.
if (cached && !rejected.has(cached)) {
// Held a few minutes here: KV is read again now and then, so a token
// another isolate replaced is picked up.
heldTokens.set(key, { token: cached, until: Date.now() + HELD_TOKEN_MS });
return { ok: true, token: cached };
}
const url = `${FEISHU_API}/open-apis/auth/v3/tenant_access_token/internal`;
try {
@@ -135,6 +268,7 @@ export async function getTenantAccessToken(env: Env): Promise<TokenResult> {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ app_id: appId, app_secret: appSecret }),
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
});
const data = (await res.json().catch(() => null)) as {
code?: number;
@@ -146,8 +280,24 @@ export async function getTenantAccessToken(env: Env): Promise<TokenResult> {
const err = data?.msg ?? `HTTP ${res.status}`;
return { ok: false, error: err, errorCode: "FEISHU_TOKEN", status: res.status };
}
const ttl = Math.max(60, (data.expire ?? 7200) - 60);
await env.KV.put(TOKEN_KEY, data.tenant_access_token, { expirationTtl: ttl });
// Kept in KV until a held copy (read from KV just before it expires
// there, then held HELD_TOKEN_MS) still ends ahead of the token itself.
const ttl = Math.max(60, (data.expire ?? 7200) - 60 - HELD_TOKEN_MS / 1000);
heldTokens.set(key, {
token: data.tenant_access_token,
until: Date.now() + Math.min(HELD_TOKEN_MS, ttl * 1000),
});
// Shared with other isolates when it can be: a failed KV write (say, the
// daily write quota spent) never throws away a valid token — it is then
// held here until it nears expiry, instead of being fetched (and its
// write retried) again every few minutes.
const token = data.tenant_access_token;
// Feishu handing out a token refused before means it is valid after all.
rejected.delete(token);
cleared.delete(token);
await env.KV.put(key, token, { expirationTtl: ttl }).catch(() => {
heldTokens.set(key, { token, until: Date.now() + ttl * 1000 });
});
return { ok: true, token: data.tenant_access_token };
} catch (err) {
return {
@@ -162,12 +312,17 @@ export async function sendMessage(
token: string,
chatId: string,
content: Record<string, unknown>,
sendKey?: string,
): Promise<SendResult> {
const url = `${FEISHU_API}/open-apis/im/v1/messages?receive_id_type=chat_id`;
// One `uuid` for every attempt of this send (of the delivery, given a
// `sendKey`): Feishu delivers at most one message per uuid within an hour,
// so a retry after a lost response never posts a duplicate.
const body: Record<string, unknown> = {
receive_id: chatId,
msg_type: "interactive",
content: JSON.stringify(content),
uuid: (sendKey ?? crypto.randomUUID()).slice(0, 50),
};
return feishuRequest(url, "POST", token, body, chatId);
}
@@ -188,6 +343,7 @@ export async function sendText(token: string, chatId: string, text: string): Pro
receive_id: chatId,
msg_type: "text",
content: JSON.stringify({ text }),
uuid: crypto.randomUUID(),
};
return feishuRequest(url, "POST", token, body, chatId);
}
@@ -197,7 +353,8 @@ export async function updateCard(
cardToken: string,
card: Record<string, unknown>,
): Promise<SendResult> {
const url = `${FEISHU_API}/open-apis/interactive/v1/config/update`;
// Delayed update of an interactive card after a callback (`token` from it).
const url = `${FEISHU_API}/open-apis/interactive/v1/card/update`;
const body: Record<string, unknown> = { token: cardToken, card };
return feishuRequest(url, "POST", token, body, "card");
}
+348 -72
View File
@@ -5,13 +5,31 @@ import {
commentAsUser,
mergePullRequestAsUser,
closePullRequestAsUser,
BeforeGitHubError,
CommandTakenError,
githubRefused,
} from "../../github/oauth";
import { getFeishuLink, removeFeishuLink } from "../../github/store";
import { getTenantAccessToken, sendText, updateCard } from "./rest";
import { getFeishuLink, unlinkAsCommand } from "../../github/store";
import { acceptedKey, claimOnce, confirmClaim, idempotencyStore } from "../../lib/idempotency";
import { forgetRejectedToken, getTenantAccessToken, sendText, updateCard } from "./rest";
const FEISHU_API = "https://open.feishu.cn";
const MAX_BODY_SIZE = 1024 * 1024;
/** Handled event ids are remembered this long; replays of older requests are refused. */
const EVENT_CLAIM_SECONDS = 86400;
/** An event is held this long while it runs (then a retry may take it). */
const EVENT_PROCESSING_SECONDS = 300;
/**
* Oldest signed request accepted. It stays well inside the event claim, so a
* request old enough for its claim to have expired can never run again,
* while Feishu's own retries (spread over hours) still pass.
*/
const MAX_REQUEST_AGE_SECONDS = 12 * 3600;
const MAX_CLOCK_SKEW_SECONDS = 300;
const GITHUB_TARGET_RE = /github\.com\/([^/\s]+)\/([^/\s]+)\/(?:issues|pull)\/(\d+)/;
// github.com itself only (never e.g. `evilgithub.com`): actions hit the GitHub API.
const GITHUB_TARGET_RE =
/https?:\/\/(?:www\.)?github\.com\/([^/\s]+)\/([^/\s]+)\/(?:issues|pull)\/(\d+)/;
const BTN_PREFIX = "ghpr|";
interface FeishuSender {
@@ -19,6 +37,8 @@ interface FeishuSender {
}
interface FeishuMessage {
chat_id?: string;
/** "p2p" for the private chat with the bot, "group" otherwise. */
chat_type?: string;
content?: string;
}
interface FeishuAction {
@@ -29,9 +49,22 @@ interface FeishuEvent {
sender?: FeishuSender;
message?: FeishuMessage;
action?: FeishuAction;
operator?: { operator_id?: { open_id?: string } };
/** `card.action.trigger` (schema 2.0): the clicking user is `operator.open_id`. */
operator?: { open_id?: string; operator_id?: { open_id?: string } };
token?: string;
open_message_id?: string;
context?: { open_message_id?: string };
}
/** Tracks whether a command reached GitHub, so a failed run knows if it may be retried. */
interface CommandRun {
acted: boolean;
/** Records the command as done right before its GitHub write. */
beforeWrite?: () => Promise<void>;
/** This run wrote the "done" (only then may it give it back). */
markedDone?: boolean;
/** The command's "done" record, for writes committed together with it. */
done?: { key: string; ttl: number };
}
function splitThree(rest: string): [string, string, string] {
@@ -51,31 +84,53 @@ function timingSafeEqual(a: string, b: string): boolean {
return diff === 0;
}
async function hmacSha256Base64(secret: string, payload: string): Promise<string> {
const key = await crypto.subtle.importKey(
"raw",
new TextEncoder().encode(secret),
{ name: "HMAC", hash: "SHA-256" },
false,
["sign"],
);
const sig = await crypto.subtle.sign("HMAC", key, new TextEncoder().encode(payload));
let binary = "";
const bytes = new Uint8Array(sig);
for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i] ?? 0);
return btoa(binary);
async function sha256Hex(data: string): Promise<string> {
const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(data));
return Array.from(new Uint8Array(digest))
.map((b) => b.toString(16).padStart(2, "0"))
.join("");
}
/**
* Feishu event signature (when an Encrypt Key is configured): `X-Lark-Signature`
* is the hex SHA-256 of `timestamp + nonce + encryptKey + rawBody`, with the
* timestamp and nonce from `X-Lark-Request-Timestamp` / `X-Lark-Request-Nonce`.
*/
export async function verifyFeishuSignature(
appSecret: string,
encryptKey: string,
timestamp: string,
nonce: string,
body: string,
signature: string,
): Promise<boolean> {
const raw = `${timestamp}\n${nonce}\n${body}`;
const computed = await hmacSha256Base64(appSecret, raw);
return timingSafeEqual(computed, signature);
if (!signature) return false;
const computed = await sha256Hex(`${timestamp}${nonce}${encryptKey}${body}`);
return timingSafeEqual(computed, signature.toLowerCase());
}
function base64ToBytes(value: string): Uint8Array {
const bin = atob(value);
const bytes = new Uint8Array(bin.length);
for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
return bytes;
}
/**
* Decrypt an `{"encrypt": "..."}` body: AES-256-CBC with key SHA-256(encryptKey)
* and the IV in the first 16 bytes of the base64-decoded ciphertext.
*/
export async function decryptFeishuPayload(encryptKey: string, encrypted: string): Promise<string> {
const keyBytes = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(encryptKey));
const data = base64ToBytes(encrypted);
const key = await crypto.subtle.importKey("raw", keyBytes, { name: "AES-CBC" }, false, [
"decrypt",
]);
const plain = await crypto.subtle.decrypt(
{ name: "AES-CBC", iv: data.slice(0, 16) },
key,
data.slice(16),
);
return new TextDecoder().decode(plain);
}
function describeError(err: unknown): string {
@@ -98,22 +153,50 @@ function extractTarget(text: string): { owner: string; repo: string; number: num
return { owner: m[1] ?? "", repo: m[2] ?? "", number };
}
/** Whether a failed reply may succeed later (then the event is given back). */
function replyRetryable(result: { status?: number; errorCode?: string }): boolean {
if (!result.status || result.status >= 500 || result.status === 429) return true;
return result.errorCode === "FEISHU_TOKEN" || result.errorCode === "RATE_LIMITED";
}
/**
* Send a reply. One that could not be sent for now throws, so the event is
* retried rather than confirmed unanswered (a login link never sent); a
* refusal that a retry cannot change is logged.
*/
async function replyToken(env: Env, chatId: string, text: string): Promise<void> {
const tokenRes = await getTenantAccessToken(env);
if (!tokenRes.ok || !tokenRes.token) {
log.warn({ err: tokenRes.error }, "Feishu token unavailable for reply");
return;
throw new Error("Feishu token unavailable for reply");
}
await sendText(tokenRes.token, chatId, text);
const result = await forgetRejectedToken(
await sendText(tokenRes.token, chatId, text),
env,
tokenRes.token,
);
if (result.ok) return;
log.warn({ chatId, err: result.error }, "Feishu reply failed");
if (replyRetryable(result)) throw new Error(`Feishu reply failed: ${result.error ?? "unknown"}`);
}
async function handleLogin(env: Env, openId: string, chatId: string): Promise<void> {
async function handleLogin(
env: Env,
openId: string,
chatId: string,
chatType: string | undefined,
): Promise<void> {
// An authorization link posted in a group can be opened by anyone there;
// binding happens in the private chat with the bot only.
if (chatType !== "p2p") {
await replyToken(env, chatId, "请私聊机器人发送 /gh login 绑定 GitHub 账号。");
return;
}
const state = crypto.randomUUID();
const pending = {
redirectTo: "/admin",
expiresAt: Date.now() + 10 * 60 * 1000,
feishuUserId: openId,
feishuChatId: chatId,
link: { platform: "feishu", userId: openId, chatId },
};
await env.KV.put(`state:${state}`, JSON.stringify(pending), { expirationTtl: 600 });
const url = getOAuthURL(env.GITHUB_CLIENT_ID ?? "", state);
@@ -128,13 +211,21 @@ async function handleLogin(env: Env, openId: string, chatId: string): Promise<vo
);
}
async function handleLogout(env: Env, openId: string, chatId: string): Promise<void> {
async function handleLogout(
env: Env,
openId: string,
chatId: string,
run: CommandRun,
): Promise<void> {
const linked = await getFeishuLink(env.DB, openId);
if (!linked) {
await replyToken(env, chatId, "你还没有绑定 GitHub 账号。");
return;
}
await removeFeishuLink(env.DB, openId);
await unlinkAsCommand(env, "feishu", openId, run.done);
// Done from here: should the reply fail, a retry must not unlink again
// (by then it could remove a link made since).
run.acted = true;
await replyToken(env, chatId, "已解绑 GitHub 账号。");
}
@@ -143,6 +234,7 @@ async function handleComment(
openId: string,
chatId: string,
text: string,
run: CommandRun,
): Promise<void> {
const githubUserId = await getFeishuLink(env.DB, openId);
if (!githubUserId) {
@@ -159,25 +251,37 @@ async function handleComment(
return;
}
const ghIdx = text.indexOf("/gh");
const rest = text.slice(ghIdx + 3).trim();
const parts = rest.split(/\s+/);
const linkIdx = parts.findIndex((p) => p.includes("github.com"));
const body =
parts
.slice(linkIdx + 1)
.join(" ")
.trim() || "(来自飞书)";
// Only leading space goes: a comment's trailing lines are its own.
const rest = text.slice(ghIdx + 3).replace(/^\s+/, "");
// The comment is what follows the link, as written (line breaks and
// indentation kept).
const link = /\S*github\.com\S*/.exec(rest);
const written = (link ? rest.slice(link.index + link[0].length) : rest).replace(
/^[ \t]*\r?\n?/,
"",
);
const body = written.trim() ? written : "(来自飞书)";
try {
const res = await commentAsUser(
env.KV,
env,
githubUserId,
target.owner,
target.repo,
target.number,
body,
{ beforeWrite: run.beforeWrite },
);
run.acted = true;
await replyToken(env, chatId, `✅ 已评论:[查看](${res.htmlUrl})(@${res.login})`);
} catch (err) {
// Nothing reached GitHub yet: give the event back for a retry.
if (err instanceof BeforeGitHubError) throw err;
// Done, only its reply failed: the event must not run again.
if (run.acted) throw err;
// GitHub was called and may have acted (its answer lost): never run
// again. A refusal did nothing: should its reply fail, the command is
// given back and runs again.
if (!githubRefused(err)) run.acted = true;
await replyToken(env, chatId, `❌ ${describeError(err)}`);
}
}
@@ -188,6 +292,7 @@ async function handleMergeOrClose(
chatId: string,
action: "merge" | "close",
body: string,
run: CommandRun,
): Promise<void> {
const githubUserId = await getFeishuLink(env.DB, openId);
if (!githubUserId) {
@@ -207,25 +312,48 @@ async function handleMergeOrClose(
}
try {
if (action === "merge") {
await mergePullRequestAsUser(env.KV, githubUserId, target.owner, target.repo, target.number);
await mergePullRequestAsUser(
env,
githubUserId,
target.owner,
target.repo,
target.number,
undefined,
{
beforeWrite: run.beforeWrite,
},
);
} else {
await closePullRequestAsUser(env.KV, githubUserId, target.owner, target.repo, target.number);
await closePullRequestAsUser(env, githubUserId, target.owner, target.repo, target.number, {
beforeWrite: run.beforeWrite,
});
}
run.acted = true;
await replyToken(
env,
chatId,
`✅ 已${action === "merge" ? "合并" : "关闭"} ${target.owner}/${target.repo}#${target.number}`,
);
} catch (err) {
// Nothing reached GitHub yet: give the event back for a retry.
if (err instanceof BeforeGitHubError) throw err;
// Done, only its reply failed: the event must not run again.
if (run.acted) throw err;
// GitHub was called and may have acted (its answer lost): never run
// again. A refusal did nothing: should its reply fail, the command is
// given back and runs again.
if (!githubRefused(err)) run.acted = true;
await replyToken(env, chatId, `❌ ${describeError(err)}`);
}
}
async function handleMessage(env: Env, event: FeishuEvent): Promise<void> {
const sender = event.sender ?? {};
const openId = sender.sender_id?.open_id ?? sender.sender_id?.union_id ?? "";
/**
* The `/gh` command a message carries, or null: only a message that is a
* command — `/gh` first, after any @mentions of the bot — runs one; quoting
* or discussing a command never does.
*/
function commandOf(event: FeishuEvent): string | null {
const message = event.message ?? {};
const chatId = message.chat_id ?? "";
let text = "";
if (message.content) {
try {
@@ -235,25 +363,34 @@ async function handleMessage(env: Env, event: FeishuEvent): Promise<void> {
text = "";
}
}
const idx = text.indexOf("/gh");
if (idx < 0) return;
const rest = text.slice(idx + 3).trim();
const command = text.replace(/^\s*(?:@\S+\s+)*/, "");
return /^\/gh(?:\s|$)/.test(command) ? command : null;
}
async function handleMessage(env: Env, event: FeishuEvent, run: CommandRun): Promise<void> {
const sender = event.sender ?? {};
const openId = sender.sender_id?.open_id ?? sender.sender_id?.union_id ?? "";
const message = event.message ?? {};
const chatId = message.chat_id ?? "";
const command = commandOf(event);
if (!command) return;
const rest = command.slice(3).trim();
const sub = rest.split(/\s+/)[0] ?? "";
switch (sub) {
case "login":
await handleLogin(env, openId, chatId);
await handleLogin(env, openId, chatId, message.chat_type);
break;
case "logout":
await handleLogout(env, openId, chatId);
await handleLogout(env, openId, chatId, run);
break;
case "comment":
await handleComment(env, openId, chatId, text);
await handleComment(env, openId, chatId, command, run);
break;
case "merge":
await handleMergeOrClose(env, openId, chatId, "merge", text);
await handleMergeOrClose(env, openId, chatId, "merge", command, run);
break;
case "close":
await handleMergeOrClose(env, openId, chatId, "close", text);
await handleMergeOrClose(env, openId, chatId, "close", command, run);
break;
default:
await replyToken(
@@ -276,13 +413,21 @@ function buildResultCard(ok: boolean, message: string): Record<string, unknown>
};
}
async function handleCardAction(env: Env, payload: Record<string, unknown>): Promise<void> {
async function handleCardAction(
env: Env,
payload: Record<string, unknown>,
run: CommandRun,
): Promise<void> {
const event = (payload.event ?? {}) as FeishuEvent;
const action = event.action ?? {};
const actionId = action.action_id ?? action.value?.v ?? "";
const openId = event.operator?.operator_id?.open_id ?? event.sender?.sender_id?.open_id ?? "";
const openId =
event.operator?.open_id ??
event.operator?.operator_id?.open_id ??
event.sender?.sender_id?.open_id ??
"";
const cardToken = event.token ?? "";
const openMessageId = event.open_message_id ?? "";
const openMessageId = event.context?.open_message_id ?? event.open_message_id ?? "";
if (!actionId.startsWith(BTN_PREFIX)) return;
@@ -303,16 +448,28 @@ async function handleCardAction(env: Env, payload: Record<string, unknown>): Pro
}
try {
if (isMerge) {
await mergePullRequestAsUser(env.KV, githubUserId, owner, repo, number);
await mergePullRequestAsUser(env, githubUserId, owner, repo, number, undefined, {
beforeWrite: run.beforeWrite,
});
} else {
await closePullRequestAsUser(env.KV, githubUserId, owner, repo, number);
await closePullRequestAsUser(env, githubUserId, owner, repo, number, {
beforeWrite: run.beforeWrite,
});
}
run.acted = true;
await replyCard(
env,
cardToken,
buildResultCard(true, `${isMerge ? "已合并" : "已关闭"} ${owner}/${repo}#${number}`),
);
} catch (err) {
if (err instanceof BeforeGitHubError) throw err;
// Done, only its card update failed: the event must not run again.
if (run.acted) throw err;
// GitHub was called and may have acted (its answer lost): never run
// again. A refusal did nothing: should its reply fail, the command is
// given back and runs again.
if (!githubRefused(err)) run.acted = true;
await replyCard(env, cardToken, buildResultCard(false, describeError(err)));
}
return;
@@ -329,12 +486,40 @@ async function replyCard(
card: Record<string, unknown>,
): Promise<void> {
const tokenRes = await getTenantAccessToken(env);
if (!tokenRes.ok || !tokenRes.token) return;
await updateCard(tokenRes.token, cardToken, card);
if (!tokenRes.ok || !tokenRes.token) {
log.warn({ err: tokenRes.error }, "Feishu token unavailable for card update");
throw new Error("Feishu token unavailable for card update");
}
const result = await forgetRejectedToken(
await updateCard(tokenRes.token, cardToken, card),
env,
tokenRes.token,
);
if (result.ok) return;
log.warn({ err: result.error }, "Failed to update the Feishu card");
if (replyRetryable(result))
throw new Error(`Feishu card update failed: ${result.error ?? "unknown"}`);
}
/**
* Inbound Feishu callbacks (events and card actions). Every request must be
* authenticated by the app's own credentials before anything runs:
* - with `FEISHU_ENCRYPT_KEY`, the `X-Lark-Signature` must verify and an
* encrypted body is decrypted with that key;
* - with `FEISHU_VERIFICATION_TOKEN`, the token inside the payload must match.
* At least one of them is required; without either the endpoint refuses all
* requests, since an unauthenticated one could act as any linked user.
*/
export async function handleFeishuWebhookRequest(request: Request, env: Env): Promise<Response> {
// Unauthenticated until verified below: bound what is read and parsed.
const contentLength = Number(request.headers.get("content-length") ?? 0);
if (contentLength > MAX_BODY_SIZE) {
return new Response("Request too large", { status: 413 });
}
const rawBody = await request.text();
if (rawBody.length > MAX_BODY_SIZE) {
return new Response("Request too large", { status: 413 });
}
let payload: Record<string, unknown>;
try {
payload = JSON.parse(rawBody);
@@ -342,20 +527,54 @@ export async function handleFeishuWebhookRequest(request: Request, env: Env): Pr
return new Response("invalid json", { status: 400 });
}
const encryptKey = env.FEISHU_ENCRYPT_KEY?.trim() ?? "";
const verificationToken = env.FEISHU_VERIFICATION_TOKEN?.trim() ?? "";
if (!encryptKey && !verificationToken) {
log.warn("Feishu callback refused: FEISHU_ENCRYPT_KEY / FEISHU_VERIFICATION_TOKEN not set");
return new Response("Feishu callback verification is not configured", { status: 403 });
}
if (encryptKey && typeof payload.encrypt === "string") {
try {
payload = JSON.parse(await decryptFeishuPayload(encryptKey, payload.encrypt));
} catch (err) {
log.warn({ err: String(err) }, "Feishu payload decryption failed");
return new Response("invalid payload", { status: 400 });
}
}
const header = payload.header as Record<string, unknown> | undefined;
const eventPart = payload.event as Record<string, unknown> | undefined;
const type = (payload.type as string) ?? header?.event_type ?? eventPart?.type ?? "";
// The verification token travels inside the (decrypted) payload: in
// `header.token` for schema 2.0 events, at the top level otherwise.
const token = String(header?.token ?? payload.token ?? "");
if (verificationToken && !timingSafeEqual(token, verificationToken)) {
log.warn("Feishu verification token mismatch");
return new Response("invalid token", { status: 401 });
}
if (type === "url_verification") {
return Response.json({ challenge: (payload.challenge as string) ?? "" });
}
const appSecret = env.FEISHU_APP_SECRET?.trim() ?? "";
const signature = request.headers.get("x-lark-signature") ?? "";
const timestamp = request.headers.get("x-lark-timestamp") ?? "";
const nonce = request.headers.get("x-lark-nonce") ?? "";
if (appSecret && signature) {
const ok = await verifyFeishuSignature(appSecret, timestamp, nonce, rawBody, signature);
if (encryptKey) {
const timestamp = request.headers.get("x-lark-request-timestamp") ?? "";
// The signature covers the timestamp: a stale one marks a replay that
// may outlive the event claim below.
const age = Math.floor(Date.now() / 1000) - Number(timestamp);
if (!Number.isFinite(age) || age > MAX_REQUEST_AGE_SECONDS || age < -MAX_CLOCK_SKEW_SECONDS) {
log.warn({ timestamp }, "Feishu callback refused: request timestamp out of range");
return new Response("stale request", { status: 401 });
}
const ok = await verifyFeishuSignature(
encryptKey,
timestamp,
request.headers.get("x-lark-request-nonce") ?? "",
rawBody,
request.headers.get("x-lark-signature") ?? "",
);
if (!ok) {
log.warn("Feishu signature verification failed");
return new Response("invalid signature", { status: 401 });
@@ -363,16 +582,73 @@ export async function handleFeishuWebhookRequest(request: Request, env: Env): Pr
}
const eventType = (header?.event_type as string) ?? eventPart?.type ?? type;
try {
if (eventType === "im.message.receive_v1" || eventType === "message") {
await handleMessage(env, (payload.event ?? payload) as FeishuEvent);
} else if (eventType === "card.action.trigger") {
await handleCardAction(env, payload);
}
} catch (err) {
log.error({ err }, "Feishu webhook handling failed");
const isMessage = eventType === "im.message.receive_v1" || eventType === "message";
// Nothing to do for an ordinary chat message (no `/gh` command) or an
// event type not handled: answered at once, with no dedup records written
// (each would be D1 writes for an event that has no side effect).
if (
(isMessage && !commandOf((payload.event ?? payload) as FeishuEvent)) ||
(!isMessage && eventType !== "card.action.trigger")
) {
return new Response("ok", { status: 200 });
}
// Feishu retries a callback it considers failed; run each event once. An
// event whose earlier run is unconfirmed is retried later, never dropped.
const eventId = String(header?.event_id ?? payload.uuid ?? "");
const claims = idempotencyStore(env.DB, env.KV);
const claimKey = `bot:feishu:${eventId}`;
if (eventId) {
const state = await claimOnce(claims, claimKey, EVENT_PROCESSING_SECONDS);
if (state === "done") return new Response("ok", { status: 200 });
if (state === "pending") return new Response("temporarily unavailable", { status: 503 });
}
const run: CommandRun = { acted: false };
// Done on record before GitHub is written to: should the confirmation
// after it fail, a retry still never repeats the write.
if (eventId) {
run.done = { key: acceptedKey(claimKey), ttl: EVENT_CLAIM_SECONDS };
run.beforeWrite = async (): Promise<void> => {
// Taken: a run that outlived its claim finds another run's "done" here
// and writes nothing.
if (!(await claims.claim(acceptedKey(claimKey), EVENT_CLAIM_SECONDS)))
throw new CommandTakenError();
run.markedDone = true;
};
}
try {
if (isMessage) {
await handleMessage(env, (payload.event ?? payload) as FeishuEvent, run);
} else if (eventType === "card.action.trigger") {
await handleCardAction(env, payload, run);
}
} catch (err) {
// Finished by another run meanwhile: nothing to give back or repeat.
if (err instanceof CommandTakenError) return new Response("ok", { status: 200 });
log.error({ err }, "Feishu webhook handling failed");
// Failed before reaching GitHub: give the event back and answer with an
// error, so Feishu's retry runs it. After GitHub acted it stays claimed.
if (!run.acted) {
if (eventId) {
// An early "done" this run wrote (right before a write GitHub
// refused) too — never one another run wrote meanwhile.
if (run.markedDone) {
await claims.release(acceptedKey(claimKey)).catch(() => undefined);
}
await claims.release(claimKey).catch(() => undefined);
}
return new Response("temporarily unavailable", { status: 503 });
}
}
// A "done" already written before the GitHub write needs no second write.
if (eventId && !run.markedDone) {
// Best-effort: unconfirmed, the claim still holds off retries until it expires.
await confirmClaim(claims, claimKey, EVENT_CLAIM_SECONDS).catch((err: unknown) =>
log.warn({ err, eventId }, "Failed to confirm a Feishu event"),
);
}
return new Response("ok", { status: 200 });
}
+17
View File
@@ -16,3 +16,20 @@ export function getDriver(target: RouteTarget): PlatformDriver {
if (!driver) throw new Error(`No driver for platform "${platform}"`);
return driver;
}
/**
* Where a target's messages actually go: two targets naming the same place
* (a Discord thread given alone, or with its parent channel) are one
* destination, and get one copy of each message.
*/
export function destinationOf(target: RouteTarget): string {
const platform = target.platform ?? "discord";
if (platform === "telegram") {
// Read as the driver sends it (a number): `01` and `1` are one topic.
const topic = target.topicId ?? "";
const id = topic && Number.isFinite(Number(topic)) ? String(Number(topic)) : topic;
return `telegram:${target.chatId ?? ""}/${id}`;
}
if (platform === "feishu") return `feishu:${target.chatId ?? ""}`;
return `discord:${target.threadId || target.channelId || ""}`;
}
+238 -30
View File
@@ -4,13 +4,25 @@ import {
commentAsUser,
mergePullRequestAsUser,
closePullRequestAsUser,
BeforeGitHubError,
CommandTakenError,
githubRefused,
} from "../../github/oauth";
import { getTelegramLink, removeTelegramLink } from "../../github/store";
import { getTelegramLink, unlinkAsCommand } from "../../github/store";
import { acceptedKey, claimOnce, confirmClaim, idempotencyStore } from "../../lib/idempotency";
import type { Env } from "../../types";
import { sendMessage } from "./rest";
import { telegramWebhookSecret } from "./secret";
const GITHUB_ISSUE_RE = /github\.com\/([^/\s]+)\/([^/\s]+)\/(?:issues|pull)\/(\d+)/;
const GITHUB_PR_RE = /github\.com\/([^/\s]+)\/([^/\s]+)\/pull\/(\d+)/;
// github.com itself only (never e.g. `evilgithub.com`): actions hit the GitHub API.
/** An update is held this long while its command runs (then a redelivery may take it). */
const COMMAND_CLAIM_SECONDS = 300;
/** Handled updates are remembered this long. */
const COMMAND_DONE_SECONDS = 86400;
const GITHUB_ISSUE_RE =
/^https?:\/\/(?:www\.)?github\.com\/([^/\s]+)\/([^/\s]+)\/(?:issues|pull)\/(\d+)/;
const GITHUB_PR_RE = /^https?:\/\/(?:www\.)?github\.com\/([^/\s]+)\/([^/\s]+)\/pull\/(\d+)/;
interface TelegramMessage {
message_id?: number;
@@ -23,6 +35,26 @@ interface TelegramMessage {
reply_to_message?: TelegramMessage;
}
let knownBotUsername: { token: string; name: string } | null = null;
/** This bot's username (lower case), from `getMe`; remembered per isolate. */
async function botUsername(env: Env): Promise<string> {
const token = env.TELEGRAM_TOKEN ?? "";
if (knownBotUsername?.token === token) return knownBotUsername.name;
const res = await fetch(`https://api.telegram.org/bot${token}/getMe`, {
signal: AbortSignal.timeout(15_000),
});
const data = (await res.json().catch(() => null)) as {
result?: { username?: string };
} | null;
const name = data?.result?.username;
// Unknown: thrown, so Telegram redelivers the update rather than the
// command being dropped.
if (!res.ok || !name) throw new Error(`Telegram getMe failed (HTTP ${res.status})`);
knownBotUsername = { token, name: name.toLowerCase() };
return knownBotUsername.name;
}
interface Target {
owner: string;
repo: string;
@@ -54,13 +86,23 @@ function extractTarget(msg: TelegramMessage, prOnly = false): Target | null {
return null;
}
/**
* Send a reply. One that could not be sent for now throws, so the update is
* redelivered rather than confirmed unanswered (a login link never sent);
* a refusal that a retry cannot change (e.g. the bot was blocked) is logged.
*/
async function reply(
env: Env,
chatId: string,
topicId: string | undefined,
text: string,
): Promise<void> {
await sendMessage(env.TELEGRAM_TOKEN ?? "", chatId, text, topicId);
const result = await sendMessage(env.TELEGRAM_TOKEN ?? "", chatId, text, topicId);
if (result.ok) return;
log.warn({ chatId, err: result.error }, "Telegram reply failed");
if (!result.status || result.status >= 500 || result.status === 429) {
throw new Error(`Telegram reply failed: ${result.error ?? "unknown"}`);
}
}
function errText(err: unknown): string {
@@ -77,17 +119,23 @@ async function cmdLogin(env: Env, msg: TelegramMessage): Promise<void> {
if (!chatId || !telegramUserId) return;
const topicId = msg.message_thread_id != null ? String(msg.message_thread_id) : undefined;
// An authorization link posted in a group can be opened by anyone there;
// binding happens in the private chat with the bot only.
if (msg.chat?.type !== "private") {
return reply(env, chatId, topicId, "请私聊机器人发送 /gh login 绑定 GitHub 账号。");
}
const clientId = env.GITHUB_CLIENT_ID;
if (!clientId)
return reply(env, chatId, topicId, "服务器未配置 GitHub OAuth(GITHUB_CLIENT_ID)。");
const name = msg.from?.username ? `@${msg.from.username}` : msg.from?.first_name;
const state = crypto.randomUUID().replace(/-/g, "");
await env.KV.put(
`state:${state}`,
JSON.stringify({
redirectTo: "/",
telegramUserId,
telegramChatId: chatId,
link: { platform: "telegram", userId: telegramUserId, name, chatId },
expiresAt: Date.now() + 600_000,
}),
{ expirationTtl: 600 },
@@ -97,20 +145,39 @@ async function cmdLogin(env: Env, msg: TelegramMessage): Promise<void> {
env,
chatId,
topicId,
`点击链接授权 GitHub,即可用**本人身份**评论(10 分钟内有效):\n${url}`,
`点击链接授权 GitHub,即可用<b>本人身份</b>评论(10 分钟内有效):\n${url}`,
);
}
async function cmdLogout(env: Env, msg: TelegramMessage): Promise<void> {
async function cmdLogout(env: Env, msg: TelegramMessage, run: CommandRun): Promise<void> {
const chatId = chatIdOf(msg);
const telegramUserId = userIdOf(msg);
if (!chatId || !telegramUserId) return;
const topicId = msg.message_thread_id != null ? String(msg.message_thread_id) : undefined;
await removeTelegramLink(env.DB, telegramUserId);
await unlinkAsCommand(env, "telegram", telegramUserId, run.done);
// Done from here: should the reply fail, a redelivery must not unlink again
// (by then it could remove a link made since).
run.acted = true;
await reply(env, chatId, topicId, "已解绑你的 GitHub 账号。");
}
async function cmdComment(env: Env, msg: TelegramMessage, body: string): Promise<void> {
/** Tracks whether a command reached GitHub, so a failed run knows if it may be retried. */
interface CommandRun {
acted: boolean;
/** Records the command as done right before its GitHub write. */
beforeWrite?: () => Promise<void>;
/** This run wrote the "done" (only then may it give it back). */
markedDone?: boolean;
/** The command's "done" record, for writes committed together with it. */
done?: { key: string; ttl: number };
}
async function cmdComment(
env: Env,
msg: TelegramMessage,
body: string,
run: CommandRun,
): Promise<void> {
const chatId = chatIdOf(msg);
const telegramUserId = userIdOf(msg);
if (!chatId || !telegramUserId) return;
@@ -134,20 +201,35 @@ async function cmdComment(env: Env, msg: TelegramMessage, body: string): Promise
try {
const { htmlUrl, login } = await commentAsUser(
env.KV,
env,
githubUserId,
target.owner,
target.repo,
target.number,
body,
{ beforeWrite: run.beforeWrite },
);
run.acted = true;
await reply(env, chatId, topicId, `已以 **@${login}** 身份评论:${htmlUrl}`);
} catch (err) {
// Nothing reached GitHub yet: give the update back for a retry.
if (err instanceof BeforeGitHubError) throw err;
// Done, only its reply failed: the update must not run again.
if (run.acted) throw err;
// GitHub was called and may have acted (its answer lost): never run
// again. A refusal did nothing: should its reply fail, the command is
// given back and runs again.
if (!githubRefused(err)) run.acted = true;
await reply(env, chatId, topicId, errText(err));
}
}
async function cmdMergeClose(env: Env, msg: TelegramMessage, op: "merge" | "close"): Promise<void> {
async function cmdMergeClose(
env: Env,
msg: TelegramMessage,
op: "merge" | "close",
run: CommandRun,
): Promise<void> {
const chatId = chatIdOf(msg);
const telegramUserId = userIdOf(msg);
if (!chatId || !telegramUserId) return;
@@ -171,10 +253,23 @@ async function cmdMergeClose(env: Env, msg: TelegramMessage, op: "merge" | "clos
try {
if (op === "merge") {
await mergePullRequestAsUser(env.KV, githubUserId, target.owner, target.repo, target.number);
await mergePullRequestAsUser(
env,
githubUserId,
target.owner,
target.repo,
target.number,
undefined,
{
beforeWrite: run.beforeWrite,
},
);
} else {
await closePullRequestAsUser(env.KV, githubUserId, target.owner, target.repo, target.number);
await closePullRequestAsUser(env, githubUserId, target.owner, target.repo, target.number, {
beforeWrite: run.beforeWrite,
});
}
run.acted = true;
const label = op === "merge" ? "合并" : "关闭";
await reply(
env,
@@ -183,6 +278,14 @@ async function cmdMergeClose(env: Env, msg: TelegramMessage, op: "merge" | "clos
`✅ 已${label} PR ${target.owner}/${target.repo}#${target.number}`,
);
} catch (err) {
// Nothing reached GitHub yet: give the update back for a retry.
if (err instanceof BeforeGitHubError) throw err;
// Done, only its reply failed: the update must not run again.
if (run.acted) throw err;
// GitHub was called and may have acted (its answer lost): never run
// again. A refusal did nothing: should its reply fail, the command is
// given back and runs again.
if (!githubRefused(err)) run.acted = true;
await reply(env, chatId, topicId, errText(err));
}
}
@@ -193,21 +296,45 @@ export async function syncTelegramWebhook(env: Env): Promise<void> {
if (!token) return;
const baseUrl = env.BASE_URL;
if (!baseUrl) return;
const secret = env.TELEGRAM_WEBHOOK_SECRET;
// Always registered with a secret (explicit or derived from the bot token),
// so the webhook endpoint can authenticate every call.
const secret = await telegramWebhookSecret(env);
const registration = {
url: `${baseUrl.replace(/\/$/, "")}/telegram/webhook`,
secret_token: secret,
allowed_updates: ["message"],
};
// Registered again only when the registration changed (or once a day, to
// repair a webhook changed elsewhere): this runs every few minutes, and
// each run would otherwise be a subrequest. One KV read per run; a write
// only when it registers.
const digest = [
...new Uint8Array(
await crypto.subtle.digest(
"SHA-256",
new TextEncoder().encode(`${token}|${JSON.stringify(registration)}`),
),
),
]
.slice(0, 12)
.map((b) => b.toString(16).padStart(2, "0"))
.join("");
// One record per bot holding what it is registered with now: switching
// A -> B -> A registers A again (a record per digest would still hold A's).
const marker = `telegram:webhook:${token.split(":")[0]}`;
if ((await env.KV.get(marker).catch(() => null)) === digest) return;
const res = await fetch(`https://api.telegram.org/bot${token}/setWebhook`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
url: `${baseUrl.replace(/\/$/, "")}/telegram/webhook`,
secret_token: secret || undefined,
allowed_updates: ["message"],
}),
body: JSON.stringify(registration),
});
if (!res.ok) {
const err = await res.text();
log.error({ status: res.status, err }, "Failed to set Telegram webhook");
return;
}
await env.KV.put(marker, digest, { expirationTtl: 86400 }).catch(() => undefined);
}
/** Handle a single Telegram update (message). Called from the webhook route. */
@@ -215,32 +342,113 @@ export async function handleTelegramUpdate(env: Env, update: unknown): Promise<v
const message = (update as { message?: TelegramMessage })?.message;
if (!message?.text) return;
const text = message.text.trim();
const m = text.match(/^\/gh(?:\s+|$)(.*)$/s);
// Only leading space goes: a comment's trailing lines are its own.
const text = message.text.replace(/^\s+/, "");
// `/gh@BotName ...` is how group chats address one bot among several: run
// it only when the name is this bot's.
const m = text.match(/^\/gh(?:@(\w+))?(?:\s+|$)(.*)$/s);
if (!m) return;
if (m[1] && m[1].toLowerCase() !== (await botUsername(env))) return;
const rest = (m[1] ?? "").trim();
const [sub, ...args] = rest.split(/\s+/);
const body = args.join(" ").trim();
// Telegram redelivers an update until the webhook answers 200; a command
// that already ran (e.g. its reply failed afterwards) must not run again,
// and one whose run is unconfirmed is retried later, never dropped.
const updateId = (update as { update_id?: number }).update_id;
const claims = idempotencyStore(env.DB, env.KV);
// Update ids count per bot: the key names the bot (the id before the
// token's ":", stable across token changes), so another bot's update with
// the same number is never taken for this one.
const botId = (env.TELEGRAM_TOKEN ?? "").split(":")[0] ?? "";
const claimKey = `bot:telegram:${botId}:${updateId}`;
if (updateId != null) {
// Handled under the key used before bots were told apart (kept one day).
if (await claims.has(acceptedKey(`bot:telegram:${updateId}`))) return;
const state = await claimOnce(claims, claimKey, COMMAND_CLAIM_SECONDS);
if (state === "done") return;
if (state === "pending") throw new Error("Telegram update is already being processed");
}
const rest = (m[2] ?? "").replace(/^\s+/, "");
const sub = rest.split(/\s/, 1)[0] ?? "";
// The comment as written: line breaks and indentation kept (only the
// separator after the subcommand goes — up to its first line break).
const body = rest.slice(sub.length).replace(/^[ \t]*\r?\n?/, "");
const run: CommandRun = { acted: false };
// Done on record before GitHub is written to: should the confirmation
// after it fail, a redelivery still never repeats the write.
if (updateId != null) {
run.done = { key: acceptedKey(claimKey), ttl: COMMAND_DONE_SECONDS };
run.beforeWrite = async (): Promise<void> => {
// Taken: a run that outlived its claim finds another run's "done" here
// and writes nothing.
if (!(await claims.claim(acceptedKey(claimKey), COMMAND_DONE_SECONDS)))
throw new CommandTakenError();
run.markedDone = true;
};
}
try {
await runCommand(env, message, sub, body, text, run);
} catch (err) {
// Finished by another run meanwhile: nothing to give back or repeat.
if (err instanceof CommandTakenError) return;
// Failed before reaching GitHub (e.g. a storage error) or refused by it:
// give the update back (an early "done" too), so Telegram's redelivery
// runs it. After GitHub may have acted, it is done for good — running
// again would repeat the comment / merge.
if (updateId != null) {
if (run.acted) {
// Done for good; a "done" written before the GitHub write needs no
// second write.
if (!run.markedDone) await confirmDone();
} else {
// Only a "done" this run wrote: one another run wrote while this one
// outlived its claim stays.
if (run.markedDone) {
await claims.release(acceptedKey(claimKey)).catch(() => undefined);
}
await claims.release(claimKey).catch(() => undefined);
}
}
throw err;
}
// A "done" already written before the GitHub write needs no second write.
if (updateId != null && !run.markedDone) await confirmDone();
/** Best-effort: unconfirmed, the claim still holds off redeliveries until it expires. */
async function confirmDone(): Promise<void> {
await confirmClaim(claims, claimKey, COMMAND_DONE_SECONDS).catch((err: unknown) =>
log.warn({ err, updateId }, "Failed to confirm a Telegram update"),
);
}
}
async function runCommand(
env: Env,
message: TelegramMessage,
sub: string | undefined,
body: string,
text: string,
run: CommandRun,
): Promise<void> {
switch (sub) {
case "login":
return cmdLogin(env, message);
case "logout":
return cmdLogout(env, message);
return cmdLogout(env, message, run);
case "comment":
if (!body) {
if (!body.trim()) {
const chatId = chatIdOf(message);
const topicId =
message.message_thread_id != null ? String(message.message_thread_id) : undefined;
if (chatId) await reply(env, chatId, topicId, "请附上评论内容:/gh comment 你的评论");
return;
}
return cmdComment(env, message, body);
return cmdComment(env, message, body, run);
case "merge":
return cmdMergeClose(env, message, "merge");
return cmdMergeClose(env, message, "merge", run);
case "close":
return cmdMergeClose(env, message, "close");
return cmdMergeClose(env, message, "close", run);
default:
log.info({ text }, "Unhandled /gh command from Telegram");
}
+25 -21
View File
@@ -1,7 +1,7 @@
import MarkdownIt from "markdown-it";
import type { RouteTarget, Env, NeutralMessage } from "../../types";
import type { PlatformDriver, SendResult } from "../types";
import { sendMessage, sendPhoto, editMessageText, editMessageCaption } from "./rest";
import { sendMessage, editMessageText } from "./rest";
import { renderNeutralMessage } from "./render";
const markdown = new MarkdownIt({ html: false, linkify: false, breaks: false, typographer: false });
@@ -23,6 +23,19 @@ function richHeaderUrl(message: NeutralMessage, avatar: string, host: string): s
return `${host.replace(/\/+$/, "")}/api/richheader?${params.toString()}`;
}
/**
* The avatar shown above the text as a link preview: the rich-header card
* when a host serves it, the plain (small) avatar image otherwise. Messages
* are always text messages (never photos with captions), so every edit is an
* `editMessageText` with the full 4096-character budget.
*/
function previewUrl(message: NeutralMessage, env: Env): string | undefined {
const avatar = message.author?.iconUrl;
if (!avatar) return undefined;
const richHeaderHost = env.TELEGRAM_RICH_HEADER_HOST ?? env.BASE_URL;
return richHeaderHost ? richHeaderUrl(message, avatar, richHeaderHost) : smallAvatar(avatar);
}
export class TelegramDriver implements PlatformDriver {
readonly id = "telegram";
@@ -33,16 +46,7 @@ export class TelegramDriver implements PlatformDriver {
}
const token = env.TELEGRAM_TOKEN ?? "";
const text = renderNeutralMessage(message);
const avatar = message.author?.iconUrl;
const richHeaderHost = env.TELEGRAM_RICH_HEADER_HOST ?? env.BASE_URL;
if (avatar && richHeaderHost) {
const rhUrl = richHeaderUrl(message, avatar, richHeaderHost);
return sendMessage(token, chatId, text, target.topicId, rhUrl);
}
if (avatar) {
return sendPhoto(token, chatId, smallAvatar(avatar), text, target.topicId);
}
return sendMessage(token, chatId, text, target.topicId);
return sendMessage(token, chatId, text, target.topicId, previewUrl(message, env));
}
async edit(
@@ -57,15 +61,15 @@ export class TelegramDriver implements PlatformDriver {
}
const token = env.TELEGRAM_TOKEN ?? "";
const text = renderNeutralMessage(message);
const avatar = message.author?.iconUrl;
const richHeaderHost = env.TELEGRAM_RICH_HEADER_HOST ?? env.BASE_URL;
if (avatar && richHeaderHost) {
const rhUrl = richHeaderUrl(message, avatar, richHeaderHost);
return editMessageText(token, chatId, messageId, text, target.topicId, rhUrl);
}
if (avatar) {
return editMessageCaption(token, chatId, messageId, text, target.topicId);
}
return editMessageText(token, chatId, messageId, text, target.topicId);
// A photo sent by an older version has no text to edit; Telegram's error
// then makes dispatch send a fresh message instead.
return editMessageText(
token,
chatId,
messageId,
text,
target.topicId,
previewUrl(message, env),
);
}
}
+47 -17
View File
@@ -19,6 +19,9 @@ md.renderer.rules.strong_open = (): string => "<b>";
md.renderer.rules.strong_close = (): string => "</b>";
md.renderer.rules.em_open = (): string => "<i>";
md.renderer.rules.em_close = (): string => "</i>";
// Telegram's HTML has no <br>: a Markdown hard break ("two spaces, newline")
// would get the whole message rejected. A plain newline breaks the line there.
md.renderer.rules.hardbreak = (): string => "\n";
function mdToHtml(s: string): string {
return md.renderInline(s);
@@ -37,26 +40,53 @@ function formatTimestamp(ts?: string): string {
* an incomplete trailing tag and closes every tag still open at the cut
* point, so parse_mode HTML never rejects the message.
*/
/**
* Clamp HTML to `max` characters as Telegram counts them — the text shown,
* tags left out and an entity (`&amp;`) as one character — keeping tags and
* entities whole and closing the tags still open (an ellipsis marks the cut).
*/
function capHtml(text: string, max: number): string {
if (text.length <= max) return text;
let cut = text.slice(0, max).replace(/<[^>]*$/u, "");
const stack: string[] = [];
const re = /<(\/?)([a-z]+)(?:\s[^>]*)?>/g;
let m: RegExpExecArray | null;
while ((m = re.exec(cut)) !== null) {
const tag = m[2]!;
if (m[1] === "/") {
const open = stack.lastIndexOf(tag);
if (open >= 0) stack.splice(open);
} else {
stack.push(tag);
let shown = 0;
let i = 0;
while (i < text.length) {
if (text[i] === "<") {
const end = text.indexOf(">", i);
if (end < 0) break;
const m = /^<(\/?)([a-z]+)/.exec(text.slice(i, end + 1));
if (m) {
const tag = m[2]!;
if (m[1] === "/") {
const open = stack.lastIndexOf(tag);
if (open >= 0) stack.splice(open);
} else {
stack.push(tag);
}
}
i = end + 1;
continue;
}
const entity =
text[i] === "&" ? /^&(?:#\d+|#x[0-9a-f]+|[a-z]+);/i.exec(text.slice(i, i + 12)) : null;
const code = text.charCodeAt(i);
// A surrogate pair counts (and is cut) as the two code units it is.
const step = entity ? entity[0].length : code >= 0xd800 && code <= 0xdbff ? 2 : 1;
const weight = entity ? 1 : step;
// One character is kept for the ellipsis.
if (shown + weight > max - 1) {
const rest = text.slice(i).replace(/<[^>]*>/g, "");
// What is left fits after all (no ellipsis needed).
if (shown + rest.replace(/&(?:#\d+|#x[0-9a-f]+|[a-z]+);/gi, "x").length <= max) return text;
const closers = [...stack]
.reverse()
.map((tag) => `</${tag}>`)
.join("");
return `${text.slice(0, i)}…${closers}`;
}
shown += weight;
i += step;
}
const closers = stack
.reverse()
.map((tag) => `</${tag}>`)
.join("");
return `${cut}…${closers}`;
return text;
}
export function renderNeutralMessage(message: NeutralMessage): string {
@@ -76,7 +106,7 @@ export function renderNeutralMessage(message: NeutralMessage): string {
if (message.author) {
const name = mdToHtml(message.author.name);
const author = message.author.url ? `<a href="${esc(message.author.url)}">${name}</a>` : name;
parts.push(`👤 ${author}`);
parts.push(message.emoji === false ? author : `👤 ${author}`);
}
if (message.description) {
+42 -2
View File
@@ -7,8 +7,21 @@ interface TelegramResponse {
ok?: boolean;
description?: string;
result?: { message_id?: number };
parameters?: { retry_after?: number };
}
/**
* Longest rate-limit wait (seconds) slept inside one request; a longer one is
* returned as a 429 failure so the delivery queue retries it later instead.
*/
const MAX_INLINE_RETRY_AFTER = 10;
/**
* Longest a single API call may take. Bounded, so a message edit always ends
* well within the update-chain lock it runs under (see `deliverUpdate`).
*/
export const REQUEST_TIMEOUT_MS = 15_000;
async function post(
token: string,
method: string,
@@ -25,13 +38,25 @@ async function post(
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(body),
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
});
lastStatus = res.status;
const data = (await res.json().catch(() => null)) as TelegramResponse | null;
if (res.status === 429) {
const retryAfter = (data as { retry_after?: number })?.retry_after ?? 1;
// Telegram reports the wait in `parameters.retry_after` (seconds).
const raw = Number(data?.parameters?.retry_after);
const retryAfter = Number.isFinite(raw) && raw > 0 ? raw : 1;
lastError = data?.description ?? `Rate limited (retry_after=${retryAfter})`;
if (retryAfter > MAX_INLINE_RETRY_AFTER || attempt === 2) {
return {
ok: false,
error: lastError,
errorCode: "RATE_LIMITED",
status: 429,
attempts: attempt + 1,
};
}
await new Promise((r) => setTimeout(r, retryAfter * 1000));
continue;
}
@@ -51,10 +76,25 @@ async function post(
};
}
// Accepted, but the answer is unreadable (cut off, timed out): the
// message may well exist, and Telegram has no idempotency key, so a
// retry could post it twice — a visible failure, not retried (and not
// a success without a message id, which an update chain would lose).
if (data?.ok !== true) {
log.error({ status: res.status, chatId }, "Unreadable Telegram response");
return {
ok: false,
error: "Telegram accepted the request, but its answer could not be read",
errorCode: "TELEGRAM_UNCERTAIN",
status: res.status,
attempts: attempt + 1,
};
}
return {
ok: true,
status: res.status,
messageId: data?.result?.message_id != null ? String(data.result.message_id) : undefined,
messageId: data.result?.message_id != null ? String(data.result.message_id) : undefined,
attempts: attempt + 1,
};
} catch (err) {
+22
View File
@@ -0,0 +1,22 @@
import type { Env } from "../../types";
/**
* The `secret_token` Telegram must send with every webhook call. It is
* `TELEGRAM_WEBHOOK_SECRET` when set, otherwise derived from the bot token
* (which is itself secret), so the webhook endpoint is never left
* unauthenticated. Empty when no bot token is configured.
*/
export async function telegramWebhookSecret(env: Env): Promise<string> {
const explicit = env.TELEGRAM_WEBHOOK_SECRET?.trim();
if (explicit) return explicit;
const token = env.TELEGRAM_TOKEN?.trim();
if (!token) return "";
const digest = await crypto.subtle.digest(
"SHA-256",
new TextEncoder().encode(`webhooker:telegram-webhook:${token}`),
);
// Telegram allows 1-256 characters of A-Z, a-z, 0-9, `_` and `-`.
return Array.from(new Uint8Array(digest))
.map((b) => b.toString(16).padStart(2, "0"))
.join("");
}
+12 -6
View File
@@ -1,5 +1,7 @@
import type { Env } from "../../types";
import { log } from "../../lib/log";
import { handleTelegramUpdate } from "./commands";
import { telegramWebhookSecret } from "./secret";
const MAX_BODY_SIZE = 1024 * 1024;
@@ -19,12 +21,16 @@ export async function handleTelegramWebhookRequest(request: Request, env: Env):
return new Response("Request too large", { status: 413 });
}
const secret = env.TELEGRAM_WEBHOOK_SECRET;
if (secret) {
const token = request.headers.get("X-Telegram-Bot-Api-Secret-Token");
if (!token || !timingSafeEqual(token, secret)) {
return new Response("Invalid secret", { status: 401 });
}
// Every call must carry the webhook secret. Without a bot token there is
// no secret to check and nothing to act with, so the endpoint stays closed.
const secret = await telegramWebhookSecret(env);
if (!secret) {
log.warn("Telegram webhook called but TELEGRAM_TOKEN is not configured");
return new Response("Telegram is not configured", { status: 403 });
}
const token = request.headers.get("X-Telegram-Bot-Api-Secret-Token");
if (!token || !timingSafeEqual(token, secret)) {
return new Response("Invalid secret", { status: 401 });
}
const rawBody = await request.text();
+135 -28
View File
@@ -1,40 +1,93 @@
import { RE2JS } from "re2js";
import type { WebhookEvent, Filter, FilterNode, FilterOp } from "../types";
const regexCache = new Map<string, RegExp>();
/** A compiled user pattern; `test` searches (like `RegExp#test`). */
interface Pattern {
test(text: string): boolean;
}
const patternCache = new Map<string, Pattern | null>();
const keywordBodyCache = new WeakMap<WebhookEvent, string>();
const MAX_PATTERN_LENGTH = 200;
const MAX_CACHED_PATTERNS = 500;
function isWrappedRegex(pattern: string): boolean {
return pattern.length >= 2 && pattern.startsWith("/") && pattern.endsWith("/");
}
function compileRegex(pattern: string): RegExp | null {
if (pattern.length > MAX_PATTERN_LENGTH) return null;
const cached = regexCache.get(pattern);
if (cached) return cached;
/**
* Compile with RE2 (re2js): matching runs in time linear in the input, so no
* pattern — however it nests or repeats — can stall a Worker on a crafted
* payload. RE2 has no backreferences or lookaround; such patterns are invalid.
*/
function compileRe2(source: string): Pattern {
const re = RE2JS.compile(source, RE2JS.CASE_INSENSITIVE);
return { test: (text: string): boolean => re.matcher(text).find() };
}
/**
* Why a user-supplied regex cannot be used, or null when it is fine. Patterns
* run on the linear-time RE2 engine, so only the length limit and the RE2
* syntax (no backreferences or lookaround) constrain them.
*/
export function regexSafetyIssue(source: string): string | null {
if (source.length > MAX_PATTERN_LENGTH) return "pattern is too long";
try {
const re = new RegExp(pattern, "i");
regexCache.set(pattern, re);
return re;
} catch {
compileRe2(source);
return null;
} catch {
return "invalid or unsupported regular expression (RE2 syntax: no backreferences or lookaround)";
}
}
function compileGlob(pattern: string, anchored: boolean): RegExp | null {
if (pattern.length > MAX_PATTERN_LENGTH) return null;
const cacheKey = `${anchored ? "a" : "s"}:${pattern}`;
const cached = regexCache.get(cacheKey);
if (cached) return cached;
const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&");
const body = escaped.replace(/\*/g, ".*").replace(/\?/g, ".");
try {
const re = new RegExp(anchored ? `^${body}$` : body, "i");
regexCache.set(cacheKey, re);
return re;
} catch {
return null;
function cached(key: string, build: () => Pattern | null): Pattern | null {
if (patternCache.has(key)) {
// Most recently used last: the oldest unused pattern is evicted first.
const hit = patternCache.get(key) ?? null;
patternCache.delete(key);
patternCache.set(key, hit);
return hit;
}
// Bounded: patterns come from every group's routes and test requests. One
// pattern makes room at a time — clearing everything would recompile every
// pattern in use once the limit is reached.
if (patternCache.size >= MAX_CACHED_PATTERNS) {
patternCache.delete(patternCache.keys().next().value!);
}
const re = build();
patternCache.set(key, re);
return re;
}
function compileRegex(pattern: string): Pattern | null {
// Regex and glob entries live under distinct prefixes: the regex `a:foo*`
// and the anchored glob `foo*` must never share a cache slot.
return cached(`r:${pattern}`, () => {
if (pattern.length > MAX_PATTERN_LENGTH) return null;
try {
return compileRe2(pattern);
} catch {
return null;
}
});
}
function compileGlob(pattern: string, anchored: boolean): Pattern | null {
if (pattern.length > MAX_PATTERN_LENGTH) return null;
// Repeated `*` match the same as one; an unanchored search needs no leading
// or trailing `*` at all.
let glob = pattern.replace(/\*+/g, "*");
if (!anchored) glob = glob.replace(/^\*+|\*+$/g, "");
return cached(`${anchored ? "a" : "s"}:${glob}`, () => {
const escaped = glob.replace(/[.+^${}()|[\]\\]/g, "\\$&");
// One pass: a sequential replace would rewrite the `?` of `(?s:` again.
const body = escaped.replace(/[*?]/g, (c) => (c === "*" ? "(?s:.*)" : "(?s:.)"));
try {
return compileRe2(anchored ? `^${body}$` : body);
} catch {
return null;
}
});
}
function isGlob(pattern: string): boolean {
@@ -77,7 +130,7 @@ function extractBranch(event: WebhookEvent): string | undefined {
const p = event.payload;
switch (event.event) {
case "push":
return typeof p.ref === "string" ? p.ref.replace("refs/heads/", "") : undefined;
return typeof p.ref === "string" ? p.ref.replace(/^refs\/heads\//, "") : undefined;
case "pull_request":
case "pull_request_review":
case "pull_request_review_comment":
@@ -89,14 +142,22 @@ function extractBranch(event: WebhookEvent): string | undefined {
return (p.workflow_run as { head_branch?: string } | undefined)?.head_branch;
case "check_suite":
return (p.check_suite as { head_branch?: string } | undefined)?.head_branch;
case "check_run":
return (p.check_run as { check_suite?: { head_branch?: string } } | undefined)?.check_suite
?.head_branch;
case "workflow_job":
return (p.workflow_job as { head_branch?: string } | undefined)?.head_branch;
case "deployment":
return typeof p.ref === "string" ? p.ref.replace("refs/heads/", "") : undefined;
case "deployment_status": {
// The deployed ref lives on the deployment object (a top-level `ref` is
// only read as a fallback).
const ref = (p.deployment as { ref?: unknown } | undefined)?.ref ?? p.ref;
return typeof ref === "string" ? ref.replace(/^refs\/heads\//, "") : undefined;
}
case "commit_comment":
return undefined;
case "code_scanning_alert":
return typeof p.ref === "string" ? p.ref : undefined;
return typeof p.ref === "string" ? p.ref.replace(/^refs\/heads\//, "") : undefined;
default:
return undefined;
}
@@ -111,7 +172,8 @@ function stringify(value: unknown): string {
if (typeof value === "string") return value;
if (value === undefined || value === null) return "";
if (typeof value === "number" || typeof value === "boolean") return String(value);
return JSON.stringify(value);
// Payload values are JSON; anything else (should it appear) compares as "".
return JSON.stringify(value) ?? "";
}
function toNumber(value: unknown): number | null {
@@ -142,8 +204,11 @@ function walkPath(node: unknown, parts: string[], index: number, out: unknown[])
return;
}
if (node === null || node === undefined || typeof node !== "object") return;
const next = (node as Record<string, unknown>)[parts[index]!];
walkPath(next, parts, index + 1, out);
// Own properties only: `constructor`, `__proto__`, `toString`... are not
// payload data and must never reach the comparison.
const key = parts[index]!;
if (!Object.prototype.hasOwnProperty.call(node, key)) return;
walkPath((node as Record<string, unknown>)[key], parts, index + 1, out);
}
function valueList(filter: Filter, event: WebhookEvent): unknown[] {
@@ -220,6 +285,48 @@ function matchFilter(filter: Filter, event: WebhookEvent, keywordBody?: string):
return filter.exclude ? !matches : matches;
}
/**
* Whether a filter tree has the shape the evaluator assumes: every node is a
* leaf (`type`, no `all`/`any`/`not`) or exactly one combinator holding nodes
* of the same shape. A node mixing kinds (`{"all": 0, "any": [...]}`) passes
* a lenient schema but makes the evaluator throw.
*/
export function filterNodeWellFormed(node: unknown): boolean {
if (!node || typeof node !== "object" || Array.isArray(node)) return false;
const n = node as Record<string, unknown>;
const kinds = (["all", "any", "not"] as const).filter((k) => k in n);
if (kinds.length > 1) return false;
const kind = kinds[0];
if (kind === "all" || kind === "any") {
const list = n[kind];
return Array.isArray(list) && list.every(filterNodeWellFormed);
}
if (kind === "not") return filterNodeWellFormed(n.not);
return typeof n.type === "string";
}
/**
* Whether every pattern in a filter tree can actually run the way the matcher
* reads it — a regex compiles on the engine in use, a glob compiles within
* the length limit. One that cannot (written for another engine, e.g. with
* lookahead, or too long) would quietly never match — and flip an `exclude`
* into a pass. Literal comparisons (`contains`, `startsWith`, ...) are not
* patterns and always run.
*/
export function filterPatternsUsable(node: FilterNode): boolean {
if ("all" in node) return node.all.every(filterPatternsUsable);
if ("any" in node) return node.any.every(filterPatternsUsable);
if ("not" in node) return filterPatternsUsable(node.not);
const op = node.type === "keyword" ? "keyword" : (node.op ?? "eq");
return toPatterns(node).every((p) => {
if (op === "regex") return compileRegex(p) !== null;
if (op !== "eq" && op !== "in" && op !== "ne" && op !== "keyword") return true;
if (isWrappedRegex(p)) return compileRegex(p.slice(1, -1)) !== null;
if (isGlob(p)) return compileGlob(p, op !== "keyword") !== null;
return true;
});
}
export function containsKeyword(node: FilterNode): boolean {
if ("all" in node) return node.all.some(containsKeyword);
if ("any" in node) return node.any.some(containsKeyword);
+12 -1
View File
@@ -184,7 +184,13 @@ export function formatCheckRun(
details_url?: string;
status?: string;
output?: { title?: string; summary?: string };
check_suite?: { id?: number };
app?: { id?: number };
};
// Phases of one check (a fresh run id each, as Cloudflare Pages sends them)
// share a suite; two checks of the same name from different workflows or
// apps do not — they never share a message.
const owner = checkRun.check_suite?.id ?? checkRun.app?.id;
const status = workflowStatus(checkRun.status, checkRun.conclusion);
const emoji = WORKFLOW_CONCLUSION_EMOJI[status] ?? "⏳";
@@ -226,9 +232,14 @@ export function formatCheckRun(
url: checkRun.html_url,
color: GITHUB_COLORS[colorKey],
fields,
// Chains tracked before the suite joined the key.
legacyUpdateKey:
repo && checkRun.name && checkRun.head_sha && owner != null
? `check_run:${repo}:${checkRun.name}:${checkRun.head_sha}`
: undefined,
updateKey:
repo && checkRun.name && checkRun.head_sha
? `check_run:${repo}:${checkRun.name}:${checkRun.head_sha}`
? `check_run:${repo}:${owner != null ? `${owner}:` : ""}${checkRun.name}:${checkRun.head_sha}`
: repo && checkRun.id != null
? `check_run:${repo}:${checkRun.id}`
: undefined,
+6 -2
View File
@@ -8,6 +8,7 @@ import {
type T,
buildMessage,
repoBaseUrl,
shortRef,
} from "./helpers";
/** Shared "branch/tag + commit" fields for deployment events. */
@@ -21,7 +22,10 @@ function addDeploymentRefFields(
const isTag = deployment.ref.startsWith("refs/tags/");
fields.push({
name: t("fields.branch_tag"),
value: isTag ? tagLink(baseUrl, deployment.ref) : branchLink(baseUrl, deployment.ref),
// A deployment's ref may be a full ref: its one leading prefix goes.
value: isTag
? tagLink(baseUrl, shortRef(deployment.ref))
: branchLink(baseUrl, shortRef(deployment.ref)),
inline: true,
});
}
@@ -133,7 +137,7 @@ export function formatDeploymentStatus(
fields.push({
name: t("fields.status"),
value: `${em(emoji)}${status}`,
value: `${em(emoji)}${state}`,
inline: true,
});
+104 -15
View File
@@ -67,30 +67,42 @@ export function commitLink(baseUrl: string | undefined, sha: string, short?: str
return baseUrl ? `[\`${label}\`](${baseUrl}/commit/${encodeRefPath(sha)})` : `\`${label}\``;
}
/** Inline code + hyperlink for a branch (or bare ref), e.g. [`main`](.../tree/main). */
/**
* A ref's short name: only one leading `refs/heads/` or `refs/tags/` goes —
* a branch may itself contain such a path (`feature/refs/tags/test`).
*/
export function shortRef(ref: string): string {
return ref.replace(/^refs\/(?:heads|tags)\//, "");
}
/**
* Inline code + hyperlink for a branch, e.g. [`main`](.../tree/main). Takes
* the short name, as written (a branch may well be called `refs/heads/x`);
* a full ref is shortened by the caller (`shortRef`).
*/
export function branchLink(baseUrl: string | undefined, branch: string, label?: string): string {
const clean = branch.replace("refs/heads/", "").replace("refs/tags/", "");
const clean = branch;
const display = label ?? clean;
return baseUrl ? `[\`${display}\`](${baseUrl}/tree/${encodeRefPath(clean)})` : `\`${display}\``;
}
/** Inline code + hyperlink for a tag, e.g. [`v1.0`](.../releases/tag/v1.0). */
/** Inline code + hyperlink for a tag (its short name), e.g. [`v1.0`](.../releases/tag/v1.0). */
export function tagLink(baseUrl: string | undefined, tag: string, label?: string): string {
const clean = tag.replace("refs/tags/", "");
const clean = tag;
const display = label ?? clean;
return baseUrl
? `[\`${display}\`](${baseUrl}/releases/tag/${encodeRefPath(clean)})`
: `\`${display}\``;
}
/** URL for a branch or tag, when the repository base URL is available. */
/** URL for a branch or tag (its short name), when the repository base URL is available. */
export function refUrl(
baseUrl: string | undefined,
ref: string,
type: "branch" | "tag",
): string | undefined {
if (!baseUrl) return undefined;
const clean = ref.replace("refs/heads/", "").replace("refs/tags/", "");
const clean = ref;
return type === "tag"
? `${baseUrl}/releases/tag/${encodeRefPath(clean)}`
: `${baseUrl}/tree/${encodeRefPath(clean)}`;
@@ -110,28 +122,105 @@ export function cap(text: string, max: number): string {
return text.length > max ? text.slice(0, max) : text;
}
/** A numeric character reference, kept verbatim when it names no valid code point. */
function codePoint(value: number, original: string): string {
if (!Number.isInteger(value) || value < 0 || value > 0x10ffff) return original;
if (value >= 0xd800 && value <= 0xdfff) return original;
return String.fromCodePoint(value);
}
/**
* Markdown code and autolinks, which are text as written (see `htmlToText`):
* - a fenced block — ``` or ~~~ (3 or more) at a line start, closed by a
* fence of that same character, at least as long, or by the end of the text;
* - an indented code block — lines indented by 4 spaces or a tab;
* - an inline span — a run of backticks closed by a run of the same length
* within 1000 characters (a shorter or longer run inside is content),
* possibly across lines — bounded, so unmatched runs cost little;
* - an autolink — `<scheme:…>` or an e-mail address `<name@host>`.
*/
const MARKDOWN_VERBATIM =
/^ {0,3}(?<fence>(?<fc>[`~])\k<fc>{2,})[^\n]*(?:\n[\s\S]*?(?:\n {0,3}\k<fence>\k<fc>*[ \t]*(?=\n|$(?![\s\S]))|$(?![\s\S]))|$(?![\s\S]))|^(?: {4}|\t)[^\n]*(?:\n(?: {4}|\t)[^\n]*)*|(?<!`)(?<tick>`+)(?!`)[\s\S]{0,1000}?(?<!`)\k<tick>(?!`)|<[a-z][a-z0-9+.-]*:[^\s<>]*>|<[^\s<>@]+@[^\s<>@]+>/gim;
/**
* Most of a body cleaned: callers keep a few hundred characters of the
* result, and an unbounded body (megabytes of markup) must not cost an
* unbounded amount of CPU on every delivery and retry.
*/
const MAX_HTML_INPUT = 10_000;
/**
* Recent results: one event's body is summarized for every route it matches
* (each route formats its own message), so the same text is cleaned once.
*/
const htmlToTextCache = new Map<string, string>();
const HTML_TO_TEXT_CACHE_SIZE = 32;
export function htmlToText(input: string): string {
if (!input) return "";
const cached = htmlToTextCache.get(input);
if (cached !== undefined) return cached;
const result = htmlToTextUncached(input);
if (htmlToTextCache.size >= HTML_TO_TEXT_CACHE_SIZE) {
htmlToTextCache.delete(htmlToTextCache.keys().next().value!);
}
htmlToTextCache.set(input, result);
return result;
}
function htmlToTextUncached(input: string): string {
const text = input.length > MAX_HTML_INPUT ? input.slice(0, MAX_HTML_INPUT) : input;
// Markdown code and autolinks are text as written: `<T>` or
// `<MyComponent />` there is content, not HTML to strip, an entity there is
// literal, and its spacing (a leading indent included) is its own. Only the
// text around them is cleaned and trimmed.
const parts: Array<{ text: string; verbatim: boolean }> = [];
let last = 0;
for (const m of text.matchAll(MARKDOWN_VERBATIM)) {
parts.push({ text: clean(text.slice(last, m.index)), verbatim: false });
parts.push({ text: m[0], verbatim: true });
last = m.index + m[0].length;
}
parts.push({ text: clean(text.slice(last)), verbatim: false });
const first = parts.findIndex((p) => p.text.trim() !== "");
if (first < 0) return "";
const kept = parts.slice(first);
while (kept.length && kept[kept.length - 1]!.text.trim() === "") kept.pop();
if (!kept[0]!.verbatim) kept[0]!.text = kept[0]!.text.trimStart();
const end = kept[kept.length - 1]!;
if (!end.verbatim) end.text = end.text.trimEnd();
return kept.map((p) => p.text).join("");
}
function clean(text: string): string {
return stripHtml(text)
.replace(/[ \t]+\n/g, "\n")
.replace(/\n{3,}/g, "\n\n");
}
function stripHtml(input: string): string {
// A tag never spans another `<`: each `<` is looked at once, so text full
// of unclosed `<` stays linear.
let out = input
.replace(/<br\s*\/?>/gi, "\n")
.replace(/<table[^>]*>/gi, "\n")
.replace(/<table[^<>]*>/gi, "\n")
.replace(/<\/(p|div|tr|h[1-6]|ul|ol|table|blockquote)>/gi, "\n")
.replace(/<\/t[dh]>/gi, " ")
.replace(/<li[^>]*>/gi, "\n• ")
.replace(/<[^>]+>/g, "");
.replace(/<li[^<>]*>/gi, "\n• ")
// A prefix such as `<script` is still dangerous when the closing `>` is
// absent or is supplied later by a renderer. Remove the opening bracket
// itself after known HTML has been converted to text.
.replace(/<[^<>]+>/g, "")
.replace(/</g, "");
out = out
.replace(/&nbsp;/gi, " ")
.replace(/&lt;/gi, "<")
.replace(/&gt;/gi, ">")
.replace(/&quot;/gi, '"')
.replace(/&apos;/gi, "'")
.replace(/&#x([0-9a-f]+);/gi, (_m, hex: string) => String.fromCodePoint(parseInt(hex, 16)))
.replace(/&#(\d+);/g, (_m, dec: string) => String.fromCodePoint(Number(dec)))
.replace(/&#x([0-9a-f]+);/gi, (m, hex: string) => codePoint(parseInt(hex, 16), m))
.replace(/&#(\d+);/g, (m, dec: string) => codePoint(Number(dec), m))
.replace(/&amp;/gi, "&");
out = out
.replace(/[ \t]+\n/g, "\n")
.replace(/\n{3,}/g, "\n\n")
.trim();
return out;
}
+16 -3
View File
@@ -28,8 +28,21 @@ export function formatEvent(
const ctx: FormatContext = { payload, repo, repoUrl, author, t, showEmoji };
return (
const message =
findFormatter(eventType)?.format(ctx) ??
formatGeneric(eventType, payload, repo, author, t, showEmoji)
);
formatGeneric(eventType, payload, repo, author, t, showEmoji);
// Action buttons (merge / close) call the github.com API with the clicker's
// token: on any other forge they would act on a same-named GitHub repository.
if (message.actions && !isGitHubDotCom(event.provider, repoUrl)) delete message.actions;
return message;
}
function isGitHubDotCom(provider: string | undefined, repoUrl: string | undefined): boolean {
if ((provider ?? "github") !== "github") return false;
if (!repoUrl) return true;
try {
return new URL(repoUrl).hostname.toLowerCase() === "github.com";
} catch {
return false;
}
}
+13 -6
View File
@@ -9,6 +9,8 @@ import {
type T,
buildMessage,
repoBaseUrl,
senderProfileUrl,
shortRef,
} from "./helpers";
export function formatPush(
@@ -20,7 +22,8 @@ export function formatPush(
): NeutralMessage {
const rawRef = (payload.ref as string) ?? "";
const isTagPush = rawRef.startsWith("refs/tags/");
const ref = rawRef.replace("refs/heads/", "").replace("refs/tags/", "tag: ");
// One prefix, read once: a branch may itself be named `refs/tags/x`.
const ref = rawRef.startsWith("refs/tags/") ? `tag: ${shortRef(rawRef)}` : shortRef(rawRef);
const commits = (payload.commits ?? []) as Array<{
id?: string;
message?: string;
@@ -52,7 +55,7 @@ export function formatPush(
type: refType,
ref,
}),
url: refUrl(baseUrl, rawRef, refType),
url: refUrl(baseUrl, shortRef(rawRef), refType),
color: GITHUB_COLORS.delete,
},
t,
@@ -65,7 +68,7 @@ export function formatPush(
// instead of a confusing "0 commits pushed".
if (created && count === 0) {
const refType = isTagPush ? "tag" : "branch";
const cleanRef = rawRef.replace("refs/heads/", "").replace("refs/tags/", "");
const cleanRef = shortRef(rawRef);
return buildMessage(
{
author,
@@ -75,7 +78,7 @@ export function formatPush(
type: refType,
ref: cleanRef,
}),
url: refUrl(baseUrl, cleanRef, refType),
url: refUrl(baseUrl, shortRef(rawRef), refType),
color: GITHUB_COLORS.create,
},
t,
@@ -105,7 +108,9 @@ export function formatPush(
if (commitAuthor === "invalid-email-address") {
if (commitAuthorName) msg += ` - ${commitAuthorName}`;
} else {
msg += ` - [${commitAuthor}](https://github.com/${encodeURIComponent(commitAuthor)})`;
// On the forge the push came from (a Gitea author is not the
// GitHub account of the same name).
msg += ` - [${commitAuthor}](${senderProfileUrl(baseUrl ?? undefined, commitAuthor)})`;
}
}
const url = baseUrl && c.id ? `${baseUrl}/commit/${c.id}` : null;
@@ -151,7 +156,9 @@ export function formatPush(
count,
s: count !== 1 ? "s" : "",
repo: repo ?? t("common.repository"),
ref: isTagPush ? tagLink(baseUrl, rawRef, ref) : branchLink(baseUrl, rawRef, ref),
ref: isTagPush
? tagLink(baseUrl, shortRef(rawRef), ref)
: branchLink(baseUrl, shortRef(rawRef), ref),
}),
url: compareUrl,
color: GITHUB_COLORS.push,
+9 -2
View File
@@ -11,12 +11,19 @@ export function formatCodeScanningAlert(
): NeutralMessage {
const action = (payload.action as string) ?? "created";
const alert = payload.alert as {
rule?: { id?: string; severity?: string; description?: string };
rule?: {
id?: string;
severity?: string;
security_severity_level?: string;
description?: string;
};
most_recent_instance?: { location?: { path?: string } };
state?: string;
};
const severity = alert.rule?.severity ?? "warning";
// The security severity (critical/high/medium/low) where the rule has one;
// `severity` is the tool's own level (error/warning/note).
const severity = alert.rule?.security_severity_level ?? alert.rule?.severity ?? "warning";
const colorKey =
severity === "critical"
? "code_scanning_critical"
+80 -48
View File
@@ -1,5 +1,13 @@
import type { FilterNode } from "./types";
import { log } from "./lib/log";
import { canUseD1 } from "./storage/d1";
import {
ConfigConflictError,
groupVersion,
groupVersionBump,
groupVersionGuard,
runGuarded,
} from "./storage/config-store";
export interface NamedFragment {
id: string;
@@ -15,59 +23,83 @@ interface D1FragmentRow {
node: string;
}
export async function loadFragments(db: D1Database): Promise<NamedFragment[]> {
try {
const stmt = db.prepare("SELECT id, group_id, name, node FROM d1_fragments ORDER BY id");
if (typeof stmt.all !== "function") return [];
const { results } = await stmt.all<D1FragmentRow>();
if (!results || results.length === 0) return [];
return results.map((r) => ({
id: r.id,
groupId: r.group_id || undefined,
name: r.name,
node: JSON.parse(r.node) as FilterNode,
}));
} catch (err) {
log.warn({ err }, "Failed to load fragments from D1");
return [];
}
/**
* Named filter fragments, optionally of one group. Without a usable D1
* binding there are none; a D1 error propagates, because callers replace a
* group's whole list and must never build that list from a failed read. An
* unreadable row is skipped (logged) — or, with `strict`, an error: a write
* replacing the group's list must never silently delete a row nobody saw.
*/
export async function loadFragments(
db: D1Database,
groupId?: string,
strict = false,
): Promise<NamedFragment[]> {
if (!canUseD1(db)) return [];
const stmt =
groupId === undefined
? db.prepare("SELECT id, group_id, name, node FROM d1_fragments ORDER BY id")
: db
.prepare(
"SELECT id, group_id, name, node FROM d1_fragments WHERE group_id = ? ORDER BY id",
)
.bind(groupId);
const { results } = await stmt.all<D1FragmentRow>();
return (results ?? []).flatMap((r) => {
try {
return [
{
id: r.id,
groupId: r.group_id || undefined,
name: r.name,
node: JSON.parse(r.node) as FilterNode,
},
];
} catch (err) {
if (strict) {
throw new Error(`filter fragment "${r.id}" is unreadable; repair the row before saving`);
}
log.warn({ err, id: r.id }, "Ignoring malformed filter fragment");
return [];
}
});
}
export async function saveFragments(db: D1Database, fragments: NamedFragment[]): Promise<void> {
/**
* Replace one group's fragments in a single transaction; other groups are
* untouched. `expect` sees the stored fragments first and may throw; the write
* then commits only if nothing changed the group in between (version guard).
*/
export async function saveGroupFragments(
db: D1Database,
groupId: string,
fragments: NamedFragment[],
opts?: {
expect?: (current: NamedFragment[]) => void | Promise<void>;
/** Guard statements committed first in the write (see `GuardedWrite.guards`). */
guards?: (db: D1Database) => D1PreparedStatement[];
},
): Promise<NamedFragment[]> {
if (!canUseD1(db)) throw new Error("Filter fragments require the D1 binding");
const version = await groupVersion(db, groupId);
if (version == null) throw new ConfigConflictError(`group "${groupId}" does not exist`);
// Loaded strictly even without `expect`: an unreadable row aborts here.
const stored = await loadFragments(db, groupId, true);
await opts?.expect?.(stored);
const now = Date.now();
// Load existing fragments to determine what to delete
const existing = await loadFragments(db);
const existingKeys = new Set(existing.map((f) => `${f.id}:${f.groupId ?? ""}`));
const newKeys = new Set(fragments.map((f) => `${f.id}:${f.groupId ?? ""}`));
const statements: D1PreparedStatement[] = [];
// Delete fragments that are no longer present
for (const key of existingKeys) {
if (!newKeys.has(key)) {
const [id, groupId] = key.split(":");
statements.push(
db.prepare("DELETE FROM d1_fragments WHERE id = ? AND group_id = ?").bind(id, groupId),
);
}
}
// Upsert all fragments
for (const f of fragments) {
statements.push(
await runGuarded(db, [
groupVersionGuard(db, groupId, version),
...(opts?.guards?.(db) ?? []),
groupVersionBump(db, groupId),
db.prepare("DELETE FROM d1_fragments WHERE group_id = ?").bind(groupId),
...fragments.map((f) =>
db
.prepare(
`INSERT INTO d1_fragments (id, group_id, name, node, version, created_at, updated_at)
VALUES (?, ?, ?, ?, 1, ?, ?)
ON CONFLICT(id, group_id) DO UPDATE SET
name = excluded.name,
node = excluded.node,
updated_at = excluded.updated_at`,
VALUES (?, ?, ?, ?, 1, ?, ?)`,
)
.bind(f.id, f.groupId ?? "", f.name, JSON.stringify(f.node), now, now),
);
}
await db.batch(statements);
.bind(f.id, groupId, f.name, JSON.stringify(f.node), now, now),
),
]);
return loadFragments(db, groupId);
}
+72 -2
View File
@@ -1,25 +1,89 @@
import { createAppJwt } from "./oauth";
import { sharedFlight, type Flight } from "../lib/shared-flight";
export const GITHUB_API_VERSION = "2022-11-28";
/**
* Longest an optional enrichment call (installation token, jobs, check runs)
* may take, its body included: a hanging one must not hold up the delivery.
*/
export const ENRICH_TIMEOUT_MS = 10_000;
export const GITHUB_USER_AGENT = "WebHooker (https://github.com/Elysola/WebHooker)";
/**
* A payload-supplied API URL (`jobs_url`, `check_runs_url`) parsed only when
* it points at the GitHub REST API's repository endpoints. Anything else — a
* forged payload aiming a token at its own server — yields null.
*/
export function githubApiUrl(value: string | undefined): URL | null {
if (!value) return null;
try {
const url = new URL(value);
if (url.protocol !== "https:" || url.host !== "api.github.com") return null;
if (!url.pathname.startsWith("/repos/") || url.username || url.password) return null;
return url;
} catch {
return null;
}
}
/**
* Mint a short-lived installation access token for a GitHub App installation.
* Returns undefined when the App credentials are missing or the exchange
* fails. Best-effort: callers must not depend on it.
*/
/**
* Installation tokens this isolate holds (until shortly before they expire),
* and exchanges in flight: one per installation, not one per event — each is
* a subrequest plus a JWT signature.
*/
const installationTokens = new Map<string, { token: string; until: number }>();
const exchanging = new Map<string, Flight<string | undefined>>();
/**
* Installations whose exchange just failed, until when: enrichment is
* optional, so the events right after go without it instead of each signing
* a JWT and asking GitHub again.
*/
const failedExchanges = new Map<string, number>();
const FAILED_EXCHANGE_MS = 60_000;
export async function getInstallationToken(
appId: string | undefined,
privateKey: string | undefined,
installationId: number | undefined,
): Promise<string | undefined> {
if (!appId || !privateKey || !installationId) return undefined;
const key = `${appId}:${installationId}`;
const held = installationTokens.get(key);
if (held && Date.now() < held.until) return held.token;
if ((failedExchanges.get(key) ?? 0) > Date.now()) return undefined;
return sharedFlight(
exchanging,
key,
async () => {
const token = await exchangeInstallationToken(appId, privateKey, installationId);
if (token) failedExchanges.delete(key);
else {
if (failedExchanges.size >= 1000) failedExchanges.clear();
failedExchanges.set(key, Date.now() + FAILED_EXCHANGE_MS);
}
return token;
},
ENRICH_TIMEOUT_MS + 5_000,
);
}
async function exchangeInstallationToken(
appId: string,
privateKey: string,
installationId: number,
): Promise<string | undefined> {
try {
const jwt = await createAppJwt(appId, privateKey);
const res = await fetch(
`https://api.github.com/app/installations/${installationId}/access_tokens`,
{
method: "POST",
signal: AbortSignal.timeout(ENRICH_TIMEOUT_MS),
headers: {
Authorization: `Bearer ${jwt}`,
Accept: "application/vnd.github+json",
@@ -29,8 +93,14 @@ export async function getInstallationToken(
},
);
if (!res.ok) return undefined;
const { token } = (await res.json()) as { token?: string };
return token ?? undefined;
const { token, expires_at } = (await res.json()) as { token?: string; expires_at?: string };
if (!token) return undefined;
const expires = Date.parse(expires_at ?? "");
// Tokens last an hour; given back 5 minutes early (a request in flight
// never carries one that just expired).
const until = (Number.isFinite(expires) ? expires : Date.now() + 3600_000) - 5 * 60_000;
if (until > Date.now()) installationTokens.set(`${appId}:${installationId}`, { token, until });
return token;
} catch {
return undefined;
}
+13 -3
View File
@@ -1,4 +1,10 @@
import { getInstallationToken, GITHUB_API_VERSION, GITHUB_USER_AGENT } from "./app-token";
import {
getInstallationToken,
githubApiUrl,
ENRICH_TIMEOUT_MS,
GITHUB_API_VERSION,
GITHUB_USER_AGENT,
} from "./app-token";
const CLOUDFLARE_SLUG = "cloudflare-workers-and-pages";
@@ -18,12 +24,16 @@ export async function getCheckSuiteBuildLogUrl(
privateKey: string | undefined,
installationId: number | undefined,
): Promise<string | undefined> {
if (!checkRunsUrl || !appId || !privateKey || !installationId) return undefined;
const url = githubApiUrl(checkRunsUrl);
if (!url || !appId || !privateKey || !installationId) return undefined;
try {
const token = await getInstallationToken(appId, privateKey, installationId);
if (!token) return undefined;
const runsRes = await fetch(checkRunsUrl, {
// No redirects: a token must never follow a response to another host.
const runsRes = await fetch(url.toString(), {
redirect: "manual",
signal: AbortSignal.timeout(ENRICH_TIMEOUT_MS),
headers: {
Authorization: `Bearer ${token}`,
Accept: "application/vnd.github+json",
+169 -31
View File
@@ -1,8 +1,9 @@
import { Octokit } from "octokit";
import { saveToken, getToken } from "./store";
import { saveToken, getToken, type TokenSource } from "./store";
export function getOAuthURL(clientId: string, state: string): string {
return `https://github.com/login/oauth/authorize?client_id=${clientId}&scope=repo&state=${state}`;
// `read:org` lets the install flow check organization admin rights.
return `https://github.com/login/oauth/authorize?client_id=${clientId}&scope=${encodeURIComponent("repo read:org")}&state=${state}`;
}
function b64url(buf: ArrayBuffer | string): string {
@@ -40,31 +41,82 @@ export async function createAppJwt(appId: string, privateKey: string): Promise<s
return `${data}.${b64url(sig)}`;
}
export interface InstallationAccount {
login: string;
id: number;
/** "User" or "Organization". */
type: string;
}
const API_HEADERS = {
Accept: "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
"User-Agent": "WebHooker",
};
/**
* Look up the account (org/user login) that owns a GitHub App installation,
* using an App JWT. Returns null when the App credentials are missing or the
* lookup fails (the caller falls back to an anonymous installation group).
* Look up the account (org/user) that owns a GitHub App installation, using
* an App JWT. Returns null when the App credentials are missing or the lookup
* fails.
*/
export async function getInstallationDetails(
appId: string,
privateKey: string,
installationId: number,
): Promise<InstallationAccount | null> {
if (!appId || !privateKey) return null;
try {
const jwt = await createAppJwt(appId, privateKey);
const res = await fetch(`https://api.github.com/app/installations/${installationId}`, {
headers: { ...API_HEADERS, Authorization: `Bearer ${jwt}` },
});
if (!res.ok) return null;
const data = (await res.json()) as {
account?: { login?: string; id?: number; type?: string };
};
if (!data.account?.login) return null;
// An account without an id can still be named, but never matches a user.
return { login: data.account.login, id: data.account.id ?? -1, type: data.account.type ?? "" };
} catch {
return null;
}
}
/** The login of the account owning an installation (null when unknown). */
export async function getInstallationAccount(
appId: string,
privateKey: string,
installationId: number,
): Promise<string | null> {
if (!appId || !privateKey) return null;
return (await getInstallationDetails(appId, privateKey, installationId))?.login ?? null;
}
/**
* Whether a GitHub user may administer an App installation. The Setup URL's
* `installation_id` is attacker-controlled, so binding it to a group requires
* proof from the user's own token: the user is the installation's account, or
* an active admin of its organization. Merely having access to it (being
* listed in `/user/installations` as a collaborator) is not enough.
*/
export async function userManagesInstallation(
token: string,
userId: string,
_installationId: number,
account: InstallationAccount | null,
): Promise<boolean> {
const headers = { ...API_HEADERS, Authorization: `Bearer ${token}` };
try {
const jwt = await createAppJwt(appId, privateKey);
const res = await fetch(`https://api.github.com/app/installations/${installationId}`, {
headers: {
Authorization: `Bearer ${jwt}`,
Accept: "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
},
});
if (!res.ok) return null;
const data = (await res.json()) as { account?: { login?: string } };
return data.account?.login ?? null;
if (!account) return false;
if (account.type === "User") return String(account.id) === userId;
const res = await fetch(
`https://api.github.com/user/memberships/orgs/${encodeURIComponent(account.login)}`,
{ headers },
);
if (!res.ok) return false;
const membership = (await res.json()) as { state?: string; role?: string };
return membership.state === "active" && membership.role === "admin";
} catch {
return null;
return false;
}
}
@@ -74,6 +126,7 @@ export async function handleOAuthCallback(
code: string,
_state: string,
kv: KVNamespace,
db?: D1Database,
): Promise<{ userId: string; login: string } | null> {
const res = await fetch("https://github.com/login/oauth/access_token", {
method: "POST",
@@ -102,15 +155,18 @@ export async function handleOAuthCallback(
const octokit = new Octokit({ auth: data.access_token });
const { data: user } = await octokit.rest.users.getAuthenticated();
await saveToken(kv, user.id.toString(), data.access_token, 3600 * 10);
await saveToken(kv, user.id.toString(), data.access_token, 3600 * 10, db);
return { userId: user.id.toString(), login: user.login };
}
export async function getUserOctokit(userId: string, kv: KVNamespace): Promise<Octokit | null> {
export async function getUserOctokit(userId: string, kv: TokenSource): Promise<Octokit | null> {
const token = await getToken(kv, userId);
if (!token) return null;
return new Octokit({ auth: token });
// No automatic retries: the user's actions (a comment, a merge) are not
// idempotent, and a retry after a lost response would act twice. A failure
// is reported to the user instead, who can try again knowingly.
return new Octokit({ auth: token, retry: { enabled: false }, throttle: { enabled: false } });
}
/**
@@ -125,8 +181,84 @@ function mapGitHubError(err: unknown): Error {
return err instanceof Error ? err : new Error(String(err));
}
async function requireOctokit(kv: KVNamespace, githubUserId: string): Promise<Octokit> {
const octokit = await getUserOctokit(githubUserId, kv);
/**
* The immutable user id GitHub has for `login` right now, or null when it is
* unknown (no such user, no answer). Looked up with the caller's own token.
*/
export async function resolveGitHubUserId(token: string, login: string): Promise<string | null> {
try {
const res = await fetch(`https://api.github.com/users/${encodeURIComponent(login)}`, {
headers: {
Authorization: `Bearer ${token}`,
Accept: "application/vnd.github+json",
"User-Agent": "WebHooker",
},
redirect: "error",
signal: AbortSignal.timeout(5000),
});
if (!res.ok) return null;
const data = (await res.json()) as { id?: unknown; login?: unknown };
const sameLogin =
typeof data.login === "string" && data.login.toLowerCase() === login.toLowerCase();
return sameLogin && typeof data.id === "number" ? String(data.id) : null;
} catch {
return null;
}
}
/**
* A failure before anything was sent to GitHub (the user's token could not be
* read). Bot commands give such a command back for the platform to retry,
* instead of consuming it with an error reply.
*/
export class BeforeGitHubError extends Error {}
/**
* Another run finished this command meanwhile (this one outlived its claim):
* nothing is written, and that run's "done" stays.
*/
export class CommandTakenError extends BeforeGitHubError {}
/**
* GitHub refused the request (or none could be made for want of a token):
* nothing was done, so running the command again is safe. Anything else (a
* network failure, a 5xx) may have acted.
*/
export function githubRefused(err: unknown): boolean {
const message = err instanceof Error ? err.message : "";
if (["GITHUB_TOKEN_EXPIRED", "GITHUB_FORBIDDEN", "GITHUB_NOT_FOUND"].includes(message)) {
return true;
}
const status = (err as { status?: unknown })?.status;
return typeof status === "number" && status >= 400 && status < 500;
}
/**
* `beforeWrite` runs once only the write itself is left (a bot records the
* command as done there, so a lost confirmation can never run the write
* again); its failure aborts with nothing done.
*/
export interface WriteHooks {
beforeWrite?: () => Promise<void>;
}
async function beforeWrite(hooks: WriteHooks | undefined): Promise<void> {
if (!hooks?.beforeWrite) return;
try {
await hooks.beforeWrite();
} catch (err) {
if (err instanceof BeforeGitHubError) throw err;
throw new BeforeGitHubError(err instanceof Error ? err.message : String(err));
}
}
async function requireOctokit(kv: TokenSource, githubUserId: string): Promise<Octokit> {
let octokit: Octokit | null;
try {
octokit = await getUserOctokit(githubUserId, kv);
} catch (err) {
throw new BeforeGitHubError(err instanceof Error ? err.message : String(err));
}
if (!octokit) throw new Error("GITHUB_TOKEN_EXPIRED");
return octokit;
}
@@ -136,23 +268,25 @@ async function requireOctokit(kv: KVNamespace, githubUserId: string): Promise<Oc
* so the comment shows up under their own identity instead of the bot.
*/
export async function commentAsUser(
kv: KVNamespace,
kv: TokenSource,
githubUserId: string,
owner: string,
repo: string,
issueNumber: number,
body: string,
hooks?: WriteHooks,
): Promise<{ htmlUrl: string; login: string }> {
const octokit = await requireOctokit(kv, githubUserId);
await beforeWrite(hooks);
try {
const { data: me } = await octokit.rest.users.getAuthenticated();
// The author comes with the created comment: no separate `/user` request.
const res = await octokit.rest.issues.createComment({
owner,
repo,
issue_number: issueNumber,
body,
});
return { htmlUrl: res.data.html_url, login: me.login };
return { htmlUrl: res.data.html_url, login: res.data.user?.login ?? "" };
} catch (err) {
throw mapGitHubError(err);
}
@@ -160,7 +294,7 @@ export async function commentAsUser(
/** Fetch a single issue comment's current body (used to prefill the edit modal). */
export async function getCommentAsUser(
kv: KVNamespace,
kv: TokenSource,
githubUserId: string,
owner: string,
repo: string,
@@ -177,7 +311,7 @@ export async function getCommentAsUser(
/** Edit an existing issue/PR comment. GitHub enforces permission (403 if not allowed). */
export async function editCommentAsUser(
kv: KVNamespace,
kv: TokenSource,
githubUserId: string,
owner: string,
repo: string,
@@ -200,7 +334,7 @@ export async function editCommentAsUser(
/** Delete an existing issue/PR comment. GitHub enforces permission (403 if not allowed). */
export async function deleteCommentAsUser(
kv: KVNamespace,
kv: TokenSource,
githubUserId: string,
owner: string,
repo: string,
@@ -216,14 +350,16 @@ export async function deleteCommentAsUser(
/** Merge a pull request as the linked GitHub user. GitHub enforces permission (403 if not allowed). */
export async function mergePullRequestAsUser(
kv: KVNamespace,
kv: TokenSource,
githubUserId: string,
owner: string,
repo: string,
pullNumber: number,
method: "merge" | "squash" | "rebase" = "squash",
hooks?: WriteHooks,
): Promise<void> {
const octokit = await requireOctokit(kv, githubUserId);
await beforeWrite(hooks);
try {
await octokit.rest.pulls.merge({
owner,
@@ -238,13 +374,15 @@ export async function mergePullRequestAsUser(
/** Close a pull request as the linked GitHub user. GitHub enforces permission (403 if not allowed). */
export async function closePullRequestAsUser(
kv: KVNamespace,
kv: TokenSource,
githubUserId: string,
owner: string,
repo: string,
pullNumber: number,
hooks?: WriteHooks,
): Promise<void> {
const octokit = await requireOctokit(kv, githubUserId);
await beforeWrite(hooks);
try {
await octokit.rest.pulls.update({
owner,
+212 -4
View File
@@ -1,7 +1,16 @@
import { idempotencyStore, recordStatement, unrecordedGuard } from "../lib/idempotency";
import { canUseD1, isMissingTable } from "../storage/d1";
interface StoredToken {
userId: string;
accessToken: string;
expiresAt: number;
/**
* Valid only while D1 records it as the user's live token (see
* `markLive`). Unset on tokens saved before that record existed, or
* without the D1 table: those follow the revocation records alone.
*/
live?: boolean;
}
async function hashToken(token: string): Promise<string> {
@@ -12,11 +21,93 @@ async function hashToken(token: string): Promise<string> {
.join("");
}
/**
* Tokens are revoked (on removal and rotation) in the idempotency store (D1)
* as well as deleted from KV, whose deletes take a while to reach every
* location: a revoked token must stop authenticating at once.
*/
function revokedTokenKey(tokenHash: string): string {
return `token-revoked:${tokenHash}`;
}
/**
* The user's live token, in D1. KV can hand a node a stale copy of the user's
* token, so a removal or rotation working from that copy would miss the
* newer token: dropping every live record of the user covers it whatever
* copy was read.
*/
function liveTokenKey(userId: string, tokenHash: string): string {
return `${liveTokenPrefix(userId)}${tokenHash}`;
}
function liveTokenPrefix(userId: string): string {
return `token-live:${userId}:`;
}
/** Statement dropping the user's live records (all, or all but `keep`). */
function dropLive(db: D1Database, userId: string, keep = ""): D1PreparedStatement {
// A range over the prefix (`;` follows `:`), never a LIKE pattern.
return db
.prepare("DELETE FROM dedup_keys WHERE key >= ? AND key < ? AND key <> ?")
.bind(liveTokenPrefix(userId), `token-live:${userId};`, keep);
}
/**
* Record `tokenHash` as the user's only live token, in one D1 batch. Returns
* false where the D1 table does not exist yet (the token then follows the
* revocation records alone).
*/
async function markLive(
db: D1Database,
userId: string,
tokenHash: string,
ttlSeconds: number,
): Promise<boolean> {
const now = Date.now();
const key = liveTokenKey(userId, tokenHash);
try {
await db.batch([
dropLive(db, userId, key),
db
.prepare(
`INSERT INTO dedup_keys (key, claimed_at, expires_at) VALUES (?, ?, ?)
ON CONFLICT(key) DO UPDATE SET
claimed_at = excluded.claimed_at, expires_at = excluded.expires_at`,
)
.bind(key, now, now + ttlSeconds * 1000),
]);
return true;
} catch (err) {
if (isMissingTable(err)) return false;
throw err;
}
}
/** Whether a stored token may still be used (expiry aside). */
async function usable(
db: D1Database | undefined,
kv: KVNamespace,
t: StoredToken,
tokenHash: string,
): Promise<boolean> {
if (!db) return true;
const claims = idempotencyStore(db, kv);
if (await claims.has(revokedTokenKey(tokenHash))) return false;
if (t.live && !(await claims.has(liveTokenKey(t.userId, tokenHash)))) return false;
return true;
}
async function revokeToken(db: D1Database, kv: KVNamespace, t: StoredToken): Promise<void> {
const ttl = Math.max(60, Math.ceil((t.expiresAt - Date.now()) / 1000));
await idempotencyStore(db, kv).claim(revokedTokenKey(await hashToken(t.accessToken)), ttl);
}
export async function saveToken(
kv: KVNamespace,
userId: string,
accessToken: string,
expiresInSeconds: number,
db?: D1Database,
): Promise<void> {
const token: StoredToken = {
userId,
@@ -24,13 +115,36 @@ export async function saveToken(
expiresAt: Date.now() + expiresInSeconds * 1000,
};
const ttl = Math.max(Math.floor(expiresInSeconds * 0.9), 60);
const tokenHash = await hashToken(accessToken);
// Rotation: the previous token's reverse index must stop resolving to this
// user, or the old token would keep acting with the new one.
// A failed read aborts the rotation: the old token must be revoked, and
// "none" cannot be assumed.
const previous = await kv.get<StoredToken>(`token:${userId}`, "json");
const rotated = previous?.accessToken && previous.accessToken !== accessToken;
// Revoked before it is overwritten: should this fail, the old token is
// still on record and a retry revokes it then.
if (rotated && db) await revokeToken(db, kv, previous);
// The only live token from here on — whichever older one some node's KV
// still shows.
if (db && canUseD1(db) && (await markLive(db, userId, tokenHash, expiresInSeconds))) {
token.live = true;
}
await kv.put(`token:${userId}`, JSON.stringify(token), { expirationTtl: ttl });
const tokenHash = await hashToken(accessToken);
await kv.put(`token-reverse:${tokenHash}`, userId, { expirationTtl: ttl });
if (rotated) await kv.delete(`token-reverse:${await hashToken(previous.accessToken)}`);
}
export async function getToken(kv: KVNamespace, userId: string): Promise<string | null> {
/** Where tokens are read from: KV alone, or KV with D1 (revocations honored). */
export type TokenSource = KVNamespace | { KV: KVNamespace; DB?: D1Database };
function sourceOf(source: TokenSource): { kv: KVNamespace; db?: D1Database } {
return "KV" in source ? { kv: source.KV, db: source.DB } : { kv: source };
}
export async function getToken(source: TokenSource, userId: string): Promise<string | null> {
const { kv, db } = sourceOf(source);
const raw = await kv.get(`token:${userId}`, "json");
if (!raw) return null;
const t = raw as StoredToken;
@@ -38,25 +152,119 @@ export async function getToken(kv: KVNamespace, userId: string): Promise<string
await kv.delete(`token:${userId}`);
return null;
}
// Revoked or no longer live, even where KV still holds it (writes and
// deletes take a while to spread).
if (!(await usable(db, kv, t, await hashToken(t.accessToken)))) return null;
return t.accessToken;
}
export async function removeToken(kv: KVNamespace, userId: string): Promise<void> {
export async function removeToken(kv: KVNamespace, userId: string, db?: D1Database): Promise<void> {
// First, and whatever KV shows here: a newer token than the copy read below
// stops too.
if (db && canUseD1(db)) {
try {
await dropLive(db, userId).run();
} catch (err) {
if (!isMissingTable(err)) throw err;
}
}
const raw = await kv.get(`token:${userId}`, "json");
if (raw) {
const t = raw as StoredToken;
if (db) await revokeToken(db, kv, t);
const tokenHash = await hashToken(t.accessToken);
await kv.delete(`token-reverse:${tokenHash}`);
}
await kv.delete(`token:${userId}`);
}
/**
* The GitHub user whose *current* stored token is `accessToken`. The reverse
* index only narrows the lookup; the stored token must match too, so a token
* that was rotated, revoked or expired never authenticates (even while a stale
* reverse key is still around).
*/
export async function findUserIdByToken(
kv: KVNamespace,
accessToken: string,
db?: D1Database,
): Promise<string | null> {
const tokenHash = await hashToken(accessToken);
return await kv.get(`token-reverse:${tokenHash}`, "text");
if (db && (await idempotencyStore(db, kv).has(revokedTokenKey(tokenHash)))) return null;
const userId = await kv.get(`token-reverse:${tokenHash}`, "text");
if (!userId) return null;
const stored = await kv.get<StoredToken>(`token:${userId}`, "json");
if (!stored || Date.now() >= stored.expiresAt) return null;
if ((await hashToken(stored.accessToken)) !== tokenHash) return null;
if (!(await usable(db, kv, stored, tokenHash))) return null;
return userId;
}
export type LinkPlatform = "discord" | "telegram" | "feishu";
const LINK_TABLES: Record<LinkPlatform, { table: string; column: string }> = {
discord: { table: "discord_links", column: "discord_user_id" },
telegram: { table: "telegram_links", column: "telegram_user_id" },
feishu: { table: "feishu_links", column: "feishu_user_id" },
};
/** Statement linking a chat account to a GitHub account (for a batch). */
export function linkStatement(
db: D1Database,
platform: LinkPlatform,
chatUserId: string,
githubUserId: string,
): D1PreparedStatement {
const { table, column } = LINK_TABLES[platform];
return db
.prepare(`INSERT OR REPLACE INTO ${table} (${column}, github_user_id) VALUES (?, ?)`)
.bind(chatUserId, githubUserId);
}
/** Statement removing a chat account's link (for a batch). */
export function unlinkStatement(
db: D1Database,
platform: LinkPlatform,
chatUserId: string,
): D1PreparedStatement {
const { table, column } = LINK_TABLES[platform];
return db.prepare(`DELETE FROM ${table} WHERE ${column} = ?`).bind(chatUserId);
}
/**
* Unlink as a bot command: the command's "done" record (`done`) commits in
* the same D1 transaction, so an unlink that went through — even one whose
* answer was lost — is never run again by a redelivery (which by then could
* remove a link made since).
*/
export async function unlinkAsCommand(
env: { DB: D1Database; KV: KVNamespace },
platform: LinkPlatform,
chatUserId: string,
done?: { key: string; ttl: number },
): Promise<void> {
if (!done || !canUseD1(env.DB)) {
await unlinkStatement(env.DB, platform, chatUserId).run();
return;
}
try {
await env.DB.batch([
// Done already (another run took over this command after ours outlived
// its claim): nothing is removed — the link may be a new one by now.
unrecordedGuard(env.DB, done.key),
unlinkStatement(env.DB, platform, chatUserId),
recordStatement(env.DB, done.key, done.ttl),
]);
} catch (err) {
// Recorded — committed here after all (its answer lost), or by another
// run before: done either way.
if (
!(await idempotencyStore(env.DB, env.KV)
.has(done.key)
.catch(() => false))
)
throw err;
}
}
/**
+15 -4
View File
@@ -1,4 +1,10 @@
import { getInstallationToken, GITHUB_API_VERSION, GITHUB_USER_AGENT } from "./app-token";
import {
getInstallationToken,
githubApiUrl,
ENRICH_TIMEOUT_MS,
GITHUB_API_VERSION,
GITHUB_USER_AGENT,
} from "./app-token";
export interface WorkflowJob {
name?: string;
@@ -24,10 +30,10 @@ export async function getWorkflowRunJobs(
privateKey: string | undefined,
installationId: number | undefined,
): Promise<WorkflowJob[] | undefined> {
if (!jobsUrl) return undefined;
const url = githubApiUrl(jobsUrl);
if (!url) return undefined;
try {
const token = await getInstallationToken(appId, privateKey, installationId);
const url = new URL(jobsUrl);
url.searchParams.set("per_page", "100");
const headers: Record<string, string> = {
Accept: "application/vnd.github+json",
@@ -35,7 +41,12 @@ export async function getWorkflowRunJobs(
"User-Agent": GITHUB_USER_AGENT,
};
if (token) headers.Authorization = `Bearer ${token}`;
const res = await fetch(url.toString(), { headers });
// No redirects: a token must never follow a response to another host.
const res = await fetch(url.toString(), {
headers,
redirect: "manual",
signal: AbortSignal.timeout(ENRICH_TIMEOUT_MS),
});
if (!res.ok) return undefined;
const data = (await res.json()) as { jobs?: WorkflowJob[] };
return data.jobs?.length ? data.jobs : undefined;
+50 -17
View File
@@ -1,4 +1,6 @@
import { groupScopeClause, type GroupScope } from "./send-log";
import { log } from "./log";
import { isMissingTable } from "../storage/d1";
export interface AuditEntry {
id?: number;
@@ -10,6 +12,11 @@ export interface AuditEntry {
targetType?: string;
targetId?: string;
groupId?: string;
/**
* Immutable instance of the group (write-only): the entry is filed under
* whatever id that instance has when it is written.
*/
groupInstance?: string;
/** Free-form metadata. Never include secrets or message bodies. */
detail?: Record<string, unknown>;
ip?: string;
@@ -46,13 +53,20 @@ function toEntry(r: AuditRow): AuditEntry {
};
}
/** Best-effort write; failures never break the business flow. */
/**
* Best-effort write; failures never break the business flow.
*
* The group is resolved when the row is written, not when the action was
* decided: an entry landing after its group was renamed is filed under the
* new id (with the instance known), after a delete or a rename under none —
* never under an id another group may take over later.
*/
export async function recordAudit(db: D1Database, entry: AuditEntry): Promise<void> {
try {
await db
const insert = (groupSql: string, ...groupArgs: Array<string | null>): D1PreparedStatement =>
db
.prepare(
`INSERT INTO audit_logs (ts, actor_id, actor_login, action, target_type, target_id, group_id, detail, ip)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
VALUES (?, ?, ?, ?, ?, ?, ${groupSql}, ?, ?)`,
)
.bind(
entry.ts,
@@ -61,11 +75,31 @@ export async function recordAudit(db: D1Database, entry: AuditEntry): Promise<vo
entry.action,
entry.targetType ?? null,
entry.targetId ?? null,
entry.groupId ?? null,
...groupArgs,
entry.detail ? JSON.stringify(entry.detail) : null,
entry.ip ?? null,
)
.run();
);
try {
try {
if (entry.groupInstance) {
// The id given is tried first (a primary-key lookup); the groups are
// scanned only when it no longer holds that instance (renamed).
await insert(
"COALESCE((SELECT id FROM d1_groups WHERE id = ? AND json_valid(data) AND json_extract(data, '$.instance') = ?), (SELECT id FROM (SELECT id, CASE WHEN json_valid(data) THEN data ELSE '{}' END AS data FROM d1_groups) WHERE json_extract(data, '$.instance') = ?))",
entry.groupId ?? "",
entry.groupInstance,
entry.groupInstance,
).run();
} else if (entry.groupId) {
await insert("(SELECT id FROM d1_groups WHERE id = ?)", entry.groupId).run();
} else {
await insert("?", null).run();
}
} catch (err) {
// Groups not in D1 (an unmigrated database): file it as given.
if (!isMissingTable(err) || !entry.groupId) throw err;
await insert("?", entry.groupId).run();
}
} catch (err) {
log.warn({ err, action: entry.action }, "Failed to record audit entry");
}
@@ -73,23 +107,22 @@ export async function recordAudit(db: D1Database, entry: AuditEntry): Promise<vo
export async function getAuditLog(
db: D1Database,
opts: { groupId?: string; limit?: number } = {},
opts: { groupId?: GroupScope; limit?: number } = {},
): Promise<AuditEntry[]> {
const limit = Math.min(Math.max(opts.limit ?? 50, 1), 200);
try {
if (opts.groupId) {
const { results } = await db
.prepare(`SELECT ${COLUMNS} FROM audit_logs WHERE group_id = ? ORDER BY ts DESC LIMIT ?`)
.bind(opts.groupId, limit)
.all<AuditRow>();
return results.map(toEntry);
}
const clause = groupScopeClause(opts.groupId);
const { results } = await db
.prepare(`SELECT ${COLUMNS} FROM audit_logs ORDER BY ts DESC LIMIT ?`)
.bind(limit)
.prepare(
`SELECT ${COLUMNS} FROM audit_logs${clause ? ` WHERE ${clause.sql}` : ""} ORDER BY ts DESC LIMIT ?`,
)
.bind(...(clause?.args ?? []), limit)
.all<AuditRow>();
return results.map(toEntry);
} catch (err) {
// Only a table not created yet means "nothing yet"; any other failure
// must reach the caller, not pass for an empty history.
if (!isMissingTable(err)) throw err;
log.warn({ err }, "Failed to load audit log");
return [];
}
+20
View File
@@ -1,5 +1,6 @@
import { en } from "./locales/en";
import { zh } from "./locales/zh";
import { sharedFlight, type Flight } from "./shared-flight";
type NestedStrings = { [key: string]: string | NestedStrings };
export type Translations = NestedStrings;
@@ -22,6 +23,13 @@ function interpolate(template: string, params: Record<string, string | number>):
const cache = new Map<string, Translations>();
/** Languages found without an override, until when (a read failure is not one). */
const missing = new Map<string, number>();
const MISSING_RECHECK_MS = 5 * 60_000;
const MAX_MISSING = 100;
/** Override reads in flight: concurrent asks for one language share a KV read. */
const loading = new Map<string, Flight<Translations>>();
export async function loadTranslations(
lang: string,
kv?: { get<T>(key: string, type: "json"): Promise<T | null> },
@@ -31,9 +39,19 @@ export async function loadTranslations(
const cached = cache.get(lang);
if (cached) return cached;
// A language with no override is not looked up again for a while: every
// dispatch asks for each group's language, and each ask is a KV read.
if ((missing.get(lang) ?? 0) > Date.now()) return en;
if (!kv) return en;
return sharedFlight(loading, lang, () => readOverride(lang, kv), 10_000);
}
async function readOverride(
lang: string,
kv: { get<T>(key: string, type: "json"): Promise<T | null> },
): Promise<Translations> {
try {
const stored = await kv.get<Partial<Translations>>(`i18n:${lang}`, "json");
if (stored) {
@@ -41,6 +59,8 @@ export async function loadTranslations(
cache.set(lang, merged);
return merged;
}
if (missing.size >= MAX_MISSING) missing.clear();
missing.set(lang, Date.now() + MISSING_RECHECK_MS);
} catch {
// KV read failed, fall back to EN
}
+197 -11
View File
@@ -1,15 +1,24 @@
import { canUseD1 } from "../storage/d1";
import { canUseD1, isMissingTable, kvFallbackActive, noteKvFallback } from "../storage/d1";
import { log } from "./log";
/**
* Delivery idempotency: a small, reusable abstraction over "has this key been
* seen / can I claim it once" so webhook dedup, nonce replay protection and
* future delivery retry all share one semantics. Backed by D1 when the binding
* is present (atomic `INSERT OR IGNORE`), falling back to a best-effort
* get-then-put KV claim otherwise.
* seen / can I claim it once" so webhook dedup, nonce replay protection,
* invite consumption and bot-command dedup all share one semantics. Backed by
* D1 when the binding is present (atomic UPSERT), falling back to a
* best-effort get-then-put KV claim otherwise — and also when the D1 table
* does not exist yet (an unmigrated database). Other D1 failures propagate.
*/
export interface IdempotencyStore {
has(key: string): Promise<boolean>;
claim(key: string, ttlSeconds: number): Promise<boolean>;
/** Give a claim back (the claimed work failed and may be retried). */
release(key: string): Promise<void>;
}
/** KV rejects TTLs under 60 seconds. */
function kvTtl(ttlSeconds: number): number {
return Math.max(60, Math.ceil(ttlSeconds));
}
export function kvIdempotencyStore(kv: KVNamespace): IdempotencyStore {
@@ -21,8 +30,15 @@ export function kvIdempotencyStore(kv: KVNamespace): IdempotencyStore {
async claim(key, ttlSeconds): Promise<boolean> {
const value = await kv.get(key);
if (value !== null && value !== undefined) return false;
await kv.put(key, "1", { expirationTtl: ttlSeconds });
return true;
// KV has no compare-and-set: write a unique value and read it back, so
// of two racing claimers the one whose write was overwritten backs off
// (best effort — D1 is the atomic store).
const mine = crypto.randomUUID();
await kv.put(key, mine, { expirationTtl: kvTtl(ttlSeconds) });
return (await kv.get(key)) === mine;
},
async release(key): Promise<void> {
await kv.delete(key);
},
};
}
@@ -49,21 +65,191 @@ export function d1IdempotencyStore(db: D1Database): IdempotencyStore {
.run();
return (result.meta?.changes ?? 0) > 0;
},
async release(key): Promise<void> {
await db.prepare("DELETE FROM dedup_keys WHERE key = ?").bind(key).run();
},
};
}
/** Run against D1, degrading to KV only while the D1 table is missing. */
function withKvFallback(
primary: IdempotencyStore,
fallback: IdempotencyStore,
kvNs: KVNamespace,
): IdempotencyStore {
async function run<T>(op: string, d1: () => Promise<T>, kv: () => Promise<T>): Promise<T> {
try {
return await d1();
} catch (err) {
if (!isMissingTable(err)) throw err;
log.warn({ err, op }, "D1 dedup table missing, falling back to KV");
await noteKvFallback(kvNs);
return kv();
}
}
return {
has: (key) =>
run(
"has",
async () =>
(await primary.has(key)) || ((await kvFallbackActive(kvNs)) && fallback.has(key)),
() => fallback.has(key),
),
claim: (key, ttl) =>
run(
"claim",
async () => {
// Claimed in KV while the table was missing: still taken.
if ((await kvFallbackActive(kvNs)) && (await fallback.has(key))) return false;
return primary.claim(key, ttl);
},
() => fallback.claim(key, ttl),
),
release: (key) =>
run(
"release",
async () => {
await primary.release(key);
// A copy claimed in KV while the table was missing must go too.
if (await kvFallbackActive(kvNs)) await fallback.release(key);
},
() => fallback.release(key),
),
};
}
export function idempotencyStore(db: D1Database, kv: KVNamespace): IdempotencyStore {
return canUseD1(db) ? d1IdempotencyStore(db) : kvIdempotencyStore(kv);
if (canUseD1(db)) return withKvFallback(d1IdempotencyStore(db), kvIdempotencyStore(kv), kv);
// Without D1, claims live in KV: marked, so a D1 binding added later still
// honors the ones not expired yet.
const store = kvIdempotencyStore(kv);
return {
...store,
async claim(key, ttl): Promise<boolean> {
await noteKvFallback(kv);
return store.claim(key, ttl);
},
};
}
/**
* Canonical dedup key for a webhook delivery: provider + tenant + delivery id.
* The legacy global endpoint has no tenant, so `groupId` is "global".
* Key segment naming a group instance. The `@` keeps it apart from every group
* id (ids are `[a-z0-9-]`): an id chosen to equal some instance string can
* never address that instance's keys, and no id-prefix scan reaches them.
*/
export function instanceScope(instance: string): string {
return `@i:${instance}`;
}
/**
* Scope segment of a tenant: the group's immutable instance when known (a
* group re-created under a former id never shares state with its
* predecessor, and a renamed group keeps its own), else the group id
* (messages from before instances), or a marker no group id can take for the
* legacy global endpoint.
*/
export function tenantScope(groupId: string | undefined, instance?: string): string {
if (groupId && instance) return instanceScope(instance);
return groupId ?? "@global";
}
/**
* Marker that the work a claim guarded was accepted for good. A claim alone
* may be left over from an attempt that failed and could not give it back;
* only a confirmed one proves a retry is a duplicate. The prefix keeps it
* apart from every claim key (none starts with it), whatever ids they embed.
*/
export function acceptedKey(key: string): string {
return `accepted:${key}`;
}
/**
* - `claimed`: this caller runs the work (and must confirm or release it);
* - `done`: an earlier run was confirmed — a duplicate;
* - `pending`: another run holds the claim unconfirmed (still running, or it
* failed and could not give the claim back): retry later, after `ttl` at most.
*/
export type ClaimState = "claimed" | "done" | "pending";
/**
* Claim one-time work: a short processing claim (`ttlSeconds`), confirmed
* with `confirmClaim` once the work is done for good. A failed run that also
* fails to give its claim back blocks retries only until the claim expires,
* and is never mistaken for a finished one.
*/
export async function claimOnce(
store: IdempotencyStore,
key: string,
ttlSeconds: number,
): Promise<ClaimState> {
if (await store.has(acceptedKey(key))) return "done";
if (await store.claim(key, ttlSeconds)) return "claimed";
return (await store.has(acceptedKey(key))) ? "done" : "pending";
}
const CONFIRM_ATTEMPTS = 3;
/**
* Record that claimed work was done; duplicates are recognized for
* `ttlSeconds`. Tried a few times: once the work happened, a lost
* confirmation leaves only the short processing claim between a late
* redelivery and running the work again.
*/
export async function confirmClaim(
store: IdempotencyStore,
key: string,
ttlSeconds: number,
): Promise<void> {
for (let attempt = 1; ; attempt++) {
try {
await store.claim(acceptedKey(key), ttlSeconds);
return;
} catch (err) {
if (attempt >= CONFIRM_ATTEMPTS) throw err;
await new Promise((r) => setTimeout(r, 200 * attempt));
}
}
}
/**
* Statement recording `key` (as `claim` would) in the same D1 transaction as
* other writes: the record and the writes commit together, so a failure
* whose answer was lost can be told apart afterwards (`has(key)`).
*/
export function recordStatement(
db: D1Database,
key: string,
ttlSeconds: number,
): D1PreparedStatement {
const now = Date.now();
return db
.prepare(
`INSERT INTO dedup_keys (key, claimed_at, expires_at) VALUES (?, ?, ?)
ON CONFLICT(key) DO UPDATE SET
claimed_at = excluded.claimed_at, expires_at = excluded.expires_at`,
)
.bind(key, now, now + ttlSeconds * 1000);
}
/**
* Statement aborting its D1 transaction while `key` is recorded (unexpired):
* a write done once can never be done again by a replay.
*/
export function unrecordedGuard(db: D1Database, key: string): D1PreparedStatement {
return db
.prepare(
`SELECT CASE WHEN EXISTS (SELECT 1 FROM dedup_keys WHERE key = ? AND expires_at > ?)
THEN json('already recorded') ELSE 1 END AS ok`,
)
.bind(key, Date.now());
}
/** Canonical dedup key for a webhook delivery: provider + tenant + delivery id. */
export function deliveryKey(
provider: string,
groupId: string | undefined,
deliveryId: string,
instance?: string,
): string {
return `delivery:${provider}:${groupId ?? "global"}:${deliveryId}`;
return `delivery:${provider}:${tenantScope(groupId, instance)}:${deliveryId}`;
}
+1
View File
@@ -208,5 +208,6 @@ export const en = {
delivery: "Delivery",
route_ok: "{route} → {target}",
route_fail: "{route} → {target}: {error}",
route_earlier: "{route} → {target}: handled by an earlier attempt",
},
};
+1
View File
@@ -209,5 +209,6 @@ export const zh = {
delivery: "投递",
route_ok: "{route} → {target}",
route_fail: "{route} → {target}: {error}",
route_earlier: "{route} → {target}:已在之前的尝试中处理",
},
};
+37 -6
View File
@@ -1,10 +1,41 @@
/** An Error as JSON keeps its message, stack and cause (it would print as `{}`). */
function plain(value: unknown, depth = 0): unknown {
if (value instanceof Error) {
return {
name: value.name,
message: value.message,
stack: value.stack,
...(value.cause !== undefined && depth < 3 ? { cause: plain(value.cause, depth + 1) } : {}),
};
}
if (value && typeof value === "object" && !Array.isArray(value) && depth < 3) {
return Object.fromEntries(
Object.entries(value as Record<string, unknown>).map(([k, v]) => [k, plain(v, depth + 1)]),
);
}
return value;
}
/**
* One JSON line: `log.x("text")`, or `log.x({ ...fields }, "text")` — the
* fields (errors included, with their message and stack) beside the text.
*/
function line(level: string, first: string | object, args: unknown[]): string {
if (typeof first === "string") return JSON.stringify({ level, msg: first, ...args });
const [text, ...rest] = args;
return JSON.stringify({
level,
msg: typeof text === "string" ? text : undefined,
...(plain(first) as Record<string, unknown>),
...(typeof text === "string" ? rest : args),
});
}
export const log = {
info: (msg: string | object, ...args: unknown[]): void =>
console.log(JSON.stringify({ level: "info", msg, ...args })),
warn: (msg: string | object, ...args: unknown[]): void =>
console.warn(JSON.stringify({ level: "warn", msg, ...args })),
info: (msg: string | object, ...args: unknown[]): void => console.log(line("info", msg, args)),
warn: (msg: string | object, ...args: unknown[]): void => console.warn(line("warn", msg, args)),
error: (msg: string | object, ...args: unknown[]): void =>
console.error(JSON.stringify({ level: "error", msg, ...args })),
console.error(line("error", msg, args)),
fatal: (msg: string | object, ...args: unknown[]): void =>
console.error(JSON.stringify({ level: "fatal", msg, ...args })),
console.error(line("fatal", msg, args)),
};
+248 -21
View File
@@ -1,40 +1,107 @@
import { canUseD1 } from "../storage/d1";
import { canUseD1, isMissingTable, kvFallbackActive, noteKvFallback } from "../storage/d1";
import { log } from "./log";
export interface MessageTracker {
get(eventId: string, targetId: string): Promise<string | null>;
set(eventId: string, targetId: string, messageId: string): Promise<void>;
delete(eventId: string, targetId: string): Promise<void>;
/**
* A tracked platform message: its id plus the version (event time, ms) of the
* update it currently shows, so an older event arriving late never overwrites
* a newer state. Entries written before versions existed carry `version: 0`.
*/
export interface TrackedMessage {
messageId: string;
version: number;
}
const MESSAGE_KEY_TTL_SECONDS = 86400;
export interface MessageTracker {
get(eventId: string, targetId: string): Promise<TrackedMessage | null>;
/** Store (or refresh) an entry; refreshing also renews its retention. */
set(eventId: string, targetId: string, entry: TrackedMessage): Promise<void>;
delete(eventId: string, targetId: string): Promise<void>;
/**
* Re-file an entry under a new key, dropping the old one — in one D1
* transaction, so the entry is never left under both (two owners).
*/
move(
from: { eventId: string; targetId: string },
to: { eventId: string; targetId: string },
entry: TrackedMessage,
): Promise<void>;
}
/** KV entries outlive the D1 retention of message_tracking rows (30 days). */
const MESSAGE_KEY_TTL_SECONDS = 30 * 86400;
/** Stored value: JSON `{ id, v }`, or a bare message id from older versions. */
export function decodeTracked(value: string | null | undefined): TrackedMessage | null {
if (!value) return null;
if (value.startsWith("{")) {
try {
const parsed = JSON.parse(value) as { id?: unknown; v?: unknown };
if (typeof parsed.id === "string" && parsed.id) {
return { messageId: parsed.id, version: Number(parsed.v) || 0 };
}
} catch {
// fall through: treat as a bare id
}
}
return { messageId: value, version: 0 };
}
export function encodeTracked(entry: TrackedMessage): string {
return JSON.stringify({ id: entry.messageId, v: entry.version });
}
/**
* What a moved entry leaves under its old key in D1: a tombstone. Without it,
* a KV copy of the old entry (written while the table was missing, its delete
* not yet seen everywhere) could be read back and adopted a second time.
*/
const MOVED = '{"moved":true}';
export function kvMessageTracker(kv: KVNamespace): MessageTracker {
const key = (eventId: string, targetId: string): string => `msg:${eventId}:${targetId}`;
return {
async get(eventId: string, targetId: string): Promise<string | null> {
return kv.get(key(eventId, targetId));
async get(eventId, targetId): Promise<TrackedMessage | null> {
return decodeTracked(await kv.get(key(eventId, targetId)));
},
async set(eventId: string, targetId: string, messageId: string): Promise<void> {
await kv.put(key(eventId, targetId), messageId, {
async set(eventId, targetId, entry): Promise<void> {
await kv.put(key(eventId, targetId), encodeTracked(entry), {
expirationTtl: MESSAGE_KEY_TTL_SECONDS,
});
},
async delete(eventId: string, targetId: string): Promise<void> {
async delete(eventId, targetId): Promise<void> {
await kv.delete(key(eventId, targetId));
},
async move(from, to, entry): Promise<void> {
// No transactions in KV: the old key goes first, so a failure leaves
// the entry lost (a new message later) rather than owned twice.
await kv.delete(key(from.eventId, from.targetId));
await kv.put(key(to.eventId, to.targetId), encodeTracked(entry), {
expirationTtl: MESSAGE_KEY_TTL_SECONDS,
});
},
};
}
/** A D1 lookup: the entry, and whether its key was moved away (a tombstone). */
async function d1Lookup(
db: D1Database,
eventId: string,
targetId: string,
): Promise<{ entry: TrackedMessage | null; moved: boolean }> {
const row = await db
.prepare("SELECT message_id FROM message_tracking WHERE event_id = ? AND target_id = ?")
.bind(eventId, targetId)
.first<{ message_id: string }>();
if (row?.message_id === MOVED) return { entry: null, moved: true };
return { entry: decodeTracked(row?.message_id), moved: false };
}
export function d1MessageTracker(db: D1Database): MessageTracker {
return {
async get(eventId: string, targetId: string): Promise<string | null> {
const row = await db
.prepare("SELECT message_id FROM message_tracking WHERE event_id = ? AND target_id = ?")
.bind(eventId, targetId)
.first<{ message_id: string }>();
return row?.message_id ?? null;
async get(eventId, targetId): Promise<TrackedMessage | null> {
return (await d1Lookup(db, eventId, targetId)).entry;
},
async set(eventId: string, targetId: string, messageId: string): Promise<void> {
async set(eventId, targetId, entry): Promise<void> {
await db
.prepare(
`INSERT INTO message_tracking (event_id, target_id, message_id, updated_at)
@@ -42,18 +109,178 @@ export function d1MessageTracker(db: D1Database): MessageTracker {
ON CONFLICT(event_id, target_id) DO UPDATE SET
message_id = excluded.message_id, updated_at = excluded.updated_at`,
)
.bind(eventId, targetId, messageId, Date.now())
.bind(eventId, targetId, encodeTracked(entry), Date.now())
.run();
},
async delete(eventId: string, targetId: string): Promise<void> {
async delete(eventId, targetId): Promise<void> {
await db
.prepare("DELETE FROM message_tracking WHERE event_id = ? AND target_id = ?")
.bind(eventId, targetId)
.run();
},
async move(from, to, entry): Promise<void> {
await db.batch([
db
.prepare(
`INSERT INTO message_tracking (event_id, target_id, message_id, updated_at)
VALUES (?, ?, ?, ?)
ON CONFLICT(event_id, target_id) DO UPDATE SET
message_id = excluded.message_id, updated_at = excluded.updated_at`,
)
.bind(to.eventId, to.targetId, encodeTracked(entry), Date.now()),
// The old key keeps a tombstone (pruned with the other rows).
db
.prepare(
`INSERT INTO message_tracking (event_id, target_id, message_id, updated_at)
VALUES (?, ?, ?, ?)
ON CONFLICT(event_id, target_id) DO UPDATE SET
message_id = excluded.message_id, updated_at = excluded.updated_at`,
)
.bind(from.eventId, from.targetId, MOVED, Date.now()),
]);
},
};
}
/** D1 first; degrades to KV only while the D1 table is missing (unmigrated). */
function withKvFallback(
primary: MessageTracker,
fallback: MessageTracker,
kvNs: KVNamespace,
db: D1Database,
): MessageTracker {
async function run<T>(op: string, d1: () => Promise<T>, kv: () => Promise<T>): Promise<T> {
try {
return await d1();
} catch (err) {
if (!isMissingTable(err)) throw err;
log.warn({ err, op }, "D1 message_tracking table missing, falling back to KV");
await noteKvFallback(kvNs);
return kv();
}
}
return {
get: (e, t) =>
run(
"get",
async () => {
const { entry, moved } = await d1Lookup(db, e, t);
// A moved key never falls back to its KV copy.
if (entry || moved) return entry;
return (await kvFallbackActive(kvNs)) ? fallback.get(e, t) : null;
},
() => fallback.get(e, t),
),
set: (e, t, entry) =>
run(
"set",
() => primary.set(e, t, entry),
() => fallback.set(e, t, entry),
),
delete: (e, t) =>
run(
"delete",
async () => {
await primary.delete(e, t);
// Otherwise the KV copy from the missing-table period would resurface.
if (await kvFallbackActive(kvNs)) await fallback.delete(e, t);
},
() => fallback.delete(e, t),
),
move: (from, to, entry) =>
run(
"move",
async () => {
// D1 first (the new entry and the old key's tombstone, together):
// from then on the tombstone keeps any KV copy from being adopted
// again. The KV copy goes after — removing it first would lose the
// entry altogether should the D1 write fail.
await primary.move(from, to, entry);
// Cleanup only (the tombstone already guards): a failure here never
// fails the move that went through.
try {
if (await kvFallbackActive(kvNs)) await fallback.delete(from.eventId, from.targetId);
} catch {
// left to expire
}
},
() => fallback.move(from, to, entry),
),
};
}
export function messageTracker(db: D1Database, kv: KVNamespace): MessageTracker {
return canUseD1(db) ? d1MessageTracker(db) : kvMessageTracker(kv);
if (canUseD1(db)) return withKvFallback(d1MessageTracker(db), kvMessageTracker(kv), kv, db);
// Without D1, tracking lives in KV: marked, so a D1 binding added later
// still finds the messages to edit.
const tracker = kvMessageTracker(kv);
return {
...tracker,
async set(eventId, targetId, entry): Promise<void> {
await noteKvFallback(kv);
return tracker.set(eventId, targetId, entry);
},
async move(from, to, entry): Promise<void> {
await noteKvFallback(kv);
return tracker.move(from, to, entry);
},
};
}
/**
* Serializes the updates of one message chain. With D1 the lock is a
* `dedup_keys` row taken by an atomic conditional UPSERT (taken over only once
* expired) and released only by its holder, identified by the claim time it
* wrote. Without D1 it degrades to a best-effort KV marker.
*/
export interface ChainLock {
/** A holder token, or null while someone else holds the lock. */
acquire(key: string, ttlSeconds: number): Promise<string | null>;
release(key: string, token: string): Promise<void>;
}
export function chainLock(db: D1Database, kv: KVNamespace): ChainLock {
const kvLock: ChainLock = {
async acquire(key, ttlSeconds) {
if (await kv.get(key)) return null;
const token = crypto.randomUUID();
await kv.put(key, token, { expirationTtl: Math.max(60, ttlSeconds) });
return token;
},
async release(key, token) {
if ((await kv.get(key)) === token) await kv.delete(key);
},
};
if (!canUseD1(db)) return kvLock;
return {
async acquire(key, ttlSeconds): Promise<string | null> {
const now = Date.now();
try {
const result = await db
.prepare(
`INSERT INTO dedup_keys (key, claimed_at, expires_at) VALUES (?, ?, ?)
ON CONFLICT(key) DO UPDATE SET
claimed_at = excluded.claimed_at, expires_at = excluded.expires_at
WHERE dedup_keys.expires_at < excluded.claimed_at`,
)
.bind(key, now, now + ttlSeconds * 1000)
.run();
return (result.meta?.changes ?? 0) > 0 ? String(now) : null;
} catch (err) {
if (!isMissingTable(err)) throw err;
return kvLock.acquire(key, ttlSeconds);
}
},
async release(key, token): Promise<void> {
try {
await db
.prepare("DELETE FROM dedup_keys WHERE key = ? AND claimed_at = ?")
.bind(key, Number(token))
.run();
} catch (err) {
if (!isMissingTable(err)) throw err;
await kvLock.release(key, token);
}
},
};
}
+65 -30
View File
@@ -1,38 +1,73 @@
import type { SendRecord } from "./send-log";
import { log } from "./log";
import { isMissingTable } from "../storage/d1";
/**
* Best-effort write of a dispatch's send logs, in one D1 transaction. A
* failed write is tried once more before it is given up (and logged).
*/
/**
* The group a log row belongs to now, found by its instance: the id it had
* at dispatch is tried first (one primary-key lookup — the usual case); only
* when it was renamed or deleted meanwhile are the groups scanned.
*/
const GROUP_BY_INSTANCE =
"COALESCE((SELECT id FROM d1_groups WHERE id = ? AND json_valid(data) AND json_extract(data, '$.instance') = ?), (SELECT id FROM (SELECT id, CASE WHEN json_valid(data) THEN data ELSE '{}' END AS data FROM d1_groups) WHERE json_extract(data, '$.instance') = ?))";
export async function recordSendBatch(db: D1Database, records: SendRecord[]): Promise<void> {
if (records.length === 0) return;
try {
const stmts = records.map((r) =>
db
.prepare(
`INSERT INTO send_logs (ts, route_id, group_id, event, repo, target, ok, error, status, message_id, delivery_id, platform, actor, action, duration_ms, error_code, attempts, detail)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
)
.bind(
r.ts,
r.routeId,
r.groupId ?? null,
r.event,
r.repo ?? null,
r.target,
r.ok ? 1 : 0,
r.error ?? null,
r.status ?? null,
r.messageId ?? null,
r.deliveryId ?? null,
r.platform ?? null,
r.actor ?? null,
r.action ?? null,
r.durationMs ?? null,
r.errorCode ?? null,
r.attempts ?? null,
r.detail ? JSON.stringify(r.detail) : null,
),
// With the group's instance known, the row is filed under whatever id
// that instance has at insert time: a delivery finishing after its group
// was renamed lands under the new id, after a delete under none — never
// under an id another group may own by then. Without groups in D1 (an
// unmigrated database) rows are filed under the id as given.
const write = (byInstance: boolean): Promise<unknown> =>
db.batch(
records.map((r) => {
const resolve = byInstance && r.groupInstance != null;
return db
.prepare(
`INSERT INTO send_logs (ts, route_id, group_id, event, repo, target, ok, error, status, message_id, delivery_id, platform, actor, action, duration_ms, error_code, attempts, detail)
VALUES (?, ?, ${resolve ? GROUP_BY_INSTANCE : "?"}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
)
.bind(
r.ts,
r.routeId,
...(resolve
? [r.groupId ?? "", r.groupInstance, r.groupInstance]
: [r.groupId ?? null]),
r.event,
r.repo ?? null,
r.target,
r.ok ? 1 : 0,
r.error ?? null,
r.status ?? null,
r.messageId ?? null,
r.deliveryId ?? null,
r.platform ?? null,
r.actor ?? null,
r.action ?? null,
r.durationMs ?? null,
r.errorCode ?? null,
r.attempts ?? null,
r.detail ? JSON.stringify(r.detail) : null,
);
}),
);
await db.batch(stmts);
} catch (err) {
log.warn({ err, count: records.length }, "Failed to record send log batch");
let byInstance = true;
for (let attempt = 1; ; attempt++) {
try {
await write(byInstance);
return;
} catch (err) {
if (byInstance && isMissingTable(err) && /d1_groups/.test(String(err))) {
byInstance = false;
continue;
}
if (attempt >= 2 || isMissingTable(err)) {
log.warn({ err, count: records.length }, "Failed to record send log batch");
return;
}
}
}
}
+54 -15
View File
@@ -1,10 +1,13 @@
import { log } from "./log";
import { isMissingTable } from "../storage/d1";
export interface SendRecord {
id?: number;
ts: number;
routeId: string;
groupId?: string;
/** Immutable instance of the group (write-only: resolves `group_id` at insert time). */
groupInstance?: string;
event: string;
repo?: string;
target: string;
@@ -104,14 +107,44 @@ export async function recordSend(db: D1Database, record: SendRecord): Promise<vo
}
}
export async function getSendLog(db: D1Database, limit = 50): Promise<SendRecord[]> {
/**
* Rows visible to a reader: every group (undefined), one group (a string) or
* a set of groups (an array — empty matches nothing). The condition is applied
* in SQL, before ordering and LIMIT, so another group's rows can neither leak
* into nor crowd out the reader's own.
*/
export type GroupScope = string | string[] | undefined;
export function groupScopeClause(scope: GroupScope): { sql: string; args: string[] } | null {
if (scope === undefined) return null;
const ids = typeof scope === "string" ? [scope] : scope;
if (ids.length === 0) return { sql: "0", args: [] };
if (ids.length === 1) return { sql: "group_id = ?", args: ids };
// One JSON parameter, however many groups: D1 binds at most 100 per statement.
return {
sql: "group_id IN (SELECT value FROM json_each(?))",
args: [JSON.stringify(ids)],
};
}
export async function getSendLog(
db: D1Database,
limit = 50,
scope?: GroupScope,
): Promise<SendRecord[]> {
try {
const clause = groupScopeClause(scope);
const { results } = await db
.prepare(`SELECT ${COLUMNS} FROM send_logs ORDER BY ts DESC LIMIT ?`)
.bind(limit)
.prepare(
`SELECT ${COLUMNS} FROM send_logs${clause ? ` WHERE ${clause.sql}` : ""} ORDER BY ts DESC LIMIT ?`,
)
.bind(...(clause?.args ?? []), limit)
.all<LogRow>();
return results.map(toRecord);
} catch (err) {
// Only a table not created yet means "nothing yet"; any other failure
// must reach the caller, not pass for an empty history.
if (!isMissingTable(err)) throw err;
log.warn({ err }, "Failed to load send log");
return [];
}
@@ -126,6 +159,9 @@ export async function getSendLogById(db: D1Database, id: number): Promise<SendRe
const row = results[0];
return row ? toRecord(row) : null;
} catch (err) {
// Only a table not created yet means "nothing yet"; any other failure
// must reach the caller, not pass for an empty history.
if (!isMissingTable(err)) throw err;
log.warn({ err, id }, "Failed to load send log entry");
return null;
}
@@ -142,37 +178,40 @@ export async function getSendLogByDelivery(
.all<LogRow>();
return results.map(toRecord);
} catch (err) {
// Only a table not created yet means "nothing yet"; any other failure
// must reach the caller, not pass for an empty history.
if (!isMissingTable(err)) throw err;
log.warn({ err, deliveryId }, "Failed to load send log by delivery");
return [];
}
}
/** The latest failed sends, optionally of one group (rows map with `toRecord`). */
/** The latest failed sends within a group scope (rows map with `toRecord`). */
export function failedSendLogStatement(
db: D1Database,
limit = 20,
groupId?: string,
scope?: GroupScope,
): D1PreparedStatement {
return groupId
? db
.prepare(
`SELECT ${COLUMNS} FROM send_logs WHERE ok = 0 AND group_id = ? ORDER BY ts DESC LIMIT ?`,
)
.bind(groupId, limit)
: db
.prepare(`SELECT ${COLUMNS} FROM send_logs WHERE ok = 0 ORDER BY ts DESC LIMIT ?`)
.bind(limit);
const clause = groupScopeClause(scope);
return db
.prepare(
`SELECT ${COLUMNS} FROM send_logs WHERE ok = 0${clause ? ` AND ${clause.sql}` : ""} ORDER BY ts DESC LIMIT ?`,
)
.bind(...(clause?.args ?? []), limit);
}
export async function getFailedSendLog(
db: D1Database,
limit = 20,
groupId?: string,
groupId?: GroupScope,
): Promise<SendRecord[]> {
try {
const { results } = await failedSendLogStatement(db, limit, groupId).all<LogRow>();
return results.map(toRecord);
} catch (err) {
// Only a table not created yet means "nothing yet"; any other failure
// must reach the caller, not pass for an empty history.
if (!isMissingTable(err)) throw err;
log.warn({ err }, "Failed to load failed send log");
return [];
}
+81
View File
@@ -0,0 +1,81 @@
/**
* In-flight work shared by concurrent callers in one isolate (one token
* exchange, one marker read, one setup run instead of one each).
*
* A run is started by one request and awaited by others. In Workers the
* starting request can end or be canceled before its I/O settles; its
* promise then never settles and its cleanup never runs. So another request
* waits for a run only until `maxWaitMs` after it started, and a run that
* old is never joined: the first caller to find it stuck starts a
* replacement and links it from the stuck run (`replacedBy`), so every other
* waiter of that run follows the link — even once the replacement has
* finished and left the map — and a stuck run is replaced once, not once per
* waiter. Nothing waits forever on runs that cannot finish: a caller that
* found `MAX_JOINS` runs stuck runs alone.
*/
export interface Flight<T> {
promise: Promise<T>;
startedAt: number;
replacedBy?: Flight<T>;
}
export interface FlightMap<K, T> {
get(key: K): Flight<T> | undefined;
set(key: K, value: Flight<T>): unknown;
delete(key: K): boolean;
}
const STALE = Symbol("stale");
/** Runs a caller waits on (and finds stuck) before it runs alone. */
const MAX_JOINS = 3;
export async function sharedFlight<K, T>(
flights: FlightMap<K, T>,
key: K,
run: () => Promise<T>,
maxWaitMs: number,
): Promise<T> {
let current = flights.get(key);
for (let joined = 1; current; joined++) {
const left = current.startedAt + maxWaitMs - Date.now();
if (left > 0) {
let timer: ReturnType<typeof setTimeout> | undefined;
const timeout = new Promise<typeof STALE>((resolve) => {
timer = setTimeout(() => resolve(STALE), left);
});
try {
const result = await Promise.race([current.promise, timeout]);
if (result !== STALE) return result as T;
} finally {
clearTimeout(timer);
}
}
// `current` is stuck.
if (joined >= MAX_JOINS) return run();
if (current.replacedBy) {
current = current.replacedBy;
continue;
}
const flight = start(flights, key, run, current);
current.replacedBy = flight;
return flight.promise;
}
return start(flights, key, run).promise;
}
/** Start a run and publish it, unless another run took the key meanwhile. */
function start<K, T>(
flights: FlightMap<K, T>,
key: K,
run: () => Promise<T>,
replacing?: Flight<T>,
): Flight<T> {
const flight: Flight<T> = { promise: run(), startedAt: Date.now() };
const published = flights.get(key);
if (!published || published === replacing) flights.set(key, flight);
const done = (): void => {
if (flights.get(key) === flight) flights.delete(key);
};
flight.promise.then(done, done);
return flight;
}
+32 -33
View File
@@ -1,5 +1,13 @@
import { isMissingTable } from "../storage/d1";
import { log } from "../lib/log";
import { failedSendLogStatement, toRecord, type LogRow, type SendRecord } from "../lib/send-log";
import {
failedSendLogStatement,
groupScopeClause,
toRecord,
type GroupScope,
type LogRow,
type SendRecord,
} from "../lib/send-log";
export interface DeliveryMetrics {
total: number;
@@ -18,6 +26,9 @@ export interface DeliveryMetrics {
interface TotalsRow {
total: number;
ok: number;
avg: number | null;
attempts_total: number;
attempts_avg: number | null;
}
interface BreakdownRow {
@@ -31,15 +42,6 @@ interface StatusRow {
count: number;
}
interface AvgRow {
avg: number | null;
}
interface AttemptsRow {
total: number;
avg: number | null;
}
const EMPTY: DeliveryMetrics = {
total: 0,
ok: 0,
@@ -56,19 +58,26 @@ const EMPTY: DeliveryMetrics = {
export async function getDeliveryMetrics(
db: D1Database,
groupId?: string,
groupId?: GroupScope,
): Promise<DeliveryMetrics> {
const where = groupId ? " WHERE group_id = ?" : "";
const and = groupId ? " AND group_id = ?" : "";
const clause = groupScopeClause(groupId);
const where = clause ? ` WHERE ${clause.sql}` : "";
const and = clause ? ` AND ${clause.sql}` : "";
const bindGroup = (stmt: D1PreparedStatement): D1PreparedStatement =>
groupId ? stmt.bind(groupId) : stmt;
clause && clause.args.length > 0 ? stmt.bind(...clause.args) : stmt;
try {
// One round trip for all seven queries (each D1 call is a network hop), and
// One round trip for all five queries (each D1 call is a network hop), and
// a batch runs as one transaction, so the totals and breakdowns agree.
const results = await db.batch([
// Totals, average duration and attempts in one scan (AVG and SUM skip
// NULLs, as the separate filtered queries did).
bindGroup(
db.prepare(`SELECT COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok FROM send_logs${where}`),
db.prepare(
`SELECT COUNT(*) AS total, COALESCE(SUM(ok), 0) AS ok, AVG(duration_ms) AS avg,
COALESCE(SUM(attempts), 0) AS attempts_total, AVG(attempts) AS attempts_avg
FROM send_logs${where}`,
),
),
bindGroup(
db.prepare(
@@ -88,17 +97,6 @@ export async function getDeliveryMetrics(
WHERE status IS NOT NULL${and} GROUP BY status`,
),
),
bindGroup(
db.prepare(
`SELECT AVG(duration_ms) AS avg FROM send_logs WHERE duration_ms IS NOT NULL${and}`,
),
),
bindGroup(
db.prepare(
`SELECT COALESCE(SUM(attempts), 0) AS total, AVG(attempts) AS avg
FROM send_logs WHERE attempts IS NOT NULL${and}`,
),
),
failedSendLogStatement(db, 20, groupId),
]);
const rows = <T>(i: number): T[] => (results[i]?.results ?? []) as T[];
@@ -106,9 +104,7 @@ export async function getDeliveryMetrics(
const byPlatform = rows<BreakdownRow>(1);
const byEvent = rows<BreakdownRow>(2);
const byStatus = rows<StatusRow>(3);
const duration = rows<AvgRow>(4);
const attempts = rows<AttemptsRow>(5);
const recentFailures = rows<LogRow>(6).map(toRecord);
const recentFailures = rows<LogRow>(4).map(toRecord);
const total = Number(totals[0]?.total) || 0;
const ok = Number(totals[0]?.ok) || 0;
@@ -132,12 +128,15 @@ export async function getDeliveryMetrics(
status: String(r.status),
count: Number(r.count) || 0,
})),
avgDurationMs: Number(duration[0]?.avg) || 0,
totalAttempts: Number(attempts[0]?.total) || 0,
avgAttempts: Number(attempts[0]?.avg) || 0,
avgDurationMs: Number(totals[0]?.avg) || 0,
totalAttempts: Number(totals[0]?.attempts_total) || 0,
avgAttempts: Number(totals[0]?.attempts_avg) || 0,
recentFailures,
};
} catch (err) {
// Only a table not created yet means "nothing yet"; any other failure
// must reach the caller, not pass for an empty history.
if (!isMissingTable(err)) throw err;
log.warn({ err }, "Failed to compute delivery metrics");
return EMPTY;
}
+47 -10
View File
@@ -2,9 +2,28 @@ import type { Env, WebhookEvent } from "../../types";
import type { Provider } from "../types";
import { hmacSha256Hex, timingSafeEqual } from "../hmac";
import { verifySignature } from "../github/verify";
import { claimOnce, idempotencyStore } from "../../lib/idempotency";
const REPLAY_WINDOW_SECONDS = 300;
const NONCE_TTL_SECONDS = 600;
/** How long an accepted nonce is remembered (longer than the signature window). */
export const NONCE_TTL_SECONDS = 600;
/**
* How long a nonce stays held while its request is processed. Short, so a
* request whose attempt failed (and could not give the nonce back) can be
* retried well within the signature window.
*/
export const NONCE_CLAIM_SECONDS = 120;
/**
* The request cannot be judged now: its nonce is held by an attempt not
* confirmed as accepted (still running, or failed without giving the nonce
* back), or the nonce store is unavailable. Not a proven replay: retry later.
*/
export class ReplayPendingError extends Error {
constructor(message = "Replay nonce is held by an unconfirmed attempt") {
super(message);
}
}
function parseTimestamp(value: string | undefined): number | null {
if (!value) return null;
@@ -17,14 +36,18 @@ function parseTimestamp(value: string | undefined): number | null {
* (Unix seconds) and `X-WebHooker-Nonce` and signs
* `timestamp + "." + nonce + "." + body` instead of the bare body. We reject
* timestamps outside a ±5min window and reject any nonce already seen (nonces
* live in KV for 10min). Senders without those headers keep the legacy
* are kept for 10min). Senders without those headers keep the legacy
* body-only signature so existing integrations continue to work.
*
* Nonces are scoped to the signing secret, so two tenants picking the same
* nonce never block each other, and claimed atomically (D1 when available),
* so two concurrent replays cannot both pass.
*/
async function verifyReplayProtected(
body: string,
headers: Record<string, string>,
secret: string,
kv: KVNamespace,
env: Env,
): Promise<boolean> {
const timestamp = parseTimestamp(headers["x-webhooker-timestamp"]);
const nonce = headers["x-webhooker-nonce"];
@@ -37,11 +60,23 @@ async function verifyReplayProtected(
const expected = `sha256=${await hmacSha256Hex(secret, `${timestamp}.${nonce}.${body}`)}`;
if (!timingSafeEqual(signature ?? "", expected)) return false;
const nonceKey = `nonce:${nonce}`;
const seen = await kv.get(nonceKey);
if (seen !== null && seen !== undefined) return false;
await kv.put(nonceKey, "1", { expirationTtl: NONCE_TTL_SECONDS });
return true;
const key = await replayNonceKey(secret, nonce);
let state;
try {
state = await claimOnce(idempotencyStore(env.DB, env.KV), key, NONCE_CLAIM_SECONDS);
} catch {
// A storage failure says nothing about the signature.
throw new ReplayPendingError("Replay protection is temporarily unavailable");
}
// Only a nonce whose request was accepted makes this a replay.
if (state === "pending") throw new ReplayPendingError();
return state === "claimed";
}
/** Claim key of a replay-protection nonce, scoped to the signing secret. */
export async function replayNonceKey(secret: string, nonce: string): Promise<string> {
const scope = (await hmacSha256Hex(secret, "webhooker:nonce-scope")).slice(0, 16);
return `nonce:${scope}:${nonce}`;
}
/**
@@ -72,7 +107,7 @@ export const customProvider: Provider = {
headers["x-webhooker-timestamp"] !== undefined &&
headers["x-webhooker-nonce"] !== undefined
) {
return verifyReplayProtected(body, headers, secret, env.KV);
return verifyReplayProtected(body, headers, secret, env);
}
return verifySignature(body, headers["x-webhooker-signature"], secret);
},
@@ -80,7 +115,9 @@ export const customProvider: Provider = {
parse(body, _headers): WebhookEvent | null {
try {
const payload = JSON.parse(body) as Record<string, unknown>;
if (!payload || typeof payload !== "object") return null;
// An object only: a top-level array (or scalar) could not be parked and
// restored like every other payload, so it is refused up front.
if (!payload || typeof payload !== "object" || Array.isArray(payload)) return null;
// Optional id for sender-side dedup (retries from monitoring systems).
const deliveryId =
typeof payload.deliveryId === "string" && payload.deliveryId
+369 -60
View File
@@ -1,103 +1,412 @@
import type { Config, Env, Group, WebhookEvent, WebhookProvider } from "../types";
import { dispatchEvent } from "../core/dispatch";
import { loadConfig } from "../config";
import { loadGroups } from "../web/groups";
import { dispatchEvent, type DispatchOptions } from "../core/dispatch";
import { loadConfigSnapshot, loadGroupFresh } from "../config";
import { groupAliasChain } from "../web/group-lifecycle";
import { log } from "../lib/log";
import { recordSendBatch } from "../lib/send-log-batch";
import {
DELIVERY_DLQ,
PayloadUnavailableError,
type DeliveryMessage,
type DispatchSummary,
acquireDelivery,
classifyDelivery,
finishLease,
getDeliveryState,
renewLease,
type DeliveryStatus,
deliveryStateKey,
discardPayload,
getDeliveryState,
hadPermanentFailure,
isRetryableFailure,
markDeliveryDead,
markTargetsDone,
resolvePayload,
retryDelay,
setDeliveryState,
targetDoneChecker,
} from "./delivery";
type Dispatch = (
config: Config,
event: WebhookEvent,
env: Env,
groups?: Group[],
options?: DispatchOptions,
) => Promise<DispatchSummary>;
export async function handleQueueBatch(
batch: MessageBatch<DeliveryMessage>,
env: Env,
dispatch: (
config: Config,
event: WebhookEvent,
env: Env,
groups?: Group[],
) => Promise<DispatchSummary> = dispatchEvent,
dispatch: Dispatch = dispatchEvent,
budgetMs = BATCH_BUDGET_MS,
): Promise<void> {
// Shared by the whole batch: the limits it guards (D1 queries and
// subrequests) are per invocation.
const budget: BatchBudget = { deadline: Date.now() + budgetMs, sendsLeft: BATCH_SEND_LIMIT };
for (const message of batch.messages) {
const body = message.body;
if (batch.queue === DELIVERY_DLQ) {
await markDead(env, body);
message.ack();
continue;
try {
if (batch.queue === DELIVERY_DLQ) {
if (await markDead(env, body)) message.ack();
// Another copy holds a live lease and may still need the payload.
else message.retry({ delaySeconds: BUSY_RETRY_SECONDS });
continue;
}
// Out of time or sends before it started (earlier messages took them):
// back to the queue for a later invocation — a retry (one more queue
// read), not a copy (a write and a delete more). Waiting is not a
// failure, though: a delivery never started whose retry would use up
// its last one (sending it to the DLQ untried) is copied instead. One
// that was started has real failures among its attempts; a copy would
// start their count over, so it keeps the retry.
if (spent(budget)) {
if (message.attempts > QUEUE_MAX_RETRIES && env.QUEUE && (await neverStarted(env, body))) {
try {
await env.QUEUE.send(body, { delaySeconds: CONTINUE_DELAY_SECONDS });
message.ack();
continue;
} catch (err) {
log.warn({ deliveryId: body.deliveryId, err }, "Failed to queue a delivery for later");
}
}
message.retry({ delaySeconds: CONTINUE_DELAY_SECONDS });
continue;
}
await processMessage(env, body, message, dispatch, budget);
} catch (err) {
// One message's storage failure must not fail (and replay) the batch.
log.error({ deliveryId: body.deliveryId, err }, "Queue message handling failed");
message.retry({ delaySeconds: retryDelay(message.attempts) });
}
await processMessage(env, body, message, dispatch);
}
}
/** Renew a held lease when it is older than this (the lease lasts five minutes). */
const LEASE_RENEW_MS = 60_000;
const FINAL: ReadonlySet<DeliveryStatus> = new Set(["delivered", "failed", "dead"]);
/** Wait for an unexpired lease of another consumer: its full length plus a margin. */
const BUSY_RETRY_SECONDS = 310;
/**
* Sending time of one batch: a consumer invocation may run 15 minutes, and a
* target takes under a minute at worst (bounded platform calls), so sending
* stops in time to record what was done.
*/
const BATCH_BUDGET_MS = 12 * 60_000;
/**
* Targets one batch sends at most: each costs D1 queries (its ledger record,
* an update chain's lock and tracking) and subrequests, and one invocation
* may make only 50 of each on Workers Free (with ~7 D1 queries per delivery
* besides; batches are small, see wrangler.jsonc). A message left unstarted
* goes back to the queue (a retry: one queue read); a message cut short is
* continued by a new one only when it made progress.
*/
const BATCH_SEND_LIMIT = 20;
/** What a batch may still spend: its time and its sends. */
interface BatchBudget {
deadline: number;
sendsLeft: number;
}
function spent(budget: BatchBudget): boolean {
return Date.now() >= budget.deadline || budget.sendsLeft <= 0;
}
const CONTINUE_DELAY_SECONDS = 5;
/** The delivery queue's `max_retries` (`wrangler.jsonc`). */
const QUEUE_MAX_RETRIES = 5;
/** No attempt has taken the delivery's lease yet (its attempts were all waits). */
async function neverStarted(env: Env, body: DeliveryMessage): Promise<boolean> {
const key = deliveryStateKey(body.provider, body.groupId, body.deliveryId, body.groupInstance);
const state = await getDeliveryState(env, key);
return state == null || state === "pending";
}
/** Clean up after a terminal state; a failure must not reopen the delivery. */
async function discardQuietly(env: Env, body: DeliveryMessage): Promise<void> {
await discardPayload(env, body).catch((err) =>
log.warn({ deliveryId: body.deliveryId, err }, "Failed to discard a parked payload"),
);
}
async function processMessage(
env: Env,
body: DeliveryMessage,
message: Message<DeliveryMessage>,
dispatch: (
config: Config,
event: WebhookEvent,
env: Env,
groups?: Group[],
) => Promise<DispatchSummary>,
dispatch: Dispatch,
budget: BatchBudget,
): Promise<void> {
const key = deliveryStateKey(body.provider, body.groupId, body.deliveryId);
const prior = await getDeliveryState(env, key);
if (prior === "delivered" || prior === "dead") {
const key = deliveryStateKey(body.provider, body.groupId, body.deliveryId, body.groupInstance);
if (await finishedUnderLegacyKey(env, body)) {
message.ack();
return;
}
const lease = await acquireDelivery(env, key);
if (lease.status === "finished") {
message.ack();
return;
}
if (lease.status === "busy") {
// Another consumer holds this delivery (or a holder vanished): one retry
// after the lease surely expired, instead of spending the retry budget
// polling a lease that is still valid.
message.retry({ delaySeconds: BUSY_RETRY_SECONDS });
return;
}
await setDeliveryState(env, key, "processing");
const payload = await resolvePayload(env, body);
const event: WebhookEvent = {
event: body.event,
payload,
deliveryId: body.deliveryId,
provider: body.provider as WebhookProvider,
installationId: body.installationId,
};
try {
const config = await loadConfig(env);
if (body.groupId) {
config.routes = config.routes.filter((r) => r.groupId === body.groupId);
}
const groups = await loadGroups(env.KV);
const summary = await dispatch(config, event, env, groups);
const { failed, retryable } = classifyDelivery(summary);
if (!failed) {
await setDeliveryState(env, key, "delivered");
await discardPayload(env, body);
message.ack();
return;
}
if (retryable) {
await setDeliveryState(env, key, "retrying");
let token = lease.token;
let renewedAt = Date.now();
let leaseLost = false;
/** A renewal failed with an unknown outcome: stop sending, but never assume a new holder. */
let leaseUncertain = false;
let renewing: Promise<boolean> | null = null;
/** The batch's budget ran out: the rest is left to a continuation message. */
let outOfTime = false;
/** End this attempt with `status`; acks instead when the lease was lost. */
const finish = async (status: DeliveryStatus, then: "ack" | "retry"): Promise<void> => {
const owned = !leaseLost && (await finishLease(env, key, token, status));
if (owned && then === "retry") {
message.retry({ delaySeconds: retryDelay(message.attempts) });
return;
}
await setDeliveryState(env, key, "failed");
await discardPayload(env, body);
if (!owned && !leaseLost) {
// The outcome could not be written, yet no new holder is known (our own
// renewal may have gone through unseen, or the lease lapsed unnoticed):
// come back once any lease surely expired; the state then decides.
message.retry({ delaySeconds: BUSY_RETRY_SECONDS });
return;
}
// Lost lease: the new holder owns the delivery (and its outcome).
message.ack();
if (owned && FINAL.has(status)) await discardQuietly(env, body);
};
let summary: DispatchSummary;
// Targets recorded as sent the moment they were sent (see `afterTarget`).
const recorded = new Set<string>();
try {
const payload = await resolvePayload(env, body);
const event: WebhookEvent = {
event: body.event,
payload,
deliveryId: body.deliveryId,
provider: body.provider as WebhookProvider,
installationId: body.installationId,
};
// Read fresh, not from this isolate's cache: a group (or route) created
// moments ago elsewhere must not be taken for missing — for a tenant
// delivery that verdict is final.
const { config, groups } = await loadConfigSnapshot(env, { fresh: true });
// A tenant delivery belongs to the group instance that received it — under
// whatever id it has now — and never to another group that took over the
// id since. A message from before instances belongs to the group that had
// its id back then (`legacyId`), also when that group was renamed before
// instances existed (followed through the rename aliases) — never to a
// group created since or renamed to the id.
let tenantId = body.groupId;
if (tenantId && body.groupInstance) {
tenantId = groups.find((g) => g.instance === body.groupInstance)?.id;
} else if (tenantId) {
const owner = (id: string): string | undefined => groups.find((g) => g.legacyId === id)?.id;
let found = owner(tenantId);
for (const id of found ? [] : (await groupAliasChain(env, tenantId)).slice(1)) {
found ??= owner(id);
}
tenantId = found;
}
if (body.groupId && (!tenantId || !groups.some((g) => g.id === tenantId))) {
// A group row that cannot be read may be the tenant: retry until it is
// repaired rather than give the delivery up as one of a deleted group.
if (config.unreadableGroups?.length) {
log.error(
{ deliveryId: body.deliveryId, unreadable: config.unreadableGroups },
"Tenant unresolved while some group configuration is unreadable",
);
await finish("retrying", "retry");
return;
}
// The group was deleted: nothing can receive the event any more.
log.warn({ deliveryId: body.deliveryId, groupId: body.groupId }, "Tenant group is gone");
await finish("failed", "ack");
return;
}
const options: DispatchOptions = {
tenantId,
// Always consult the ledger: another queue message of the same delivery
// (a sender redelivery) may already have sent some targets.
skipTarget: targetDoneChecker(env, key),
// Keep the lease alive during a long dispatch; once it is lost, stop
// sending (the new holder takes over the remaining targets).
beforeTarget: async () => {
if (leaseLost || leaseUncertain || outOfTime) return false;
if (spent(budget)) {
outOfTime = true;
return false;
}
budget.sendsLeft--;
if (Date.now() - renewedAt < LEASE_RENEW_MS) return true;
// Routes run in parallel: they share one renewal. Two renewals with
// the same token would race, and the loser would take its own
// successful renewal for a lost lease.
renewing ??= (async (): Promise<boolean> => {
try {
token = await renewLease(env, key, token);
renewedAt = Date.now();
leaseLost = token == null && lease.token != null;
return !leaseLost;
} catch (err) {
log.warn({ deliveryId: body.deliveryId, err }, "Lease renewal failed");
leaseUncertain = true;
return false;
} finally {
renewing = null;
}
})();
return renewing;
},
afterTarget: async (targetKey) => {
await markTargetsDone(env, key, [targetKey]);
recorded.add(targetKey);
},
};
const scoped = tenantId
? { ...config, routes: config.routes.filter((r) => r.groupId === tenantId) }
: config;
summary = await dispatch(scoped, event, env, groups, options);
} catch (err) {
if (err instanceof PayloadUnavailableError) {
// Nothing left to dispatch in this message: drop it, loudly, without
// retrying forever or pretending the delivery succeeded. The delivery
// stays open (not final): the sender's redelivery, with its own payload,
// can still deliver it.
log.error({ deliveryId: body.deliveryId, err: err.message }, "Queued payload unavailable");
// On record where admins look (the group's send log and failure
// metrics): the event needs a redelivery from its sender.
await recordSendBatch(env.DB, [
{
ts: Date.now(),
routeId: "",
groupId: body.groupId,
groupInstance: body.groupInstance,
event: body.event,
target: "",
ok: false,
error: `${err.message}; redeliver the webhook to send it`,
errorCode: "PAYLOAD_MISSING",
deliveryId: body.deliveryId,
},
]);
await finish("retrying", "ack");
return;
}
log.error({ deliveryId: body.deliveryId, err }, "Queue delivery failed");
await setDeliveryState(env, key, "retrying");
message.retry({ delaySeconds: retryDelay(message.attempts) });
await finish("retrying", "retry");
return;
}
const { failed, retryable } = classifyDelivery(summary);
// Record finished targets first, whatever comes next: if a later write
// fails and the message is retried, these targets are never sent again.
const permanent = summary.failures.filter((f) => !isRetryableFailure(f));
try {
await markTargetsDone(
env,
key,
[...(summary.succeeded ?? []), ...permanent.flatMap((f) => f.key ?? [])].filter(
(k) => !recorded.has(k),
),
{ permanentFailure: permanent.length > 0 },
);
} catch (err) {
// The outcome is not on record: retry (giving the lease back now) rather
// than finish the delivery with a forgotten permanent failure.
log.error({ deliveryId: body.deliveryId, err }, "Failed to record delivery targets");
await finish("retrying", "retry");
return;
}
if (failed && retryable) {
// Stopped for time, not for a failure: continued by a new message (sent
// before the lease is given back; a lease still held only delays it) —
// only when this attempt finished targets. One that made no progress
// (its targets kept failing until time ran out) is an ordinary retry and
// counts against the retry limit, so it cannot loop forever.
const progressed = (summary.succeeded?.length ?? 0) > 0 || permanent.length > 0;
if (outOfTime && progressed && !leaseLost && env.QUEUE) {
try {
await env.QUEUE.send(body, { delaySeconds: CONTINUE_DELAY_SECONDS });
await finish("retrying", "ack");
return;
} catch (err) {
log.warn({ deliveryId: body.deliveryId, err }, "Failed to queue the rest of a delivery");
}
}
await finish("retrying", "retry");
return;
}
// Targets that failed for good in an earlier attempt still count.
const anyPermanent = failed || (await hadPermanentFailure(env, key));
await finish(anyPermanent ? "failed" : "delivered", "ack");
}
async function markDead(env: Env, body: DeliveryMessage): Promise<void> {
const key = deliveryStateKey(body.provider, body.groupId, body.deliveryId);
await setDeliveryState(env, key, "dead");
/**
* The delivery already finished under the state key used before deliveries
* were scoped by `@global` / group instance (`delivery-state:{provider}:
* {groupId | "global"}:{id}`, kept 7 days): a sender's redelivery of an event
* handled before the upgrade is then not sent again. The old key is only
* trusted where its owner is certain — for a tenant, the group that had the
* id back then (its `legacyId`); for the global endpoint, while no group named
* "global" (which used the same key) exists or was deleted or renamed away
* recently.
*/
async function finishedUnderLegacyKey(env: Env, body: DeliveryMessage): Promise<boolean> {
// The global endpoint's old key was shared with any tenant once named
// "global" — even one deleted before the upgrade, which nothing records:
// its owner is never certain, so it is never trusted (a redelivery is sent
// again rather than an event dropped). Nothing to read for it, then.
if (!body.groupId || body.groupId === "global") return false;
// The store's one-time setup (seeding from KV, which also turns on reading
// older KV state) must have run before state is read: one group row reads
// it in, not the whole configuration (read below only when it matters).
const { group } = await loadGroupFresh(env, body.groupId);
// The message's own group, still under this id, with no older id of the
// same name (a group created since): nothing it could have finished under
// the old key, so that key is not read.
if (
body.groupInstance &&
group?.instance === body.groupInstance &&
group.legacyId !== body.groupId
) {
return false;
}
// Custom delivery ids got a namespace since (`d:`; `n:` for nonce
// identities, which did not exist before) — in messages queued since. A
// message queued before carries the id as the sender sent it, prefix-like
// text included.
const namespaced = body.provider === "custom" && (body.stateVersion ?? 0) >= 2;
if (namespaced && body.deliveryId.startsWith("n:")) return false;
const id =
namespaced && body.deliveryId.startsWith("d:") ? body.deliveryId.slice(2) : body.deliveryId;
const legacy = `delivery-state:${body.provider}:${body.groupId ?? "global"}:${id}`;
const state = await getDeliveryState(env, legacy);
if (!state || !FINAL.has(state)) return false;
// (A tenant named "global" shared this very key with the global endpoint:
// refused above.)
const { groups } = await loadConfigSnapshot(env, { fresh: true });
const owner = body.groupInstance
? groups.find((g) => g.instance === body.groupInstance)
: groups.find((g) => g.legacyId === body.groupId);
return owner?.legacyId === body.groupId;
}
/**
* Settle a message that left the DLQ; false while another copy of the
* delivery holds a live lease (its state and payload are left alone then).
*/
async function markDead(env: Env, body: DeliveryMessage): Promise<boolean> {
const key = deliveryStateKey(body.provider, body.groupId, body.deliveryId, body.groupInstance);
// A delivery another message of it finished (or is sending) keeps its state.
if ((await markDeliveryDead(env, key)) === "busy") return false;
await discardPayload(env, body);
return true;
}
+413 -61
View File
@@ -1,13 +1,22 @@
import type { Env } from "../types";
import { r2PayloadStore } from "../storage/payload";
import { canUseD1 } from "../storage/d1";
import { canUseD1, isMissingTable, kvFallbackActive, noteKvFallback } from "../storage/d1";
import { idempotencyStore, kvIdempotencyStore, tenantScope } from "../lib/idempotency";
import { log } from "../lib/log";
export type DeliveryStatus =
"pending" | "processing" | "delivered" | "retrying" | "failed" | "dead";
export interface DeliveryMessage {
deliveryId: string;
/** Tenant group of a per-group ingress (`/webhook/{groupId}`); absent for the global endpoint. */
groupId?: string;
/**
* The tenant group's immutable instance at ingress. The consumer delivers
* only to the group holding it now (following renames), never to another
* group that later took over the id.
*/
groupInstance?: string;
provider: string;
event: string;
payload?: Record<string, unknown>;
@@ -18,9 +27,17 @@ export interface DeliveryMessage {
installationId?: number;
receivedAt: number;
requestId?: string;
/**
* 2 for messages whose state keys use the current scopes (`@global`,
* instances). Absent on messages queued before: only those may have state
* under the old `...:global:` key.
*/
stateVersion?: number;
}
export interface DispatchFailure {
/** Stable identity of the route × target, used to skip it on a retry. */
key?: string;
target: string;
error?: string;
errorCode?: string;
@@ -30,31 +47,79 @@ export interface DispatchFailure {
export interface DispatchSummary {
attempts: number;
failures: DispatchFailure[];
/** Keys of the route × targets that were sent successfully. */
succeeded?: string[];
}
export const DELIVERY_QUEUE = "webhooker-delivery";
export const DELIVERY_DLQ = "webhooker-delivery-dlq";
const MAX_QUEUE_MESSAGE_BYTES = 100_000;
const PAYLOAD_KV_TTL_SECONDS = 3600;
const STATE_KV_TTL_SECONDS = 3600;
/**
* Queue messages may be 128 KB (128 000 bytes, a little of it taken by the
* queue's own metadata); measured in UTF-8 bytes of the JSON. Close to the
* limit, so only payloads that cannot go in the message are parked — each
* parked one costs a KV write, read and delete (scarce on Workers Free).
*/
const MAX_QUEUE_MESSAGE_BYTES = 120_000;
/**
* Parked payloads must outlive every retry and the DLQ hand-off. Queue
* messages are retained up to four days, so the payload is too.
*/
export const PAYLOAD_TTL_SECONDS = 4 * 86400;
/**
* What a delivery finished (state, done targets, permanent failures) is kept
* as long as delivery state (pruned after 7 days): a redelivery within that
* window — queued or inline — never sends a finished target again.
*/
const LEDGER_TTL_SECONDS = 7 * 86400;
const STATE_KV_TTL_SECONDS = LEDGER_TTL_SECONDS;
/** How long a just-parked KV payload may still be invisible (eventual consistency). */
const KV_VISIBILITY_GRACE_MS = 10 * 60_000;
/** A consumer holds a delivery this long before another one may take over. */
const LEASE_MS = 5 * 60_000;
const RETRYABLE_ERROR_CODES = new Set(["DISCORD_5XX", "TELEGRAM_5XX", "NETWORK", "RETRIES"]);
const RETRYABLE_ERROR_CODES = new Set([
"DISCORD_5XX",
"TELEGRAM_5XX",
"FEISHU_5XX",
"FEISHU_TOKEN",
"NETWORK",
"RETRIES",
"RATE_LIMITED",
]);
const RETRY_DELAYS_SECONDS = [5, 30, 120, 600];
/** The stored payload of a delivery is gone or unreadable: it cannot be dispatched. */
export class PayloadUnavailableError extends Error {
constructor(message: string) {
super(message);
this.name = "PayloadUnavailableError";
}
}
export function isRetryableError(code?: string): boolean {
return code == null || RETRYABLE_ERROR_CODES.has(code);
}
/** Transient by code, or by HTTP status (5xx, 429) whatever the platform's code. */
export function isRetryableFailure(failure: DispatchFailure): boolean {
if (failure.status != null && (failure.status >= 500 || failure.status === 429)) return true;
return isRetryableError(failure.errorCode);
}
/**
* A delivery is retried when any target failed transiently. Targets that
* succeeded or failed permanently are recorded first (see `markTargetsDone`)
* so the retry only re-sends the transient ones: one bad target never cancels
* the retries of the others.
*/
export function classifyDelivery(summary: DispatchSummary): {
failed: boolean;
retryable: boolean;
} {
if (summary.failures.length === 0) return { failed: false, retryable: false };
const retryable = summary.failures.every((f) => isRetryableError(f.errorCode));
const permanent = summary.failures.some((f) => !isRetryableError(f.errorCode));
return { failed: true, retryable: retryable && !permanent };
return { failed: true, retryable: summary.failures.some(isRetryableFailure) };
}
export function retryDelay(attempt: number): number {
@@ -63,20 +128,32 @@ export function retryDelay(attempt: number): number {
return RETRY_DELAYS_SECONDS[idx] ?? 600;
}
function scopeKey(provider: string, groupId: string | undefined, deliveryId: string): string {
return `${provider}:${groupId ?? "global"}:${deliveryId}`;
function scopeKey(
provider: string,
groupId: string | undefined,
deliveryId: string,
instance?: string,
): string {
return `${provider}:${tenantScope(groupId, instance)}:${deliveryId}`;
}
/** State key of a delivery; tenant deliveries are keyed by their group instance. */
export function deliveryStateKey(
provider: string,
groupId: string | undefined,
deliveryId: string,
instance?: string,
): string {
return `delivery-state:${scopeKey(provider, groupId, deliveryId)}`;
return `delivery-state:${scopeKey(provider, groupId, deliveryId, instance)}`;
}
/**
* KV key of a parked payload. Unique per enqueue: a redelivery of the same
* delivery parks its own copy, so cleaning up after its failed send can never
* remove the payload an earlier queue message still needs.
*/
function payloadKey(provider: string, groupId: string | undefined, deliveryId: string): string {
return `queue:payload:${scopeKey(provider, groupId, deliveryId)}`;
return `queue:payload:${scopeKey(provider, groupId, deliveryId)}:${crypto.randomUUID()}`;
}
export async function setDeliveryState(
@@ -86,16 +163,24 @@ export async function setDeliveryState(
): Promise<void> {
const db = env.DB;
if (canUseD1(db)) {
await db
.prepare(
`INSERT INTO delivery_state (key, status, updated_at)
VALUES (?, ?, ?)
ON CONFLICT(key) DO UPDATE SET status = excluded.status, updated_at = excluded.updated_at`,
)
.bind(key, status, Date.now())
.run();
return;
try {
await db
.prepare(
`INSERT INTO delivery_state (key, status, updated_at)
VALUES (?, ?, ?)
ON CONFLICT(key) DO UPDATE SET status = excluded.status, updated_at = excluded.updated_at`,
)
.bind(key, status, Date.now())
.run();
return;
} catch (err) {
if (!isMissingTable(err)) throw err;
log.warn({ err, key }, "D1 delivery_state table missing, falling back to KV");
}
}
// Marked, so D1 (missing table now, or added to the deployment later)
// still honors this state.
await noteKvFallback(env.KV);
await env.KV.put(key, JSON.stringify({ status, at: Date.now() }), {
expirationTtl: STATE_KV_TTL_SECONDS,
});
@@ -104,11 +189,17 @@ export async function setDeliveryState(
export async function getDeliveryState(env: Env, key: string): Promise<DeliveryStatus | null> {
const db = env.DB;
if (canUseD1(db)) {
const row = await db
.prepare("SELECT status FROM delivery_state WHERE key = ?")
.bind(key)
.first<{ status: DeliveryStatus }>();
return row?.status ?? null;
try {
const row = await db
.prepare("SELECT status FROM delivery_state WHERE key = ?")
.bind(key)
.first<{ status: DeliveryStatus }>();
if (row?.status || !(await kvFallbackActive(env.KV))) return row?.status ?? null;
} catch (err) {
if (!isMissingTable(err)) throw err;
log.warn({ err, key }, "D1 delivery_state table missing, falling back to KV");
await noteKvFallback(env.KV);
}
}
const raw = await env.KV.get(key);
if (!raw) return null;
@@ -119,63 +210,324 @@ export async function getDeliveryState(env: Env, key: string): Promise<DeliveryS
}
}
export type LeaseResult = "acquired" | "finished" | "busy";
/**
* A delivery lease: its status and, when acquired, the fencing token (the
* `updated_at` the holder wrote). Renewals and terminal writes only apply
* while the row still carries the holder's token, so a consumer whose lease
* was taken over can neither overwrite the new holder's state nor go on as if
* it still owned the delivery. Null token: KV fallback (no fencing).
*/
export interface Lease {
status: LeaseResult;
token: number | null;
}
const FINISHED: ReadonlySet<DeliveryStatus | null> = new Set(["delivered", "dead", "failed"]);
/**
* Take a delivery for processing. With D1 this is one conditional UPSERT: it
* succeeds only for a new delivery, one waiting for a retry, or one whose
* previous holder's lease expired — so two consumers can never dispatch the
* same delivery at the same time. `finished` deliveries are done for good;
* `busy` ones are being processed elsewhere right now.
*/
export async function acquireDelivery(env: Env, key: string): Promise<Lease> {
const db = env.DB;
if (canUseD1(db)) {
try {
// Finished while the table was missing (state lives in KV then).
if (await kvFallbackActive(env.KV)) {
const kvState = await env.KV.get(key);
if (
kvState &&
FINISHED.has((JSON.parse(kvState) as { status?: DeliveryStatus }).status ?? null)
) {
return { status: "finished", token: null };
}
}
const now = Date.now();
const result = await db
.prepare(
`INSERT INTO delivery_state (key, status, updated_at) VALUES (?, 'processing', ?)
ON CONFLICT(key) DO UPDATE SET status = 'processing', updated_at = excluded.updated_at
WHERE delivery_state.status IN ('pending', 'retrying')
OR (delivery_state.status = 'processing' AND delivery_state.updated_at < ?)`,
)
.bind(key, now, now - LEASE_MS)
.run();
if ((result.meta?.changes ?? 0) > 0) return { status: "acquired", token: now };
const status = await getDeliveryState(env, key);
return { status: FINISHED.has(status) ? "finished" : "busy", token: null };
} catch (err) {
if (!isMissingTable(err)) throw err;
log.warn({ err, key }, "D1 delivery_state table missing, falling back to KV");
await noteKvFallback(env.KV);
}
}
// KV: best effort (no atomic compare-and-set).
const prior = await getDeliveryState(env, key);
if (FINISHED.has(prior)) return { status: "finished", token: null };
await setDeliveryState(env, key, "processing");
return { status: "acquired", token: null };
}
/**
* Mark a delivery dead (its message left the DLQ) — atomically, and only
* while it is neither finished nor held by a live lease: a copy of the same
* delivery that finished or is still sending keeps its state. Without D1:
* best effort.
*/
export async function markDeliveryDead(
env: Env,
key: string,
): Promise<"dead" | "finished" | "busy"> {
if (canUseD1(env.DB)) {
try {
const now = Date.now();
const result = await env.DB.prepare(
`INSERT INTO delivery_state (key, status, updated_at) VALUES (?, 'dead', ?)
ON CONFLICT(key) DO UPDATE SET status = 'dead', updated_at = excluded.updated_at
WHERE delivery_state.status IN ('pending', 'retrying')
OR (delivery_state.status = 'processing' AND delivery_state.updated_at < ?)`,
)
.bind(key, now, now - LEASE_MS)
.run();
if ((result.meta?.changes ?? 0) > 0) return "dead";
return FINISHED.has(await getDeliveryState(env, key)) ? "finished" : "busy";
} catch (err) {
if (!isMissingTable(err)) throw err;
}
}
if (FINISHED.has(await getDeliveryState(env, key))) return "finished";
await setDeliveryState(env, key, "dead");
return "dead";
}
/**
* Extend a held lease; the new token, or null when the lease was lost (taken
* over after expiry). Without D1 the lease cannot be fenced and stays valid.
*/
export async function renewLease(
env: Env,
key: string,
token: number | null,
): Promise<number | null> {
if (token == null || !canUseD1(env.DB)) return token;
const now = Math.max(Date.now(), token + 1);
const result = await env.DB.prepare(
`UPDATE delivery_state SET updated_at = ?
WHERE key = ? AND status = 'processing' AND updated_at = ?`,
)
.bind(now, key, token)
.run();
return (result.meta?.changes ?? 0) > 0 ? now : null;
}
/**
* Write the outcome of a held lease (`retrying` or a terminal status); false
* when the lease was lost meanwhile — the new holder's state then stands.
*/
export async function finishLease(
env: Env,
key: string,
token: number | null,
status: DeliveryStatus,
): Promise<boolean> {
if (token == null || !canUseD1(env.DB)) {
await setDeliveryState(env, key, status);
return true;
}
const result = await env.DB.prepare(
`UPDATE delivery_state SET status = ?, updated_at = ?
WHERE key = ? AND status = 'processing' AND updated_at = ?`,
)
.bind(status, Math.max(Date.now(), token + 1), key, token)
.run();
return (result.meta?.changes ?? 0) > 0;
}
function targetDoneKey(stateKey: string, targetKey: string): string {
return `delivery-target:${stateKey}:${targetKey}`;
}
/**
* Remember route × targets that need no further attempt for this delivery —
* one D1 transaction for all of them (KV claims without D1 or its table).
* With `permanentFailure`, the delivery's permanent-failure marker is written
* in the same transaction: a target is never skipped as finished while the
* failure that finished it is forgotten (which would let the delivery end
* up `delivered`).
*/
export async function markTargetsDone(
env: Env,
stateKey: string,
targetKeys: string[],
opts: { permanentFailure?: boolean } = {},
): Promise<void> {
const keys = targetKeys.map((t) => targetDoneKey(stateKey, t));
// The marker goes first: in the KV fallback, a write failing midway leaves
// at worst a marker without its target (the target is simply retried).
if (opts.permanentFailure) keys.unshift(permanentFailureKey(stateKey));
if (keys.length === 0) return;
if (canUseD1(env.DB)) {
const now = Date.now();
const expires = now + LEDGER_TTL_SECONDS * 1000;
try {
await env.DB.batch(
keys.map((key) =>
env.DB.prepare(
`INSERT INTO dedup_keys (key, claimed_at, expires_at) VALUES (?, ?, ?)
ON CONFLICT(key) DO UPDATE SET expires_at = excluded.expires_at`,
).bind(key, now, expires),
),
);
return;
} catch (err) {
if (!isMissingTable(err)) throw err;
}
}
await noteKvFallback(env.KV);
const store = kvIdempotencyStore(env.KV);
for (const key of keys) {
await store.claim(key, LEDGER_TTL_SECONDS);
}
}
/**
* Whether a route × target already finished in an earlier attempt of this
* delivery. The delivery's finished targets are read in one D1 query (on the
* first check) rather than one per target: a large fan-out would otherwise
* spend the invocation's D1 queries before sending anything.
*/
export function targetDoneChecker(
env: Env,
stateKey: string,
): (targetKey: string) => Promise<boolean> {
const store = idempotencyStore(env.DB, env.KV);
let finished: Promise<Set<string> | null> | null = null;
return async (targetKey) => {
const key = targetDoneKey(stateKey, targetKey);
finished ??= finishedTargets(env, stateKey);
const known = await finished;
if (!known) return store.has(key);
if (known.has(key)) return true;
// Copies written to KV while the table was missing: looked up only for
// the targets D1 does not know as finished.
return (await kvFallbackActive(env.KV)) ? kvIdempotencyStore(env.KV).has(key) : false;
};
}
/** The delivery's finished-target keys, or null where they are read one by one. */
async function finishedTargets(env: Env, stateKey: string): Promise<Set<string> | null> {
if (!canUseD1(env.DB)) return null;
const prefix = targetDoneKey(stateKey, "");
try {
const { results } = await env.DB.prepare(
"SELECT key FROM dedup_keys WHERE key >= ? AND key < ? AND expires_at > ?",
)
.bind(prefix, `${prefix}\uffff`, Date.now())
.all<{ key: string }>();
return new Set((results ?? []).map((r) => r.key));
} catch (err) {
if (isMissingTable(err)) return null;
throw err;
}
}
function permanentFailureKey(stateKey: string): string {
return `delivery-failed:${stateKey}`;
}
/** Whether an earlier attempt of this delivery left a target failed for good. */
export function hadPermanentFailure(env: Env, stateKey: string): Promise<boolean> {
return idempotencyStore(env.DB, env.KV).has(permanentFailureKey(stateKey));
}
function byteLength(text: string): number {
return new TextEncoder().encode(text).byteLength;
}
/**
* Enqueue a verified webhook. Payloads too large for a queue message are
* parked in R2 (or KV) first; when the send itself fails, the parked copy is
* removed again and the error propagates, so the caller can tell the sender
* to retry instead of acknowledging an event that was never queued.
*/
export async function enqueueWebhook(env: Env, message: DeliveryMessage): Promise<void> {
const queue = env.QUEUE;
if (!queue) throw new Error("QUEUE binding is not configured");
const { payload, ...rest } = message;
const direct: DeliveryMessage = { ...rest, payload };
if (JSON.stringify(direct).length <= MAX_QUEUE_MESSAGE_BYTES) {
if (byteLength(JSON.stringify(direct)) <= MAX_QUEUE_MESSAGE_BYTES) {
await queue.send(direct);
return;
}
const bucket = env.PAYLOAD;
if (bucket) {
const store = r2PayloadStore(env);
const key = await store.put(JSON.stringify(payload ?? {}), PAYLOAD_KV_TTL_SECONDS);
await queue.send({ ...rest, payloadRef: key, payloadRefType: "r2" });
return;
const serialized = JSON.stringify(payload ?? {});
let parked: DeliveryMessage;
if (env.PAYLOAD) {
const key = await r2PayloadStore(env).put(serialized, PAYLOAD_TTL_SECONDS);
parked = { ...rest, payloadRef: key, payloadRefType: "r2" };
} else {
const payloadRef = payloadKey(message.provider, message.groupId, message.deliveryId);
await env.KV.put(payloadRef, serialized, { expirationTtl: PAYLOAD_TTL_SECONDS });
parked = { ...rest, payloadRef, payloadRefType: "kv" };
}
const payloadRef = payloadKey(message.provider, message.groupId, message.deliveryId);
await env.KV.put(payloadRef, JSON.stringify(payload ?? {}), {
expirationTtl: PAYLOAD_KV_TTL_SECONDS,
});
await queue.send({ ...rest, payloadRef, payloadRefType: "kv" });
// A failed send leaves the parked copy in place: the queue may have taken
// the message after all (the error can be a lost response), and deleting
// the payload would doom it. An unused copy expires (KV TTL) or is pruned
// as an orphan (R2).
await queue.send(parked);
}
function parsePayload(raw: string, ref: string): Record<string, unknown> {
try {
const parsed = JSON.parse(raw) as unknown;
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
return parsed as Record<string, unknown>;
}
} catch {
// reported below
}
throw new PayloadUnavailableError(`Parked payload ${ref} is corrupt`);
}
/**
* The event payload of a queued delivery. A parked payload that is missing or
* corrupt is an explicit error — dispatching an empty payload instead would
* match nothing and record the delivery as delivered.
*/
export async function resolvePayload(
env: Env,
message: DeliveryMessage,
): Promise<Record<string, unknown>> {
if (message.payload) return message.payload;
if (message.payloadRef) {
if (message.payloadRefType === "r2" && env.PAYLOAD) {
const store = r2PayloadStore(env);
const raw = await store.get(message.payloadRef);
if (raw) {
try {
return JSON.parse(raw) as Record<string, unknown>;
} catch {
return {};
}
}
}
const raw = await env.KV.get(message.payloadRef);
if (raw) {
try {
return JSON.parse(raw) as Record<string, unknown>;
} catch {
return {};
}
if (!message.payloadRef) throw new PayloadUnavailableError("Delivery carries no payload");
if (message.payloadRefType === "r2") {
if (!env.PAYLOAD) throw new Error("R2 binding is not configured");
const raw = await r2PayloadStore(env).get(message.payloadRef);
if (raw == null) {
throw new PayloadUnavailableError(`Parked payload ${message.payloadRef} is missing`);
}
return parsePayload(raw, message.payloadRef);
}
return {};
const raw = await env.KV.get(message.payloadRef);
if (raw == null) {
// KV writes can take a while to become visible elsewhere: a payload
// parked minutes ago is retried, not declared lost.
if (Date.now() - message.receivedAt < KV_VISIBILITY_GRACE_MS) {
throw new Error(`Parked payload ${message.payloadRef} is not visible yet`);
}
throw new PayloadUnavailableError(`Parked payload ${message.payloadRef} is missing`);
}
return parsePayload(raw, message.payloadRef);
}
export async function discardPayload(env: Env, message: DeliveryMessage): Promise<void> {
if (!message.payloadRef) return;
if (message.payloadRefType === "r2" && env.PAYLOAD) {
const store = r2PayloadStore(env);
await store.delete(message.payloadRef);
if (message.payloadRefType === "r2") {
if (env.PAYLOAD) await r2PayloadStore(env).delete(message.payloadRef);
return;
}
await env.KV.delete(message.payloadRef);
File diff suppressed because it is too large. Load diff
+89
View File
@@ -1,3 +1,5 @@
import { sharedFlight, type Flight } from "../lib/shared-flight";
/**
* True when a D1Database binding looks like the real thing. Production D1
* always exposes both `prepare` and `batch`; test harnesses range from
@@ -9,3 +11,90 @@
export function canUseD1(db: D1Database | undefined | null): boolean {
return typeof db?.prepare === "function" && typeof (db as D1Database).batch === "function";
}
/**
* The D1 error says the table does not exist yet (migrations not applied).
* Only this case may degrade a D1-backed store to KV: on any other failure
* the two stores would each hold part of the state (a claim taken in D1 is
* invisible in KV), so the error must propagate and the caller retry.
*/
export function isMissingTable(err: unknown): boolean {
return /no such table/i.test(String(err));
}
/**
* Set whenever state is written to KV instead of D1: by a D1-backed store
* whose table was missing, and by every store of a deployment without D1
* (which may get a D1 binding later). Until it expires — longer than any state
* written that way lives, counted from the last such write — reads that find
* nothing in D1 also look in KV, so that state is not forgotten. Its value is
* its own expiry (ms), so a fresh enough marker is not rewritten.
*/
const KV_FALLBACK_MARKER = "storage:kv-fallback";
const KV_FALLBACK_TTL = 35 * 86400;
const MARKER_RECHECK_MS = 5 * 60_000;
const MARKER_ACTIVE_RECHECK_MS = 3600_000;
/** A fallback write refreshes the marker at most this often (per isolate). */
const MARKER_REFRESH_MS = 3600_000;
const fallbackMarkers = new WeakMap<
KVNamespace,
{ active: boolean; until: number; writtenAt?: number }
>();
/** Marker reads in flight: concurrent checks share one KV read. */
const markerReads = new WeakMap<KVNamespace, Flight<boolean>>();
/** Marker refreshes in flight: concurrent fallbacks share one read and write. */
const markerWrites = new WeakMap<KVNamespace, Flight<void>>();
/** How long another request's marker read or write is waited for. */
const MARKER_FLIGHT_MS = 10_000;
export function noteKvFallback(kv: KVNamespace): Promise<void> {
const cached = fallbackMarkers.get(kv);
if (cached?.writtenAt != null && Date.now() - cached.writtenAt < MARKER_REFRESH_MS) {
return Promise.resolve();
}
return sharedFlight(markerWrites, kv, () => writeFallbackMarker(kv), MARKER_FLIGHT_MS);
}
async function writeFallbackMarker(kv: KVNamespace): Promise<void> {
const stored = Number(await kv.get(KV_FALLBACK_MARKER));
// Rewritten at most about daily, whatever the number of isolates.
if (!(stored - Date.now() > (KV_FALLBACK_TTL - 86400) * 1000)) {
await kv.put(KV_FALLBACK_MARKER, String(Date.now() + KV_FALLBACK_TTL * 1000), {
expirationTtl: KV_FALLBACK_TTL,
});
}
// Remembered only once it is stored: a failed write is tried again by the
// next fallback instead of being taken for done.
fallbackMarkers.set(kv, {
active: true,
until: Date.now() + MARKER_ACTIVE_RECHECK_MS,
writtenAt: Date.now(),
});
}
export async function kvFallbackActive(kv: KVNamespace): Promise<boolean> {
// "No marker" is remembered for a few minutes too: this check runs on
// nearly every claim, and reading KV each time would spend the KV read
// quota (Workers Free) on a marker that is almost always absent. The cost
// is a short window after another isolate first sets it (accepted).
// A seen marker is read again hourly: once it has expired, the KV
// lookups it turns on stop too (a long-lived isolate would otherwise keep
// reading KV for state long gone).
const cached = fallbackMarkers.get(kv);
if (cached && Date.now() < cached.until) return cached.active;
return sharedFlight(
markerReads,
kv,
async () => {
const active = (await kv.get(KV_FALLBACK_MARKER)) != null;
fallbackMarkers.set(kv, {
active,
until: Date.now() + (active ? MARKER_ACTIVE_RECHECK_MS : MARKER_RECHECK_MS),
writtenAt: fallbackMarkers.get(kv)?.writtenAt,
});
return active;
},
MARKER_FLIGHT_MS,
);
}
+80 -2
View File
@@ -16,6 +16,14 @@ export interface Env {
TELEGRAM_RICH_HEADER_HOST?: string;
FEISHU_APP_ID?: string;
FEISHU_APP_SECRET?: string;
/**
* Feishu event "Encrypt Key": verifies `X-Lark-Signature` and decrypts
* encrypted callbacks. This or FEISHU_VERIFICATION_TOKEN is required for
* inbound Feishu commands and card buttons.
*/
FEISHU_ENCRYPT_KEY?: string;
/** Feishu event "Verification Token", checked against every callback payload. */
FEISHU_VERIFICATION_TOKEN?: string;
/**
* When enabled ("1"/"true"), GitHub users without any group access get a
* personal group on first login instead of being blocked.
@@ -43,6 +51,12 @@ export interface Config {
token: string;
};
routes: Route[];
/**
* Groups with a stored route that could not be read. Their events are
* reported as transient failures instead of passing for delivered while a
* route that might have matched was silently left out.
*/
unreadableGroups?: string[];
}
export interface RouteTarget {
@@ -91,11 +105,16 @@ export type GroupRole = "owner" | "admin" | "viewer";
export interface GroupMember {
/**
* GitHub login (case-insensitive). Ids are matched when stored, logins
* otherwise; identityMatches handles both.
* GitHub login (case-insensitive). An unbound all-digit entry (legacy
* `adminIds`) is read as a user id; see `memberMatches`.
*/
login: string;
role: GroupRole;
/**
* Immutable GitHub user id of the member, bound by the server (invite
* acceptance, the member's sign-in). Once bound, only that account matches.
*/
userId?: string;
}
export interface Group {
@@ -130,6 +149,46 @@ export interface Group {
* `owners` list. Empty/omitted = no installation restriction.
*/
installationId?: number;
/**
* Isolated groups (self-service personal groups) only receive events from
* their own `/webhook/{groupId}` ingress; the global endpoint dispatches to
* them only once a super admin binds a source (`installationId` or
* `owners`). Only super admins may edit this field.
*/
isolated?: boolean;
/**
* Immutable identity of this group, assigned by the server when the group is
* created and kept across renames. A group re-created under a former id gets
* a new one, so invites and delivery ledgers of the old group never apply to
* it. Clients cannot set it.
*/
instance?: string;
/**
* Set when the group got its `instance` from the one-time backfill, i.e. it
* existed before instances: only such a group may adopt state stored under
* its id (a pre-instance webhook secret or invite). Server-owned.
*/
instanceBackfilled?: boolean;
/**
* For a backfilled group: the id it had when it got its instance. State
* stored under a group id before instances (a webhook secret, an invite, a
* queued delivery) belongs to the group with that `legacyId` only — not to
* whichever group holds the id now (another one may have been renamed to
* it). Server-owned, kept across renames.
*/
legacyId?: string;
/**
* The group's webhook secret (per-group ingress). Server-owned, stored with
* the group so it changes only through a compare-and-swap that re-checks the
* writer's role; never sent to clients. `null` in a mutation removes it.
*/
webhookSecret?: string | null;
/**
* Opaque revision of the stored group, set on read by the admin API and
* echoed back on update so a stale edit is rejected instead of overwriting a
* newer one. Never persisted.
*/
revision?: string;
/**
* Whether to include emoji in messages sent through this group's routes.
* Defaults to true when omitted.
@@ -321,11 +380,28 @@ export interface NeutralMessage {
* previously sent message instead of sending a new one.
*/
updateKey?: string;
/**
* The chain key this message had before `updateKey` changed shape, used
* only to find a message tracked under the old key (then re-keyed).
*/
legacyUpdateKey?: string;
/**
* Stable identity of this message for one target, the same in every
* attempt of a delivery (set by dispatch from delivery × route × target).
* Platforms with request idempotency (Discord `nonce`, Feishu `uuid`) use
* it, so a retry after a lost response never posts the message twice.
*/
sendKey?: string;
/**
* Discord role ids to mention in the message content (set by dispatch from
* the route's `discordRoleIds`). Only used by the Discord driver.
*/
mentionRoleIds?: string[];
/**
* False when the group turned emoji off: drivers must not add decorative
* emoji of their own (e.g. Telegram's author marker). Defaults to true.
*/
emoji?: boolean;
}
export interface FormattedMessage {
@@ -353,4 +429,6 @@ export interface FormattedMessage {
custom_id?: string;
}>;
}>;
/** Which mentions in `content` may notify; set so only the route's roles ping. */
allowed_mentions?: { parse: string[]; roles?: string[] };
}
+1 -1
View File
@@ -33,7 +33,7 @@ async function userOctokit(
event: H3Event,
userId: string,
): Promise<Awaited<ReturnType<typeof getUserOctokit>>> {
return getUserOctokit(userId, cfEnv(event).KV);
return getUserOctokit(userId, cfEnv(event));
}
function fail(event: H3Event, status: number, error: string): Record<string, unknown> {
+1272 -249
View File
File diff suppressed because it is too large. Load diff
+11 -9
View File
@@ -1,7 +1,7 @@
import type { H3Event } from "h3";
import { createError, getHeader } from "h3";
import type { Group, GroupRole } from "../types";
import { getAdminSession, isDebugMode, type AdminSession } from "./session";
import { authLogin, getAdminSession, isDebugMode, type AdminSession } from "./session";
import {
loadGroups,
resolveScope,
@@ -28,14 +28,15 @@ const AUTH_KEY = "auth";
export async function loadAuth(event: H3Event): Promise<AuthContext | null> {
const env = cfEnv(event);
initConfigStore(env);
const groups = await loadGroups(env.KV);
if (isDebugMode(env)) {
const session: AdminSession = { userId: "debug", login: "debug" };
return { session, scope: resolveScope(env, groups, session.userId, session.login), groups };
}
const session = await getAdminSession(env.KV, getHeader(event, "cookie"));
// The session first: a request without one reads no configuration.
const session: AdminSession | null = isDebugMode(env)
? { userId: "debug", login: "debug" }
: await getAdminSession(env.KV, getHeader(event, "cookie"), env.DB);
if (!session) return null;
const scope = resolveScope(env, groups, session.userId, session.login);
// Access checks read the authoritative store: a membership revoked in
// another isolate must not linger in this one's cache.
const groups = await loadGroups(env.KV, { fresh: true });
const scope = resolveScope(env, groups, session.userId, authLogin(session));
return { session, scope, groups };
}
@@ -99,7 +100,8 @@ export async function bearerUserId(event: H3Event): Promise<string> {
const auth = getHeader(event, "authorization");
if (!auth?.startsWith("Bearer "))
throw createError({ statusCode: 401, statusMessage: "Missing authorization" });
const userId = await findUserIdByToken(env.KV, auth.slice(7));
// Resolves only the user's *current* token: a rotated or revoked one fails.
const userId = await findUserIdByToken(env.KV, auth.slice(7), env.DB);
if (!userId) throw createError({ statusCode: 401, statusMessage: "Invalid or expired token" });
return userId;
}
+148
View File
@@ -0,0 +1,148 @@
import type { Env } from "../types";
import { retiredGroupKey, type LifecycleStatements } from "../storage/config-store";
import { idempotencyStore, instanceScope } from "../lib/idempotency";
import { canUseD1 } from "../storage/d1";
import { log } from "../lib/log";
/**
* A deleted or renamed group id stays retired this long: queued deliveries
* still addressed to it (queue retries span minutes, the DLQ up to days) can
* then never reach an unrelated group that took over the id.
*/
const RETIRED_TTL_SECONDS = 4 * 86400;
const ALIAS_TTL_SECONDS = 4 * 86400;
const retiredKey = retiredGroupKey;
function aliasKey(id: string): string {
return `group:renamed:${id}`;
}
/** D1 alias row: `group-alias:{from} {to}` (ids never contain spaces). */
function aliasRowPrefix(from: string): string {
return `group-alias:${from} `;
}
/**
* The first key past every key starting with `prefix` (its last character
* incremented): `key >= prefix AND key < end` is a primary-key range, where
* `substr(key, …) = prefix` scans the whole table.
*/
function prefixEnd(prefix: string): string {
return prefix.slice(0, -1) + String.fromCharCode(prefix.charCodeAt(prefix.length - 1) + 1);
}
function retireStatement(db: D1Database, id: string): D1PreparedStatement {
const now = Date.now();
return db
.prepare(
`INSERT INTO dedup_keys (key, claimed_at, expires_at) VALUES (?, ?, ?)
ON CONFLICT(key) DO UPDATE SET claimed_at = excluded.claimed_at, expires_at = excluded.expires_at`,
)
.bind(retiredKey(id), now, now + RETIRED_TTL_SECONDS * 1000);
}
/**
* Statements committed with a group's deletion: its send logs and audit
* entries stay for super admins but no longer belong to any group (a later
* group with the same id must not inherit them), its message tracking is
* dropped, and the id is retired.
*/
export function deleteLifecycle(
id: string,
instance?: string,
legacyId?: string,
): LifecycleStatements {
// The group's own chains only: under its instance, and — from before
// instances — under the id it had back then. Rows under its current id may
// belong to the group that had that id before instances.
const prefixes = [
...(instance ? [`${instanceScope(instance)}:`] : []),
...(legacyId ? [`${legacyId}:`] : []),
];
return (db) => [
db.prepare("UPDATE send_logs SET group_id = NULL WHERE group_id = ?").bind(id),
db.prepare("UPDATE audit_logs SET group_id = NULL WHERE group_id = ?").bind(id),
...prefixes.map((prefix) =>
db
.prepare("DELETE FROM message_tracking WHERE event_id >= ? AND event_id < ?")
.bind(prefix, prefixEnd(prefix)),
),
retireStatement(db, id),
];
}
/**
* Statements committed with a group's rename: its history moves to the new id
* (visible to the group, to nobody else), and the old id is retired. Message
* tracking stays put: chains are keyed by the group's instance, and those from
* before instances by its `legacyId` — neither changes with a rename.
*/
export function renameLifecycle(from: string, to: string): LifecycleStatements {
return (db) => [
db.prepare("UPDATE send_logs SET group_id = ? WHERE group_id = ?").bind(to, from),
db.prepare("UPDATE audit_logs SET group_id = ? WHERE group_id = ?").bind(to, from),
retireStatement(db, from),
// Committed with the rename itself: queued deliveries for the old id can
// always find the new one.
db
.prepare(
`INSERT INTO dedup_keys (key, claimed_at, expires_at) VALUES (?, ?, ?)
ON CONFLICT(key) DO UPDATE SET claimed_at = excluded.claimed_at, expires_at = excluded.expires_at`,
)
.bind(`${aliasRowPrefix(from)}${to}`, Date.now(), Date.now() + ALIAS_TTL_SECONDS * 1000),
];
}
/** Mark a group id as recently used (KV-only deployments; D1 does it in the transaction). */
export async function retireGroupId(env: Env, id: string): Promise<void> {
if (canUseD1(env.DB)) return;
try {
await idempotencyStore(env.DB, env.KV).claim(retiredKey(id), RETIRED_TTL_SECONDS);
} catch (err) {
log.warn({ err, id }, "Failed to retire group id");
}
}
export async function isGroupIdRetired(env: Env, id: string): Promise<boolean> {
return idempotencyStore(env.DB, env.KV).has(retiredKey(id));
}
/** Remember where a renamed group went (KV-only deployments; D1 records it in the rename). */
export async function recordGroupRename(env: Env, from: string, to: string): Promise<void> {
if (canUseD1(env.DB)) return;
await env.KV.put(aliasKey(from), to, { expirationTtl: ALIAS_TTL_SECONDS });
}
async function nextAlias(env: Env, id: string): Promise<string | null> {
if (canUseD1(env.DB)) {
const prefix = aliasRowPrefix(id);
const row = await env.DB.prepare(
`SELECT key FROM dedup_keys WHERE key >= ? AND key < ? AND expires_at > ?
ORDER BY claimed_at DESC LIMIT 1`,
)
.bind(prefix, prefixEnd(prefix), Date.now())
.first<{ key: string }>();
return row ? row.key.slice(prefix.length) : null;
}
return env.KV.get(aliasKey(id));
}
/**
* The current id of a group that may have been renamed (follows chains).
* Read errors propagate: an unknown alias must not pass for "no alias".
*/
export async function resolveGroupAlias(env: Env, id: string): Promise<string> {
return (await groupAliasChain(env, id)).at(-1)!;
}
/** Every id a group had from `id` on, oldest first (`id` itself included). */
export async function groupAliasChain(env: Env, id: string): Promise<string[]> {
const chain = [id];
for (let i = 0; i < 5; i++) {
const next = await nextAlias(env, chain.at(-1)!);
if (!next || chain.includes(next)) break;
chain.push(next);
}
return chain;
}
+261 -89
View File
@@ -1,12 +1,15 @@
import type { Env, Group, GroupMember, GroupRole } from "../types";
import { isAdminUser } from "./session";
import { log } from "../lib/log";
import { migrateGroups, validateGroups } from "../config/schema";
import { identityEntryMatches, isAdminUser } from "./session";
import { getConfigStore } from "../config";
const GROUPS_KEY = "config:groups";
const GROUPS_CACHE_TTL = 300_000;
let groupsCache: { groups: Group[]; expiresAt: number } | null = null;
import { idempotencyStore, unrecordedGuard } from "../lib/idempotency";
import { ConfigConflictError } from "../storage/config-store";
import { canUseD1 } from "../storage/d1";
import type {
GroupMutation,
GuardedWrite,
LifecycleStatements,
LoadOptions,
} from "../storage/config-store";
/**
* Normalizes a group's member list. Groups stored with the legacy `adminIds`
@@ -24,7 +27,8 @@ export function normalizeGroupMembers(group: Group): GroupMember[] {
if (!login || seen.has(login.toLowerCase())) continue;
const role: GroupRole = m?.role === "admin" || m?.role === "viewer" ? m.role : "owner";
seen.add(login.toLowerCase());
out.push({ login, role });
const userId = typeof m?.userId === "string" && /^\d+$/.test(m.userId) ? m.userId : "";
out.push(userId ? { login, role, userId } : { login, role });
}
return out;
}
@@ -40,44 +44,155 @@ export function normalizeGroupMembers(group: Group): GroupMember[] {
return out;
}
export async function loadGroups(kv: KVNamespace): Promise<Group[]> {
const store = getConfigStore(kv);
if (store) return store.loadGroups();
if (groupsCache && Date.now() < groupsCache.expiresAt) {
return groupsCache.groups;
}
try {
const stored = await kv.get<Group[]>(GROUPS_KEY, "json");
const groups = Array.isArray(stored) ? validateGroups(migrateGroups(stored)) : [];
groupsCache = { groups, expiresAt: Date.now() + GROUPS_CACHE_TTL };
return groups;
} catch (err) {
log.warn({ err }, "Failed to load groups from KV");
}
return [];
export async function loadGroups(kv: KVNamespace, opts?: LoadOptions): Promise<Group[]> {
return getConfigStore(kv).loadGroups(opts);
}
/**
* Replace the full group set (super-admin bulk write): groups left out are
* deleted with their routes and fragments, only changed groups are rewritten.
*/
export async function saveGroups(kv: KVNamespace, groups: Group[]): Promise<void> {
const store = getConfigStore(kv);
if (store) {
await store.saveGroups(groups);
groupsCache = null;
return;
}
await kv.put(GROUPS_KEY, JSON.stringify(groups));
groupsCache = null;
await getConfigStore(kv).saveGroups(groups);
}
export function invalidateGroupsCache(): void {
groupsCache = null;
/** Insert a group unless its id is taken; true when it was created. */
export async function createGroup(kv: KVNamespace, group: Group): Promise<boolean> {
return getConfigStore(kv).createGroup(group);
}
/** Case-insensitive match of a GitHub userId or login against a list of ids/logins. */
/**
* Atomically read-modify-write one group. `mutate` gets the current stored
* group and returns the replacement, or null to keep it unchanged.
*/
export async function updateGroup(
kv: KVNamespace,
id: string,
mutate: GroupMutation,
opts?: Pick<GuardedWrite<Group>, "guards" | "after">,
): Promise<Group | null> {
return getConfigStore(kv).updateGroup(id, mutate, opts);
}
/** Delete one group with its routes and filter fragments. */
export async function deleteGroup(
kv: KVNamespace,
id: string,
opts?: GuardedWrite<Group> & { extra?: LifecycleStatements; version?: number },
): Promise<void> {
await getConfigStore(kv).deleteGroup(id, opts);
}
/** Move a group (with its routes and fragments) to a new id atomically. */
export async function renameGroup(
kv: KVNamespace,
from: string,
to: string,
opts?: GuardedWrite<Group> & { extra?: LifecycleStatements; transform?: (g: Group) => Group },
): Promise<Group> {
return getConfigStore(kv).renameGroup(from, to, opts);
}
/** Kept for callers that reset state between tests; the store owns its caches. */
export function invalidateGroupsCache(): void {}
/** Whether any entry names the user: all-digit entries are user ids, others logins. */
export function identityMatches(ids: string[], userId: string, login: string): boolean {
const wanted = ids.map((s) => s.trim()).filter(Boolean);
if (wanted.length === 0) return false;
return wanted.some((id) => id === userId || id.toLowerCase() === login.toLowerCase());
return ids.some((id) => identityEntryMatches(id, userId, login));
}
/**
* Whether a member entry is this user. A member bound to a GitHub user id is
* that account only — a login can be renamed and then claimed by someone
* else, an id never changes. Unbound entries match like `identityMatches`.
*/
export function memberMatches(member: GroupMember, userId: string, login: string): boolean {
if (member.userId) return member.userId === userId;
return identityEntryMatches(member.login, userId, login);
}
/**
* A guard statement that aborts its transaction unless the user holds one of
* `roles` in group `gid` as stored at that moment. Member entries match the
* way `memberMatches` does: a bound entry by user id, an all-digit one by
* user id, any other by `login` (pass "" to match by id only); groups still
* on legacy `adminIds` count those as owners.
*/
export function memberRoleStatement(
db: D1Database,
gid: string,
userId: string,
login: string,
roles: GroupRole[],
): D1PreparedStatement {
return db
.prepare(
`SELECT CASE WHEN EXISTS (
SELECT 1 FROM (SELECT id, CASE WHEN json_valid(data) THEN data ELSE '{}' END AS data FROM d1_groups) g, json_each(g.data, '$.members') m
WHERE g.id = ? AND json_extract(m.value, '$.role') IN (${roles.map(() => "?").join(", ")})
AND (json_extract(m.value, '$.userId') = ?
OR (json_extract(m.value, '$.userId') IS NULL AND (
(json_extract(m.value, '$.login') NOT GLOB '*[^0-9]*'
AND json_extract(m.value, '$.login') = ?)
OR (json_extract(m.value, '$.login') GLOB '*[^0-9]*'
AND lower(json_extract(m.value, '$.login')) = lower(?)))))
) OR (? IN (${roles.map(() => "?").join(", ")}) AND EXISTS (
SELECT 1 FROM (SELECT id, CASE WHEN json_valid(data) THEN data ELSE '{}' END AS data FROM d1_groups) g, json_each(g.data, '$.adminIds') a
WHERE g.id = ? AND coalesce(json_array_length(g.data, '$.members'), 0) = 0
AND ((a.value NOT GLOB '*[^0-9]*' AND a.value = ?)
OR (a.value GLOB '*[^0-9]*' AND lower(a.value) = lower(?)))
)) THEN 1 ELSE json('role revoked') END AS ok`,
)
.bind(gid, ...roles, userId, userId, login, "owner", ...roles, gid, userId, login);
}
/**
* Carry the user ids already bound to members over to a submitted member
* list (matched by login): clients edit logins and roles, never the binding.
*/
export function carryMemberIds(next: Group, prev: Group | undefined): Group {
if (!prev || !Array.isArray(next.members)) return next;
const bound = new Map(
normalizeGroupMembers(prev)
.filter((m) => m.userId)
.map((m) => [m.login.toLowerCase(), m.userId!]),
);
if (bound.size === 0) return next;
return {
...next,
members: next.members.map((m) => {
const userId = bound.get(m.login.trim().toLowerCase());
return userId ? { ...m, userId } : m;
}),
};
}
/**
* Bind the signed-in user's id to the member entries that name them only by
* login, so a later owner of that login (after a rename) never inherits the
* membership. Best-effort; each group is a compare-and-swap.
*/
export async function bindMemberIds(
kv: KVNamespace,
groups: Group[],
userId: string,
login: string,
): Promise<void> {
const unbound = (g: Group): boolean =>
normalizeGroupMembers(g).some(
(m) => !m.userId && !/^\d+$/.test(m.login) && memberMatches(m, userId, login),
);
for (const group of groups.filter(unbound)) {
await updateGroup(kv, group.id, (g) => {
if (!unbound(g)) return null;
const members = normalizeGroupMembers(g).map((m) =>
!m.userId && !/^\d+$/.test(m.login) && memberMatches(m, userId, login)
? { ...m, userId }
: m,
);
return { ...g, members };
});
}
}
export function isGroupAdmin(group: Group, userId: string, login: string): boolean {
@@ -89,12 +204,8 @@ export function isGroupAdmin(group: Group, userId: string, login: string): boole
* `adminIds` groups resolve to "owner" for every listed admin.
*/
export function memberRole(group: Group, userId: string, login: string): GroupRole | undefined {
const members = normalizeGroupMembers(group);
const wanted = members.filter((m) => {
const candidate = m.login.trim().toLowerCase();
return candidate === login.toLowerCase() || candidate === userId;
});
return wanted.length > 0 ? wanted[0]!.role : undefined;
const member = normalizeGroupMembers(group).find((m) => memberMatches(m, userId, login));
return member?.role;
}
/**
@@ -130,6 +241,27 @@ export function groupAcceptsInstallation(group: Group, installationId?: number):
return installationId != null && group.installationId === installationId;
}
/**
* Whether the group may receive an event that arrived on the legacy global
* endpoint (`POST /webhook`). Isolated groups (self-service personal groups)
* only take events from their own `/webhook/{groupId}` ingress, unless a super
* admin bound them to a source: a GitHub App installation or an `owners` list.
*/
/**
* Whether a group is isolated. Personal groups (`u-{userId}`) created before
* the flag existed count as isolated too, unless a super admin explicitly set
* `isolated: false`.
*/
export function isIsolated(group: Group): boolean {
return group.isolated ?? /^u-\d+$/.test(group.id);
}
export function groupAcceptsGlobalIngress(group: Group): boolean {
const isolated = isIsolated(group);
if (!isolated) return true;
return group.installationId != null || (group.owners ?? []).some((o) => o.trim().length > 0);
}
/**
* Auto-provision a GitHub App installation on `installation.created`:
* 1. No-op when a group is already bound to this installation id.
@@ -137,63 +269,103 @@ export function groupAcceptsInstallation(group: Group, installationId?: number):
* account login (so existing org groups light up automatically).
* 3. Otherwise create a dedicated `inst-{installationId}` group bound to the
* installation. Returns the group that now owns the installation.
*
* Every write is an atomic create-if-absent or compare-and-swap update, so
* concurrent deliveries converge on the same group without a lock.
*/
/** An automatic installation binding is remembered as long as delivery state (7 days). */
const AUTO_BOUND_TTL_SECONDS = 7 * 86400;
export async function ensureInstallationGroup(
kv: KVNamespace,
installationId: number,
accountLogin: string,
db?: D1Database,
): Promise<Group | null> {
// A per-installation lock serializes concurrent `installation.created`
// webhooks: without it two requests both pass the "no group bound" check,
// both create `inst-{id}`, and the second save overwrites the first.
const lockKey = `inst:lock:${installationId}`;
for (let attempt = 0; attempt < 5; attempt++) {
const holder = await kv.get(lockKey);
if (!holder) break;
const groups = await loadGroups(kv);
const existing = groups.find((g) => g.installationId === installationId);
if (existing) return existing;
await new Promise((r) => setTimeout(r, 40 * (attempt + 1)));
}
// Each automatic binding is recorded (per installation and group) in the
// transaction that makes it: a retry binds the groups an earlier attempt
// did not reach, never again one it bound — an admin may have undone that.
const claims = db ? idempotencyStore(db, kv) : null;
const boundKey = (g: Group): string =>
`installation-bound:${installationId}:${g.instance ?? g.id}`;
const groups = await loadGroups(kv, { fresh: true });
// Already bound somewhere (an earlier attempt): the matching groups still
// unbound are bound all the same — that attempt may have failed midway.
const existing = groups.find((g) => g.installationId === installationId);
let locked = false;
try {
await kv.put(lockKey, "1", { expirationTtl: 60 });
locked = true;
const groups = await loadGroups(kv);
const existing = groups.find((g) => g.installationId === installationId);
if (existing) return existing;
const login = accountLogin.trim().toLowerCase();
const candidates = groups.filter(
const login = accountLogin.trim().toLowerCase();
const candidates = groups.filter(
(g) =>
g.installationId == null &&
login.length > 0 &&
(g.owners ?? []).some((o) => o.trim().toLowerCase() === login),
);
let bound: Group | null = null;
for (const candidate of candidates) {
if (claims && (await claims.has(boundKey(candidate)))) continue;
const key = boundKey(candidate);
const now = Date.now();
// Re-checked on the group as it is at the write: the same group (not one
// that took over the id), still unbound, still owned by this account —
// an `owners` list cleared meanwhile must not be reopened by a binding.
const updated = await updateGroup(
kv,
candidate.id,
(g) =>
g.instance === candidate.instance &&
g.installationId == null &&
login.length > 0 &&
(g.owners ?? []).some((o) => o.trim().toLowerCase() === login),
);
if (candidates.length > 0) {
const next = groups.map((g) => (candidates.includes(g) ? { ...g, installationId } : g));
await saveGroups(kv, next);
return next.find((g) => g.id === candidates[0]!.id) ?? null;
(g.owners ?? []).some((o) => o.trim().toLowerCase() === login)
? { ...g, installationId }
: null,
db && canUseD1(db)
? {
// Checked again at the write: an attempt that read before another
// one bound (and an admin unbound) the group never binds it again.
guards: (d): D1PreparedStatement[] => [unrecordedGuard(d, key)],
// Recorded only when the binding was written (the statement before).
after: (d): D1PreparedStatement[] => [
d
.prepare(
`INSERT INTO dedup_keys (key, claimed_at, expires_at) SELECT ?, ?, ? WHERE changes() > 0
ON CONFLICT(key) DO UPDATE SET
claimed_at = excluded.claimed_at, expires_at = excluded.expires_at`,
)
.bind(key, now, now + AUTO_BOUND_TTL_SECONDS * 1000),
],
}
: undefined,
).catch((err: unknown) => {
// Refused at the write (bound before by another attempt, or still
// contended): this group is left as it is.
if (err instanceof ConfigConflictError) return null;
throw err;
});
if (updated?.installationId === installationId) {
bound ??= updated;
// Without D1 transactions, recorded right after (best effort).
if (claims && !(db && canUseD1(db))) await claims.claim(key, AUTO_BOUND_TTL_SECONDS);
}
const gid = `inst-${installationId}`;
const dedicated = groups.find((g) => g.id === gid);
const group: Group = {
id: gid,
name: accountLogin.trim() || `Installation ${installationId}`,
adminIds: [],
installationId,
};
await saveGroups(
kv,
dedicated ? groups.map((g) => (g.id === gid ? { ...g, ...group } : g)) : [...groups, group],
);
return group;
} finally {
if (locked) await kv.delete(lockKey);
}
if (existing) return existing;
if (bound) return bound;
const gid = `inst-${installationId}`;
const group: Group = {
id: gid,
name: accountLogin.trim() || `Installation ${installationId}`,
adminIds: [],
installationId,
};
if (await createGroup(kv, group)) {
// As stored (with the instance the server gave it), for its audit.
return (await loadGroups(kv, { fresh: true })).find((g) => g.id === gid) ?? group;
}
// The id exists already: only a group already bound to this installation
// (a concurrent delivery) is used. An unbound group that merely carries the
// name is never bound — its members did not prove they own the installation.
const current = (await loadGroups(kv, { fresh: true })).find((g) => g.id === gid);
return current?.installationId === installationId ? current : null;
}
export interface AccessScope {
+407 -79
View File
@@ -1,4 +1,21 @@
import { loadGroups, saveGroups, identityMatches, normalizeGroupMembers } from "./groups";
import type { Env, Group } from "../types";
import {
loadGroups,
updateGroup,
memberMatches,
memberRole,
memberRoleStatement,
normalizeGroupMembers,
} from "./groups";
import {
acceptedKey,
claimOnce,
confirmClaim,
idempotencyStore,
recordStatement,
} from "../lib/idempotency";
import { canUseD1 } from "../storage/d1";
import { ConfigConflictError } from "../storage/config-store";
import { log } from "../lib/log";
export interface Invite {
@@ -8,9 +25,46 @@ export interface Invite {
expiresAt: number;
createdBy: string;
note?: string;
/**
* The invited group's immutable `instance` id. A group later re-created
* under the same id has another instance, so the invite never admits anyone
* to it. Absent on invites created before instances existed.
*/
groupInstance?: string;
/**
* Who issued it: a super admin, or a member who must still be an owner of
* the group when the invite is used — an owner removed meanwhile cannot
* have it let anyone in. Absent on older invites.
*/
issuer?: { super: true } | { userId: string; login: string };
}
const INVITE_TTL = 24 * 3600;
/** The invite belongs to an earlier group that had the same id. */
class StaleInviteError extends Error {}
/**
* The login is still on a member entry bound to another account (a member
* who renamed since). Adding a second entry with that login would be dropped
* as a duplicate — a "successful" join without access — so the owner must
* clear the stale entry first.
*/
class MemberConflictError extends Error {}
function revokedKey(token: string): string {
return `invite-revoked:${token}`;
}
/** How long an invite stays claimed while one acceptance is being processed. */
const INVITE_CLAIM_SECONDS = 120;
/** Invites are valid for seven days; their KV keys live exactly that long. */
export const INVITE_LIFETIME_MS = 7 * 86400_000;
const INDEX_TTL = INVITE_LIFETIME_MS / 1000;
/** KV TTL (seconds) for an invite expiring at `expiresAt` (KV minimum: 60). */
function ttlFor(expiresAt: number): number {
return Math.max(60, Math.ceil((expiresAt - Date.now()) / 1000));
}
function generateInviteToken(): string {
const bytes = new Uint8Array(32);
@@ -33,91 +87,208 @@ function indexKey(groupId: string): string {
return `invite:group:${groupId}`;
}
/**
* Tokens of a group's index. A failed read propagates: taken for an empty
* index, it would be written back without the existing tokens, leaving those
* invites valid but unlisted (and so impossible to revoke from the console).
*/
async function readIndex(kv: KVNamespace, groupId: string): Promise<string[]> {
try {
const raw = await kv.get<string[]>(indexKey(groupId), "json");
return Array.isArray(raw) ? raw : [];
} catch (err) {
log.warn({ err, groupId }, "Failed to read invite index");
return [];
}
const raw = await kv.get<string[]>(indexKey(groupId), "json");
return Array.isArray(raw) ? raw : [];
}
async function writeIndex(kv: KVNamespace, groupId: string, tokens: string[]): Promise<void> {
await kv.put(indexKey(groupId), JSON.stringify(tokens), { expirationTtl: INVITE_TTL });
await kv.put(indexKey(groupId), JSON.stringify(tokens), { expirationTtl: INDEX_TTL });
}
async function removeFromIndex(kv: KVNamespace, groupId: string, token: string): Promise<void> {
const tokens = (await readIndex(kv, groupId)).filter((t) => t !== token);
await writeIndex(kv, groupId, tokens);
/**
* Per-invite membership key, written on its own (no read-modify-write): two
* invites created at once can both lose their index entry to the other's
* index write, but never their marker. Listing reads both — the index for
* fresh writes, the markers (an eventually consistent `list`) for durability.
*/
function markerPrefix(groupId: string): string {
return `invite:by-group:${groupId}:`;
}
function listMarkers(kv: KVNamespace, groupId: string): Promise<string[]> {
return listPrefix(kv, markerPrefix(groupId));
}
/**
* Per-invite key under the issuing group's instance: it never moves, so the
* group finds its invites even when moving them to a new id after a rename
* failed halfway.
*/
function instancePrefix(instance: string): string {
return `invite:by-instance:${instance}:`;
}
async function listPrefix(kv: KVNamespace, prefix: string): Promise<string[]> {
const tokens: string[] = [];
let cursor: string | undefined;
do {
const page = await kv.list({ prefix, cursor });
for (const k of page.keys) {
if (k.name.startsWith(prefix)) tokens.push(k.name.slice(prefix.length));
}
cursor = page.list_complete ? undefined : (page as { cursor?: string }).cursor;
} while (cursor);
return tokens;
}
/** Every token listed for a group: by its index, its id markers or its instance markers. */
async function groupTokens(kv: KVNamespace, groupId: string, instance?: string): Promise<string[]> {
const [indexed, marked, owned] = await Promise.all([
readIndex(kv, groupId),
listMarkers(kv, groupId),
instance ? listPrefix(kv, instancePrefix(instance)) : Promise.resolve([]),
]);
return [...new Set([...indexed, ...marked, ...owned])];
}
/**
* Write the index without the given tokens — only when it lists any of them:
* KV writes are the scarcest quota (Workers Free), and an unchanged index
* needs no rewrite.
*/
async function dropFromIndex(kv: KVNamespace, groupId: string, gone: string[]): Promise<void> {
const current = await readIndex(kv, groupId);
const kept = current.filter((t) => !gone.includes(t));
if (kept.length !== current.length) await writeIndex(kv, groupId, kept);
}
/** Remove every listing link of an invite that is gone (index, id and instance markers). */
async function removeFromIndex(kv: KVNamespace, invite: Invite, token: string): Promise<void> {
await kv.delete(`${markerPrefix(invite.groupId)}${token}`);
if (invite.groupInstance) await kv.delete(`${instancePrefix(invite.groupInstance)}${token}`);
await dropFromIndex(kv, invite.groupId, [token]);
}
export async function createInvite(kv: KVNamespace, invite: Invite): Promise<string> {
const token = generateInviteToken();
await kv.put(inviteKey(token), JSON.stringify(invite), { expirationTtl: INVITE_TTL });
const ttl = { expirationTtl: ttlFor(invite.expiresAt) };
await kv.put(inviteKey(token), JSON.stringify(invite), ttl);
await kv.put(`${markerPrefix(invite.groupId)}${token}`, "1", ttl);
if (invite.groupInstance) {
await kv.put(`${instancePrefix(invite.groupInstance)}${token}`, "1", ttl);
}
const tokens = await readIndex(kv, invite.groupId);
tokens.push(token);
await writeIndex(kv, invite.groupId, tokens);
return token;
}
/** The invite, or null when it does not exist (any more). Read errors propagate. */
export async function getInvite(kv: KVNamespace, token: string): Promise<Invite | null> {
try {
const raw = await kv.get<Invite>(inviteKey(token), "json");
if (!raw) return null;
if (Date.now() > raw.expiresAt) {
await kv.delete(inviteKey(token));
await removeFromIndex(kv, raw.groupId, token);
return null;
}
return raw;
} catch (err) {
log.warn({ err }, "Failed to load invite");
const raw = await kv.get<Invite>(inviteKey(token), "json");
if (!raw) return null;
if (Date.now() > raw.expiresAt) {
await kv.delete(inviteKey(token));
await removeFromIndex(kv, raw, token);
return null;
}
return raw;
}
export async function consumeInvite(kv: KVNamespace, token: string): Promise<void> {
const raw = await kv.get<Invite>(inviteKey(token), "json");
await kv.delete(inviteKey(token));
if (raw) await removeFromIndex(kv, raw.groupId, token);
if (raw) await removeFromIndex(kv, raw, token);
}
/** All pending (unexpired) invites of a group, via the per-group index. */
/**
* Whether an invite was issued by this very group, not by an earlier group
* that had the same id (its index may outlive a failed cleanup). An invite from
* before instances belongs to the group that had its id back then (`legacyId`).
*/
function issuedBy(
invite: Invite,
group: Pick<Group, "instance" | "legacyId"> | undefined,
): boolean {
if (!group?.instance) return true;
if (invite.groupInstance) return invite.groupInstance === group.instance;
return group.legacyId === invite.groupId;
}
/**
* All pending (unexpired) invites of a group, via the per-group index. With
* the group given, invites of an earlier group under the same id are left
* out (and removed): their links must not reach the new group's owners.
*/
export async function listInvites(
kv: KVNamespace,
groupId: string,
group?: Pick<Group, "instance" | "legacyId">,
): Promise<Array<Invite & { token: string }>> {
try {
const tokens = await readIndex(kv, groupId);
const out: Array<Invite & { token: string }> = [];
const stale: string[] = [];
for (const token of tokens) {
const invite = await getInvite(kv, token);
if (invite && invite.groupId === groupId) {
out.push({ ...invite, token });
} else {
stale.push(token);
// Read errors propagate (the console shows a failure, not "no invites"),
// and only tokens whose invite is known to be gone are pruned.
// A pre-instance group may still have invites filed under the id it had
// back then (a rename whose invite move failed).
const formerId = group?.legacyId && group.legacyId !== groupId ? group.legacyId : undefined;
const tokens = [
...new Set([
...(await groupTokens(kv, groupId, group?.instance)),
...(formerId ? await groupTokens(kv, formerId) : []),
]),
];
const out: Array<Invite & { token: string }> = [];
const stale: string[] = [];
for (const token of tokens) {
const invite = await getInvite(kv, token);
// Also the group's own invites still filed under a former id (a rename
// whose invite move failed): they are usable, so they must be listed.
const ours =
invite != null &&
issuedBy(invite, group) &&
(invite.groupId === groupId ||
(group?.instance != null && invite.groupInstance === group.instance) ||
(formerId != null && !invite.groupInstance && invite.groupId === formerId));
if (invite && ours) {
out.push({ ...invite, token });
} else {
// Not this group's (gone, or issued by a former holder of the id — which
// may live on under another id): only the listing link to this id goes;
// the invite itself is not ours to revoke and expires on its own.
await kv.delete(`${markerPrefix(groupId)}${token}`);
// A gone invite's marker under this group's instance goes too, or every
// later listing would find (and clean up after) it again until it expires.
if (!invite && group?.instance) {
await kv.delete(`${instancePrefix(group.instance)}${token}`);
}
stale.push(token);
}
if (stale.length > 0) {
await writeIndex(
kv,
groupId,
tokens.filter((t) => !stale.includes(t)),
);
}
return out;
} catch (err) {
log.warn({ err }, "Failed to list invites");
return [];
}
// Re-read: an invite created meanwhile must stay listed.
if (stale.length > 0) await dropFromIndex(kv, groupId, stale);
return out;
}
export async function revokeInvite(kv: KVNamespace, token: string): Promise<void> {
/**
* Revoke an invite. With `db`, the revocation is first recorded in the
* idempotency store (for the invite's lifetime): an acceptance already past
* its KV read checks it inside its member write, so it cannot still join.
*/
export async function revokeInvite(
kv: KVNamespace,
token: string,
db?: D1Database,
guards?: (db: D1Database) => D1PreparedStatement[],
): Promise<void> {
const raw = await kv.get<Invite>(inviteKey(token), "json");
if (db) {
const ttl = ttlFor(raw?.expiresAt ?? Date.now() + INVITE_LIFETIME_MS);
if (guards && canUseD1(db)) {
// The revocation record commits only while the guards hold (e.g. the
// revoking owner still is one): a request authorized before a role
// change can no longer act after it.
await db.batch([...guards(db), recordStatement(db, revokedKey(token), ttl)]);
} else {
await idempotencyStore(db, kv).claim(revokedKey(token), ttl);
}
}
await kv.delete(inviteKey(token));
if (raw) await removeFromIndex(kv, raw.groupId, token);
if (raw) await removeFromIndex(kv, raw, token);
}
/**
@@ -125,9 +296,14 @@ export async function revokeInvite(kv: KVNamespace, token: string): Promise<void
* Best-effort: a failure leaves the old invites in place (they will be
* rejected as group-missing after the rename).
*/
export async function migrateInvites(kv: KVNamespace, from: string, to: string): Promise<void> {
export async function migrateInvites(
kv: KVNamespace,
from: string,
to: string,
group?: Pick<Group, "instance" | "legacyId">,
): Promise<void> {
try {
const tokens = await readIndex(kv, from);
const tokens = await groupTokens(kv, from);
if (tokens.length === 0) {
await kv.delete(indexKey(from));
return;
@@ -135,59 +311,211 @@ export async function migrateInvites(kv: KVNamespace, from: string, to: string):
const moved: string[] = [];
for (const token of tokens) {
const invite = await kv.get<Invite>(inviteKey(token), "json");
if (invite && invite.groupId === from) {
await kv.put(inviteKey(token), JSON.stringify({ ...invite, groupId: to }), {
expirationTtl: INVITE_TTL,
});
// Only the renamed group's own invites move (a former group's leftovers
// stay behind); they get its instance, so they keep working under the
// new id even when they date from before instances.
if (invite && invite.groupId === from && issuedBy(invite, group)) {
const groupInstance = invite.groupInstance ?? group?.instance;
const ttl = { expirationTtl: ttlFor(invite.expiresAt) };
await kv.put(
inviteKey(token),
JSON.stringify({ ...invite, groupId: to, groupInstance }),
ttl,
);
await kv.put(`${markerPrefix(to)}${token}`, "1", ttl);
if (groupInstance) await kv.put(`${instancePrefix(groupInstance)}${token}`, "1", ttl);
await kv.delete(`${markerPrefix(from)}${token}`);
moved.push(token);
}
}
await kv.put(indexKey(to), JSON.stringify(moved), { expirationTtl: INVITE_TTL });
const kept = (await readIndex(kv, to)).filter((t) => !moved.includes(t));
await kv.put(indexKey(to), JSON.stringify([...kept, ...moved]), {
expirationTtl: INDEX_TTL,
});
await kv.delete(indexKey(from));
} catch (err) {
log.warn({ err, from, to }, "Failed to migrate invites on group rename");
}
}
/** Revoke every pending invite of a deleted group. */
export async function deleteGroupInvites(
kv: KVNamespace,
groupId: string,
group?: Pick<Group, "instance" | "legacyId">,
): Promise<void> {
const instance = group?.instance;
try {
for (const token of await groupTokens(kv, groupId, instance)) {
// Only the deleted group's own invites are revoked: one listed under
// its id may have been issued by a group that lives on under another.
const invite = await kv.get<Invite>(inviteKey(token), "json");
if (
invite &&
issuedBy(invite, group) &&
(invite.groupId === groupId || (instance != null && invite.groupInstance === instance))
) {
await kv.delete(inviteKey(token));
}
await kv.delete(`${markerPrefix(groupId)}${token}`);
if (instance) await kv.delete(`${instancePrefix(instance)}${token}`);
}
await kv.delete(indexKey(groupId));
} catch (err) {
log.warn({ err, groupId }, "Failed to delete invites of a removed group");
}
}
/**
* Adds the accepting user to the invited group (or upgrades their role when
* they are already a viewer) and consumes the invite. The invited role is
* never an owner — ownership stays with the inviter's discretion.
*
* The invite is claimed atomically before the membership changes, so two
* people accepting the same link at once cannot both join; the member write is
* a compare-and-swap on the group, so it never reverts a concurrent edit.
*/
export async function acceptInvite(
kv: KVNamespace,
env: Pick<Env, "KV" | "DB">,
token: string,
userId: string,
login: string,
/** The login identity checks may use ("" when it is not trusted as this user's). */
matchLogin = login,
): Promise<
| { ok: true; groupId: string; role: "admin" | "viewer" }
| { ok: false; reason: "invalid" | "group-missing" }
| { ok: true; groupId: string; groupInstance?: string; role: "admin" | "viewer" }
| { ok: false; reason: "invalid" | "group-missing" | "conflict" }
> {
const kv = env.KV;
const invite = await getInvite(kv, token);
if (!invite) return { ok: false, reason: "invalid" };
const groups = await loadGroups(kv);
const group = groups.find((g) => g.id === invite.groupId);
if (!group) return { ok: false, reason: "group-missing" };
const members = normalizeGroupMembers(group);
const idx = members.findIndex((m) => identityMatches([m.login], userId, login));
if (idx >= 0) {
if (invite.role === "admin" && members[idx]!.role === "viewer") {
members[idx]!.role = "admin";
}
} else {
members.push({ login, role: invite.role });
const claims = idempotencyStore(env.DB, kv);
const claimKey = `invite-used:${token}`;
// A short claim while joining, confirmed for the invite's lifetime once
// used: an attempt that fails and cannot give the claim back blocks the
// invite for a couple of minutes, not until it expires.
if ((await claimOnce(claims, claimKey, INVITE_CLAIM_SECONDS)) !== "claimed") {
return { ok: false, reason: "invalid" };
}
const next = groups.map((g) =>
g.id === group.id
? {
...g,
if (await claims.has(revokedKey(token))) {
await claims.release(claimKey).catch(() => undefined);
return { ok: false, reason: "invalid" };
}
let joined = false;
let joinedId: string | undefined;
let joinedInstance: string | undefined;
/** The member who issued the invite is no longer an owner (it is void then). */
const issuerGone = async (): Promise<boolean> => {
const by = invite.issuer && "userId" in invite.issuer ? invite.issuer : null;
if (!by) return false;
const group = (await loadGroups(kv, { fresh: true })).find((g) =>
invite.groupInstance ? g.instance === invite.groupInstance : g.legacyId === invite.groupId,
);
return !group || memberRole(group, by.userId, by.login) !== "owner";
};
try {
// The invited group is the instance the invite was issued for, under
// whatever id it has now. An invite from before instances only applies to
// a group that existed back then, never to one created since.
const groups = await loadGroups(kv, { fresh: true });
const target = invite.groupInstance
? groups.find((g) => g.instance === invite.groupInstance)
: groups.find((g) => g.legacyId === invite.groupId);
// Its group instance is gone for good: the invite can never be used.
if (!target && invite.groupInstance) throw new StaleInviteError();
if (!target) return { ok: false, reason: "group-missing" };
// The invite is confirmed as used in the same transaction that adds the
// member — only when that write went through (`changes()`: the member
// update right before; a lost compare-and-swap changes nothing) and the
// member is there after it: no crash between the two can leave a member
// added and the invite still open, nor a retried write consume it unused.
const confirmWithJoin = (db: D1Database): D1PreparedStatement[] => [
db
.prepare(
`INSERT INTO dedup_keys (key, claimed_at, expires_at)
SELECT ?, ?, ? WHERE changes() > 0 AND EXISTS (
SELECT 1 FROM (SELECT id, CASE WHEN json_valid(data) THEN data ELSE '{}' END AS data FROM d1_groups) g, json_each(g.data, '$.members') m
WHERE g.id = ? AND json_extract(m.value, '$.userId') = ?)
ON CONFLICT(key) DO UPDATE SET expires_at = excluded.expires_at`,
)
.bind(acceptedKey(claimKey), Date.now(), invite.expiresAt, target.id, userId),
];
// And it only commits while the invite is unused and not revoked: an
// earlier acceptance whose answer was lost (its claim given back, its use
// committed), or a revocation that landed after the KV read, aborts it.
const issuer = invite.issuer && "userId" in invite.issuer ? invite.issuer : null;
const unusedGuard = (db: D1Database): D1PreparedStatement[] => [
db
.prepare(
"SELECT CASE WHEN EXISTS (SELECT 1 FROM dedup_keys WHERE key IN (?, ?) AND expires_at > ?) THEN json('invite used') ELSE 1 END AS ok",
)
.bind(acceptedKey(claimKey), revokedKey(token), Date.now()),
...(issuer
? [memberRoleStatement(db, target.id, issuer.userId, issuer.login, ["owner"])]
: []),
];
const updated = await updateGroup(
kv,
target.id,
(group) => {
const sameInstance = invite.groupInstance
? group.instance === invite.groupInstance
: group.legacyId === invite.groupId;
if (!sameInstance) throw new StaleInviteError();
const members = normalizeGroupMembers(group);
const idx = members.findIndex((m) => memberMatches(m, userId, matchLogin));
if (idx >= 0) {
if (invite.role === "admin" && members[idx]!.role === "viewer") {
members[idx]!.role = "admin";
}
members[idx]!.userId ??= userId;
} else {
if (members.some((m) => m.login.toLowerCase() === login.toLowerCase())) {
throw new MemberConflictError();
}
// Bound to the account that accepted, not to its (renamable) login.
members.push({ login, role: invite.role, userId });
}
return {
...group,
members,
adminIds: members.filter((m) => m.role === "owner").map((m) => m.login),
}
: g,
};
},
{ guards: unusedGuard, after: confirmWithJoin },
);
joined = updated != null;
joinedId = updated?.id;
joinedInstance = updated?.instance;
} catch (err) {
// The invite stays usable once the owner resolved the conflict.
if (err instanceof MemberConflictError) return { ok: false, reason: "conflict" };
// Used meanwhile (by an acceptance whose answer was lost).
if (
err instanceof ConfigConflictError &&
((await claims.has(acceptedKey(claimKey))) ||
(await claims.has(revokedKey(token))) ||
(await issuerGone()))
) {
return { ok: false, reason: "invalid" };
}
if (!(err instanceof StaleInviteError)) throw err;
await consumeInvite(kv, token);
return { ok: false, reason: "invalid" };
} finally {
if (!joined) await claims.release(claimKey).catch(() => undefined);
}
if (!joined) return { ok: false, reason: "group-missing" };
await confirmClaim(claims, claimKey, ttlFor(invite.expiresAt)).catch((err: unknown) =>
log.warn({ err }, "Failed to confirm an invite claim"),
);
await saveGroups(kv, next);
await consumeInvite(kv, token);
return { ok: true, groupId: group.id, role: invite.role };
return {
ok: true,
groupId: joinedId ?? invite.groupId,
groupInstance: joinedInstance,
role: invite.role,
};
}
+316 -111
View File
@@ -11,19 +11,32 @@ import {
import {
getOAuthURL,
handleOAuthCallback as handleGithubOAuthCallback,
getInstallationAccount,
getInstallationDetails,
userManagesInstallation,
} from "../github/oauth";
import { removeToken, saveDiscordLink, saveTelegramLink, saveFeishuLink } from "../github/store";
import { createAdminSession, adminCookie, getAdminSession } from "./session";
import { getToken, linkStatement, removeToken } from "../github/store";
import {
createAdminSession,
adminCookie,
getAdminSession,
isAdminUser,
authLogin,
claimLogin,
} from "./session";
import {
loadGroups,
saveGroups,
createGroup,
updateGroup,
resolveScope,
hasAnyAccess,
ensureInstallationGroup,
normalizeGroupMembers,
roleAt,
bindMemberIds,
memberRole,
} from "./groups";
import { log } from "../lib/log";
import { idempotencyStore, recordStatement, unrecordedGuard } from "../lib/idempotency";
import { clientIp } from "./auth";
import { recordAudit } from "../lib/audit";
import { sendMessage } from "../drivers/telegram/rest";
@@ -32,9 +45,22 @@ import { cfEnv } from "../cf";
import { initConfigStore } from "../config";
import type { Env, Group } from "../types";
/** Which chat account a bot `/gh login` link binds, with a name to show. */
interface PendingLink {
platform: "discord" | "telegram" | "feishu";
userId: string;
/** Display name of the chat account, shown on the confirmation page. */
name?: string;
/** Chat to notify once the link is confirmed (Telegram / Feishu). */
chatId?: string;
}
interface PendingState {
redirectTo: string;
expiresAt: number;
/** Set when the flow was started by a bot command instead of a browser login. */
link?: PendingLink;
// Legacy shapes of `link`, still accepted for states created before it.
discordUserId?: string;
telegramUserId?: string;
telegramChatId?: string;
@@ -42,8 +68,75 @@ interface PendingState {
feishuChatId?: string;
}
function linkedPage(login: string): string {
return `<!doctype html><html lang="zh"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>绑定成功</title><style>body{margin:0;min-height:100vh;display:flex;align-items:center;justify-content:center;font-family:system-ui,-apple-system,'Segoe UI',sans-serif;background:#f6f7f9;color:#1f2328}.card{background:#fff;border:1px solid #e5e7eb;border-radius:12px;padding:32px 40px;text-align:center;box-shadow:0 1px 3px rgba(0,0,0,.06)}.ok{color:#16a34a;font-size:40px}h1{font-size:18px;margin:12px 0 4px}p{color:#57606a;font-size:14px;margin:0}</style></head><body><div class="card"><div class="ok">✓</div><h1>GitHub 账号已绑定</h1><p>已连接为 <b>@${login}</b>,现在可以回到 Discord 用 GitHub 评论了。</p></div></body></html>`;
/** A confirmed OAuth identity waiting for the user to approve a bot link. */
interface LinkConfirmation extends PendingLink {
githubUserId: string;
login: string;
}
const STATE_COOKIE = "wh_oauth_state";
const STATE_TTL = 600;
const LINK_TTL = 600;
function stateCookie(state: string): string {
return `${STATE_COOKIE}=${state}; Path=/auth/github; HttpOnly; Secure; SameSite=Lax; Max-Age=${STATE_TTL}`;
}
function clearStateCookie(): string {
return `${STATE_COOKIE}=; Path=/auth/github; HttpOnly; Secure; SameSite=Lax; Max-Age=0`;
}
function cookieValue(header: string | undefined, name: string): string | undefined {
for (const part of (header ?? "").split(";")) {
const idx = part.indexOf("=");
if (idx !== -1 && part.slice(0, idx).trim() === name) return part.slice(idx + 1).trim();
}
return undefined;
}
function pendingLink(pending: PendingState): PendingLink | null {
if (pending.link) return pending.link;
if (pending.discordUserId) return { platform: "discord", userId: pending.discordUserId };
if (pending.telegramUserId) {
return {
platform: "telegram",
userId: pending.telegramUserId,
chatId: pending.telegramChatId,
};
}
if (pending.feishuUserId) {
return { platform: "feishu", userId: pending.feishuUserId, chatId: pending.feishuChatId };
}
return null;
}
const PAGE_STYLE =
"body{margin:0;min-height:100vh;display:flex;align-items:center;justify-content:center;font-family:system-ui,-apple-system,'Segoe UI',sans-serif;background:#f6f7f9;color:#1f2328}.card{background:#fff;border:1px solid #e5e7eb;border-radius:12px;padding:28px 32px;width:min(480px,92vw);box-sizing:border-box;text-align:center;box-shadow:0 1px 3px rgba(0,0,0,.06)}.ok{color:#16a34a;font-size:40px}h1{font-size:18px;margin:12px 0 6px}p{color:#57606a;font-size:14px;margin:6px 0}b{color:#1f2328}.btn{display:inline-block;margin-top:16px;background:#1f2328;color:#fff;border:0;border-radius:8px;padding:10px 18px;font-size:14px;cursor:pointer}.btn:hover{background:#32383f}.warn{color:#b45309;font-size:13px}";
const PLATFORM_NAMES: Record<PendingLink["platform"], string> = {
discord: "Discord",
telegram: "Telegram",
feishu: "飞书",
};
function linkedPage(login: string, platform: PendingLink["platform"]): string {
return `<!doctype html><html lang="zh"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>绑定成功</title><style>${PAGE_STYLE}</style></head><body><div class="card"><div class="ok">✓</div><h1>GitHub 账号已绑定</h1><p>已连接为 <b>@${escapeHtml(login)}</b>,现在可以回到 ${PLATFORM_NAMES[platform]} 用 GitHub 评论了。</p></div></body></html>`;
}
/**
* Asks the GitHub user to approve the binding before it happens: a `/gh
* login` link may have been forwarded, so the page names both accounts and
* nothing is linked until this user confirms.
*/
function confirmLinkPage(nonce: string, link: LinkConfirmation): string {
const chatAccount = link.name
? `${escapeHtml(link.name)}(ID ${escapeHtml(link.userId)})`
: `ID ${escapeHtml(link.userId)}`;
return `<!doctype html><html lang="zh"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>确认绑定</title><style>${PAGE_STYLE}</style></head><body><div class="card"><h1>确认绑定 GitHub 账号</h1><p>将 GitHub 账号 <b>@${escapeHtml(link.login)}</b></p><p>绑定到 ${PLATFORM_NAMES[link.platform]} 账号 <b>${chatAccount}</b>?</p><p class="warn">绑定后,该 ${PLATFORM_NAMES[link.platform]} 账号可以用你的 GitHub 身份评论、合并或关闭 PR。若这个链接不是你本人发起的,请直接关闭本页。</p><form method="post" action="/auth/github/link"><input type="hidden" name="nonce" value="${escapeHtml(nonce)}"><button class="btn" type="submit">确认绑定</button></form></div></body></html>`;
}
function messagePage(title: string, text: string): string {
return `<!doctype html><html lang="zh"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>${escapeHtml(title)}</title><style>${PAGE_STYLE}</style></head><body><div class="card"><h1>${escapeHtml(title)}</h1><p>${escapeHtml(text)}</p></div></body></html>`;
}
function generateRandomHex(length: number): string {
@@ -54,12 +147,26 @@ function generateRandomHex(length: number): string {
.join("");
}
function safeRedirectPath(value: string | undefined): string {
export function safeRedirectPath(value: string | undefined): string {
if (!value) return "/";
if (!value.startsWith("/")) return "/";
if (value.startsWith("//")) return "/";
if (/^\/\\/.test(value)) return "/";
return value;
// Browsers drop tabs and line breaks from URLs (`/\t/evil.example` becomes
// `//evil.example`): refused, and the path is checked as a browser reads
// it — it must stay on this origin, and only that normalized path is used.
if (/[\u0000-\u001f\u007f]/.test(value)) return "/";
const base = "https://webhooker.invalid";
let url: URL;
try {
url = new URL(value, base);
} catch {
return "/";
}
// Dot segments can normalize into `//host` (`/a/..//evil.example`): read
// again, that would leave the site.
if (url.origin !== base || url.pathname.startsWith("//")) return "/";
return `${url.pathname}${url.search}${url.hash}`;
}
function escapeHtml(s: string): string {
@@ -79,19 +186,22 @@ function selfSignupEnabled(env: Env): boolean {
* Opt-in self service: users without any group access get a personal group
* they own, so they can configure their own routing without a super admin.
* The group id is deterministic (`u-{userId}`), so it is created at most once.
* Personal groups are isolated: only their own `/webhook/{groupId}` ingress
* dispatches to them, never the operator's global endpoint.
*/
async function ensurePersonalGroup(env: Env, userId: string, login: string): Promise<boolean> {
if (!selfSignupEnabled(env)) return false;
const groups = await loadGroups(env.KV);
const gid = `u-${userId}`;
if (groups.some((g) => g.id === gid)) return true;
const personal: Group = {
id: gid,
name: `@${login}`,
members: [{ login, role: "owner" }],
members: [{ login, role: "owner", userId }],
adminIds: [login],
isolated: true,
};
await saveGroups(env.KV, [...groups, personal]);
if (!(await createGroup(env.KV, personal))) return true;
// Filed under the instance the server gave the group (it follows renames).
const stored = (await loadGroups(env.KV, { fresh: true })).find((g) => g.id === gid);
await recordAudit(env.DB, {
ts: Date.now(),
actorId: userId,
@@ -100,6 +210,7 @@ async function ensurePersonalGroup(env: Env, userId: string, login: string): Pro
targetType: "group",
targetId: gid,
groupId: gid,
groupInstance: stored?.instance,
detail: { auto: true },
});
return true;
@@ -131,6 +242,29 @@ function installPage(opts: {
return `<!doctype html><html lang="zh"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>安装 GitHub App</title><style>body{margin:0;min-height:100vh;display:flex;align-items:center;justify-content:center;font-family:system-ui,-apple-system,'Segoe UI',sans-serif;background:#f6f7f9;color:#1f2328}.card{background:#fff;border:1px solid #e5e7eb;border-radius:12px;padding:28px 32px;width:min(480px,92vw);box-shadow:0 1px 3px rgba(0,0,0,.06)}h1{font-size:17px;margin:0 0 4px}p{color:#57606a;font-size:13.5px;margin:6px 0}code{background:#f0f1f3;border-radius:4px;padding:1px 5px;font-size:12.5px}.opt{display:flex;align-items:flex-start;gap:8px;padding:9px 10px;border:1px solid #e5e7eb;border-radius:8px;margin-top:8px;cursor:pointer}.opt:hover{background:#fafbfc}.hint{font-size:12.5px;color:#8b949e;margin-top:10px}.btn{display:inline-block;margin-top:14px;background:#1f2328;color:#fff;border:0;border-radius:8px;padding:10px 18px;font-size:14px;cursor:pointer}.btn:hover{background:#32383f}.skip{margin-left:12px;color:#8b949e;font-size:13px;text-decoration:none}</style></head><body><div class="card"><h1>GitHub App 安装成功</h1>${accountLine}<p>将安装绑定到哪个分组?建议直接创建新分组,之后可以在控制台添加路由与成员。</p><form method="post" action="/auth/github/install/bind"><input type="hidden" name="installation_id" value="${installationId}"><label class="opt"><input type="radio" name="group" value="" checked><span><b>创建新分组</b> <code>inst-${installationId}</code></span></label>${ownedNote}${ownedOptions}<button class="btn" type="submit">确定</button></form></div></body></html>`;
}
/**
* Whether the signed-in user may bind this installation. Super admins (the
* operator) may; everyone else must prove access with their own GitHub token,
* because the Setup URL's `installation_id` can be set to anything.
*/
async function canManageInstallation(
env: Env,
session: { userId: string; login: string },
installationId: number,
): Promise<{ ok: boolean; accountLogin: string }> {
const account = await getInstallationDetails(
env.GITHUB_APP_ID ?? "",
env.GITHUB_PRIVATE_KEY ?? "",
installationId,
);
const accountLogin = account?.login ?? "";
if (isAdminUser(env, session.userId, authLogin(session))) return { ok: true, accountLogin };
const token = await getToken(env, session.userId);
if (!token) return { ok: false, accountLogin };
const ok = await userManagesInstallation(token, session.userId, installationId, account);
return { ok, accountLogin };
}
/** GET /auth/github — start the OAuth flow. */
export async function handleOAuthStart(event: H3Event): Promise<void> {
const env = cfEnv(event);
@@ -140,9 +274,12 @@ export async function handleOAuthStart(event: H3Event): Promise<void> {
const state = generateRandomHex(16);
const pending: PendingState = {
redirectTo,
expiresAt: Date.now() + 10 * 60 * 1000,
expiresAt: Date.now() + STATE_TTL * 1000,
};
await env.KV.put(`state:${state}`, JSON.stringify(pending), { expirationTtl: 600 });
await env.KV.put(`state:${state}`, JSON.stringify(pending), { expirationTtl: STATE_TTL });
// Binds the flow to this browser: the callback only signs in the browser
// that started it (no login CSRF with a callback URL from another session).
appendResponseHeader(event, "Set-Cookie", stateCookie(state));
await sendRedirect(event, getOAuthURL(env.GITHUB_CLIENT_ID ?? "", state));
}
@@ -156,29 +293,28 @@ export async function handleInstallPage(event: H3Event): Promise<string | void>
if (!rawId || !Number.isInteger(installationId) || installationId <= 0) {
throw createError({ statusCode: 400, statusMessage: "Missing installation_id" });
}
const session = await getAdminSession(env.KV, getHeader(event, "cookie"));
const session = await getAdminSession(env.KV, getHeader(event, "cookie"), env.DB);
if (!session) {
const target = `/auth/github/install?installation_id=${installationId}`;
await sendRedirect(event, `/auth/github?redirect=${encodeURIComponent(target)}`);
return;
}
const accountLogin =
(await getInstallationAccount(
env.GITHUB_APP_ID ?? "",
env.GITHUB_PRIVATE_KEY ?? "",
installationId,
)) ?? "";
const groups = await loadGroups(env.KV);
const scope = resolveScope(env, groups, session.userId, session.login);
const access = await canManageInstallation(env, session, installationId);
if (!access.ok) {
await sendRedirect(event, "/admin?error=install-forbidden");
return;
}
const groups = await loadGroups(env.KV, { fresh: true });
const scope = resolveScope(env, groups, session.userId, authLogin(session));
const owned = groups.filter((g) => roleAt(scope, g.id) === "owner");
return installPage({ installationId, accountLogin, owned });
return installPage({ installationId, accountLogin: access.accountLogin, owned });
}
/** POST /auth/github/install/bind — provision the chosen binding. */
export async function handleInstallBind(event: H3Event): Promise<void> {
const env = cfEnv(event);
initConfigStore(env);
const session = await getAdminSession(env.KV, getHeader(event, "cookie"));
const session = await getAdminSession(env.KV, getHeader(event, "cookie"), env.DB);
if (!session) {
await sendRedirect(event, "/admin?error=forbidden");
return;
@@ -189,49 +325,60 @@ export async function handleInstallBind(event: H3Event): Promise<void> {
if (!Number.isInteger(installationId) || installationId <= 0) {
throw createError({ statusCode: 400, statusMessage: "Missing installation_id" });
}
const access = await canManageInstallation(env, session, installationId);
if (!access.ok) {
await sendRedirect(event, "/admin?error=install-forbidden");
return;
}
const accountLogin = access.accountLogin;
const chosenGroupId = String(body["group"] ?? "").trim();
const groups = await loadGroups(env.KV);
const scope = resolveScope(env, groups, session.userId, session.login);
const groups = await loadGroups(env.KV, { fresh: true });
const scope = resolveScope(env, groups, session.userId, authLogin(session));
const bind = async (groupId: string, group: Group | null): Promise<void> => {
if (!group) {
if (chosenGroupId) {
// Binding to an existing group requires owner permission on it.
if (!groups.some((g) => g.id === chosenGroupId) || roleAt(scope, chosenGroupId) !== "owner") {
await sendRedirect(event, "/admin?error=forbidden");
return;
}
// Still an owner of the group as it is now (the write is a CAS on it).
let revoked = false;
const bound = await updateGroup(env.KV, chosenGroupId, (g) => {
const owner =
isAdminUser(env, session.userId, authLogin(session)) ||
memberRole(g, session.userId, authLogin(session)) === "owner";
if (!owner) {
revoked = true;
return null;
}
return { ...g, installationId };
});
if (revoked) {
await sendRedirect(event, "/admin?error=forbidden");
return;
}
if (!bound) {
await sendRedirect(event, "/admin?error=install");
return;
}
const next = groups.map((g) => (g.id === group.id ? { ...g, installationId } : g));
await saveGroups(env.KV, next);
await recordAudit(env.DB, {
ts: Date.now(),
actorId: session.userId,
actorLogin: session.login,
action: "installation.bind",
targetType: "group",
targetId: groupId,
groupId,
targetId: chosenGroupId,
groupId: chosenGroupId,
groupInstance: bound.instance,
detail: { installationId },
ip: clientIp(event),
});
await sendRedirect(event, "/admin?install=ok");
};
if (chosenGroupId) {
// Binding to an existing group requires owner permission on it.
const group = groups.find((g) => g.id === chosenGroupId);
if (!group || roleAt(scope, chosenGroupId) !== "owner") {
await sendRedirect(event, "/admin?error=forbidden");
return;
}
return bind(chosenGroupId, group);
return;
}
// Default: auto-create a dedicated inst-{id} group.
const accountLogin =
(await getInstallationAccount(
env.GITHUB_APP_ID ?? "",
env.GITHUB_PRIVATE_KEY ?? "",
installationId,
)) ?? "";
const group = await ensureInstallationGroup(env.KV, installationId, accountLogin);
const group = await ensureInstallationGroup(env.KV, installationId, accountLogin, env.DB);
if (!group) {
await sendRedirect(event, "/admin?error=install");
return;
@@ -244,27 +391,29 @@ export async function handleInstallBind(event: H3Event): Promise<void> {
targetType: "group",
targetId: group.id,
groupId: group.id,
groupInstance: group.instance,
detail: { source: "setup_url", account: accountLogin || undefined },
ip: clientIp(event),
});
// Self-service SaaS: the installer manages their own auto-created group.
if (selfSignupEnabled(env)) {
const members = normalizeGroupMembers(group);
const alreadyMember = members.some(
(m) => m.login.toLowerCase() === session.login.toLowerCase() || m.login === session.userId,
);
if (!alreadyMember) {
const updated: Group = {
...group,
members: [...members, { login: session.login, role: "owner" }],
adminIds: [...new Set([...(group.adminIds ?? []), session.login])],
// Self-service SaaS: the verified installer manages the group created for
// their installation — only the dedicated `inst-{id}` group, only while it is
// still bound to this installation, and only while nobody manages it yet.
// Any other group (an org group bound via `owners`, an operator group, one
// claimed earlier) is never handed to a new owner this way.
if (selfSignupEnabled(env) && group.id === `inst-${installationId}`) {
let added = false;
await updateGroup(env.KV, group.id, (g) => {
const members = normalizeGroupMembers(g);
if (g.installationId !== installationId || members.length > 0) return null;
added = true;
return {
...g,
members: [{ login: session.login, role: "owner", userId: session.userId }],
adminIds: [session.login],
};
const all = await loadGroups(env.KV);
await saveGroups(
env.KV,
all.map((g) => (g.id === group.id ? updated : g)),
);
});
if (added) {
await recordAudit(env.DB, {
ts: Date.now(),
actorId: session.userId,
@@ -273,6 +422,7 @@ export async function handleInstallBind(event: H3Event): Promise<void> {
targetType: "group",
targetId: group.id,
groupId: group.id,
groupInstance: group.instance,
detail: { login: session.login, role: "owner", auto: true },
ip: clientIp(event),
});
@@ -300,12 +450,19 @@ export async function handleOAuthCallback(event: H3Event): Promise<unknown> {
return { error: "Invalid or expired state" };
}
const pending = raw as PendingState;
await env.KV.delete(`state:${state}`);
if (Date.now() > pending.expiresAt) {
await env.KV.delete(`state:${state}`);
setResponseStatus(event, 400);
return { error: "Invalid or expired state" };
}
await env.KV.delete(`state:${state}`);
const link = pendingLink(pending);
// A browser sign-in must come back to the browser that started it. Bot
// links are opened elsewhere by design; they are confirmed explicitly below.
if (!link && cookieValue(getHeader(event, "cookie"), STATE_COOKIE) !== state) {
setResponseStatus(event, 400);
return { error: "Invalid or expired state" };
}
if (!link) appendResponseHeader(event, "Set-Cookie", clearStateCookie());
const result = await handleGithubOAuthCallback(
env.GITHUB_CLIENT_ID ?? "",
@@ -313,47 +470,24 @@ export async function handleOAuthCallback(event: H3Event): Promise<unknown> {
code,
state,
env.KV,
env.DB,
);
if (!result) {
setResponseStatus(event, 400);
return { error: "OAuth failed" };
}
// Discord account-linking flow: bind the Discord user to this GitHub account.
if (pending.discordUserId) {
await saveDiscordLink(env.DB, pending.discordUserId, result.userId);
const isBrowserLink = (getHeader(event, "accept") ?? "").includes("text/html");
if (isBrowserLink) return linkedPage(result.login);
return { ok: true, discordUserId: pending.discordUserId, login: result.login };
}
// Telegram account-linking flow: bind the Telegram user to this GitHub account.
if (pending.telegramUserId) {
await saveTelegramLink(env.DB, pending.telegramUserId, result.userId);
if (pending.telegramChatId && env.TELEGRAM_TOKEN) {
await sendMessage(
env.TELEGRAM_TOKEN,
pending.telegramChatId,
`✅ GitHub 账号已绑定:**@${result.login}**。现在可以用 /gh comment 评论了。`,
).catch(() => undefined);
}
return { ok: true, telegramUserId: pending.telegramUserId, login: result.login };
}
// Feishu account-linking flow: bind the Feishu user to this GitHub account.
if (pending.feishuUserId) {
await saveFeishuLink(env.DB, pending.feishuUserId, result.userId);
if (pending.feishuChatId) {
const tokenRes = await getTenantAccessToken(env);
if (tokenRes.ok && tokenRes.token) {
await sendFeishuText(
tokenRes.token,
pending.feishuChatId,
`✅ GitHub 账号已绑定:**@${result.login}**。现在可以用 /gh comment 评论了。`,
).catch(() => undefined);
}
}
return { ok: true, feishuUserId: pending.feishuUserId, login: result.login };
// Bot account linking: never bind on the callback itself. The person who
// authorized GitHub confirms which chat account gets their identity.
if (link) {
const nonce = generateRandomHex(16);
const confirmation: LinkConfirmation = {
...link,
githubUserId: result.userId,
login: result.login,
};
await env.KV.put(`link:${nonce}`, JSON.stringify(confirmation), { expirationTtl: LINK_TTL });
return confirmLinkPage(nonce, confirmation);
}
const isBrowser = (getHeader(event, "accept") ?? "").includes("text/html");
@@ -365,14 +499,17 @@ export async function handleOAuthCallback(event: H3Event): Promise<unknown> {
pending.redirectTo.startsWith("/admin/invite") ||
pending.redirectTo.startsWith("/admin/invite?");
let groups = await loadGroups(env.KV);
let scope = resolveScope(env, groups, result.userId, result.login);
let groups = await loadGroups(env.KV, { fresh: true });
// A login first seen with another account grants nothing by name.
const trusted = await claimLogin(env.KV, result.userId, result.login);
const name = trusted ? result.login : "";
let scope = resolveScope(env, groups, result.userId, name);
if (!hasAnyAccess(scope) && !isInviteFlow) {
const created = await ensurePersonalGroup(env, result.userId, result.login);
if (created) {
groups = await loadGroups(env.KV);
scope = resolveScope(env, groups, result.userId, result.login);
groups = await loadGroups(env.KV, { fresh: true });
scope = resolveScope(env, groups, result.userId, name);
}
}
if (!hasAnyAccess(scope) && !isInviteFlow) {
@@ -380,7 +517,13 @@ export async function handleOAuthCallback(event: H3Event): Promise<unknown> {
return;
}
const sessionId = await createAdminSession(env.KV, result.userId, result.login);
// Tie the member entries naming this login to the account itself.
if (trusted) {
await bindMemberIds(env.KV, groups, result.userId, result.login).catch((err: unknown) =>
log.warn({ err, userId: result.userId }, "Failed to bind member ids"),
);
}
const sessionId = await createAdminSession(env.KV, result.userId, result.login, trusted);
appendResponseHeader(event, "Set-Cookie", adminCookie(sessionId));
await recordAudit(env.DB, {
ts: Date.now(),
@@ -400,15 +543,77 @@ export async function handleOAuthCallback(event: H3Event): Promise<unknown> {
};
}
/** DELETE /auth/token/:userId — revoke a stored user token. */
/** POST /auth/github/link — the GitHub user approved a bot account link. */
export async function handleLinkConfirm(event: H3Event): Promise<string> {
const env = cfEnv(event);
const body = (await readBody(event).catch(() => ({}))) as Record<string, string | undefined>;
const nonce = String(body["nonce"] ?? "");
const raw = /^[0-9a-f]{32}$/.test(nonce) ? await env.KV.get(`link:${nonce}`, "json") : null;
// One use, recorded in D1: a replayed confirmation (or a stale KV copy
// elsewhere) can never restore a link that was undone or replaced since.
const claims = idempotencyStore(env.DB, env.KV);
const usedKey = `link-used:${nonce}`;
const expired = (): string => {
setResponseStatus(event, 400);
return messagePage("链接已失效", "请回到聊天里重新发送 /gh login。");
};
if (!raw || (await claims.has(usedKey))) return expired();
const link = raw as LinkConfirmation;
const notice = `✅ GitHub 账号已绑定:@${link.login}。现在可以用 /gh comment 评论了。`;
// The link and its one-use record commit in one transaction, refused once
// the record exists: a replay can never restore a link undone or replaced
// since — not even after a write whose answer was lost (the record then
// tells it went through). A failure before the commit leaves the pending
// link, so submitting it again works.
try {
await env.DB.batch([
unrecordedGuard(env.DB, usedKey),
recordStatement(env.DB, usedKey, LINK_TTL),
linkStatement(env.DB, link.platform, link.userId, link.githubUserId),
]);
} catch (err) {
if (!(await claims.has(usedKey).catch(() => false))) throw err;
// Recorded: committed here (its answer lost), or by a concurrent submit.
if (/already recorded|malformed JSON/i.test(String(err))) return expired();
}
await env.KV.delete(`link:${nonce}`).catch(() => undefined);
if (link.platform === "telegram") {
if (link.chatId && env.TELEGRAM_TOKEN) {
await sendMessage(env.TELEGRAM_TOKEN, link.chatId, escapeHtml(notice)).catch(() => undefined);
}
} else if (link.platform === "feishu") {
if (link.chatId) {
const tokenRes = await getTenantAccessToken(env);
if (tokenRes.ok && tokenRes.token) {
await sendFeishuText(tokenRes.token, link.chatId, notice).catch(() => undefined);
}
}
}
await recordAudit(env.DB, {
ts: Date.now(),
actorId: link.githubUserId,
actorLogin: link.login,
action: "account.link",
targetType: link.platform,
targetId: link.userId,
ip: clientIp(event),
});
return linkedPage(link.login, link.platform);
}
/** DELETE /auth/token/:userId — revoke a stored user token (own token, or any for super admins). */
export async function handleTokenDelete(event: H3Event, userId: string): Promise<unknown> {
const env = cfEnv(event);
const session = await getAdminSession(env.KV, getHeader(event, "cookie"));
const session = await getAdminSession(env.KV, getHeader(event, "cookie"), env.DB);
if (!session) {
setResponseStatus(event, 401);
return { error: "Unauthorized" };
}
await removeToken(env.KV, userId);
if (session.userId !== userId && !isAdminUser(env, session.userId, authLogin(session))) {
setResponseStatus(event, 403);
return { error: "Forbidden" };
}
await removeToken(env.KV, userId, env.DB);
await recordAudit(env.DB, {
ts: Date.now(),
actorId: session.userId,
+60 -7
View File
@@ -1,4 +1,5 @@
import type { Env } from "../types";
import { idempotencyStore } from "../lib/idempotency";
const SESSION_COOKIE = "wh_admin_session";
const SESSION_TTL = 24 * 3600;
@@ -6,6 +7,33 @@ const SESSION_TTL = 24 * 3600;
export interface AdminSession {
userId: string;
login: string;
/**
* False when the login was first seen with another GitHub account (a name
* released by a rename and taken over since): identity checks then match
* this session by its user id only, never by the login.
*/
loginTrusted?: boolean;
}
/** The login identity checks may use for this session ("" — by user id only). */
export function authLogin(session: AdminSession): string {
return session.loginTrusted === false ? "" : session.login;
}
/**
* Whether `login` belongs to `userId` as far as this deployment knows: the
* first account seen with a login owns it here (`login-owner:{login}`). A
* GitHub login can be renamed away and registered by someone else; that
* newcomer then never inherits what was granted to the login (an admin
* entry, a member entry not yet bound to an id).
*/
export async function claimLogin(kv: KVNamespace, userId: string, login: string): Promise<boolean> {
const key = `login-owner:${login.toLowerCase()}`;
const owner = await kv.get(key);
if (owner) return owner === userId;
await kv.put(key, userId);
// Read back: of two first sign-ins racing, only the one stored wins.
return (await kv.get(key)) === userId;
}
export function isDebugMode(env: Env): boolean {
@@ -20,14 +48,24 @@ export function isDebugMode(env: Env): boolean {
return false;
}
/**
* Whether one configured identity names this user. An all-digit entry is a
* GitHub user id, anything else a login (case-insensitive) — one string is
* never matched both ways, so a user whose login is digits can never pass for
* the user with that id.
*/
export function identityEntryMatches(entry: string, userId: string, login: string): boolean {
const e = entry.trim();
if (!e) return false;
if (/^\d+$/.test(e)) return e === userId;
return e.toLowerCase() === login.toLowerCase();
}
export function isAdminUser(env: Env, userId: string, login: string): boolean {
if (isDebugMode(env)) return true;
const ids = (env.ADMIN_USER_IDS ?? "")
return (env.ADMIN_USER_IDS ?? "")
.split(",")
.map((s) => s.trim())
.filter(Boolean);
if (ids.length === 0) return false;
return ids.includes(userId) || ids.some((id) => id.toLowerCase() === login.toLowerCase());
.some((entry) => identityEntryMatches(entry, userId, login));
}
function generateSessionId(): string {
@@ -42,9 +80,10 @@ export async function createAdminSession(
kv: KVNamespace,
userId: string,
login: string,
loginTrusted = true,
): Promise<string> {
const sessionId = generateSessionId();
const session: AdminSession = { userId, login };
const session: AdminSession = loginTrusted ? { userId, login } : { userId, login, loginTrusted };
await kv.put(`session:${sessionId}`, JSON.stringify(session), {
expirationTtl: SESSION_TTL,
});
@@ -70,21 +109,35 @@ function parseCookies(header: string | undefined): Record<string, string> {
return out;
}
/**
* Sessions are revoked in the idempotency store (D1) as well as deleted from
* KV: a KV delete takes a while to reach every location, and a stolen cookie
* must stop working at once.
*/
function revokedSessionKey(sessionId: string): string {
return `session-revoked:${sessionId}`;
}
export async function getAdminSession(
kv: KVNamespace,
cookieHeader: string | undefined,
db?: D1Database,
): Promise<AdminSession | null> {
const sessionId = parseCookies(cookieHeader)[SESSION_COOKIE];
if (!sessionId) return null;
const raw = await kv.get<AdminSession>(`session:${sessionId}`, "json");
if (!raw) return null;
if (db && (await idempotencyStore(db, kv).has(revokedSessionKey(sessionId)))) return null;
return raw;
}
export async function destroyAdminSession(
kv: KVNamespace,
cookieHeader: string | undefined,
db?: D1Database,
): Promise<void> {
const sessionId = parseCookies(cookieHeader)[SESSION_COOKIE];
if (sessionId) await kv.delete(`session:${sessionId}`);
if (!sessionId) return;
if (db) await idempotencyStore(db, kv).claim(revokedSessionKey(sessionId), SESSION_TTL);
await kv.delete(`session:${sessionId}`);
}
+86 -13
View File
@@ -1,12 +1,28 @@
import { log } from "../lib/log";
import type { Group } from "../types";
import { updateGroup } from "./groups";
/**
* Per-group webhook tenant secrets. A group can opt into its own webhook
* ingress (`POST /webhook/{groupId}`) with an independent secret, so SaaS
* users can configure their own GitHub/Gitea/custom webhooks without sharing
* (or even knowing) the operator's global secrets.
*
* Secrets are keyed by the group's immutable `instance`: a rename keeps the
* secret without moving anything, and a group re-created under a former id
* never inherits the old group's secret. A secret stored before instances
* existed (`tenant:{groupId}`) is still read — only by the group that had that
* id back then (`legacyId`) — until the secret is regenerated or removed. It is
* never copied over: a lazy copy could race a rotation and resurrect the old
* secret.
*/
const TENANT_KEY = (groupId: string): string => `tenant:${groupId}`;
type TenantGroup = Pick<Group, "id" | "instance" | "legacyId" | "webhookSecret">;
const LEGACY_KEY = (groupId: string): string => `tenant:${groupId}`;
const INSTANCE_KEY = (instance: string): string => `tenant:i:${instance}`;
function keyOf(group: TenantGroup): string {
return group.instance ? INSTANCE_KEY(group.instance) : LEGACY_KEY(group.id);
}
/** 32 random bytes → 64 hex chars. */
export function generateTenantSecret(): string {
@@ -17,22 +33,79 @@ export function generateTenantSecret(): string {
.join("");
}
export async function getTenantSecret(kv: KVNamespace, groupId: string): Promise<string | null> {
try {
return await kv.get(TENANT_KEY(groupId), "text");
} catch (err) {
log.warn({ err, groupId }, "Failed to read tenant webhook secret");
return null;
}
/**
* The group's webhook secret, or null when it has none. Read errors propagate:
* a failed read must never pass for "no secret".
*/
export async function getTenantSecret(kv: KVNamespace, group: TenantGroup): Promise<string | null> {
// Stored with the group (how secrets are written now); KV keys are read for
// secrets set before that.
// "" is an explicit "none" (removed): no older KV copy counts then.
if (group.webhookSecret != null) return group.webhookSecret || null;
const secret = await kv.get(keyOf(group), "text");
// Only the group that had the id before instances may use a pre-instance
// secret — not a group created since, nor one renamed to that id.
if (secret || !group.instance || !group.legacyId) return secret;
return kv.get(LEGACY_KEY(group.legacyId), "text");
}
/**
* The pre-instance key the group's secret may still be found under. Keys
* under its current id are left alone: they may belong to the group that had
* that id before instances (now under another id).
*/
function legacyKeys(group: TenantGroup): string[] {
return group.legacyId ? [LEGACY_KEY(group.legacyId)] : [];
}
/** Generate (or regenerate) the group's webhook secret. */
export async function setTenantSecret(kv: KVNamespace, groupId: string): Promise<string> {
export async function setTenantSecret(kv: KVNamespace, group: TenantGroup): Promise<string> {
const secret = generateTenantSecret();
await kv.put(TENANT_KEY(groupId), secret);
await kv.put(keyOf(group), secret);
// The new secret replaces the group's pre-instance one.
if (group.instance) for (const key of legacyKeys(group)) await kv.delete(key);
return secret;
}
export async function deleteTenantSecret(kv: KVNamespace, groupId: string): Promise<void> {
await kv.delete(TENANT_KEY(groupId));
/**
* Generate (or regenerate) the group's webhook secret, stored with the group
* in a compare-and-swap: `check` sees the group as it is at that moment and
* may throw (e.g. the writer is no longer an owner), so a request authorized
* earlier can never overwrite a later rotation. Older KV copies are dropped.
* Null when the group does not exist.
*/
export async function rotateTenantSecret(
kv: KVNamespace,
groupId: string,
check: (current: Group) => void,
): Promise<string | null> {
const secret = generateTenantSecret();
const saved = await updateGroup(kv, groupId, (g) => {
check(g);
return { ...g, webhookSecret: secret };
});
if (!saved) return null;
await deleteTenantSecret(kv, saved).catch(() => undefined);
return secret;
}
/** Remove the group's webhook secret (the same compare-and-swap as rotation). */
export async function removeTenantSecret(
kv: KVNamespace,
groupId: string,
check: (current: Group) => void,
): Promise<boolean> {
const saved = await updateGroup(kv, groupId, (g) => {
check(g);
return { ...g, webhookSecret: null };
});
if (!saved) return false;
await deleteTenantSecret(kv, saved);
return true;
}
/** Drop the KV copies of a group's secret (from before secrets lived with the group). */
export async function deleteTenantSecret(kv: KVNamespace, group: TenantGroup): Promise<void> {
await kv.delete(keyOf(group));
for (const key of legacyKeys(group)) await kv.delete(key);
}
+322 -83
View File
@@ -1,22 +1,46 @@
import type { H3Event } from "h3";
import { getHeader, readRawBody, setResponseStatus } from "h3";
import type { Env } from "./types";
import type { Env, Group } from "./types";
import { detectProvider } from "./providers";
import { dispatchEvent } from "./core/dispatch";
import { loadConfig, initConfigStore } from "./config";
import { loadGroups, ensureInstallationGroup } from "./web/groups";
import { loadConfigSnapshot, loadGroupFresh, initConfigStore } from "./config";
import { ensureInstallationGroup } from "./web/groups";
import { getTenantSecret } from "./web/tenants";
import { NONCE_TTL_SECONDS, ReplayPendingError, replayNonceKey } from "./providers/custom";
import { recordAudit } from "./lib/audit";
import { cfEnv, cfWaitUntil, headersFrom } from "./cf";
import { log } from "./lib/log";
import { deliveryKey, idempotencyStore } from "./lib/idempotency";
import { claimOnce, confirmClaim, deliveryKey, idempotencyStore } from "./lib/idempotency";
import { newCorrelationId } from "./lib/correlation";
import { enqueueWebhook, type DeliveryMessage } from "./queue/delivery";
import {
deliveryStateKey,
enqueueWebhook,
hadPermanentFailure,
isRetryableFailure,
markTargetsDone,
targetDoneChecker,
type DeliveryMessage,
} from "./queue/delivery";
const MAX_BODY_SIZE = 1024 * 1024;
/** How long a delivery's dedup claim lasts when it is queued. */
const QUEUED_CLAIM_SECONDS = 120;
/** How long it lasts for an inline dispatch, which may run for minutes. */
const INLINE_CLAIM_SECONDS = 600;
/** An installation's auto-provisioning is remembered as long as delivery state (7 days). */
const PROVISIONED_TTL_SECONDS = 7 * 86400;
/** Sending time of one inline dispatch: within its claim, with a margin. */
const INLINE_SEND_BUDGET_MS = 8 * 60_000;
/** Result of an inline dispatch: whether the sender should retry. */
interface InlineOutcome {
retry: boolean;
/** A target failed for good (in this or an earlier attempt). */
failed: boolean;
}
export interface WebhookResult {
status: 200 | 400 | 401 | 404 | 413;
status: 200 | 400 | 401 | 404 | 413 | 503;
body: unknown;
}
@@ -27,6 +51,10 @@ export interface WebhookResult {
* `POST /webhook/{groupId}`) the group's own secret is used for verification
* (GITHUB_WEBHOOK_SECRET/GITEA_WEBHOOK_SECRET are overridden per request) and
* only that group's routes are eligible.
*
* A tenant-signed payload is only as trustworthy as the tenant: its claimed
* GitHub App installation is dropped, so it can neither borrow the operator's
* App credentials nor provision or rebind installation groups.
*/
export async function processWebhook(
env: Env,
@@ -38,27 +66,39 @@ export async function processWebhook(
const requestId = newCorrelationId();
let effectiveEnv = env;
initConfigStore(env);
const groups = await loadGroups(env.KV);
// Until the request is verified, nothing but the receiving group's own row
// is read (fresh): an unsigned flood must not read the whole configuration
// each time. The full snapshot is read once the event is accepted.
const provider = detectProvider(headers);
if (!provider) {
return { status: 400, body: { error: "Unknown webhook provider" } };
}
let tenant: Group | undefined;
if (tenantId) {
if (!groups.some((g) => g.id === tenantId)) {
const found = await loadGroupFresh(env, tenantId);
if (!found.group) {
// Unreadable is not missing: the sender is asked to retry.
if (found.unreadable) {
return { status: 503, body: { error: "Group configuration unreadable; retry later" } };
}
return { status: 404, body: { error: "Group not found" } };
}
const secret = await getTenantSecret(env.KV, tenantId);
tenant = found.group;
const secret = await getTenantSecret(env.KV, tenant);
if (!secret) {
return { status: 404, body: { error: "Webhook disabled for this group" } };
}
effectiveEnv = { ...env, GITHUB_WEBHOOK_SECRET: secret, GITEA_WEBHOOK_SECRET: secret };
}
const provider = detectProvider(headers);
if (!provider) {
return { status: 400, body: { error: "Unknown webhook provider" } };
}
let verified = false;
try {
verified = await provider.verify(body, headers, effectiveEnv);
} catch (err) {
if (err instanceof ReplayPendingError) {
// The nonce is held by an attempt not (yet) accepted: not a replay.
return { status: 503, body: { error: "Request is already being processed; retry later" } };
}
// A malformed secret or an unavailable crypto implementation must fail as
// a clean 401, never an uncaught 500.
log.warn(
@@ -85,82 +125,281 @@ export async function processWebhook(
return { status: 401, body: { error: "Invalid signature" } };
}
const event = provider.parse(body, headers);
if (!event) {
return { status: 400, body: { error: "Invalid event" } };
// From here on, one-time claims are held (the custom-webhook nonce taken
// by `verify`, the delivery dedup key). Only once the event is accepted
// (queued, or dispatched without a transient failure) are they confirmed;
// otherwise they are given back, so the sender's retry is not refused. A
// claim that could not be given back is never mistaken for an accepted
// event: a retry finds it unconfirmed and is asked to come back later.
const claims = idempotencyStore(env.DB, env.KV);
const held: { key: string; ttl: number }[] = [];
const nonce = headers["x-webhooker-nonce"];
if (provider.id === "custom" && nonce && headers["x-webhooker-timestamp"]) {
held.push({
key: await replayNonceKey(effectiveEnv.GITHUB_WEBHOOK_SECRET, nonce),
ttl: NONCE_TTL_SECONDS,
});
}
// Auto-provision GitHub App installations so tenant isolation is configured
// without manual id entry: a group is created (or existing matching groups
// are bound) before the event is dispatched.
if (
provider.id === "github" &&
event.event === "installation" &&
event.payload.action === "created" &&
event.installationId != null
) {
const install = event.payload.installation as { account?: { login?: string } } | undefined;
const account = install?.account?.login ?? "";
try {
const group = await ensureInstallationGroup(env.KV, event.installationId, account);
if (group) {
await recordAudit(env.DB, {
ts: Date.now(),
actorLogin: account || undefined,
action: "installation.created",
targetType: "group",
targetId: group.id,
groupId: group.id,
});
}
} catch (err) {
log.warn(
{ err, installationId: event.installationId },
"Failed to auto-provision installation group",
);
const releaseClaims = async (): Promise<void> => {
for (const { key } of held.splice(0)) {
await claims.release(key).catch((err: unknown) => {
log.warn({ requestId, key, err }, "Failed to release a webhook claim");
});
}
}
};
const confirmClaims = async (): Promise<void> => {
for (const { key, ttl } of held.splice(0)) {
// Unconfirmed, a claim only delays a duplicate until it expires.
await confirmClaim(claims, key, ttl).catch((err: unknown) => {
log.warn({ requestId, key, err }, "Failed to confirm a webhook claim");
});
}
};
const retryLater = (error: string): WebhookResult => ({
status: 503,
body: { error, requestId },
});
try {
const event = provider.parse(body, headers);
if (!event) {
await releaseClaims();
return { status: 400, body: { error: "Invalid event" } };
}
if (tenantId) event.installationId = undefined;
if (event.deliveryId) {
// Dedup via the idempotency store: a provider/tenant-scoped key means
// different accounts may reuse a delivery id without colliding, while
// retries of the same delivery never dispatch twice.
const store = idempotencyStore(env.DB, env.KV);
const key = deliveryKey(provider.id, tenantId, event.deliveryId);
if (!(await store.claim(key, 120))) {
return { status: 200, body: { ok: true, duplicate: true, requestId } };
// retries of the same delivery never dispatch twice. An inline dispatch
// holds its claim for as long as it may run.
// A custom sender's ids are namespaced (`d:`) as its delivery identity is
// below: no id it picks can take the key of a nonce-identified request.
const dedupKey = event.deliveryId
? deliveryKey(
provider.id,
tenantId,
provider.id === "custom" ? `d:${event.deliveryId}` : event.deliveryId,
tenant?.instance,
)
: null;
if (dedupKey) {
const ttl = env.QUEUE ? QUEUED_CLAIM_SECONDS : INLINE_CLAIM_SECONDS;
const state = await claimOnce(claims, dedupKey, ttl);
if (state === "done") {
// Already accepted: this request's own nonce is used up as well.
await confirmClaims();
return { status: 200, body: { ok: true, duplicate: true, requestId } };
}
if (state === "pending") {
// Still in flight elsewhere, or left over from an attempt that was
// not accepted: the event may not be on its way yet.
await releaseClaims();
return retryLater("Delivery is already being processed; retry later");
}
held.push({ key: dedupKey, ttl });
}
// Provisioning runs only for an event not yet accepted (a duplicate
// returned above): a redelivery must not redo a binding an admin undid.
// Auto-provision GitHub App installations so tenant isolation is
// configured without manual id entry: a group is created (or existing
// matching groups are bound) before the event is dispatched. Only events
// signed with the operator's own secret (the global endpoint) may do this.
if (
!tenantId &&
provider.id === "github" &&
event.event === "installation" &&
event.payload.action === "created" &&
event.installationId != null
) {
const install = event.payload.installation as { account?: { login?: string } } | undefined;
const account = install?.account?.login ?? "";
// Done once per installation, on record as long as delivery state is
// kept: a redelivery after the dedup claim lapsed (minutes) must not
// redo a binding an admin undid since. Recorded only once it succeeded,
// so a provisioning that failed midway is completed by the retry.
const doneKey = `installation-provisioned:${event.installationId}`;
try {
if (!(await claims.has(doneKey))) {
const group = await ensureInstallationGroup(
env.KV,
event.installationId,
account,
env.DB,
);
await claims.claim(doneKey, PROVISIONED_TTL_SECONDS);
if (group) {
await recordAudit(env.DB, {
ts: Date.now(),
actorLogin: account || undefined,
action: "installation.created",
targetType: "group",
targetId: group.id,
groupId: group.id,
groupInstance: group.instance,
});
}
}
} catch (err) {
// Not accepted: the consumer never provisions, so a sender retry is
// the only way the group gets created (provisioning is idempotent).
log.warn(
{ err, installationId: event.installationId },
"Failed to auto-provision installation group",
);
await releaseClaims();
return retryLater("Installation provisioning failed; retry later");
}
}
// A stable identity across the sender's retries: the delivery id, else the
// replay-protection nonce of a signed request. Queued or inline, it keys
// the delivery's state and target ledger, so a retry of a request whose
// first attempt went through after all is recognized, not sent again.
// A custom sender picks its delivery ids freely: they get a namespace of
// their own, so none can coincide with a nonce-derived identity.
const nonceKey = held.find((h) => h.key.startsWith("nonce:"))?.key;
const identity =
event.deliveryId != null
? provider.id === "custom"
? `d:${event.deliveryId}`
: event.deliveryId
: nonceKey
? `n:${nonceKey}`
: undefined;
// Inline, a request identified by its nonce alone is held for the whole
// dispatch, as a delivery id is above: its nonce claim is short (shorter
// than a long dispatch), and a copy arriving meanwhile must not send the
// targets not yet on the ledger a second time.
if (!env.QUEUE && event.deliveryId == null && identity) {
const key = deliveryKey(provider.id, tenantId, identity, tenant?.instance);
const state = await claimOnce(claims, key, INLINE_CLAIM_SECONDS);
if (state === "done") {
await confirmClaims();
return { status: 200, body: { ok: true, duplicate: true, requestId } };
}
if (state === "pending") {
await releaseClaims();
return retryLater("Delivery is already being processed; retry later");
}
held.push({ key, ttl: INLINE_CLAIM_SECONDS });
}
if (env.QUEUE) {
const message: DeliveryMessage = {
deliveryId: identity ?? requestId,
groupId: tenantId,
groupInstance: tenant?.instance,
provider: provider.id,
event: event.event,
payload: event.payload,
installationId: event.installationId,
receivedAt: Date.now(),
requestId,
stateVersion: 2,
};
// Acknowledge only once the event is durably queued.
await enqueueWebhook(env, message);
await confirmClaims();
return { status: 200, body: { ok: true, requestId } };
}
// Inline dispatch (no queue): there is no durable retry, so the sender
// is answered only after the dispatch, and asked to retry while a target
// failed transiently. A delivery id gets a target ledger, so targets
// already sent are skipped when the sender redelivers.
// One consistent snapshot of routes and groups serves the dispatch — read
// fresh: a route or target removed moments ago in another isolate must
// not still be dispatched to from this isolate's cache.
const snapshot = await loadConfigSnapshot(env, { fresh: true });
if (tenantId) {
const current = snapshot.groups.find((g) => g.id === tenantId);
if (!current || current.instance !== tenant?.instance) {
// Gone or replaced since its row was read: not this group's event now.
await releaseClaims();
if (snapshot.config.unreadableGroups?.includes(tenantId)) {
return retryLater("Group configuration unreadable; retry later");
}
return { status: 404, body: { error: "Group not found" } };
}
}
const scoped = tenantId
? {
...snapshot.config,
routes: snapshot.config.routes.filter((r) => r.groupId === tenantId),
}
: snapshot.config;
const ledgerKey = identity
? deliveryStateKey(provider.id, tenantId, identity, tenant?.instance)
: null;
const run = (async (): Promise<InlineOutcome> => {
const recorded = new Set<string>();
// Dispatched under the delivery's identity (as a queued copy is), so the
// platform idempotency keys stay the same when the sender retries.
const delivered = identity ? { ...event, deliveryId: identity } : event;
// Sending stops well before the claim held for this dispatch expires
// (a target takes under a minute at worst): a copy arriving after that
// could otherwise send the targets still in flight. What is left is a
// transient failure, so the sender retries it (the ledger skips the rest).
const sendUntil = Date.now() + INLINE_SEND_BUDGET_MS;
const summary = await dispatchEvent(scoped, delivered, env, snapshot.groups, {
tenantId,
beforeTarget: async (): Promise<boolean> => Date.now() < sendUntil,
skipTarget: ledgerKey ? targetDoneChecker(env, ledgerKey) : undefined,
afterTarget: ledgerKey
? async (targetKey): Promise<void> => {
await markTargetsDone(env, ledgerKey, [targetKey]);
recorded.add(targetKey);
}
: undefined,
});
const permanent = summary.failures.filter((f) => !isRetryableFailure(f));
if (ledgerKey) {
await markTargetsDone(
env,
ledgerKey,
[...(summary.succeeded ?? []), ...permanent.flatMap((f) => f.key ?? [])].filter(
(k) => !recorded.has(k),
),
{ permanentFailure: permanent.length > 0 },
);
}
// A transient failure is always retried: without a ledger the retry
// sends the other targets again too — duplicates rather than a loss.
return {
retry: summary.failures.some(isRetryableFailure),
// Targets that failed for good in an earlier attempt still count.
failed:
permanent.length > 0 ||
(ledgerKey != null && (await hadPermanentFailure(env, ledgerKey))),
};
})();
// The dispatch keeps running should the sender hang up meanwhile.
waitUntil(run.catch(() => undefined));
let outcome: InlineOutcome;
try {
outcome = await run;
} catch (err) {
log.error({ requestId, err }, "Dispatch failed");
await releaseClaims();
return retryLater("Dispatch failed; retry later");
}
if (outcome.retry) {
log.warn({ requestId }, "Inline dispatch failed transiently");
await releaseClaims();
return retryLater("Some targets failed temporarily; retry later");
}
await confirmClaims();
// Accepted for good; a target that can never take the event is reported.
return outcome.failed
? { status: 200, body: { ok: true, requestId, failed: true } }
: { status: 200, body: { ok: true, requestId } };
} catch (err) {
log.error({ requestId, err }, "Failed to accept webhook");
await releaseClaims();
return retryLater("Temporarily unable to accept webhook");
}
const config = await loadConfig(env);
if (tenantId) {
config.routes = config.routes.filter((r) => r.groupId === tenantId);
}
if (env.QUEUE) {
const message: DeliveryMessage = {
deliveryId: event.deliveryId ?? requestId,
groupId: tenantId,
provider: provider.id,
event: event.event,
payload: event.payload,
installationId: event.installationId,
receivedAt: Date.now(),
requestId,
};
const enqueue = enqueueWebhook(env, message).catch((err) =>
log.error({ requestId, err }, "Failed to enqueue webhook"),
);
waitUntil(enqueue);
return { status: 200, body: { ok: true, requestId } };
}
const dispatch = dispatchEvent(config, event, env, groups).catch((err) =>
log.error({ requestId, err }, "Dispatch failed"),
);
waitUntil(dispatch);
return { status: 200, body: { ok: true, requestId } };
}
/** h3 wrapper for `POST /webhook` / `POST /webhook/:groupId`. */
+13
View File
@@ -0,0 +1,13 @@
import type { Env } from "../lib/types";
import { initConfigStore } from "../lib/config";
/**
* Bind the D1-backed configuration store before any handler runs, so no
* request path (invite acceptance, OAuth callbacks, bot commands, ...) can
* fall back to the KV-only store and read or write a configuration that
* diverges from D1.
*/
export default defineEventHandler((event) => {
const env = (event.context.cloudflare as { env?: Env } | undefined)?.env;
if (env?.KV) initConfigStore(env);
});
+2
View File
@@ -1,8 +1,10 @@
import type { Env } from "../lib/types";
import { handleQueueBatch } from "../lib/queue/consumer";
import { initConfigStore } from "../lib/config";
export default defineNitroPlugin((nitroApp) => {
nitroApp.hooks.hook("cloudflare:queue", async ({ batch, env }) => {
initConfigStore(env as Env);
await handleQueueBatch(batch, env as Env);
});
});
+8
View File
@@ -9,6 +9,9 @@ import {
adminApiMe,
adminApiGroupsGet,
adminApiGroupsPut,
adminGroupCreate,
adminGroupUpdate,
adminGroupDelete,
adminApiRoutesGet,
adminApiRoutesPut,
adminApiLogs,
@@ -40,6 +43,11 @@ export default defineEventHandler((event) => {
if (seg[0] === "groups" && seg.length === 1) {
if (method === "GET") return adminApiGroupsGet(event);
if (method === "PUT") return adminApiGroupsPut(event);
if (method === "POST") return adminGroupCreate(event);
}
if (seg[0] === "groups" && seg.length === 2) {
if (method === "PUT") return adminGroupUpdate(event, seg[1]!);
if (method === "DELETE") return adminGroupDelete(event, seg[1]!);
}
if (seg[0] === "routes" && seg.length === 1) {
if (method === "GET") return adminApiRoutesGet(event);
+3
View File
@@ -0,0 +1,3 @@
import { handleLinkConfirm } from "../../../lib/web/oauth";
export default defineEventHandler((event) => handleLinkConfirm(event));
+2 -1
View File
@@ -1,12 +1,13 @@
import { readRawBody } from "h3";
import { handleInteractionRequest } from "../../lib/drivers/discord/interactions";
import { cfEnv, rawRequest } from "../../lib/cf";
import { cfEnv, declaresBodyOver, rawRequest } from "../../lib/cf";
export default defineEventHandler(async (event) => {
// The raw Request is provided for headers/URL, but nitro consumes its body
// while buffering the payload — rebuild a fresh Request with the body from
// h3's internal buffers so the handler can read it (signature verification
// needs the exact raw bytes).
if (declaresBodyOver(event)) return new Response("Request too large", { status: 413 });
const source = rawRequest(event);
const body = await readRawBody(event, "utf8");
const request = new Request(source.url, {
+2 -1
View File
@@ -1,8 +1,9 @@
import { readRawBody } from "h3";
import { handleFeishuWebhookRequest } from "../../lib/drivers/feishu/updates";
import { cfEnv, rawRequest } from "../../lib/cf";
import { cfEnv, declaresBodyOver, rawRequest } from "../../lib/cf";
export default defineEventHandler(async (event) => {
if (declaresBodyOver(event)) return new Response("Request too large", { status: 413 });
const source = rawRequest(event);
const body = await readRawBody(event, "utf8");
const request = new Request(source.url, {
+2 -1
View File
@@ -1,8 +1,9 @@
import { readRawBody } from "h3";
import { handleTelegramWebhookRequest } from "../../lib/drivers/telegram/updates";
import { cfEnv, rawRequest } from "../../lib/cf";
import { cfEnv, declaresBodyOver, rawRequest } from "../../lib/cf";
export default defineEventHandler(async (event) => {
if (declaresBodyOver(event)) return new Response("Request too large", { status: 413 });
const source = rawRequest(event);
const body = await readRawBody(event, "utf8");
const request = new Request(source.url, {
+23 -2
View File
@@ -1,13 +1,33 @@
import { cfEnv } from "../lib/cf";
import { log } from "../lib/lib/log";
import { PAYLOAD_TTL_SECONDS } from "../lib/queue/delivery";
const DELIVERY_STATE_RETENTION_MS = 7 * 24 * 3600 * 1000;
const MESSAGE_TRACKING_RETENTION_MS = 30 * 24 * 3600 * 1000;
/** Parked payloads older than every possible retry are orphans (R2 has no TTL). */
const PAYLOAD_RETENTION_MS = PAYLOAD_TTL_SECONDS * 1000 + 24 * 3600 * 1000;
const PAYLOAD_PRUNE_BATCH = 500;
/**
* Delete parked queue payloads left behind in R2 (a send that failed after
* the upload, a consumer that crashed before discarding). Keys sort by upload
* date (`webhooks/YYYY/MM/DD/...`), so each run looks at the oldest objects
* first and removes those past the retention window.
*/
async function prunePayloads(bucket: R2Bucket, now: number): Promise<number> {
const listed = await bucket.list({ prefix: "webhooks/", limit: PAYLOAD_PRUNE_BATCH });
const stale = listed.objects
.filter((o) => o.uploaded.getTime() < now - PAYLOAD_RETENTION_MS)
.map((o) => o.key);
if (stale.length > 0) await bucket.delete(stale);
return stale.length;
}
/**
* Scheduled: purge expired rows from the D1-backed ephemeral stores
* (dedup_keys, delivery_state, message_tracking). These replaced the KV keys
* that relied on per-entry TTL; D1 rows must be removed by this task instead.
* (dedup_keys, delivery_state, message_tracking) and orphaned R2 payloads.
* These replaced the KV keys that relied on per-entry TTL; D1 rows and R2
* objects must be removed by this task instead.
*/
export default defineTask({
meta: { name: "storage:prune", description: "Prune D1 dedup/delivery/message-tracking rows" },
@@ -28,6 +48,7 @@ export default defineTask({
.bind(now - MESSAGE_TRACKING_RETENTION_MS)
.run();
removed += messages.meta?.changes ?? 0;
if (env.PAYLOAD) removed += await prunePayloads(env.PAYLOAD, now);
if (removed > 0) log.info({ removed }, "Pruned D1 storage rows");
return { result: "ok", removed };
} catch (err) {