From c853f2910f77950842b22648503e1dfdc964e73c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BF=85=E8=86=80?= Date: Fri, 23 Oct 2026 23:23:25 +0800 Subject: [PATCH] fix(config): transact per-group configuration Whole-table writes and cached snapshots can overwrite other groups or resurrect removed configuration. Use per-group guarded writes, one-time identity migration and explicit unreadable-row failures. --- server/lib/config.ts | 123 ++- server/lib/config/schema.ts | 6 + server/lib/storage/config-store.ts | 1479 ++++++++++++++++++++++------ server/lib/types.ts | 46 + server/middleware/config-store.ts | 13 + server/plugins/queue-consumer.ts | 2 + tests/config-store.test.ts | 682 ++++++++----- 7 files changed, 1764 insertions(+), 587 deletions(-) create mode 100644 server/middleware/config-store.ts diff --git a/server/lib/config.ts b/server/lib/config.ts index 9c6fc30..69f2faf 100644 --- a/server/lib/config.ts +++ b/server/lib/config.ts @@ -1,19 +1,31 @@ -import type { Env, Config, Route } from "./types"; -import { log } from "./lib/log"; -import { migrateRoutes, validateRoutes } from "./config/schema"; -import { d1ConfigStore, type ConfigStore } from "./storage/config-store"; +import type { Env, Config, Group, Route } from "./types"; +import { + createConfigStore, + kvConfigStore, + type ConfigStore, + type GuardedWrite, + type LoadOptions, +} from "./storage/config-store"; +import { canUseD1 } from "./storage/d1"; -const CONFIG_CACHE_TTL = 300_000; -const ROUTES_KEY = "config:routes"; -let configCache: { config: Config; expiresAt: number } | null = null; const configStores = new WeakMap(); -export function getConfigStore(kv: KVNamespace): ConfigStore | null { - return configStores.get(kv) ?? null; +/** + * The configuration store bound to this KV namespace. Every request, queue + * batch and scheduled task registers the D1-backed store first (see + * `initConfigStore`); a namespace that was never registered — a test, or a + * deployment without D1 — reads and writes the KV-only store. + */ +export function getConfigStore(kv: KVNamespace): ConfigStore { + return configStores.get(kv) ?? kvConfigStore(kv); } export function initConfigStore(env: Env): ConfigStore { - return ensureStore(env); + const existing = configStores.get(env.KV); + if (existing && (existing.kind === "d1" || !canUseD1(env.DB))) return existing; + const store = createConfigStore(env.DB, env.KV); + configStores.set(env.KV, store); + return store; } export function resetConfigStore(kv?: KVNamespace): void { @@ -23,52 +35,64 @@ export function resetConfigStore(kv?: KVNamespace): void { } } -function ensureStore(env: Env): ConfigStore { - let store = configStores.get(env.KV); - if (!store) { - store = d1ConfigStore(env.DB, env.KV); - configStores.set(env.KV, store); - } - return store; -} - -export async function loadRoutes(kv: KVNamespace): Promise { - const store = configStores.get(kv); - if (store) return store.loadRoutes(); - - try { - const stored = await kv.get(ROUTES_KEY, "json"); - if (stored) return validateRoutes(migrateRoutes(stored)); - } catch (err) { - log.warn({ err }, "Failed to load routes from KV"); - } - return []; +export async function loadRoutes(kv: KVNamespace, opts?: LoadOptions): Promise { + return getConfigStore(kv).loadRoutes(opts); } export async function saveRoutes(kv: KVNamespace, routes: Route[]): Promise { - const store = configStores.get(kv); - if (store) { - await store.saveRoutes(routes); - configCache = null; - return; - } - await kv.put(ROUTES_KEY, JSON.stringify(routes)); - configCache = null; + await getConfigStore(kv).saveRoutes(routes); } -export function invalidateConfigCache(): void { - configCache = null; +export async function saveGroupRoutes( + kv: KVNamespace, + groupId: string, + routes: Route[], + opts?: GuardedWrite, +): Promise { + return getConfigStore(kv).saveGroupRoutes(groupId, routes, opts); } +/** Drop every cached configuration snapshot held by this isolate. */ +export function invalidateConfigCache(kv?: KVNamespace): void { + if (kv) configStores.get(kv)?.invalidateCache(); +} + +/** One group, read fresh (its row alone): see `ConfigStore.loadGroup`. */ +export function loadGroupFresh( + env: Env, + id: string, +): Promise<{ group: Group | null; unreadable: boolean }> { + return initConfigStore(env).loadGroup(id); +} + +/** + * Configuration and groups of one consistent moment, for a dispatch that + * needs both (routes of a renamed group must never meet its old id, nor new + * targets meet stale source restrictions). + */ +export async function loadConfigSnapshot( + env: Env, + opts?: LoadOptions, +): Promise<{ config: Config; groups: Group[] }> { + const store = initConfigStore(env); + const { routes, groups, unreadable } = await store.loadSnapshot(opts); + const config = buildConfig(env, routes); + if (unreadable?.length) config.unreadableGroups = unreadable; + return { config, groups }; +} + +/** + * The configuration for one dispatch. Each call returns a fresh object with + * its own `routes` array, so a caller narrowing the routes (tenant scoping) + * can never leak that change into another request served by this isolate. + */ export async function loadConfig(env: Env): Promise { - if (configCache && Date.now() < configCache.expiresAt) { - return configCache.config; - } + initConfigStore(env); + return buildConfig(env, await loadRoutes(env.KV)); +} - ensureStore(env); - const routes = await loadRoutes(env.KV); - - const config: Config = { +function buildConfig(env: Env, routes: Route[]): Config { + return { baseUrl: env.BASE_URL ?? "https://webhooker.example.workers.dev", github: { webhookSecret: env.GITHUB_WEBHOOK_SECRET, @@ -80,9 +104,6 @@ export async function loadConfig(env: Env): Promise { discord: { token: env.DISCORD_TOKEN ?? "", }, - routes, + routes: [...routes], }; - - configCache = { config, expiresAt: Date.now() + CONFIG_CACHE_TTL }; - return config; } diff --git a/server/lib/config/schema.ts b/server/lib/config/schema.ts index 50d2b4a..c11699d 100644 --- a/server/lib/config/schema.ts +++ b/server/lib/config/schema.ts @@ -61,6 +61,7 @@ export const routeSchema = v.object({ export const groupMemberSchema = v.object({ login: v.string(), role: v.picklist(["owner", "admin", "viewer"]), + userId: v.optional(v.string()), }); export const forgeSourceSchema = v.object({ @@ -77,6 +78,11 @@ export const groupSchema = v.object({ owners: v.optional(v.array(v.string())), providers: v.optional(v.array(v.picklist(["github", "gitea", "gitlab", "custom"]))), installationId: v.optional(v.number()), + isolated: v.optional(v.boolean()), + instance: v.optional(v.string()), + instanceBackfilled: v.optional(v.boolean()), + legacyId: v.optional(v.string()), + webhookSecret: v.optional(v.nullable(v.string())), emoji: v.optional(v.boolean()), forgeSources: v.optional(v.array(forgeSourceSchema)), lang: v.optional(v.string()), diff --git a/server/lib/storage/config-store.ts b/server/lib/storage/config-store.ts index 49e94c9..a238e34 100644 --- a/server/lib/storage/config-store.ts +++ b/server/lib/storage/config-store.ts @@ -1,11 +1,196 @@ -import type { Group, Route } from "../types"; +import type { Filter, Group, Route } from "../types"; import { log } from "../lib/log"; +import * as v from "valibot"; +import { filterNodeSchema, filterSchema, migrateGroups, migrateRoutes } from "../config/schema"; +import { filterNodeWellFormed, filterPatternsUsable } from "../events/filter-ast"; +import { canUseD1, isMissingTable, noteKvFallback } from "./d1"; +import { sharedFlight, type Flight } from "../lib/shared-flight"; + +export interface LoadOptions { + /** + * Read the authoritative store directly, skipping every cache. Write paths + * (read-modify-write) and access checks use it so they never act on a + * snapshot another isolate has already changed. + */ + fresh?: boolean; +} + +/** A conditional write lost against a concurrent change; the caller reloads. */ +export class ConfigConflictError extends Error { + constructor(message = "Configuration changed concurrently") { + super(message); + this.name = "ConfigConflictError"; + } +} + +/** Builds the replacement of a group (null: keep it); may throw to abort. */ +export type GroupMutation = (group: Group) => Group | null | Promise; + +/** A precondition on the stored value, checked before a write that then commits atomically. */ +/** Extra statements committed in a group's rename/delete transaction. */ +export type LifecycleStatements = (db: D1Database) => D1PreparedStatement[]; + +export interface GuardedWrite { + expect?: (current: T) => void | Promise; + /** + * Guard statements run first in the write's own transaction (D1): one that + * raises aborts the write (ConfigConflictError). Checks that must hold at + * commit time, across groups, go here. Ignored by the KV-only store. + */ + guards?: LifecycleStatements; + /** + * Statements committed after the write, in the same transaction (D1): a + * record that must exist exactly when the write does. In the first of them + * SQLite's `changes()` tells whether the write changed its row (a lost + * compare-and-swap does not; it is retried). Ignored by KV. + */ + after?: LifecycleStatements; + /** + * Route lists only: replace the group's routes even though some stored rows + * cannot be read — the caller's explicit repair of a group whose unreadable + * rows would otherwise block every save. Off by default. + */ + discardUnreadable?: boolean; +} + +/** + * A statement that aborts its batch (D1 batches are transactions) unless the + * group row still has `version`: it raises a JSON error otherwise. Writers that + * guard on the version and bump it serialize against each other. + */ +export function groupVersionGuard( + db: D1Database, + id: string, + version: number, +): D1PreparedStatement { + return db + .prepare( + "SELECT CASE WHEN (SELECT version FROM d1_groups WHERE id = ?) = ? THEN 1 ELSE json('version conflict') END AS ok", + ) + .bind(id, version); +} + +export function groupVersionBump(db: D1Database, id: string): D1PreparedStatement { + return db.prepare("UPDATE d1_groups SET version = version + 1 WHERE id = ?").bind(id); +} + +/** Marker row (in `dedup_keys`) of a group id recently deleted or renamed away. */ +/** KV marker of the one-time identity migration (KV-only deployments). */ +const KV_IDENTITY_MARKER = "config:identity-migrated"; + +/** D1 marker row (in `dedup_keys`) of the one-time identity migration. */ +const IDENTITY_MARKER_ROW = "config:identity-v1"; + +/** D1 marker row (in `dedup_keys`) committed with the one-time KV seed. */ +const SEED_MARKER_ROW = "config:d1-seeded"; + +export function retiredGroupKey(id: string): string { + return `group-retired:${id}`; +} + +/** Aborts its batch (like a version guard) while the id is retired. */ +function notRetiredGuard(db: D1Database, id: string): D1PreparedStatement { + return db + .prepare( + "SELECT CASE WHEN EXISTS (SELECT 1 FROM dedup_keys WHERE key = ? AND expires_at > ?) THEN json('id retired') ELSE 1 END AS ok", + ) + .bind(retiredGroupKey(id), Date.now()); +} + +/** Run a guarded batch, turning a failed guard into a ConfigConflictError. */ +export async function runGuarded( + db: D1Database, + statements: D1PreparedStatement[], +): Promise { + try { + return await db.batch(statements); + } catch (err) { + if (/malformed JSON/i.test(String(err))) throw new ConfigConflictError(); + throw err; + } +} + +/** The stored version of a group row, or null when the group does not exist. */ +export async function groupVersion(db: D1Database, id: string): Promise { + const row = await db + .prepare("SELECT version FROM d1_groups WHERE id = ?") + .bind(id) + .first<{ version: number }>(); + return row ? row.version : null; +} + +/** Routes and groups of one moment, plus the groups with an unreadable route row. */ +export interface ConfigSnapshot { + routes: Route[]; + groups: Group[]; + unreadable?: string[]; +} export interface ConfigStore { - loadRoutes(): Promise; + readonly kind: "d1" | "kv"; + loadRoutes(opts?: LoadOptions): Promise; + /** Routes and groups of one consistent moment (use it whenever both are needed). */ + loadSnapshot(opts?: LoadOptions): Promise; + /** + * Replace the full route set. Only groups whose route list actually differs + * are rewritten, so a stale snapshot of one group never touches another. + */ saveRoutes(routes: Route[]): Promise; - loadGroups(): Promise; + /** + * Replace one group's routes (in the given order); other groups are + * untouched. `expect` sees the stored routes first and may throw (e.g. a + * ConfigConflictError for a stale revision); the write then only commits if + * nothing changed the group in between. Resolves to the stored routes. + */ + saveGroupRoutes(groupId: string, routes: Route[], opts?: GuardedWrite): Promise; + loadGroups(opts?: LoadOptions): Promise; + /** + * One group, read fresh from the store — its row alone, not the whole + * configuration (a request that may still be refused reads no more). + * `unreadable`: the row exists but cannot be used. + */ + loadGroup(id: string): Promise<{ group: Group | null; unreadable: boolean }>; + /** + * Replace the full group set: groups missing from `groups` are deleted with + * their routes and fragments, changed groups are upserted, unchanged ones are + * left alone. + */ saveGroups(groups: Group[]): Promise; + /** + * Insert a group unless one with its id exists (or, with D1, the id is + * retired); true when it was created. + */ + createGroup(group: Group): Promise; + /** + * Atomically read-modify-write one group (compare-and-swap on its stored + * version, retried on contention). `mutate` returns the new group, or null to + * leave it unchanged. Resolves to the stored group, or null when it does not + * exist. Throws ConfigConflictError when contention never settles. + */ + updateGroup( + id: string, + mutate: GroupMutation, + opts?: Pick, "guards" | "after">, + ): Promise; + /** + * Delete a group together with its routes and filter fragments; `extra` + * statements (history, id retirement) commit in the same transaction, which + * fails if the group changed after `expect` checked it. + */ + deleteGroup( + id: string, + opts?: GuardedWrite & { extra?: LifecycleStatements; version?: number }, + ): Promise; + /** + * Move a group, its routes and its fragments to a new id in one transaction + * (with `extra` statements), failing if the group changed after `expect` + * checked it. Resolves to the renamed group. + */ + renameGroup( + from: string, + to: string, + opts?: GuardedWrite & { extra?: LifecycleStatements; transform?: (g: Group) => Group }, + ): Promise; invalidateCache(): void; } @@ -29,11 +214,25 @@ interface D1RouteRow { ast: string | null; } -const CACHE_TTL = 300_000; +const CACHE_TTL = 60_000; + +/** + * Shape checks of route rows already judged, by their stored content: every + * fresh snapshot judges every row, and compiling each pattern again (more + * than the shared pattern cache holds, it would be every time) spends CPU on + * rows that have not changed. Bounded; a changed row has new content. + */ +const rowChecks = new Map(); +const MAX_ROW_CHECKS = 5000; +/** Legacy KV keys: the pre-D1 source of truth, read once to seed D1. */ const KV_ROUTES_KEY = "config:routes"; const KV_GROUPS_KEY = "config:groups"; - -const KV_CACHE_TTL = 3600; +/** Read-only degrade copy, used only while D1 reads fail. */ +/** Set once D1 holds (or has been seeded with) the configuration. */ +const KV_SEEDED_KEY = "config:d1-seeded"; +/** Group created for legacy routes that never belonged to one. */ +export const UNGROUPED_ID = "ungrouped"; +const UPDATE_ATTEMPTS = 5; function parseJson(value: string, label: string): T | null { try { @@ -44,15 +243,124 @@ function parseJson(value: string, label: string): T | null { } } +/** Stored form of a group: the id is the row key, `revision` is never persisted. */ +/** + * The webhook secret a group write keeps: one the mutation set, none when it + * cleared it (`null`), else the stored one — a client's edit never carries it. + */ +function keptSecret(next: Group, prev: Group): string | undefined { + // Removed: kept as "" — an explicit "none", so no older KV copy of the + // secret is ever read again (absent would mean "never set here"). + if (next.webhookSecret === null) return ""; + return next.webhookSecret ?? prev.webhookSecret ?? undefined; +} + +/** + * The fields dispatch and access checks rely on have the shape they assume + * (a field of the wrong type would make them throw for every event). A row + * failing this counts as unreadable — isolated to its group. + */ +function groupShapeOk(g: Group): boolean { + const strings = (x: unknown): boolean => + x === undefined || (Array.isArray(x) && x.every((s) => typeof s === "string")); + const objects = (x: unknown): boolean => + x === undefined || (Array.isArray(x) && x.every((o) => !!o && typeof o === "object")); + return ( + strings(g.providers) && + strings(g.owners) && + strings(g.adminIds) && + objects(g.members) && + objects(g.forgeSources) && + (g.logTarget == null || (typeof g.logTarget === "object" && !Array.isArray(g.logTarget))) && + (g.installationId == null || typeof g.installationId === "number") && + (g.isolated === undefined || typeof g.isolated === "boolean") && + (g.instance === undefined || typeof g.instance === "string") && + (g.legacyId === undefined || typeof g.legacyId === "string") + ); +} + +/** A fresh immutable group instance id. */ +function newInstanceId(): string { + return crypto.randomUUID().replace(/-/g, ""); +} + +/** + * Instance of a group backfilled in KV-only mode: derived from its id, so + * concurrent cold starts racing on the read-modify-write of `config:groups` + * all arrive at the same value (KV has no conditional write). Only legacy + * groups without an instance get one; new groups always get a random id. + */ +async function backfilledInstanceId(id: string): Promise { + const digest = await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(`webhooker:legacy-instance:${id}`), + ); + return [...new Uint8Array(digest).slice(0, 16)] + .map((b) => b.toString(16).padStart(2, "0")) + .join(""); +} + +function storedGroup(group: Group): Group { + const { revision: _revision, ...rest } = group as Group & { revision?: unknown }; + return rest as Group; +} + +/** JSON with object keys sorted, so equal data compares equal whatever the key order. */ +function stableJson(value: unknown): string { + return JSON.stringify(value, (_key, v: unknown) => + v && typeof v === "object" && !Array.isArray(v) + ? Object.fromEntries( + Object.entries(v as Record).sort(([a], [b]) => (a < b ? -1 : 1)), + ) + : v, + ); +} + +function sameJson(a: unknown, b: unknown): boolean { + return stableJson(a) === stableJson(b); +} + +/** The columns a route is stored as, for change detection. */ +function storedRoute(r: Route): unknown { + return { + id: r.id, + name: r.name, + enabled: !!r.enabled, + filters: r.filters ?? [], + targets: r.targets ?? [], + stop: !!r.stop, + fallback: !!r.fallback, + discordRoleIds: r.discordRoleIds ?? null, + ast: r.ast ?? null, + }; +} + +function routesByGroup(routes: Route[]): Map { + const out = new Map(); + for (const r of routes) { + const gid = r.groupId ?? ""; + const list = out.get(gid); + if (list) list.push(r); + else out.set(gid, [r]); + } + return out; +} + export function d1ConfigStore(db: D1Database, kv: KVNamespace): ConfigStore { - let routesCache: { routes: Route[]; expiresAt: number } | null = null; - let groupsCache: { groups: Group[]; expiresAt: number } | null = null; + /** + * Routes and groups are cached as one snapshot read in one transaction, so a + * dispatch never combines routes of one moment with groups of another (e.g. + * a renamed group's old routes with its new id). + */ + let snapshot: (ConfigSnapshot & { expiresAt: number }) | null = null; + let seeded = false; + let instancesChecked = false; /** * Run a statement that uses the `position` column (migration 0012) and fall * back to its legacy form only when the database confirms the column is - * missing. Any other error propagates exactly as before, and nothing is - * remembered, so a database migrated later is picked up on the next call. + * missing. Any other error propagates, and nothing is remembered, so a + * database migrated later is picked up on the next call. */ async function withPosition(run: (positioned: boolean) => Promise): Promise { try { @@ -65,311 +373,928 @@ export function d1ConfigStore(db: D1Database, kv: KVNamespace): ConfigStore { } } - async function loadRoutesFromD1(): Promise { - const results = await withPosition(async (positioned) => { - const order = positioned ? "group_id, position, id" : "id"; - const stmt = db.prepare( - `SELECT id, group_id, name, enabled, filters, targets, stop, fallback, discord_role_ids, ast FROM d1_routes ORDER BY ${order}`, - ); - if (typeof stmt.all !== "function") return []; - return (await stmt.all()).results; - }); - if (!results || results.length === 0) return []; - return results.flatMap((r) => { - const filters = parseJson(r.filters, `route:${r.id}:filters`); - const targets = parseJson(r.targets, `route:${r.id}:targets`); - const discordRoleIds = r.discord_role_ids - ? parseJson(r.discord_role_ids, `route:${r.id}:discord_role_ids`) - : undefined; - const ast = r.ast ? parseJson(r.ast, `route:${r.id}:ast`) : undefined; - if (!filters || !targets || (r.discord_role_ids && !discordRoleIds) || (r.ast && !ast)) { - return []; + function rowToRoute(r: D1RouteRow): Route | null { + const filters = parseJson(r.filters, `route:${r.id}:filters`); + const targets = parseJson(r.targets, `route:${r.id}:targets`); + const discordRoleIds = r.discord_role_ids + ? parseJson(r.discord_role_ids, `route:${r.id}:discord_role_ids`) + : undefined; + const ast = r.ast ? parseJson(r.ast, `route:${r.id}:ast`) : undefined; + if (!filters || !targets || (r.discord_role_ids && !discordRoleIds) || (r.ast && !ast)) { + return null; + } + // Parsable is not enough: a field of the wrong shape would make the + // matcher throw, i.e. the route silently never match. Such a row counts + // as unreadable (the group's deliveries then fail visibly and retry). + const isObject = (x: unknown): boolean => !!x && typeof x === "object" && !Array.isArray(x); + const check = (): boolean => + Array.isArray(filters) && + filters.every( + (f) => isObject(f) && !("all" in f || "any" in f || "not" in f) && v.is(filterSchema, f), + ) && + Array.isArray(targets) && + targets.every(isObject) && + (discordRoleIds == null || + (Array.isArray(discordRoleIds) && discordRoleIds.every((d) => typeof d === "string"))) && + (ast == null || (filterNodeWellFormed(ast) && v.is(filterNodeSchema, ast))) && + // Patterns the matcher cannot run would silently never match — checked + // on what it evaluates: the AST when there is one, else the flat list. + (ast != null + ? filterPatternsUsable(ast) + : filters.every((f) => filterPatternsUsable(f as Filter))); + // Whatever a row holds, judging it never throws: one bad row is one + // unreadable route, never a failed configuration snapshot. + const checkKey = JSON.stringify([r.filters, r.targets, r.discord_role_ids, r.ast]); + let shapeOk = rowChecks.get(checkKey); + if (shapeOk === undefined) { + try { + shapeOk = check(); + } catch { + shapeOk = false; } - return [ - { - id: r.id, - groupId: r.group_id, - name: r.name, - enabled: r.enabled === 1, - filters, - targets, - stop: r.stop === 1, - fallback: r.fallback === 1, - discordRoleIds: discordRoleIds ?? undefined, - ast: ast ?? undefined, - }, - ]; + if (rowChecks.size >= MAX_ROW_CHECKS) rowChecks.clear(); + rowChecks.set(checkKey, shapeOk); + } + if (!shapeOk) { + log.warn({ routeId: r.id, groupId: r.group_id }, "Ignoring malformed route row"); + return null; + } + return { + id: r.id, + groupId: r.group_id, + name: r.name, + enabled: r.enabled === 1, + filters, + targets, + stop: r.stop === 1, + fallback: r.fallback === 1, + discordRoleIds: discordRoleIds ?? undefined, + ast: ast ?? undefined, + }; + } + + /** + * Routes in stored order. An unreadable row is skipped (logged) for reads; + * with `strict` it is an error instead — a write that replaces the group's + * routes must never silently delete a row nobody could see. + */ + async function loadRoutesFromD1(groupId?: string, strict = false): Promise { + const results = await withPosition(async (positioned) => { + const order = positioned ? "group_id, position, id" : "group_id, id"; + const where = groupId === undefined ? "" : " WHERE group_id = ?"; + const stmt = db.prepare( + `SELECT id, group_id, name, enabled, filters, targets, stop, fallback, discord_role_ids, ast FROM d1_routes${where} ORDER BY ${order}`, + ); + return (await (groupId === undefined ? stmt : stmt.bind(groupId)).all()).results; }); + return (results ?? []).flatMap((r) => { + const route = rowToRoute(r); + if (!route && strict) { + throw new Error( + `route "${r.id}" of group "${r.group_id}" is unreadable; repair the row before saving`, + ); + } + return route ? [route] : []; + }); + } + + async function loadGroupRowsFromD1(): Promise { + const { results } = await db + .prepare("SELECT id, name, data, version FROM d1_groups ORDER BY id") + .all(); + return results ?? []; + } + + function rowToGroup(r: D1GroupRow): Group | null { + const group = parseJson(r.data, `group:${r.id}`); + if (group !== null && (typeof group !== "object" || Array.isArray(group))) return null; + if (group !== null && !groupShapeOk(group)) { + log.warn({ groupId: r.id }, "Ignoring malformed group row"); + return null; + } + // The row key is authoritative: a stale `id` inside the JSON never wins. + return group ? { ...group, id: r.id } : null; } async function loadGroupsFromD1(): Promise { - const stmt = db.prepare("SELECT id, name, data, version FROM d1_groups ORDER BY id"); - if (typeof stmt.all !== "function") return []; - const { results } = await stmt.all(); - if (!results || results.length === 0) return []; - return results.flatMap((r) => { - const group = parseJson(r.data, `group:${r.id}`); - return group ? [group] : []; + return (await loadGroupRowsFromD1()).flatMap((r) => { + const g = rowToGroup(r); + return g ? [g] : []; }); } - async function loadRoutesFromKV(): Promise { - try { - const raw = await kv.get(KV_ROUTES_KEY, "json"); - if (raw) return raw; - } catch (err) { - log.warn({ err }, "Failed to load routes from KV"); - } - return []; + /** Groups and routes from one D1 batch (one transaction: a consistent pair). */ + async function loadSnapshotFromD1(): Promise { + const [groupRows, routeRows] = await withPosition(async (positioned) => { + const order = positioned ? "group_id, position, id" : "group_id, id"; + const results = await db.batch([ + db.prepare("SELECT id, name, data, version FROM d1_groups ORDER BY id"), + db.prepare( + `SELECT id, group_id, name, enabled, filters, targets, stop, fallback, discord_role_ids, ast FROM d1_routes ORDER BY ${order}`, + ), + ]); + return [ + (results[0]?.results ?? []) as D1GroupRow[], + (results[1]?.results ?? []) as D1RouteRow[], + ] as const; + }); + const unreadable = new Set(); + return { + groups: groupRows.flatMap((r) => { + const g = rowToGroup(r); + // A group row nobody can read is not a deleted group: its routes + // must fail visibly, not be skipped as routes of a missing group. + if (!g && routeRows.some((route) => route.group_id === r.id)) unreadable.add(r.id); + return g ? [g] : []; + }), + routes: routeRows.flatMap((r) => { + const route = rowToRoute(r); + if (!route) unreadable.add(r.group_id); + return route ? [route] : []; + }), + unreadable: [...unreadable], + }; } - async function loadGroupsFromKV(): Promise { - try { - const raw = await kv.get(KV_GROUPS_KEY, "json"); - if (raw) return raw; - } catch (err) { - log.warn({ err }, "Failed to load groups from KV"); - } - return []; + /** + * A configuration list from KV: the list, `null` when the key is absent. + * Read errors propagate — a failed read must never pass for "no data". + */ + async function readKvList(key: string): Promise { + const raw = await kv.get(key, "json"); + if (raw == null) return null; + if (!Array.isArray(raw)) throw new Error(`KV key ${key} does not hold a list`); + return raw; } - function routeStatements( + function routeInsert( + r: Route, + groupId: string, + position: number, + positioned: boolean, + ): D1PreparedStatement { + const now = Date.now(); + const values = [ + r.id, + groupId, + r.name, + r.enabled ? 1 : 0, + JSON.stringify(r.filters ?? []), + JSON.stringify(r.targets ?? []), + r.stop ? 1 : 0, + r.fallback ? 1 : 0, + r.discordRoleIds ? JSON.stringify(r.discordRoleIds) : null, + r.ast ? JSON.stringify(r.ast) : null, + ]; + if (positioned) { + return db + .prepare( + `INSERT INTO d1_routes (id, group_id, name, enabled, filters, targets, stop, fallback, discord_role_ids, ast, position, version, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)`, + ) + .bind(...values, position, now, now); + } + return db + .prepare( + `INSERT INTO d1_routes (id, group_id, name, enabled, filters, targets, stop, fallback, discord_role_ids, ast, version, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)`, + ) + .bind(...values, now, now); + } + + /** Statements replacing one group's routes, in order. */ + function replaceGroupStatements( + groupId: string, routes: Route[], positioned: boolean, - existingRouteKeys?: Set, ): D1PreparedStatement[] { - const now = Date.now(); - const statements: D1PreparedStatement[] = []; - // A route's position is its index among the routes of its own group, so - // saving one group never shifts the order of another. - const nextPosition = new Map(); + return [ + db.prepare("DELETE FROM d1_routes WHERE group_id = ?").bind(groupId), + ...routes.map((r, i) => routeInsert(r, groupId, i, positioned)), + ]; + } - // If we have existing routes, delete those not in the new set - if (existingRouteKeys) { - const newKeys = new Set(routes.map((r) => `${r.id}:${r.groupId ?? ""}`)); - for (const key of existingRouteKeys) { - if (!newKeys.has(key)) { - const [id, groupId] = key.split(":"); - statements.push( - db.prepare("DELETE FROM d1_routes WHERE id = ? AND group_id = ?").bind(id, groupId), - ); + function groupUpsert(g: Group): D1PreparedStatement { + const now = Date.now(); + const stored = storedGroup(g); + return db + .prepare( + `INSERT INTO d1_groups (id, name, data, version, created_at, updated_at) + VALUES (?, ?, ?, 1, ?, ?) + ON CONFLICT(id) DO UPDATE SET + name = excluded.name, + data = excluded.data, + version = d1_groups.version + 1, + updated_at = excluded.updated_at`, + ) + .bind(stored.id, stored.name, JSON.stringify(stored), now, now); + } + + function groupDeleteStatements(id: string): D1PreparedStatement[] { + return [ + db.prepare("DELETE FROM d1_routes WHERE group_id = ?").bind(id), + db.prepare("DELETE FROM d1_fragments WHERE group_id = ?").bind(id), + db.prepare("DELETE FROM d1_groups WHERE id = ?").bind(id), + ]; + } + + /** + * Seed D1 once from the legacy KV configuration. It runs only while both + * tables are empty and no seed has happened before (a KV marker), never + * overwrites a row, and is awaited — so a configuration that was emptied on + * purpose stays empty instead of being resurrected from an old copy. + */ + /** Commit the seed together with its D1 marker; false when another isolate seeded first. */ + async function runSeed(statements: D1PreparedStatement[]): Promise { + const marker = db + .prepare("INSERT INTO dedup_keys (key, claimed_at, expires_at) VALUES (?, ?, ?)") + .bind(SEED_MARKER_ROW, Date.now(), Number.MAX_SAFE_INTEGER); + try { + await db.batch([marker, ...statements]); + return true; + } catch (err) { + if (/UNIQUE constraint failed: dedup_keys/i.test(String(err))) return false; + // No marker table yet (an unmigrated database): seed without it. + if (!isMissingTable(err) || !/dedup_keys/.test(String(err))) throw err; + await db.batch(statements); + return true; + } + } + + async function ensureSeeded(): Promise { + if (seeded) return; + if (await kv.get(KV_SEEDED_KEY)) { + seeded = true; + return; + } + const counts = await db + .prepare( + "SELECT (SELECT COUNT(*) FROM d1_groups) AS groups, (SELECT COUNT(*) FROM d1_routes) AS routes", + ) + .first<{ groups: number; routes: number }>(); + if ((counts?.groups ?? 0) === 0 && (counts?.routes ?? 0) === 0) { + // Strict reads: a KV failure aborts the seed (and leaves no marker), so + // a transient error can never seed a partial configuration for good. + // Identity fields in a legacy list came from clients (the old API stored + // whatever it was sent): dropped, the backfill assigns real ones. A list + // this server already manages (KV-only deployment, migrated) keeps them. + const managed = (await kv.get(KV_IDENTITY_MARKER)) != null; + const groups = migrateGroups((await readKvList(KV_GROUPS_KEY)) ?? []).map((g) => { + if (managed) return g; + const { instance: _i, instanceBackfilled: _b, legacyId: _l, ...rest } = g; + return rest as Group; + }); + const routes = migrateRoutes((await readKvList(KV_ROUTES_KEY)) ?? []); + if (groups.length > 0 || routes.length > 0) { + const known = new Set(groups.map((g) => g.id)); + // Routes without a group go to a group of their own, under an id no + // existing group (nor route) uses: never into a tenant's group that + // happens to be called "ungrouped". + const referenced = new Set(routes.map((r) => r.groupId).filter(Boolean)); + let ungroupedId = UNGROUPED_ID; + for (let n = 2; known.has(ungroupedId) || referenced.has(ungroupedId); n++) { + ungroupedId = `${UNGROUPED_ID}-${n}`; } - } - } else { - // Backward compatibility: delete all routes if no existing set provided - statements.push(db.prepare("DELETE FROM d1_routes")); - } - - // Upsert all routes - for (const r of routes) { - const groupId = r.groupId ?? ""; - const values = [ - r.id, - groupId, - r.name, - r.enabled ? 1 : 0, - JSON.stringify(r.filters), - JSON.stringify(r.targets), - r.stop ? 1 : 0, - r.fallback ? 1 : 0, - r.discordRoleIds ? JSON.stringify(r.discordRoleIds) : null, - r.ast ? JSON.stringify(r.ast) : null, - ]; - if (positioned) { - const position = nextPosition.get(groupId) ?? 0; - nextPosition.set(groupId, position + 1); - statements.push( - db - .prepare( - `INSERT INTO d1_routes (id, group_id, name, enabled, filters, targets, stop, fallback, discord_role_ids, ast, position, version, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?) - ON CONFLICT(id, group_id) DO UPDATE SET - name = excluded.name, - enabled = excluded.enabled, - filters = excluded.filters, - targets = excluded.targets, - stop = excluded.stop, - fallback = excluded.fallback, - discord_role_ids = excluded.discord_role_ids, - ast = excluded.ast, - position = excluded.position, - updated_at = excluded.updated_at`, - ) - .bind(...values, position, now, now), - ); - continue; - } - statements.push( - db - .prepare( - `INSERT INTO d1_routes (id, group_id, name, enabled, filters, targets, stop, fallback, discord_role_ids, ast, version, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?) - ON CONFLICT(id, group_id) DO UPDATE SET - name = excluded.name, - enabled = excluded.enabled, - filters = excluded.filters, - targets = excluded.targets, - stop = excluded.stop, - fallback = excluded.fallback, - discord_role_ids = excluded.discord_role_ids, - ast = excluded.ast, - updated_at = excluded.updated_at`, - ) - .bind(...values, now, now), - ); - } - - return statements; - } - - function groupStatements(groups: Group[], existingGroupIds: string[]): D1PreparedStatement[] { - const now = Date.now(); - const newGroupIds = new Set(groups.map((g) => g.id)); - const toDelete = existingGroupIds.filter((id) => !newGroupIds.has(id)); - - const statements: D1PreparedStatement[] = []; - - // Delete routes only when a group is actually removed. Group renames move - // routes before calling saveGroups, so the new group id is no longer in - // this list and the routes survive the group update. - for (const groupId of toDelete) { - statements.push(db.prepare("DELETE FROM d1_routes WHERE group_id = ?").bind(groupId)); - } - - // Then delete the groups themselves - for (const groupId of toDelete) { - statements.push(db.prepare("DELETE FROM d1_groups WHERE id = ?").bind(groupId)); - } - - // Finally, upsert all groups (INSERT OR REPLACE) - for (const g of groups) { - statements.push( - db - .prepare( - `INSERT INTO d1_groups (id, name, data, version, created_at, updated_at) - VALUES (?, ?, ?, 1, ?, ?) - ON CONFLICT(id) DO UPDATE SET - name = excluded.name, - data = excluded.data, - updated_at = excluded.updated_at`, - ) - .bind(g.id, g.name, JSON.stringify(g), now, now), - ); - } - - return statements; - } - - async function seedRoutesToD1(routes: Route[]): Promise { - if (routes.length === 0) return; - await withPosition((positioned) => db.batch(routeStatements(routes, positioned))); - } - - async function seedGroupsToD1(groups: Group[]): Promise { - if (groups.length === 0) return; - // When seeding, there are no existing groups to delete - await db.batch(groupStatements(groups, [])); - } - - async function syncRoutesToKV(routes: Route[], ttl: number): Promise { - try { - await kv.put(KV_ROUTES_KEY, JSON.stringify(routes), { expirationTtl: ttl }); - } catch (err) { - log.warn({ err }, "Failed to sync routes to KV cache"); - } - } - - async function syncGroupsToKV(groups: Group[], ttl: number): Promise { - try { - await kv.put(KV_GROUPS_KEY, JSON.stringify(groups), { expirationTtl: ttl }); - } catch (err) { - log.warn({ err }, "Failed to sync groups to KV cache"); - } - } - - return { - async loadRoutes(): Promise { - if (routesCache && Date.now() < routesCache.expiresAt) { - return routesCache.routes; - } - - let routes: Route[] = []; - try { - routes = await loadRoutesFromD1(); - if (routes.length > 0) { - syncRoutesToKV(routes, KV_CACHE_TTL).catch(() => undefined); - } else { - routes = await loadRoutesFromKV(); - if (routes.length > 0) { - seedRoutesToD1(routes).catch((err) => - log.warn({ err }, "Failed to seed routes from KV to D1"), + const placeholders: Group[] = []; + for (const r of routes) { + const gid = r.groupId || ungroupedId; + if (!known.has(gid)) { + known.add(gid); + // Routes without a group were global before groups existed: their + // placeholder stays open. A route naming a group that is gone is + // a leftover — kept for review, but isolated (no source bound, so + // it takes no events) instead of reopened to every source. + // It gets a fresh instance right away (no `legacyId`): it is not + // the group that once had the id, so that group's secret, invites + // and queued deliveries never apply to it. + // The group for routes that had none gets its identity now too: + // nothing of a former group applies to it, and its empty + // `legacyId` lets those routes adopt their chains, which were + // tracked without a group prefix. + placeholders.push( + r.groupId + ? { id: gid, name: gid, adminIds: [], isolated: true, instance: newInstanceId() } + : { id: gid, name: gid, adminIds: [], instance: newInstanceId(), legacyId: "" }, ); + if (r.groupId) { + log.warn({ groupId: gid }, "Seeded routes of a missing group as an isolated group"); + } } } - } catch (err) { - log.warn({ err }, "D1 routes unavailable, falling back to KV"); - routes = await loadRoutesFromKV(); + const now = Date.now(); + const statements = [...groups, ...placeholders].map((g) => + db + .prepare( + `INSERT INTO d1_groups (id, name, data, version, created_at, updated_at) + VALUES (?, ?, ?, 1, ?, ?) ON CONFLICT(id) DO NOTHING`, + ) + .bind(g.id, g.name, JSON.stringify(storedGroup(g)), now, now), + ); + await withPosition(async (positioned) => { + const routeStatements: D1PreparedStatement[] = []; + for (const [gid, list] of routesByGroup(routes)) { + // A legacy list may repeat an id; the first copy wins, as it did + // when the KV list was the source of truth. + const seen = new Set(); + list + .filter((r) => !seen.has(r.id) && seen.add(r.id)) + .forEach((r, i) => + routeStatements.push(routeInsert(r, gid || ungroupedId, i, positioned)), + ); + } + // Plain INSERTs: a conflicting row means D1 already holds data, and + // the whole seed transaction is abandoned rather than merged. The + // marker row commits with the seed: an isolate that read the empty + // tables too and arrives late (perhaps after an admin deleted the + // seeded groups) conflicts on it and never re-seeds. + await runSeed([...statements, ...routeStatements]); + }); + log.info( + { groups: groups.length, routes: routes.length }, + "Seeded D1 configuration from KV", + ); + // A deployment moving from KV to D1 also left delivery state, message + // tracking and claims in KV, written before the fallback marker + // existed: marked now, so D1 reads still consult them while they live. + await noteKvFallback(kv); } + } + await kv.put(KV_SEEDED_KEY, "1"); + seeded = true; + } - routesCache = { routes, expiresAt: Date.now() + CACHE_TTL }; - return routes; + /** + * Give every group an immutable `instance` id (once per isolate; a no-op + * once all groups have one). Renames keep it, a re-created id gets a new + * one: invites and delivery ledgers bound to it never cross to another group. + */ + async function ensureInstances(): Promise { + if (instancesChecked) return; + // A backfilled group also records the id it has right now: pre-instance + // state stored under that id belongs to it, whatever its id later becomes. + await db.batch([ + // Malformed rows are skipped (CASE keeps JSON functions off them): they + // are isolated by the snapshot as unreadable, never fail this setup. + db.prepare( + `UPDATE d1_groups SET data = json_set(data, '$.instance', lower(hex(randomblob(16))), + '$.instanceBackfilled', json('true'), '$.legacyId', id), + version = version + 1 + WHERE CASE WHEN json_valid(data) THEN json_extract(data, '$.instance') IS NULL ELSE 0 END`, + ), + // Backfilled before `legacyId` existed (and not renamed since). + db.prepare( + `UPDATE d1_groups SET data = json_set(data, '$.legacyId', id), version = version + 1 + WHERE CASE WHEN json_valid(data) + THEN json_extract(data, '$.instanceBackfilled') = 1 + AND json_extract(data, '$.legacyId') IS NULL + ELSE 0 END`, + ), + ]); + instancesChecked = true; + } + + /** + * Once per database: identity fields that were stored before this server + * assigned them came from clients (the old groups API kept any field it was + * sent), so none of them is trusted. Every group then existing is a + * pre-instance group: its `legacyId` becomes its own id, and an instance it + * shares with another group is replaced. Committed with its marker row. + */ + async function migrateIdentities(): Promise { + // No stored instance is kept: a client-chosen one could have been picked + // to equal the one another group is about to receive. + const rewrite = (): D1PreparedStatement[] => [ + db.prepare( + `UPDATE d1_groups SET data = json_set(data, '$.instance', lower(hex(randomblob(16))), + '$.instanceBackfilled', json('true'), '$.legacyId', id), version = version + 1 + WHERE json_valid(data)`, + ), + ]; + try { + const done = await db + .prepare("SELECT 1 AS hit FROM dedup_keys WHERE key = ?") + .bind(IDENTITY_MARKER_ROW) + .first<{ hit: number }>(); + if (done) return; + // Identities already managed by this server (a KV-only deployment, or + // a migration done while the marker table was missing): only the marker + // is recorded — rewriting them would cut groups off their secrets, + // invites and queued deliveries. + const managed = (await kv.get(KV_IDENTITY_MARKER)) != null; + // The marker goes first, as a plain INSERT: of two isolates racing, + // the second conflicts on it and its whole batch rolls back — a stale + // run can never rewrite groups created after the migration. + await db.batch([ + db + .prepare("INSERT INTO dedup_keys (key, claimed_at, expires_at) VALUES (?, ?, ?)") + .bind(IDENTITY_MARKER_ROW, Date.now(), Number.MAX_SAFE_INTEGER), + ...(managed ? [] : rewrite()), + ]); + } catch (err) { + // Another isolate migrated first (its marker stands): nothing to do. + if (/UNIQUE constraint failed: dedup_keys/i.test(String(err))) return; + if (!isMissingTable(err)) throw err; + // No marker table: the database misses migrations. Identities can be + // neither trusted unmigrated nor migrated safely (only once, atomically) + // — so nothing is served until the migrations are applied. + throw new Error( + "D1 migrations are incomplete (table dedup_keys is missing): apply them before serving", + ); + } + } + + const readying = new Map<"setup", Flight>(); + function ensureReady(): Promise { + if (seeded && instancesChecked) return Promise.resolve(); + // Requests arriving together at a cold isolate share one setup (its KV + // read, marker query and updates) instead of each running its own; a + // failed setup is cleared, so the next request tries again. Each step is + // safe to run twice, so a request that stops waiting for another's setup + // (see `sharedFlight`) can run its own. + return sharedFlight( + readying, + "setup", + async () => { + // Identities first: the seed below inserts groups with server-made + // identities only, which the migration must not touch. + if (!instancesChecked) await migrateIdentities(); + await ensureSeeded(); + await ensureInstances(); + }, + 15_000, + ); + } + + function invalidate(): void { + snapshot = null; + } + + async function afterWrite(): Promise { + invalidate(); + } + + /** + * The configuration snapshot. D1 read failures propagate — there is no + * stale fallback copy that could resurrect revoked restrictions or targets; + * callers fail and retry instead. + */ + async function snapshotFor(opts?: LoadOptions): Promise { + if (!opts?.fresh && snapshot && Date.now() < snapshot.expiresAt) return snapshot; + await ensureReady(); + const fresh = await loadSnapshotFromD1(); + snapshot = { ...fresh, expiresAt: Date.now() + CACHE_TTL }; + return fresh; + } + + const store: ConfigStore = { + kind: "d1", + + async loadRoutes(opts?: LoadOptions): Promise { + return (await snapshotFor(opts)).routes; + }, + + async loadSnapshot(opts?: LoadOptions): Promise { + const { routes, groups, unreadable } = await snapshotFor(opts); + return { routes, groups, unreadable }; }, async saveRoutes(routes: Route[]): Promise { - try { - await withPosition((positioned) => db.batch(routeStatements(routes, positioned))); - await syncRoutesToKV(routes, KV_CACHE_TTL); - } catch (err) { - log.warn({ err }, "D1 routes unavailable, falling back to KV"); - await syncRoutesToKV(routes, 0); + await ensureReady(); + // Versions first, routes second: a group changed in between fails its guard. + const { results: versionRows } = await db + .prepare("SELECT id, version FROM d1_groups") + .all<{ id: string; version: number }>(); + const versions = new Map((versionRows ?? []).map((r) => [r.id, r.version])); + const current = routesByGroup(await loadRoutesFromD1()); + const next = routesByGroup(routes); + const changed = new Set(); + for (const gid of new Set([...current.keys(), ...next.keys()])) { + const before = (current.get(gid) ?? []).map(storedRoute); + const after = (next.get(gid) ?? []).map(storedRoute); + if (!sameJson(before, after)) changed.add(gid); } - routesCache = null; + // A group about to be rewritten must have no unreadable row to lose. + for (const gid of changed) await loadRoutesFromD1(gid, true); + if (changed.size > 0) { + await withPosition((positioned) => + runGuarded( + db, + [...changed].flatMap((gid) => { + const version = versions.get(gid); + const guard = + version == null + ? [] + : [groupVersionGuard(db, gid, version), groupVersionBump(db, gid)]; + return [...guard, ...replaceGroupStatements(gid, next.get(gid) ?? [], positioned)]; + }), + ), + ); + } + await afterWrite(); }, - async loadGroups(): Promise { - if (groupsCache && Date.now() < groupsCache.expiresAt) { - return groupsCache.groups; - } + async saveGroupRoutes( + groupId: string, + routes: Route[], + opts?: GuardedWrite, + ): Promise { + await ensureReady(); + // Version first, routes second: any route write in between bumps the + // version, so the guard below refuses rather than overwrite it. + const version = await groupVersion(db, groupId); + if (version == null) throw new Error(`group "${groupId}" does not exist`); + // Loaded strictly even without `expect`: an unreadable row aborts here + // (unless the caller explicitly discards such rows). + const stored = await loadRoutesFromD1(groupId, !opts?.discardUnreadable); + await opts?.expect?.(stored); + await withPosition((positioned) => + runGuarded(db, [ + groupVersionGuard(db, groupId, version), + ...(opts?.guards?.(db) ?? []), + groupVersionBump(db, groupId), + ...replaceGroupStatements( + groupId, + routes.map((r) => ({ ...r, groupId })), + positioned, + ), + ]), + ); + await afterWrite(); + return loadRoutesFromD1(groupId); + }, - let groups: Group[] = []; - try { - groups = await loadGroupsFromD1(); - if (groups.length > 0) { - syncGroupsToKV(groups, KV_CACHE_TTL).catch(() => undefined); - } else { - groups = await loadGroupsFromKV(); - if (groups.length > 0) { - seedGroupsToD1(groups).catch((err) => - log.warn({ err }, "Failed to seed groups from KV to D1"), - ); - } - } - } catch (err) { - log.warn({ err }, "D1 groups unavailable, falling back to KV"); - groups = await loadGroupsFromKV(); + async loadGroups(opts?: LoadOptions): Promise { + // Fresh: the groups table alone (access checks need no routes) — a + // full snapshot would read every route along on each admin request. + if (opts?.fresh) { + await ensureReady(); + return loadGroupsFromD1(); } + return (await snapshotFor(opts)).groups; + }, - groupsCache = { groups, expiresAt: Date.now() + CACHE_TTL }; - return groups; + async loadGroup(id: string): Promise<{ group: Group | null; unreadable: boolean }> { + await ensureReady(); + const row = await db + .prepare( + "SELECT id, name, CASE WHEN json_valid(data) THEN data ELSE '' END AS data, version FROM d1_groups WHERE id = ?", + ) + .bind(id) + .first(); + if (!row) return { group: null, unreadable: false }; + const group = rowToGroup(row); + return { group, unreadable: group === null }; }, async saveGroups(groups: Group[]): Promise { - try { - // Load existing group IDs to properly handle deletions - const existing = await loadGroupsFromD1(); - const existingIds = existing.map((g) => g.id); - await db.batch(groupStatements(groups, existingIds)); - await syncGroupsToKV(groups, KV_CACHE_TTL); - } catch (err) { - log.warn({ err }, "D1 groups unavailable, falling back to KV"); - await syncGroupsToKV(groups, 0); + await ensureReady(); + const current = new Map((await loadGroupsFromD1()).map((g) => [g.id, g])); + const nextIds = new Set(groups.map((g) => g.id)); + const statements: D1PreparedStatement[] = []; + for (const id of current.keys()) { + if (!nextIds.has(id)) statements.push(...groupDeleteStatements(id)); } - groupsCache = null; + for (const g of groups) { + const prev = current.get(g.id); + // The instance id is server-owned: kept for existing groups, new otherwise. + const next = { + ...g, + instance: prev?.instance ?? newInstanceId(), + instanceBackfilled: prev?.instanceBackfilled, + legacyId: prev?.legacyId, + webhookSecret: prev?.webhookSecret ?? undefined, + }; + if (!prev || !sameJson(storedGroup(prev), storedGroup(next))) { + statements.push(groupUpsert(next)); + } + } + if (statements.length > 0) await db.batch(statements); + await afterWrite(); + }, + + async createGroup(group: Group): Promise { + await ensureReady(); + const now = Date.now(); + const stored = { + ...storedGroup(group), + instance: newInstanceId(), + instanceBackfilled: undefined, + legacyId: undefined, + webhookSecret: undefined, + }; + const insert = (sql: string, ...extra: unknown[]): Promise => + db + .prepare(sql) + .bind(stored.id, stored.name, JSON.stringify(stored), now, now, ...extra) + .run(); + let result: D1Result; + try { + // A retired id is refused in the insert itself: a delete or rename + // committed after the caller's own retirement check still counts. + result = await insert( + `INSERT INTO d1_groups (id, name, data, version, created_at, updated_at) + SELECT ?, ?, ?, 1, ?, ? + WHERE NOT EXISTS (SELECT 1 FROM dedup_keys WHERE key = ? AND expires_at > ?) + ON CONFLICT(id) DO NOTHING`, + retiredGroupKey(stored.id), + now, + ); + } catch (err) { + // No retirement records yet (an unmigrated database). + if (!isMissingTable(err) || !/dedup_keys/.test(String(err))) throw err; + result = await insert( + `INSERT INTO d1_groups (id, name, data, version, created_at, updated_at) + VALUES (?, ?, ?, 1, ?, ?) ON CONFLICT(id) DO NOTHING`, + ); + } + const created = (result.meta?.changes ?? 0) > 0; + if (created) await afterWrite(); + return created; + }, + + async updateGroup( + id: string, + mutate: GroupMutation, + opts?: Pick, "guards" | "after">, + ): Promise { + await ensureReady(); + for (let attempt = 0; attempt < UPDATE_ATTEMPTS; attempt++) { + const row = await db + .prepare("SELECT id, name, data, version FROM d1_groups WHERE id = ?") + .bind(id) + .first(); + if (!row) return null; + const prev = rowToGroup(row); + if (!prev) return null; + const next = await mutate(prev); + if (!next) return prev; + // Server-owned identity fields always come from the stored group. + const stored = { + ...storedGroup(next), + id, + instance: prev.instance, + instanceBackfilled: prev.instanceBackfilled, + legacyId: prev.legacyId, + webhookSecret: keptSecret(next, prev), + }; + const update = db + .prepare( + `UPDATE d1_groups SET name = ?, data = ?, version = version + 1, updated_at = ? + WHERE id = ? AND version = ?`, + ) + .bind(stored.name, JSON.stringify(stored), Date.now(), id, row.version); + const guards = opts?.guards?.(db) ?? []; + const after = opts?.after?.(db) ?? []; + const result = + guards.length + after.length > 0 + ? (await runGuarded(db, [...guards, update, ...after]))[guards.length]! + : await update.run(); + if ((result.meta?.changes ?? 0) > 0) { + await afterWrite(); + return stored; + } + } + throw new ConfigConflictError(); + }, + + async deleteGroup( + id: string, + opts?: GuardedWrite & { extra?: LifecycleStatements; version?: number }, + ): Promise { + await ensureReady(); + const row = await db + .prepare("SELECT id, name, data, version FROM d1_groups WHERE id = ?") + .bind(id) + .first(); + if (!row) return; + // The version the deletion was confirmed on (any route or fragment + // write since moved it): anything added after the confirmation stays. + if (opts?.version != null && row.version !== opts.version) throw new ConfigConflictError(); + const group = rowToGroup(row); + if (group) await opts?.expect?.(group); + // Deletes exactly the group that was checked: a change in between aborts. + await runGuarded(db, [ + groupVersionGuard(db, id, row.version), + ...(opts?.extra?.(db) ?? []), + ...groupDeleteStatements(id), + ]); + await afterWrite(); + }, + + async renameGroup( + from: string, + to: string, + opts?: GuardedWrite & { + extra?: LifecycleStatements; + transform?: (g: Group) => Group; + }, + ): Promise { + await ensureReady(); + const row = await db + .prepare("SELECT id, name, data, version FROM d1_groups WHERE id = ?") + .bind(from) + .first(); + const group = row ? rowToGroup(row) : null; + if (!row || !group) throw new Error(`group "${from}" does not exist`); + await opts?.expect?.(group); + const now = Date.now(); + const base = storedGroup(group); + const renamed = { + ...(opts?.transform ? opts.transform(base) : base), + id: to, + instance: base.instance, + instanceBackfilled: base.instanceBackfilled, + legacyId: base.legacyId, + }; + // The copy is only valid while the source row is unchanged (guard). + // Parent first, then children, then the old parent: every statement + // satisfies the d1_routes foreign key, and the batch is one transaction, + // so a failure leaves the group exactly as it was. + await runGuarded(db, [ + groupVersionGuard(db, from, row.version), + // The target id may have been retired since the caller checked it. + notRetiredGuard(db, to), + db + .prepare( + `INSERT INTO d1_groups (id, name, data, version, created_at, updated_at) + VALUES (?, ?, ?, 1, ?, ?)`, + ) + .bind(to, renamed.name, JSON.stringify(renamed), now, now), + db.prepare("UPDATE d1_routes SET group_id = ? WHERE group_id = ?").bind(to, from), + db.prepare("UPDATE d1_fragments SET group_id = ? WHERE group_id = ?").bind(to, from), + ...(opts?.extra?.(db) ?? []), + db.prepare("DELETE FROM d1_groups WHERE id = ?").bind(from), + ]); + await afterWrite(); + return renamed; }, invalidateCache(): void { - routesCache = null; - groupsCache = null; + invalidate(); }, }; + return store; +} + +/** + * Configuration kept in KV only, for deployments (and tests) without a D1 + * binding. Each list is one permanent key; writes are read-modify-write and + * therefore best-effort under concurrency, which is why D1 is preferred. + */ +export function kvConfigStore(kv: KVNamespace): ConfigStore { + /** Absent key: empty list. A failed read propagates, never passing for "no data". */ + async function readList(key: string): Promise { + const raw = await kv.get(key, "json"); + return Array.isArray(raw) ? raw : []; + } + const loadRoutes = async (): Promise => migrateRoutes(await readList(KV_ROUTES_KEY)); + let identitiesMigrated = false; + const loadGroups = async (): Promise => { + let groups = migrateGroups(await readList(KV_GROUPS_KEY)); + // Same one-time identity migration as the D1 store: identity fields + // stored before it came from clients, so every group then existing is a + // pre-instance group under its own id (a shared instance is replaced). + if (!identitiesMigrated && !(await kv.get(KV_IDENTITY_MARKER))) { + // No stored instance is kept: a client-chosen one could equal the one + // derived for another group. + groups = await Promise.all( + groups.map(async (g) => ({ + ...g, + instance: await backfilledInstanceId(g.id), + instanceBackfilled: true, + legacyId: g.id, + })), + ); + await kv.put(KV_GROUPS_KEY, JSON.stringify(groups.map(storedGroup))); + await kv.put(KV_IDENTITY_MARKER, "1"); + } + identitiesMigrated = true; + // Same one-time instance backfill as the D1 store. + const pending = (g: Group): boolean => + !g.instance || (g.instanceBackfilled === true && !g.legacyId); + if (!groups.some(pending)) return groups; + const filled = await Promise.all( + groups.map(async (g) => + !pending(g) + ? g + : { + ...g, + instance: g.instance ?? (await backfilledInstanceId(g.id)), + instanceBackfilled: true, + legacyId: g.id, + }, + ), + ); + await kv.put(KV_GROUPS_KEY, JSON.stringify(filled.map(storedGroup))); + return filled; + }; + const putRoutes = (routes: Route[]): Promise => + kv.put(KV_ROUTES_KEY, JSON.stringify(routes)); + const putGroups = (groups: Group[]): Promise => + kv.put(KV_GROUPS_KEY, JSON.stringify(groups.map(storedGroup))); + + return { + kind: "kv", + loadRoutes, + async loadSnapshot(): Promise<{ routes: Route[]; groups: Group[] }> { + const [groups, routes] = await Promise.all([loadGroups(), loadRoutes()]); + return { groups, routes }; + }, + async saveRoutes(routes): Promise { + await putRoutes(routes); + }, + async saveGroupRoutes(groupId, routes, opts): Promise { + const all = await loadRoutes(); + await opts?.expect?.(all.filter((r) => r.groupId === groupId)); + const saved = routes.map((r) => ({ ...r, groupId })); + await putRoutes([...all.filter((r) => r.groupId !== groupId), ...saved]); + return saved; + }, + loadGroups, + async loadGroup(id: string): Promise<{ group: Group | null; unreadable: boolean }> { + return { group: (await loadGroups()).find((g) => g.id === id) ?? null, unreadable: false }; + }, + async saveGroups(groups): Promise { + const ids = new Set(groups.map((g) => g.id)); + const routes = await loadRoutes(); + const kept = routes.filter((r) => !r.groupId || ids.has(r.groupId)); + if (kept.length !== routes.length) await putRoutes(kept); + await putGroups(groups); + }, + async createGroup(group): Promise { + const groups = await loadGroups(); + if (groups.some((g) => g.id === group.id)) return false; + // Routes left under this id by an interrupted rename or delete belong + // to another group: a new group never inherits them. + const routes = await loadRoutes(); + if (routes.some((r) => r.groupId === group.id)) { + await putRoutes(routes.filter((r) => r.groupId !== group.id)); + } + await putGroups([ + ...groups, + { + ...group, + instance: newInstanceId(), + instanceBackfilled: undefined, + legacyId: undefined, + webhookSecret: undefined, + }, + ]); + return true; + }, + async updateGroup(id, mutate): Promise { + const groups = await loadGroups(); + const prev = groups.find((g) => g.id === id); + if (!prev) return null; + const next = await mutate(prev); + if (!next) return prev; + const stored = { + ...storedGroup(next), + id, + instance: prev.instance, + instanceBackfilled: prev.instanceBackfilled, + legacyId: prev.legacyId, + webhookSecret: keptSecret(next, prev), + }; + await putGroups(groups.map((g) => (g.id === id ? stored : g))); + return stored; + }, + async deleteGroup(id, opts): Promise { + const group = (await loadGroups()).find((g) => g.id === id); + if (group) await opts?.expect?.(group); + // Group first: should the second write fail, what is left are inert + // routes of a missing group, never a group whose routes were dropped. + await putGroups((await loadGroups()).filter((g) => g.id !== id)); + await putRoutes((await loadRoutes()).filter((r) => r.groupId !== id)); + }, + async renameGroup(from, to, opts): Promise { + const groups = await loadGroups(); + const group = groups.find((g) => g.id === from); + if (!group) throw new Error(`group "${from}" does not exist`); + if (groups.some((g) => g.id === to)) throw new Error(`group "${to}" already exists`); + await opts?.expect?.(group); + const moved = { + ...(opts?.transform ? opts.transform(group) : group), + id: to, + instance: group.instance, + instanceBackfilled: group.instanceBackfilled, + legacyId: group.legacyId, + }; + // No transaction across the two keys: copy the routes first, move the + // group, drop the old copies last. Stopping anywhere leaves every route + // under the group's current id (plus inert copies under the other one). + const routes = await loadRoutes(); + const copies = routes.filter((r) => r.groupId === from).map((r) => ({ ...r, groupId: to })); + await putRoutes([...routes.filter((r) => r.groupId !== to), ...copies]); + await putGroups(groups.map((g) => (g.id === from ? moved : g))); + await putRoutes((await loadRoutes()).filter((r) => r.groupId !== from)); + return storedGroup(moved); + }, + invalidateCache(): void {}, + }; +} + +/** The store for a binding pair: D1-backed when D1 is usable, KV-only otherwise. */ +export function createConfigStore(db: D1Database | undefined, kv: KVNamespace): ConfigStore { + return canUseD1(db) ? d1ConfigStore(db!, kv) : kvConfigStore(kv); } diff --git a/server/lib/types.ts b/server/lib/types.ts index 304bba6..8c4b756 100644 --- a/server/lib/types.ts +++ b/server/lib/types.ts @@ -51,6 +51,12 @@ export interface Config { token: string; }; routes: Route[]; + /** + * Groups with a stored route that could not be read. Their events are + * reported as transient failures instead of passing for delivered while a + * route that might have matched was silently left out. + */ + unreadableGroups?: string[]; } export interface RouteTarget { @@ -138,6 +144,46 @@ export interface Group { * `owners` list. Empty/omitted = no installation restriction. */ installationId?: number; + /** + * Isolated groups (self-service personal groups) only receive events from + * their own `/webhook/{groupId}` ingress; the global endpoint dispatches to + * them only once a super admin binds a source (`installationId` or + * `owners`). Only super admins may edit this field. + */ + isolated?: boolean; + /** + * Immutable identity of this group, assigned by the server when the group is + * created and kept across renames. A group re-created under a former id gets + * a new one, so invites and delivery ledgers of the old group never apply to + * it. Clients cannot set it. + */ + instance?: string; + /** + * Set when the group got its `instance` from the one-time backfill, i.e. it + * existed before instances: only such a group may adopt state stored under + * its id (a pre-instance webhook secret or invite). Server-owned. + */ + instanceBackfilled?: boolean; + /** + * For a backfilled group: the id it had when it got its instance. State + * stored under a group id before instances (a webhook secret, an invite, a + * queued delivery) belongs to the group with that `legacyId` only — not to + * whichever group holds the id now (another one may have been renamed to + * it). Server-owned, kept across renames. + */ + legacyId?: string; + /** + * The group's webhook secret (per-group ingress). Server-owned, stored with + * the group so it changes only through a compare-and-swap that re-checks the + * writer's role; never sent to clients. `null` in a mutation removes it. + */ + webhookSecret?: string | null; + /** + * Opaque revision of the stored group, set on read by the admin API and + * echoed back on update so a stale edit is rejected instead of overwriting a + * newer one. Never persisted. + */ + revision?: string; /** * Whether to include emoji in messages sent through this group's routes. * Defaults to true when omitted. diff --git a/server/middleware/config-store.ts b/server/middleware/config-store.ts new file mode 100644 index 0000000..1b082b3 --- /dev/null +++ b/server/middleware/config-store.ts @@ -0,0 +1,13 @@ +import type { Env } from "../lib/types"; +import { initConfigStore } from "../lib/config"; + +/** + * Bind the D1-backed configuration store before any handler runs, so no + * request path (invite acceptance, OAuth callbacks, bot commands, ...) can + * fall back to the KV-only store and read or write a configuration that + * diverges from D1. + */ +export default defineEventHandler((event) => { + const env = (event.context.cloudflare as { env?: Env } | undefined)?.env; + if (env?.KV) initConfigStore(env); +}); diff --git a/server/plugins/queue-consumer.ts b/server/plugins/queue-consumer.ts index 5751249..afcfa52 100644 --- a/server/plugins/queue-consumer.ts +++ b/server/plugins/queue-consumer.ts @@ -1,8 +1,10 @@ import type { Env } from "../lib/types"; import { handleQueueBatch } from "../lib/queue/consumer"; +import { initConfigStore } from "../lib/config"; export default defineNitroPlugin((nitroApp) => { nitroApp.hooks.hook("cloudflare:queue", async ({ batch, env }) => { + initConfigStore(env as Env); await handleQueueBatch(batch, env as Env); }); }); diff --git a/tests/config-store.test.ts b/tests/config-store.test.ts index 0df1575..d322c91 100644 --- a/tests/config-store.test.ts +++ b/tests/config-store.test.ts @@ -1,33 +1,20 @@ import { describe, it, expect } from "bun:test"; -import { d1ConfigStore, type ConfigStore } from "../server/lib/storage/config-store"; +import { + d1ConfigStore, + kvConfigStore, + ConfigConflictError, + UNGROUPED_ID, +} from "../server/lib/storage/config-store"; import type { Route, Group } from "../server/lib/types"; - -interface Stmt { - sql: string; - bound: unknown[]; - bind: (...args: unknown[]) => Stmt; - all: () => Promise<{ results: Array> }>; -} - -interface FakeDB { - prepare: (sql: string) => Stmt; - batch: (stmts: Stmt[]) => Promise; -} - -interface FakeDBOptions { - /** Whether `d1_routes` has the `position` column (migration 0012). */ - positionColumn?: boolean; - /** How many batches fail with a transient (non-schema) error first. */ - batchFailures?: number; -} +import { createMemoryKV, createSqliteD1 } from "./d1-sqlite"; function route(id: string, groupId: string, overrides: Partial = {}): Route { return { id, name: `route-${id}`, enabled: true, - filters: [], - targets: [], + filters: [{ type: "event", match: "push" }], + targets: [{ platform: "discord", channelId: "c1" }], groupId, stop: false, fallback: false, @@ -35,259 +22,436 @@ function route(id: string, groupId: string, overrides: Partial = {}): Rou }; } -function group(id: string): Group { - return { id, name: `group-${id}`, adminIds: [] }; +function group(id: string, overrides: Partial = {}): Group { + return { id, name: `group-${id}`, adminIds: [], ...overrides }; } -function createDB(options: FakeDBOptions = {}): { - db: FakeDB & D1Database; - routesTable: Route[]; - groupsTable: Group[]; - queries: string[]; - batches: Stmt[][]; +function setup(options: { position?: boolean } = {}): { + db: D1Database; + sqlite: ReturnType["sqlite"]; + kv: KVNamespace; + kvStore: Map; + cfg: ReturnType; } { - const routesTable: Route[] = []; - const groupsTable: Group[] = []; - const queries: string[] = []; - const batches: Stmt[][] = []; - let batchFailures = options.batchFailures ?? 0; - const db: FakeDB = { - prepare(sql: string): Stmt { - const stmt: Stmt = { - sql, - bound: [], - bind(...args: unknown[]): Stmt { - stmt.bound = args; - return stmt; - }, - async all(): Promise<{ results: Array> }> { - queries.push(sql); - if (sql.includes("position") && !options.positionColumn) { - throw new Error("D1_ERROR: no such column: position at offset 33: SQLITE_ERROR"); - } - if (sql.includes("FROM d1_routes")) { - return { - results: routesTable.map((r) => ({ - id: r.id, - group_id: r.groupId ?? "", - name: r.name, - enabled: r.enabled ? 1 : 0, - filters: JSON.stringify(r.filters), - targets: JSON.stringify(r.targets), - stop: r.stop ? 1 : 0, - fallback: r.fallback ? 1 : 0, - discord_role_ids: r.discordRoleIds ? JSON.stringify(r.discordRoleIds) : null, - ast: r.ast ? JSON.stringify(r.ast) : null, - })), - }; - } - if (sql.includes("FROM d1_groups")) { - return { - results: groupsTable.map((g) => ({ - id: g.id, - name: g.name, - data: JSON.stringify(g), - version: 1, - })), - }; - } - return { results: [] }; - }, - }; - return stmt; - }, - async batch(stmts: Stmt[]): Promise { - batches.push(stmts); - if (batchFailures > 0) { - batchFailures--; - throw new Error("D1_ERROR: network connection lost"); - } - if (!options.positionColumn && stmts.some((st) => st.sql.includes("position,"))) { - throw new Error("D1_ERROR: table d1_routes has no column named position: SQLITE_ERROR"); - } - return []; - }, - }; - return { db: db as FakeDB & D1Database, routesTable, groupsTable, queries, batches }; -} - -/** Bound `(id, position)` pairs of every route upsert in a batch. */ -function upsertPositions(stmts: Stmt[]): Array<[unknown, unknown]> { - return stmts - .filter((s) => s.sql.includes("INSERT INTO d1_routes")) - .map((s) => [s.bound[0], s.sql.includes("position") ? s.bound[10] : undefined]); -} - -function createKV(): { kv: KVNamespace; store: Map } { - const store = new Map(); - const kv = { - store, - get: async (key: string, type?: string): Promise => { - const v = store.get(key); - if (v == null) return null; - if (type === "json") return JSON.parse(v) as T; - return v as unknown as T; - }, - put: async (key: string, value: string): Promise => { - store.set(key, value); - }, - delete: async (key: string): Promise => { - store.delete(key); - }, - list: async (): Promise<{ keys: unknown[] }> => ({ keys: [] }), - }; - return { kv: kv as unknown as KVNamespace, store }; + const { db, sqlite } = createSqliteD1({ + migrations: (name) => options.position !== false || !name.startsWith("0012"), + }); + const { kv, store } = createMemoryKV(); + return { db, sqlite, kv, kvStore: store, cfg: d1ConfigStore(db, kv) }; } describe("d1ConfigStore", () => { it("loads empty when D1 and KV are empty", async () => { - const { db } = createDB(); - const { kv } = createKV(); - const store: ConfigStore = d1ConfigStore(db, kv); - expect(await store.loadRoutes()).toEqual([]); - expect(await store.loadGroups()).toEqual([]); + const { cfg } = setup(); + expect(await cfg.loadRoutes()).toEqual([]); + expect(await cfg.loadGroups()).toEqual([]); }); - it("seeds routes from KV into memory when D1 is empty and caches in KV", async () => { - const { db } = createDB(); - const { kv, store } = createKV(); - const existing = [route("r1", "g1")]; - store.set("config:routes", JSON.stringify(existing)); - const cfg: ConfigStore = d1ConfigStore(db, kv); - const loaded = await cfg.loadRoutes(); - expect(loaded).toHaveLength(1); - expect(loaded[0].id).toBe("r1"); - }); - - it("reads routes directly from D1 when populated", async () => { - const { db, routesTable } = createDB(); - const { kv } = createKV(); - routesTable.push(route("r1", "g1")); - const cfg: ConfigStore = d1ConfigStore(db, kv); - const loaded = await cfg.loadRoutes(); - expect(loaded).toHaveLength(1); - expect(loaded[0].id).toBe("r1"); - expect(loaded[0].groupId).toBe("g1"); - }); - - it("round-trips routes through D1 + KV cache on save", async () => { - const { db, routesTable } = createDB(); - const { kv, store } = createKV(); - const cfg: ConfigStore = d1ConfigStore(db, kv); - const routes = [route("r1", "g1"), route("r2", "g2")]; - await cfg.saveRoutes(routes); - expect(routesTable).toHaveLength(0); - const cached = store.get("config:routes"); - expect(cached).toBe(JSON.stringify(routes)); - const reloaded = await cfg.loadRoutes(); - expect(reloaded).toHaveLength(2); - }); - - it("round-trips groups through D1 + KV cache on save", async () => { - const { db } = createDB(); - const { kv, store } = createKV(); - const cfg: ConfigStore = d1ConfigStore(db, kv); - const groups = [group("g1"), group("g2")]; - await cfg.saveGroups(groups); - expect(store.get("config:groups")).toBe(JSON.stringify(groups)); - const reloaded = await cfg.loadGroups(); - expect(reloaded).toHaveLength(2); - }); - - it("invalidateCache forces a reload", async () => { - const { db, routesTable } = createDB(); - const { kv } = createKV(); - const cfg: ConfigStore = d1ConfigStore(db, kv); - routesTable.push(route("r1", "g1")); - const first = await cfg.loadRoutes(); - expect(first).toHaveLength(1); - routesTable.push(route("r2", "g2")); - const cached = await cfg.loadRoutes(); - expect(cached).toHaveLength(1); - cfg.invalidateCache(); - const second = await cfg.loadRoutes(); - expect(second).toHaveLength(2); - }); - - it("persists the per-group route order when the position column exists", async () => { - const { db, batches } = createDB({ positionColumn: true }); - const { kv } = createKV(); - const cfg: ConfigStore = d1ConfigStore(db, kv); - await cfg.saveRoutes([route("b", "g1"), route("x", "g2"), route("a", "g1")]); - expect(upsertPositions(batches[0]!)).toEqual([ - ["b", 0], - ["x", 0], - ["a", 1], - ]); - }); - - it("loads routes ordered by group and position when the column exists", async () => { - const { db, queries } = createDB({ positionColumn: true }); - const { kv } = createKV(); - await d1ConfigStore(db, kv).loadRoutes(); - expect(queries.some((q) => q.includes("FROM d1_routes ORDER BY group_id, position, id"))).toBe( - true, + it("seeds D1 once from the legacy KV configuration", async () => { + const { cfg, kvStore, sqlite } = setup(); + kvStore.set("config:groups", JSON.stringify([group("g1")])); + kvStore.set( + "config:routes", + JSON.stringify([ + route("r1", "g1"), + // Legacy shape: a single `target` and no group. + { ...route("r2", ""), groupId: undefined, targets: undefined, target: { channelId: "x" } }, + ]), ); - }); - - it("falls back to the legacy id order and inserts on an unmigrated database", async () => { - const { db, queries, batches } = createDB(); - const { kv } = createKV(); - const cfg: ConfigStore = d1ConfigStore(db, kv); - await cfg.loadRoutes(); - expect(queries.at(-1)).toContain("FROM d1_routes ORDER BY id"); - await cfg.saveRoutes([route("r1", "g1")]); - expect(upsertPositions(batches.at(-1)!)).toEqual([["r1", undefined]]); - }); - - it("picks up the position column once the database is migrated", async () => { - const options: FakeDBOptions = {}; - const { db, queries } = createDB(options); - const { kv } = createKV(); - const cfg: ConfigStore = d1ConfigStore(db, kv); - await cfg.loadRoutes(); - expect(queries.at(-1)).toContain("ORDER BY id"); - options.positionColumn = true; - cfg.invalidateCache(); - await cfg.loadRoutes(); - expect(queries.at(-1)).toContain("ORDER BY group_id, position, id"); - }); - - it("never retries without positions when D1 fails for another reason", async () => { - const { db, batches } = createDB({ positionColumn: true, batchFailures: 1 }); - const { kv, store } = createKV(); - const cfg: ConfigStore = d1ConfigStore(db, kv); - const routes = [route("b", "g1"), route("a", "g1")]; - await cfg.saveRoutes(routes); - // Only the positioned batch was attempted; the save took the existing - // "D1 unavailable" path instead of writing rows that lose the order. - expect(batches).toHaveLength(1); - expect(upsertPositions(batches[0]!)).toEqual([ - ["b", 0], - ["a", 1], + const loaded = await cfg.loadRoutes(); + expect(loaded.map((r) => [r.id, r.groupId])).toEqual([ + ["r1", "g1"], + ["r2", UNGROUPED_ID], ]); - expect(store.get("config:routes")).toBe(JSON.stringify(routes)); + // The legacy single target was migrated before it reached D1. + expect(loaded[1]!.targets).toEqual([{ channelId: "x" }]); + expect((await cfg.loadGroups()).map((g) => g.id)).toEqual(["g1", UNGROUPED_ID]); + expect(kvStore.get("config:d1-seeded")).toBe("1"); + expect(sqlite.query("SELECT COUNT(*) AS n FROM d1_routes").get()).toEqual({ n: 2 }); }); - it("updating groups does not cascade-delete routes", async () => { - const { db, routesTable, groupsTable } = createDB(); - const { kv } = createKV(); - const cfg: ConfigStore = d1ConfigStore(db, kv); + it("never resurrects a configuration that was emptied on purpose", async () => { + const { cfg, kvStore } = setup(); + await cfg.createGroup(group("g1")); + await cfg.saveGroupRoutes("g1", [route("r1", "g1")]); + // A stale legacy copy is still in KV. + kvStore.set("config:routes", JSON.stringify([route("old", "g1")])); + await cfg.saveGroupRoutes("g1", []); + await cfg.deleteGroup("g1"); + expect(await cfg.loadRoutes({ fresh: true })).toEqual([]); + expect(await cfg.loadGroups({ fresh: true })).toEqual([]); + }); - // Setup initial state with groups and routes - groupsTable.push(group("g1"), group("g2")); - routesTable.push(route("r1", "g1"), route("r2", "g2")); + it("saving one group's routes leaves every other group untouched", async () => { + const { cfg } = setup(); + await cfg.createGroup(group("a")); + await cfg.createGroup(group("b")); + await cfg.saveGroupRoutes("a", [route("a1", "a")]); + await cfg.saveGroupRoutes("b", [route("b1", "b"), route("b2", "b")]); + // An admin of "a" saves from a snapshot taken before "b" changed. + await cfg.saveGroupRoutes("a", [route("a1", "a", { name: "renamed" })]); + const all = await cfg.loadRoutes({ fresh: true }); + expect(all.map((r) => `${r.groupId}/${r.id}/${r.name}`)).toEqual([ + "a/a1/renamed", + "b/b1/route-b1", + "b/b2/route-b2", + ]); + }); - // Simulate updating groups (e.g., changing a group's name) - const updatedGroups = [{ ...group("g1"), name: "Updated Group 1" }, group("g2")]; + it("saveRoutes rewrites only the groups whose list changed", async () => { + const { cfg, sqlite } = setup(); + await cfg.createGroup(group("a")); + await cfg.createGroup(group("b")); + await cfg.saveGroupRoutes("a", [route("a1", "a")]); + await cfg.saveGroupRoutes("b", [route("b1", "b")]); + sqlite.exec("UPDATE d1_routes SET updated_at = 1 WHERE group_id = 'b'"); + await cfg.saveRoutes([route("a1", "a", { enabled: false }), route("b1", "b")]); + expect(sqlite.query("SELECT updated_at FROM d1_routes WHERE group_id = 'b'").get()).toEqual({ + updated_at: 1, + }); + const a = (await cfg.loadRoutes({ fresh: true })).find((r) => r.id === "a1"); + expect(a!.enabled).toBe(false); + }); - // This should not delete routes - await cfg.saveGroups(updatedGroups); + it("persists and loads the per-group route order", async () => { + const { cfg } = setup(); + await cfg.createGroup(group("g1")); + await cfg.saveGroupRoutes("g1", [route("b", "g1"), route("c", "g1"), route("a", "g1")]); + expect((await cfg.loadRoutes({ fresh: true })).map((r) => r.id)).toEqual(["b", "c", "a"]); + }); - // Routes should still exist in the table - // Note: In the fake DB, batch() doesn't actually execute the statements, - // so we can't verify the actual deletion behavior here. - // This test mainly ensures saveGroups doesn't throw an error. - expect(routesTable).toHaveLength(2); + it("works on a database without the position column", async () => { + const { cfg } = setup({ position: false }); + await cfg.createGroup(group("g1")); + await cfg.saveGroupRoutes("g1", [route("b", "g1"), route("a", "g1")]); + expect((await cfg.loadRoutes({ fresh: true })).map((r) => r.id)).toEqual(["a", "b"]); + }); + + it("serves cached routes until invalidated, fresh reads bypass the cache", async () => { + const { cfg, sqlite } = setup(); + await cfg.createGroup(group("g1")); + await cfg.saveGroupRoutes("g1", [route("r1", "g1")]); + expect(await cfg.loadRoutes()).toHaveLength(1); + sqlite.exec("DELETE FROM d1_routes"); + expect(await cfg.loadRoutes()).toHaveLength(1); + expect(await cfg.loadRoutes({ fresh: true })).toHaveLength(0); + cfg.invalidateCache(); + expect(await cfg.loadRoutes()).toHaveLength(0); + }); + + it("refuses to create or rename onto an id retired after the caller checked it", async () => { + const { cfg, sqlite } = setup(); + await cfg.createGroup(group("a")); + // A concurrent delete/rename retired "x" between the check and the write. + sqlite.exec( + `INSERT INTO dedup_keys (key, claimed_at, expires_at) VALUES ('group-retired:x', 0, ${Date.now() + 60_000})`, + ); + expect(await cfg.createGroup(group("x"))).toBe(false); + await expect(cfg.renameGroup("a", "x")).rejects.toBeInstanceOf(ConfigConflictError); + expect((await cfg.loadGroups({ fresh: true })).map((g) => g.id)).toEqual(["a"]); + }); + + it("counts a group whose own row is unreadable — and has routes — as unreadable", async () => { + const { cfg, sqlite } = setup(); + await cfg.createGroup(group("g1")); + await cfg.createGroup(group("g2")); + await cfg.saveGroupRoutes("g1", [route("r1", "g1")]); + sqlite.exec("UPDATE d1_groups SET data = 'null' WHERE id IN ('g1', 'g2')"); + const snapshot = await cfg.loadSnapshot({ fresh: true }); + expect(snapshot.unreadable).toEqual(["g1"]); + }); + + it("counts a stored regex the engine cannot run as unreadable", async () => { + const { cfg, sqlite } = setup(); + await cfg.createGroup(group("g1")); + await cfg.saveGroupRoutes("g1", [route("r1", "g1")]); + // Valid before the switch to RE2: lookahead now fails to compile. + sqlite.exec( + `UPDATE d1_routes SET filters = '[{"type":"repo","match":"/^secret(?=/)/","exclude":true}]' WHERE id = 'r1'`, + ); + const snapshot = await cfg.loadSnapshot({ fresh: true }); + expect(snapshot.unreadable).toEqual(["g1"]); + }); + + it("keeps consulting KV state written before D1 when seeding from it", async () => { + const { cfg, kvStore } = setup(); + kvStore.set("config:groups", JSON.stringify([group("g1")])); + await cfg.loadGroups({ fresh: true }); + expect(kvStore.has("storage:kv-fallback")).toBe(true); + }); + + it("counts a route with a filter of the wrong shape as unreadable", async () => { + const { cfg, sqlite } = setup(); + await cfg.createGroup(group("g1")); + await cfg.saveGroupRoutes("g1", [route("r1", "g1")]); + sqlite.exec(`UPDATE d1_routes SET filters = '[{"type":"event","match":{}}]' WHERE id = 'r1'`); + const snapshot = await cfg.loadSnapshot({ fresh: true }); + expect(snapshot.routes).toEqual([]); + expect(snapshot.unreadable).toEqual(["g1"]); + }); + + it("refuses to replace a group's routes while one of its rows is unreadable", async () => { + const { cfg, sqlite } = setup(); + await cfg.createGroup(group("g1")); + await cfg.saveGroupRoutes("g1", [route("r1", "g1"), route("r2", "g1")]); + sqlite.exec("UPDATE d1_routes SET targets = '{broken' WHERE id = 'r2'"); + // Reads skip it, but a save would silently delete it: refused instead. + expect((await cfg.loadRoutes({ fresh: true })).map((r) => r.id)).toEqual(["r1"]); + await expect(cfg.saveGroupRoutes("g1", [route("r1", "g1")])).rejects.toThrow(/unreadable/); + await expect(cfg.saveRoutes([route("r1", "g1", { name: "renamed" })])).rejects.toThrow( + /unreadable/, + ); + expect(sqlite.query("SELECT COUNT(*) AS n FROM d1_routes").get()).toEqual({ n: 2 }); + }); + + it("renames a group with its routes and fragments in one transaction", async () => { + const { cfg, sqlite } = setup(); + await cfg.createGroup(group("old", { name: "Team" })); + await cfg.saveGroupRoutes("old", [route("r1", "old"), route("r2", "old")]); + sqlite.exec( + "INSERT INTO d1_fragments (id, group_id, name, node, created_at, updated_at) VALUES ('f1', 'old', 'F', '{}', 1, 1)", + ); + await cfg.renameGroup("old", "new"); + expect((await cfg.loadGroups({ fresh: true })).map((g) => [g.id, g.name])).toEqual([ + ["new", "Team"], + ]); + expect((await cfg.loadRoutes({ fresh: true })).map((r) => `${r.groupId}/${r.id}`)).toEqual([ + "new/r1", + "new/r2", + ]); + expect(sqlite.query("SELECT group_id FROM d1_fragments").all()).toEqual([{ group_id: "new" }]); + }); + + it("a failed rename changes nothing", async () => { + const { cfg } = setup(); + await cfg.createGroup(group("a")); + await cfg.createGroup(group("b")); + await cfg.saveGroupRoutes("a", [route("r1", "a")]); + await expect(cfg.renameGroup("a", "b")).rejects.toThrow(); + expect((await cfg.loadRoutes({ fresh: true })).map((r) => r.groupId)).toEqual(["a"]); + expect((await cfg.loadGroups({ fresh: true })).map((g) => g.id)).toEqual(["a", "b"]); + }); + + it("deleting a group removes its routes and fragments only", async () => { + const { cfg, sqlite } = setup(); + await cfg.createGroup(group("a")); + await cfg.createGroup(group("b")); + await cfg.saveGroupRoutes("a", [route("r1", "a")]); + await cfg.saveGroupRoutes("b", [route("r1", "b")]); + sqlite.exec( + "INSERT INTO d1_fragments (id, group_id, name, node, created_at, updated_at) VALUES ('f1', 'a', 'F', '{}', 1, 1)", + ); + await cfg.deleteGroup("a"); + expect((await cfg.loadRoutes({ fresh: true })).map((r) => r.groupId)).toEqual(["b"]); + expect(sqlite.query("SELECT COUNT(*) AS n FROM d1_fragments").get()).toEqual({ n: 0 }); + }); + + it("saveGroups writes only changed groups and deletes the missing ones", async () => { + const { cfg, sqlite } = setup(); + await cfg.createGroup(group("a")); + await cfg.createGroup(group("b")); + await cfg.saveGroupRoutes("a", [route("r1", "a")]); + sqlite.exec("UPDATE d1_groups SET updated_at = 1"); + await cfg.saveGroups([group("a"), group("c")]); + expect((await cfg.loadGroups({ fresh: true })).map((g) => g.id)).toEqual(["a", "c"]); + expect(sqlite.query("SELECT updated_at FROM d1_groups WHERE id = 'a'").get()).toEqual({ + updated_at: 1, + }); + // Updating a group never touches its routes. + expect(await cfg.loadRoutes({ fresh: true })).toHaveLength(1); + }); + + it("createGroup never overwrites an existing group", async () => { + const { cfg } = setup(); + expect(await cfg.createGroup(group("g", { name: "first" }))).toBe(true); + expect(await cfg.createGroup(group("g", { name: "second" }))).toBe(false); + expect((await cfg.loadGroups({ fresh: true }))[0]!.name).toBe("first"); + }); + + it("updateGroup is a compare-and-swap that retries on contention", async () => { + const { cfg, sqlite } = setup(); + await cfg.createGroup(group("g", { name: "v0" })); + let calls = 0; + const updated = await cfg.updateGroup("g", (g) => { + calls++; + // A concurrent writer bumps the row between our read and our write once. + if (calls === 1) + sqlite.exec( + "UPDATE d1_groups SET version = version + 1, data = json_set(data, '$.name', 'other')", + ); + return { ...g, name: `${g.name}+mine` }; + }); + expect(calls).toBe(2); + expect(updated!.name).toBe("other+mine"); + expect((await cfg.loadGroups({ fresh: true }))[0]!.name).toBe("other+mine"); + }); + + it("updateGroup gives up with a conflict when contention never settles", async () => { + const { cfg, sqlite } = setup(); + await cfg.createGroup(group("g")); + await expect( + cfg.updateGroup("g", (g) => { + sqlite.exec("UPDATE d1_groups SET version = version + 1"); + return { ...g, name: "x" }; + }), + ).rejects.toBeInstanceOf(ConfigConflictError); + }); + + it("updateGroup returns null for a missing group and keeps a group on null", async () => { + const { cfg } = setup(); + expect(await cfg.updateGroup("missing", (g) => g)).toBeNull(); + await cfg.createGroup(group("g")); + expect((await cfg.updateGroup("g", () => null))!.id).toBe("g"); + }); + + it("never stores a client revision", async () => { + const { cfg, sqlite } = setup(); + await cfg.createGroup({ ...group("g"), revision: "abc" }); + const row = sqlite.query("SELECT data FROM d1_groups").get() as { data: string }; + expect(JSON.parse(row.data).revision).toBeUndefined(); + }); + + it("propagates D1 write failures instead of pretending to save", async () => { + const { cfg, sqlite } = setup(); + await cfg.createGroup(group("g")); + sqlite.exec("DROP TABLE d1_routes"); + await expect(cfg.saveGroupRoutes("g", [route("r1", "g")])).rejects.toThrow(); + }); + + it("propagates D1 read failures instead of serving any stale copy", async () => { + const { cfg, sqlite } = setup(); + await cfg.createGroup(group("g")); + await cfg.saveGroupRoutes("g", [route("r1", "g")]); + sqlite.exec("DROP TABLE d1_routes"); + cfg.invalidateCache(); + await expect(cfg.loadRoutes()).rejects.toThrow(); + await expect(cfg.loadGroups()).rejects.toThrow(); + }); + + it("gives every group an immutable instance id that survives renames", async () => { + const { cfg, sqlite } = setup(); + sqlite.exec( + 'INSERT INTO d1_groups (id, name, data, version, created_at, updated_at) VALUES (\'legacy\', \'L\', \'{"id":"legacy","name":"L","adminIds":[]}\', 1, 0, 0)', + ); + await cfg.createGroup({ ...group("fresh"), instance: "client-chosen" }); + const loaded = await cfg.loadGroups({ fresh: true }); + const legacy = loaded.find((g) => g.id === "legacy"); + const fresh = loaded.find((g) => g.id === "fresh"); + expect(legacy!.instance).toMatch(/^[0-9a-f]{32}$/); + expect(fresh!.instance).not.toBe("client-chosen"); + await cfg.updateGroup("fresh", (g) => ({ ...g, instance: "tampered" })); + const renamed = await cfg.renameGroup("fresh", "moved"); + expect(renamed.instance).toBe(fresh!.instance); + await cfg.deleteGroup("moved"); + await cfg.createGroup(group("moved")); + const again = (await cfg.loadGroups({ fresh: true })).find((g) => g.id === "moved")!; + expect(again.instance).not.toBe(fresh!.instance); + }); + + it("refuses a delete when the group changed after it was checked", async () => { + const { cfg } = setup(); + await cfg.createGroup(group("g")); + await expect( + cfg.deleteGroup("g", { + expect: async () => { + await cfg.updateGroup("g", (x) => ({ ...x, name: "changed" })); + }, + }), + ).rejects.toBeInstanceOf(ConfigConflictError); + expect((await cfg.loadGroups({ fresh: true })).map((g) => g.name)).toEqual(["changed"]); + }); + + it("refuses routes for a group that does not exist", async () => { + const { cfg } = setup(); + await expect(cfg.saveGroupRoutes("ghost", [route("r1", "ghost")])).rejects.toThrow(); + }); +}); + +describe("kvConfigStore", () => { + it("keeps groups and routes in permanent KV keys", async () => { + const { kv, store } = createMemoryKV(); + const cfg = kvConfigStore(kv); + await cfg.createGroup(group("a")); + await cfg.saveGroupRoutes("a", [route("r1", "a")]); + await cfg.renameGroup("a", "b"); + expect((await cfg.loadGroups()).map((g) => g.id)).toEqual(["b"]); + expect((await cfg.loadRoutes()).map((r) => r.groupId)).toEqual(["b"]); + await cfg.deleteGroup("b"); + expect(JSON.parse(store.get("config:routes")!)).toEqual([]); + }); + + it("backfills the same instance whichever cold start wins the race", async () => { + const { kv, store } = createMemoryKV(); + store.set("config:groups", JSON.stringify([group("legacy")])); + // Two isolates read the legacy list before either wrote it back. + const [a, b] = await Promise.all([ + kvConfigStore(kv).loadGroups(), + kvConfigStore(kv).loadGroups(), + ]); + expect(a[0]!.instance).toBeTruthy(); + expect(a[0]!.instance).toBe(b[0]!.instance); + expect(a[0]!.instanceBackfilled).toBe(true); + // A rename keeps the identity. + const cfg = kvConfigStore(kv); + const moved = await cfg.renameGroup("legacy", "renamed", { transform: (g) => ({ ...g }) }); + expect(moved.instance).toBe(a[0]!.instance); + expect(moved.instanceBackfilled).toBe(true); + }); +}); + +describe("d1ConfigStore guarded writes", () => { + it("lets only one of two writes from the same base commit", async () => { + const { cfg } = setup(); + await cfg.createGroup(group("g")); + let release!: () => void; + const gate = new Promise((r) => (release = r)); + const slow = cfg.saveGroupRoutes("g", [route("a", "g")], { expect: () => gate }); + // A second write sneaks in while the first waits between check and write. + await cfg.saveGroupRoutes("g", [route("b", "g")]); + release(); + await expect(slow).rejects.toBeInstanceOf(ConfigConflictError); + expect((await cfg.loadRoutes({ fresh: true })).map((r) => r.id)).toEqual(["b"]); + }); + + it("refuses a rename whose source changed after it was read", async () => { + const { cfg } = setup(); + await cfg.createGroup(group("old", { name: "v1" })); + await expect( + cfg.renameGroup("old", "new", { + expect: async () => { + await cfg.updateGroup("old", (g) => ({ ...g, name: "v2" })); + }, + }), + ).rejects.toBeInstanceOf(ConfigConflictError); + expect((await cfg.loadGroups({ fresh: true })).map((g) => [g.id, g.name])).toEqual([ + ["old", "v2"], + ]); + }); + + it("does not seed (nor mark seeded) when the legacy KV read fails", async () => { + const { db } = createSqliteD1(); + const { kv, store } = createMemoryKV(); + store.set("config:routes", JSON.stringify([route("r1", "g1")])); + const failing = { + ...kv, + get: async (key: string, type?: string) => { + if (key === "config:groups") throw new Error("KV down"); + return kv.get(key, type as never); + }, + } as unknown as KVNamespace; + const cfg = d1ConfigStore(db, failing); + await expect(cfg.loadRoutes({ fresh: true })).rejects.toThrow("KV down"); + expect(store.get("config:d1-seeded")).toBeUndefined(); + }); + + it("fails instead of serving an empty configuration when D1 and the copy are missing", async () => { + const { db, sqlite } = createSqliteD1(); + const { kv } = createMemoryKV(); + const cfg = d1ConfigStore(db, kv); + await cfg.loadRoutes(); + sqlite.exec("DROP TABLE d1_routes"); + cfg.invalidateCache(); + await kv.delete("config:cache:routes"); + await expect(cfg.loadRoutes()).rejects.toThrow(); }); });